[ { "productId": "bento", "storyId": "ab-test-campaigns", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack mentions A/B testing, subject line testing, or automatic winner selection based on engagement; the docs cover events, subscribers, transactional email, flows, and MCP integration but nothing about split testing campaigns.", "evidenceIds": [] }, { "productId": "bento", "storyId": "agent-audits-program", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bento ships an official MCP server plus REST APIs (events, subscribers) that let an AI assistant look up subscribers, create broadcasts, and track events 'through natural conversation,' which supports programmatic access to audiences and automations triggers. However, there is no direct evidence of an API/MCP capability for pulling campaign performance metrics or automation run reports (the closest is generic revenue-attribution copy), so full 'report on what's working' analytics access isn't demonstrated. Missing for 10: explicit API/MCP endpoints for campaign/broadcast analytics or automation performance reporting, and independent confirmation of MCP reliability for this reporting use case.", "evidenceIds": [ "bento-docs-5", "bento-docs-1", "bento-docs-2", "bento-docs-9", "bento-docs-6", "bento-probe-4" ] }, { "productId": "bento", "storyId": "agent-builds-segment", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Bento's MCP server can look up subscribers, create broadcasts, and track events (bento-docs-5), and a generic 'describe what you want, AI will build it' feature (bento-docs-10), but no documentation ties this to translating a plain-language description into an actual audience segment via the API/MCP surface — segment filtering is described only as manual tag/date-based filters (bento-docs-8).", "evidenceIds": [ "bento-docs-5", "bento-docs-8", "bento-docs-10" ] }, { "productId": "bento", "storyId": "agent-drives-campaign", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "The MCP server can create broadcasts via natural language, implying a draft/queued campaign can be produced by an agent, but there is no documented approval/hold-for-human-review workflow or explicit 'scheduled pending approval' state before send. missing for 10: explicit approval-gating or human-review-before-send workflow, documentation of a queued/pending state distinct from immediate send, evidence of segment-targeted broadcast drafting via the agent.", "evidenceIds": [ "bento-docs-5", "bento-probe-4", "bento-docs-9" ] }, { "productId": "bento", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Bento has a live, confirmed llms.txt (HTTP 200) and a docs.md agent-friendly documentation file, directly enabling an AI agent to be pointed at agent-oriented docs, plus a documented MCP server for deeper agent integration. Missing for 10: independent third-party confirmation that agents actually consume these files effectively in practice.", "evidenceIds": [ "bento-probe-1", "bento-probe-2", "bento-docs-7", "bento-docs-5", "bento-probe-4" ] }, { "productId": "bento", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Bento advertises AI that can 'build' flows/segments from plain text and an MCP server letting an AI assistant query subscriber/event data conversationally, which implies some AI-data interaction, but there is no explicit documentation of AI-generated insights, analytics summaries, or proactive suggestions derived from the user's data. Missing for 10: dedicated AI insights/reporting feature, evidence of proactive suggestions, and independent confirmation of this capability in use.", "evidenceIds": [ "bento-docs-10", "bento-docs-5", "bento-probe-4" ] }, { "productId": "bento", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Bento's docs describe building automation flows triggered by user behavior, events, and tags, with the events API able to trigger automations, and even AI-assisted flow building — this directly supports autonomous background automations for an AI-native marketing use case. Missing for 10: independent/hands-on corroboration of automations running unattended, and more detail on monitoring/error-handling of long-running flows.", "evidenceIds": [ "bento-docs-9", "bento-docs-1", "bento-docs-10", "bento-docs-5" ] }, { "productId": "bento", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Bento markets an in-product AI ('Describe what you want in plain text and let Bento's powerful AI build it for you') suggesting a built-in assistant for flow creation, but there's no detailed documentation of scope, task types, or interaction model beyond this marketing line. The MCP server evidence is about external AI assistants accessing Bento, not a built-in assistant, so it doesn't count here. Missing for 10: detailed docs/UI showing the built-in assistant's task range, hands-on usage examples, and independent corroboration of its capabilities.", "evidenceIds": [ "bento-docs-10" ] }, { "productId": "bento", "storyId": "agentic-headless", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Bento exposes REST-style APIs (events, subscribers) that could be scripted, and one marketing line mentions scripting a weekly job, but there is no documented CLI, headless mode, or CI/automation guide, and no OpenAPI spec was found (404s). missing for 10: explicit CI/headless documentation, CLI tool, and API spec/reference for automated pipelines.", "evidenceIds": [ "bento-docs-1", "bento-docs-4", "bento-probe-3" ] }, { "productId": "bento", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows Bento exposes its own MCP server so external AI assistants can call Bento's tools (bento-docs-5, bento-probe-4) — the reverse of what this story asks. There is no evidence Bento itself can act as an MCP client and plug in external MCP servers to use their tools.", "evidenceIds": [ "bento-docs-5", "bento-probe-4" ] }, { "productId": "bento", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Bento documents an official MCP server that gives AI assistants direct access to the account (subscriber lookup, broadcast creation, event tracking) via natural conversation, confirmed by both docs and a probe hit on the integration page. Missing for 10: independent/hands-on corroboration beyond first-party docs and more detail on setup/auth specifics.", "evidenceIds": [ "bento-docs-5", "bento-probe-4" ] }, { "productId": "bento", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Bento ships an official MCP server documented to let an AI assistant look up subscribers, create broadcasts, track events, etc. 'through natural conversation,' and separately advertises a plain-text AI builder for flows, directly supporting natural-language operation. Missing for 10: independent/hands-on corroboration of the MCP server's reliability and broader coverage of natural-language commands beyond the examples given.", "evidenceIds": [ "bento-docs-5", "bento-docs-10", "bento-probe-4" ] }, { "productId": "bento", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of an official CLI tool for Bento; documentation focuses on APIs, MCP server, and email marketing features but never mentions a command-line interface. Missing for 10: any mention of a CLI product, install instructions, or command reference.", "evidenceIds": [] }, { "productId": "bento", "storyId": "agentic-public-api", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Bento documents multiple REST-style APIs (events, subscribers, transactional email) and provides machine-readable docs (llms.txt, docs.md) explicitly aimed at AI agents, plus an MCP server for natural-language control of the account. This satisfies the 'documented public API' story, though no formal OpenAPI/Swagger spec was found (404s on all candidate paths). Missing for 10: a discoverable OpenAPI/Swagger schema and independent third-party confirmation of programmatic API usage.", "evidenceIds": [ "bento-docs-1", "bento-docs-2", "bento-docs-3", "bento-probe-1", "bento-probe-2", "bento-probe-3", "bento-docs-5" ] }, { "productId": "bento", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows an MCP server and general API docs but no mention of scoped or least-privilege API keys/credentials, token permission levels, or role-based access control for agents. No documentation of credential scoping mechanism exists in the pack.", "evidenceIds": [] }, { "productId": "bento", "storyId": "agentic-sdks", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs.md teaser mentions 'API guides, SDK references' but the evidence pack never names specific official SDKs, languages, or repos, only a generic mention plus an MCP server for AI assistants. Missing for 10: explicit list of official SDKs (e.g., language libraries), links to SDK documentation/repos, code samples, and independent developer corroboration of SDK usage.", "evidenceIds": [ "bento-probe-2", "bento-docs-5", "bento-probe-4" ] }, { "productId": "bento", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers Bento's Events API (tracking inbound events), subscriber management, and an MCP server, but there is no mention of outbound webhook subscriptions that would let an AI-native user subscribe to Bento events. Missing for 10: any documentation of a webhooks feature, webhook endpoint configuration, or event subscription mechanism.", "evidenceIds": [ "bento-docs-1", "bento-docs-4", "bento-docs-9" ] }, { "productId": "bento", "storyId": "ai-content-drafting", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The only AI-related claim (bento-docs-10) is generic ('let Bento's powerful AI build it for you') and appears tied to flow/segment building, not to drafting or refining subject lines and email copy inside a composer. No evidence describes an AI copywriting assistant for email content.", "evidenceIds": [ "bento-docs-10" ] }, { "productId": "bento", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only static docs (events/subscribers/transactional-email pages) and an llms.txt/docs.md text dump; explicit probes for OpenAPI/Swagger specs returned 404s, and no mention of an interactive, runnable API console or sandbox exists anywhere in the pack.", "evidenceIds": [ "bento-probe-3", "bento-docs-1", "bento-docs-2", "bento-probe-1", "bento-probe-2" ] }, { "productId": "bento", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe for OpenAPI/Swagger specs at all standard paths returned 404, and no evidence pack item shows a downloadable machine-readable API spec; only human-readable docs and an MCP server are documented.", "evidenceIds": [ "bento-probe-3" ] }, { "productId": "bento", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a sandbox/test environment or staging mode separate from production data; docs cover subscribers, transactional email, MCP integration, and pricing but nothing about sandbox testing.", "evidenceIds": [] }, { "productId": "bento", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of API versioning scheme or documented deprecation policy; OpenAPI spec probe returned 404s and no docs mention version numbers or deprecation timelines.", "evidenceIds": [ "bento-probe-3" ] }, { "productId": "bento", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bento's API explicitly supports batch operations on subscribers (1–1000 per request), bulk import, and filtering/segmenting for bulk targeting, and the MCP server exposes some of this to AI assistants. However, evidence doesn't show bulk operations beyond subscriber records (e.g., bulk broadcast sends, bulk event batch limits) or independent confirmation of scale/performance. Missing for 10: evidence of bulk operations across other object types (broadcasts, tags, flows), documented rate/size limits beyond subscribers, and independent/hands-on verification.", "evidenceIds": [ "bento-docs-2", "bento-docs-4", "bento-docs-8", "bento-docs-5" ] }, { "productId": "bento", "storyId": "automation-rules-engine", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Bento supports building automation flows triggered by behavior, events, and tags, with branch validation before running ('Build flows from behavior, events, and tags. Check every branch before it runs.'), plus an Events API to track activity and trigger automations. Missing for 10: independent hands-on verification of rule complexity/reliability, and detailed documentation of conditional logic depth beyond marketing copy.", "evidenceIds": [ "bento-docs-9", "bento-docs-1" ] }, { "productId": "bento", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Bento supports event/tag-triggered automation flows and scripting API jobs (e.g. 'script the job you run every week'), which implies some recurring automation capability, but there is no explicit documentation of a scheduler, cron-like recurring job configuration, or AI-driven scheduling of workflows. Missing for 10: explicit recurring/scheduled job feature, AI-native interface for setting schedules, and independent confirmation of time-based (not just event-based) automation triggers.", "evidenceIds": [ "bento-docs-4", "bento-docs-9" ] }, { "productId": "bento", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of version history, review/diff, or rollback capability for Bento automations/flows; docs mention building flows and checking branches before running, but nothing about versioning or reverting automations.", "evidenceIds": [] }, { "productId": "bento", "storyId": "campaign-performance-reporting", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence shows Bento tracks events, ties revenue to campaigns/flows, and lets you filter subscribers by tags/segments/dates, implying some analytics capability, but there is no documentation of a dedicated per-campaign or per-automation-step dashboard showing opens, clicks, unsubscribes, and conversions over time. missing for 10: explicit reporting/analytics documentation showing opens/clicks/unsubscribes/conversions broken out per campaign and per automation step over time, and any independent or hands-on confirmation of such reporting UI.", "evidenceIds": [ "bento-docs-6", "bento-docs-8", "bento-docs-9", "bento-docs-1" ] }, { "productId": "bento", "storyId": "campaign-via-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence documents APIs for events, subscribers, and transactional email, plus an MCP server that can 'create broadcasts' via AI assistant — but there is no documented public REST API endpoint for creating or scheduling campaigns/broadcasts, and OpenAPI spec probes returned 404s. Missing for 10: a documented broadcast/campaign creation and scheduling endpoint in the public API reference.", "evidenceIds": [ "bento-docs-1", "bento-docs-3", "bento-docs-5", "bento-probe-3" ] }, { "productId": "bento", "storyId": "contacts-crud-api", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs confirm a Subscribers API supporting batch create/update (1-1000 per request) and an Events API for tracking activity and updating profile fields, plus filtering by tags/segment/date, which implies some custom-field handling. However there is no explicit documentation of a suppression/unsubscribe endpoint, no explicit mention of arbitrary custom properties support, and no OpenAPI spec or rate-limit/production-scale guarantees (openapi probe returned 404s).\nmissing for 10: explicit suppression/unsubscribe API endpoint, explicit custom-properties schema support, documented rate limits or production-scale guarantees.", "evidenceIds": [ "bento-docs-2", "bento-docs-8", "bento-docs-1", "bento-probe-3" ] }, { "productId": "bento", "storyId": "create-send-campaign", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Evidence confirms subscriber segmentation (tags/segments/date filters) and broadcast creation (including via MCP), which are core pieces of the campaign-sending loop, but there is no direct documentation of the actual campaign composition UI or a 'schedule send' feature. Missing for 10: explicit docs/screenshots of composing an email campaign, choosing send-now vs schedule-for-later, and confirmation of the full end-to-end send flow.", "evidenceIds": [ "bento-docs-2", "bento-docs-8", "bento-docs-5", "bento-docs-9" ] }, { "productId": "bento", "storyId": "custom-sending-domain", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of SPF/DKIM/DMARC domain authentication, guided DNS setup, or verification flows — only unrelated features like dedicated IPs, transactional email, and subscriber management. Domain authentication is a standard, fair-to-expect capability for an email marketing platform, so its absence is 'none' rather than 'na'. missing for 10: any documentation of SPF/DKIM/DMARC setup, DNS record guidance, or domain verification UI.", "evidenceIds": [ "bento-docs-7" ] }, { "productId": "bento", "storyId": "deliverability-visibility", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows subscriber/event tracking, dedicated IPs, and general email automation features, but there is no mention of per-send bounce/complaint/blocklist reporting or deliverability health guidance. missing for 10: bounce/complaint/blocklist reporting per send, deliverability health dashboard, actionable degradation alerts/guidance.", "evidenceIds": [ "bento-docs-7", "bento-docs-2", "bento-docs-8" ] }, { "productId": "bento", "storyId": "email-template-editor", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers transactional email, subscriber/segment management, flows/automations, and an MCP integration, but contains no mention of a visual email editor, reusable templates, personalization tokens, or cross-client rendering testing — all core to this story for an email marketing platform.", "evidenceIds": [] }, { "productId": "bento", "storyId": "engagement-data-api-read", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers Bento's Events API (tracking activity, updating fields, triggering automations) and Subscribers API filters (tags, segments, last-event/date ranges), but nothing documents a way to retrieve per-contact history of what was sent, opened, and clicked via the API. No endpoint or field for message-level engagement history is described.", "evidenceIds": [ "bento-docs-1", "bento-docs-8" ] }, { "productId": "bento", "storyId": "event-triggered-flows", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs confirm an Events API that tracks user activity per subscriber and can trigger automations, and flows explicitly built from 'behavior, events, and tags' with per-branch checks, matching the story of firing a custom event to trigger an automation for a specific person. missing for 10: no hands-on/independent confirmation of exact event-to-person targeting behavior, and no code sample showing the custom event payload triggering a single-subscriber automation.", "evidenceIds": [ "bento-docs-1", "bento-docs-9" ] }, { "productId": "bento", "storyId": "flow-state-inspection", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows generic flow-building and event/segment filtering (bento-docs-9, bento-docs-8) but nothing about viewing an individual subscriber's current position inside a running automation or safely pulling/re-entering them without breaking the flow.", "evidenceIds": [ "bento-docs-9", "bento-docs-8" ] }, { "productId": "bento", "storyId": "full-list-export", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Bento's docs mention a Subscribers API that can filter by tags, segments, and dates and can be scripted to run exports (bento-docs-2, bento-docs-4, bento-docs-8), implying some self-serve data extraction is possible via API, but there is no explicit documentation of a full audience export covering suppression list, full engagement history, or a one-click open-format (CSV/JSON) export tool. Missing for 10: documented bulk export/download feature, explicit suppression list export, engagement history export, and confirmation of an open standard export format.", "evidenceIds": [ "bento-docs-2", "bento-docs-4", "bento-docs-8" ] }, { "productId": "bento", "storyId": "guided-platform-migration", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Bento documents subscriber import (bulk import, 1-1000 per request), filtering by tags/segments and unsubscribe timestamps, and explicitly lists 'Migration help when you switch' on its pricing page plus testimonials of users moving from Mailchimp/Hubspot. However there is no dedicated guided-migration wizard or documentation detailing how custom fields and unsubscribe state are specifically preserved during a platform switch. Missing for 10: a documented step-by-step migration/import wizard, explicit confirmation that unsubscribe status and custom fields are mapped/preserved on import, and independent evidence of successful full-data migrations.", "evidenceIds": [ "bento-docs-4", "bento-docs-8", "bento-docs-11", "bento-docs-13", "bento-comm-1" ] }, { "productId": "bento", "storyId": "metrics-via-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Bento has an Events API and Subscribers API, and mentions revenue attribution to email/flows as a feature, but no documentation describes an API endpoint for pulling campaign or flow performance metrics (opens, clicks, conversions) into an external warehouse or dashboard. An OpenAPI spec search also returned 404s, giving no confirmation of such an endpoint's existence or schema. Missing for 10: documented performance/metrics API endpoint, example payload for campaign/flow stats, evidence of warehouse/dashboard export support.", "evidenceIds": [ "bento-docs-1", "bento-docs-6", "bento-probe-3" ] }, { "productId": "bento", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Bento's docs show a broad API/events/subscribers/transactional-email surface (bento-docs-1,2,3,8) and an MCP server exposing subscriber lookup, broadcast creation, and event tracking via natural language (bento-docs-5, bento-probe-4), suggesting substantial API parity with the UI. However, no public OpenAPI/swagger spec was found (bento-probe-3), and UI-only features like AI-driven flow building from plain text (bento-docs-10) and flow branch testing (bento-docs-9) aren't shown as API-accessible. Missing for 10: a full OpenAPI reference, explicit API endpoints for flow/automation creation and testing, and confirmation that all UI actions (flows, broadcasts, deliverability settings) have API equivalents.", "evidenceIds": [ "bento-docs-1", "bento-docs-2", "bento-docs-3", "bento-docs-5", "bento-docs-8", "bento-docs-9", "bento-docs-10", "bento-probe-3", "bento-probe-4" ] }, { "productId": "bento", "storyId": "openness-full-export", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Bento offers a subscribers API for querying/importing records and mentions 'migration help when you switch,' implying some data portability, but there is no explicit documentation of a full data export feature or open-format (CSV/JSON) bulk export for all account data. missing for 10: explicit bulk export endpoint/tool, documented open export formats, and independent confirmation that a full account export is possible.", "evidenceIds": [ "bento-docs-4", "bento-docs-8", "bento-docs-11" ] }, { "productId": "bento", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Bento is a closed SaaS email marketing platform; no evidence of an open-source license or public source repository for the core product is provided in the evidence pack.", "evidenceIds": [] }, { "productId": "bento", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Bento is presented as a hosted email marketing SaaS platform with no evidence of any self-hosted/open-source deployment option, Docker image, or on-prem installation guide; all docs point to a cloud-based service and pricing/migration pages.", "evidenceIds": [] }, { "productId": "bento", "storyId": "outbound-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item documents an outbound webhooks feature for delivery/engagement events (sends, opens, clicks, bounces, unsubscribes) with signed payloads; the closest items reference an inbound Events API for tracking user activity, not outbound webhook subscriptions. OpenAPI/spec probes turned up nothing, and no webhook-specific docs appear anywhere in the pack.", "evidenceIds": [ "bento-docs-1", "bento-probe-3" ] }, { "productId": "bento", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack mentions data residency, region selection, or storage location options for Bento; the only infrastructure-related item is a dedicated IP address feature for sending, unrelated to data residency. missing for 10: any mention of regional data hosting, residency guarantees, or data storage location controls.", "evidenceIds": [ "bento-docs-7" ] }, { "productId": "bento", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses AI-training data usage, opt-out controls, or privacy commitments regarding model training; Bento's evidence focuses on email marketing features, pricing, and MCP integration.", "evidenceIds": [] }, { "productId": "bento", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack items describe data retention policies, subscriber deletion/export controls, or privacy/GDPR compliance tooling; the docs focus on subscriber import/filtering/automation, not on how AI-native users can control or delete stored data. Missing for 10: documented data deletion/export APIs, retention policy settings, GDPR/CCPA compliance statements.", "evidenceIds": [] }, { "productId": "bento", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses telemetry/usage-tracking opt-out for Bento itself (as opposed to its email/marketing tracking features for end customers); this is a fair privacy-posture question for any AI-native tool but no documentation or community evidence confirms such a control exists.", "evidenceIds": [] }, { "productId": "bento", "storyId": "revenue-attribution", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bento's marketing explicitly claims revenue attribution to specific emails/flows (bento-docs-6) and supports event tracking (bento-docs-1) and flow building from events/behavior (bento-docs-9), which are prerequisites for such attribution. However, there is no documentation detail on how attribution reporting/dashboards actually work, no independent or hands-on confirmation of accuracy, and no mention of signup attribution specifically. missing for 10: detailed attribution reporting docs/UI evidence, signup-specific attribution mechanics, independent verification of revenue attribution accuracy.", "evidenceIds": [ "bento-docs-6", "bento-docs-1", "bento-docs-9" ] }, { "productId": "bento", "storyId": "scoped-api-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of scoped or read-only API key/credential management in Bento's docs; API references only cover events, subscribers, and transactional email without mention of permission levels or credential scoping for third parties like contractors or agents.", "evidenceIds": [ "bento-docs-1", "bento-docs-2", "bento-docs-4", "bento-probe-3" ] }, { "productId": "bento", "storyId": "segment-builder", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs show segmentation via tags, saved segments, and date-range filters on subscriber fields (bento-docs-8), combined with behavioral event tracking (bento-docs-1) and flows built from behavior/events/tags (bento-docs-9), which together enable dynamic, attribute+behavior-based audience segments. Missing for 10: explicit documentation stating segments automatically re-evaluate/update in real time as data changes, and independent/hands-on corroboration of live segment behavior.", "evidenceIds": [ "bento-docs-8", "bento-docs-1", "bento-docs-9" ] }, { "productId": "bento", "storyId": "segments-via-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm subscriber API supports adding/updating tags and profile fields (bento-docs-1, bento-docs-4) and filtering/reading back subscribers by tag, segment, or date range (bento-docs-8), showing read/write access to tag membership. However, there's no explicit evidence of an API endpoint to programmatically create or update segment/list definitions themselves (segments appear to be built via filters/UI), and no OpenAPI spec was found to confirm full CRUD coverage (bento-probe-3). Missing for 10: explicit segment-creation/update API endpoint, documented list CRUD operations, and independent confirmation of read-back accuracy.", "evidenceIds": [ "bento-docs-1", "bento-docs-4", "bento-docs-8", "bento-probe-3" ] }, { "productId": "bento", "storyId": "stack-integrations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows generic Events/Subscribers APIs, an MCP server, and marketing copy, but no mention of maintained first-party integrations with Stripe, Shopify, Segment, or a product database — nothing names a specific connector or integrations marketplace.", "evidenceIds": [ "bento-docs-1", "bento-docs-4", "bento-probe-2" ] }, { "productId": "bento", "storyId": "suppression-compliance", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item documents automatic suppression of unsubscribes, bounces, or spam complaints, nor honoring suppression across transactional and broadcast/flow sends; docs mention filtering by unsubscribe time and dedicated IPs but nothing about compliance suppression logic.", "evidenceIds": [] }, { "productId": "bento", "storyId": "templates-as-code", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of managing email templates as code (HTML/MJML files, template versioning, or API-managed templates with variable substitution under version control); evidence only covers subscriber management, events, transactional email use-cases, and MCP integration.", "evidenceIds": [] }, { "productId": "bento", "storyId": "transactional-send-api", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Bento's docs explicitly describe a dedicated transactional email feature for one-to-one messages like password resets, receipts, login codes, and order confirmations, sent via the same platform used for marketing, and note dedicated IP options for high-volume senders. Missing for 10: explicit confirmation this uses the same unified API endpoint as marketing sends, and independent/hands-on corroboration of transactional delivery in practice.", "evidenceIds": [ "bento-docs-3", "bento-docs-7" ] }, { "productId": "bento", "storyId": "transactional-separation", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Bento explicitly documents transactional email as a distinct category (password resets, receipts, order confirmations) separate from marketing sends, and mentions dedicated IP options for high-volume senders. However, there is no explicit documentation confirming that suppression/unsubscribe rules are bypassed for transactional messages, that transactional templates are managed separately from marketing templates, or that transactional sending reputation/IP is isolated from marketing reputation. Missing for 10: explicit suppression-list bypass behavior for transactional sends, dedicated transactional template system, and documented separate sending reputation/domain infrastructure.", "evidenceIds": [ "bento-docs-3", "bento-docs-7" ] }, { "productId": "bento", "storyId": "visual-automation-builder", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm flow-building based on behavior, events, and tags with branch checking (bento-docs-9) and event/activity tracking to trigger automations (bento-docs-1), supporting core lifecycle automation building blocks, but there is no explicit documentation of delays, timed waits, or exit rules, nor named templates like welcome/onboarding/winback series. missing for 10: explicit delay/wait-step documentation, explicit exit-rule/goal documentation, named use-case templates (welcome, onboarding, winback), independent hands-on corroboration.", "evidenceIds": [ "bento-docs-9", "bento-docs-1", "bento-docs-6" ] }, { "productId": "customer-io", "storyId": "ab-test-campaigns", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack mentions A/B testing of subject lines or content variants, nor any mechanism for automatically selecting a winner based on engagement metrics; broadcasts, transactional messages, and automations are described but split-testing is never referenced.", "evidenceIds": [] }, { "productId": "customer-io", "storyId": "agent-audits-program", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Customer.io ships an official MCP server (customer-io-probe-3) that lets an AI list automations (customer-io-docs-9) and separates reads from writes for safe automated access (customer-io-docs-10); docs also describe segments/audiences (customer-io-docs-11) and broadcast/automation structures (customer-io-docs-12, customer-io-docs-13) that an agent could query. However, the story also asks for 'recent campaign results' reporting, and evidence only vaguely claims AI can 'analyze performance' (customer-io-docs-1) without showing a concrete reporting/analytics API or MCP tool for campaign metrics. Missing for 10: explicit documentation of a reporting/analytics endpoint or MCP tool for campaign results, and independent/hands-on confirmation that an agent can actually pull full program state (audiences+automations+results) end-to-end.", "evidenceIds": [ "customer-io-probe-3", "customer-io-docs-9", "customer-io-docs-10", "customer-io-docs-11", "customer-io-docs-12", "customer-io-docs-13", "customer-io-docs-1" ] }, { "productId": "customer-io", "storyId": "agent-builds-segment", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Customer.io has an MCP server and programmatic APIs (Pipelines/Track) that could plausibly let an AI agent create segments, and docs show natural-language MCP commands like listing automations or drafting emails, but no evidence explicitly shows an agent translating a plain-language description into a created segment via API/MCP. missing for 10: a concrete example of segment creation through the API/MCP, documentation of segment-definition endpoints, and independent confirmation this works end-to-end.", "evidenceIds": [ "customer-io-docs-9", "customer-io-docs-10", "customer-io-docs-11", "customer-io-probe-3", "customer-io-docs-3" ] }, { "productId": "customer-io", "storyId": "agent-drives-campaign", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Customer.io's MCP server explicitly separates reads from writes/deletes so a human can approve each category independently, and docs show AI drafting content (welcome emails, automations) and broadcasts being queued to send at a specified time to a defined recipient list/segment — this supports an approval-gated, agent-drafted campaign workflow. However, there's no direct evidence of an agent drafting a full campaign against a segment specifically and leaving it in a 'scheduled/queued pending human approval' state as one integrated flow; the approval mechanism (read/write separation) and scheduled broadcast feature are documented separately, not as a combined AI-driven approval-to-send pipeline. missing for 10: an end-to-end example of an agent creating a segment-targeted campaign draft and explicitly queuing it for human approval before send, and independent/hands-on confirmation this works as described.", "evidenceIds": [ "customer-io-docs-9", "customer-io-docs-10", "customer-io-docs-13", "customer-io-docs-16", "customer-io-probe-3" ] }, { "productId": "customer-io", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A probe confirms a working llms.txt at docs.customer.io/llms.txt returning structured documentation content, and Customer.io also publishes agent-oriented docs like the MCP get-started guide and CLI/skills for AI assistants. missing for 10: no independent third-party report of an agent successfully consuming llms.txt in practice.", "evidenceIds": [ "customer-io-probe-1", "customer-io-docs-2", "customer-io-docs-9", "customer-io-probe-3" ] }, { "productId": "customer-io", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs show a broader AI assistant that can 'analyze performance' and 'build automations' (docs-1) plus AI in Slack (docs-20) and AI-generated design elements (docs-15/16), suggesting some data-informed suggestions, but there's no dedicated analytics/insights feature that surfaces AI-generated observations or recommendations from customer data specifically. Missing for 10: a documented insights/analytics dashboard powered by AI, concrete examples of data-driven suggestions (e.g. segment or campaign optimization tips), and independent confirmation of this capability in practice.", "evidenceIds": [ "customer-io-docs-1", "customer-io-docs-20", "customer-io-docs-15", "customer-io-docs-16" ] }, { "productId": "customer-io", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Customer.io's core product is journey/automation building that runs autonomously in the background, triggered by events or segment membership, independent of user presence (docs-12, docs-13, docs-18), and this can even be managed/listed via AI/MCP tooling (docs-9). This is well-documented first-party functionality central to the platform's purpose. Missing for 10: independent hands-on verification that automations run reliably at scale without manual intervention (community evidence only discusses unrelated outages/bugs, not automation autonomy specifically).", "evidenceIds": [ "customer-io-docs-12", "customer-io-docs-13", "customer-io-docs-18", "customer-io-docs-9", "customer-io-docs-11" ] }, { "productId": "customer-io", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Customer.io documents a 'Customer.io AI' capable of building automations, drafting messages, and analyzing performance (docs-1, docs-16), with a Slack-based assistant interface (docs-20) and AI-generated design features (docs-15), suggesting some built-in assistant functionality. However, most detailed evidence describes this AI working through an external MCP server/CLI-skills setup (docs-2, docs-9, docs-10, probe-3) rather than a fully native in-product chat assistant, and there's no independent/hands-on confirmation of its reliability. Missing for 10: clear documentation of a native in-app assistant UI (vs. MCP/CLI-only access), and third-party corroboration of task delegation working well.", "evidenceIds": [ "customer-io-docs-1", "customer-io-docs-16", "customer-io-docs-20", "customer-io-docs-15", "customer-io-docs-9", "customer-io-docs-2", "customer-io-probe-3" ] }, { "productId": "customer-io", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Customer.io exposes a documented CLI, REST APIs (Track/Pipelines), an OpenAPI spec, and Postman collection, all of which allow scripting/automation outside the UI — the core building blocks for headless/CI use. However, there is no explicit documentation or example of running these in a CI pipeline (e.g., auth/token setup for automated jobs, CI examples, idempotent scripting patterns), so headless-in-CI is inferred rather than directly evidenced. Missing for 10: explicit CI/pipeline integration docs or examples, authentication guidance for automated/headless contexts, and independent confirmation of CLI use outside interactive sessions.", "evidenceIds": [ "customer-io-docs-2", "customer-io-docs-3", "customer-io-docs-7", "customer-io-probe-2", "customer-io-docs-8" ] }, { "productId": "customer-io", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The story asks whether Customer.io lets a user plug external MCP servers into it so its AI can use their tools (Customer.io acting as an MCP client). The evidence only shows the reverse: Customer.io ships its own official MCP server (customer-io-probe-3, customer-io-docs-9/10/16) for other agents to connect to and call its tools, which is a different capability. No evidence shows Customer.io's AI assistant/CLI/Slack integration can consume arbitrary external MCP servers.", "evidenceIds": [] }, { "productId": "customer-io", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Customer.io publishes an official MCP server (docs.customer.io/ai/mcp/get-started/) that lets an agent list automations, draft emails in Design Studio, and separates reads from writes/deletes for approval control, confirmed by a live probe of the docs page. Missing for 10: independent/hands-on third-party confirmation that the MCP server works as documented and broader detail on setup/auth steps.", "evidenceIds": [ "customer-io-docs-9", "customer-io-docs-10", "customer-io-docs-16", "customer-io-probe-3" ] }, { "productId": "customer-io", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Customer.io documents an official MCP server plus CLI+skills that let an AI assistant perform actions via natural language (list automations, draft emails, build automations) with read/write permission separation, and a Slack AI interface for natural-language operation. Missing for 10: independent/hands-on corroboration of these NL workflows actually working end-to-end beyond first-party docs.", "evidenceIds": [ "customer-io-docs-1", "customer-io-docs-2", "customer-io-docs-9", "customer-io-docs-10", "customer-io-docs-16", "customer-io-docs-20", "customer-io-probe-3" ] }, { "productId": "customer-io", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs explicitly mention installing 'the CLI and the skills that teach your AI assistant to use it,' confirming an official CLI exists for AI-native workflows, but no further detail (commands, docs page, install instructions) or independent corroboration is provided. Missing for 10: dedicated CLI reference/docs page, example commands, and independent/hands-on confirmation of the CLI's use.", "evidenceIds": [ "customer-io-docs-2", "customer-io-docs-1" ] }, { "productId": "customer-io", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Customer.io publishes a documented public API (Track API, Pipelines API, Postman collection) confirmed live via OpenAPI spec probe, plus additional documented programmatic surfaces (CLI, MCP server) for AI-driven use. Missing for 10: independent third-party developer corroboration of API robustness/completeness beyond docs and probes.", "evidenceIds": [ "customer-io-docs-3", "customer-io-docs-7", "customer-io-docs-8", "customer-io-probe-2", "customer-io-probe-1", "customer-io-docs-2", "customer-io-probe-3" ] }, { "productId": "customer-io", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The MCP integration separates reads from writes/deletes so users can approve each category independently, which is a form of scoped permissioning for agent actions, but there's no evidence of true API-key-level least-privilege credential issuance (e.g., scoped tokens, role-based API keys, or workspace-level permission scoping for agents) beyond this read/write approval gate. missing for 10: scoped API key/token generation, documented credential-level permission granularity, independent verification of the approval mechanism's enforcement.", "evidenceIds": [ "customer-io-docs-10", "customer-io-probe-3", "customer-io-docs-9" ] }, { "productId": "customer-io", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Customer.io documents official Track/Pipelines APIs and SDKs for ingesting data, backed by a Postman collection and machine-readable OpenAPI spec (docs-3, docs-6, docs-7, docs-8, probe-2), which supports building AI-native/agentic integrations. However, there's no explicit enumeration of language-specific SDKs (e.g., Node, Python, Java) or independent developer corroboration of SDK quality/completeness beyond one community complaint about lack of code-based flexibility (comm-3). Missing for 10: named list of official SDK languages/repos, independent hands-on corroboration of SDK usability, and versioning/maintenance details.", "evidenceIds": [ "customer-io-docs-3", "customer-io-docs-6", "customer-io-docs-7", "customer-io-docs-8", "customer-io-probe-2", "customer-io-comm-3" ] }, { "productId": "customer-io", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of a webhook subscription mechanism (e.g., outbound event notifications for opens, clicks, unsubscribes, etc.); it only covers inbound Track/Pipelines APIs, broadcasts, and the MCP server for AI agent actions. Webhooks are a plausible capability for a messaging platform like Customer.io, but nothing here documents or demonstrates the ability to subscribe to events via webhooks.", "evidenceIds": [] }, { "productId": "customer-io", "storyId": "ai-content-drafting", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs confirm broad AI drafting ability ('draft messages', 'Draft a welcome email in Design Studio', 'AI-generated global styles') and an AI assistant reachable via MCP/CLI/Slack, implying some copy-generation capability, but none of the evidence specifically describes subject-line drafting/refinement or copy editing embedded directly inside the message composer UI. Missing for 10: explicit documentation of an in-composer AI subject-line/body-copy assistant, a described 'brief-to-draft' workflow, and independent/hands-on confirmation of this specific feature.", "evidenceIds": [ "customer-io-docs-1", "customer-io-docs-16", "customer-io-docs-15", "customer-io-docs-20" ] }, { "productId": "customer-io", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Customer.io provides an OpenAPI spec (confirmed live at openapi.json) and a downloadable Postman collection for hands-on API testing, but there's no evidence of an in-browser interactive reference (e.g., Swagger-style 'try it' console) with runnable examples embedded directly in the docs site. Missing for 10: an embedded interactive try-it-now console in the docs, independent confirmation of usability/coverage of the OpenAPI spec.", "evidenceIds": [ "customer-io-docs-8", "customer-io-probe-2" ] }, { "productId": "customer-io", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A probe confirms a live, machine-readable OpenAPI spec at https://docs.customer.io/openapi.json returning HTTP 200 with an 'openapi' key, plus a Postman collection is also provided for API onboarding. Missing for 10: independent/community corroboration of developers actually consuming the OpenAPI spec (e.g., via SDK generation).", "evidenceIds": [ "customer-io-probe-2", "customer-io-docs-8" ] }, { "productId": "customer-io", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of a sandbox/staging environment or test workspace separate from production; the evidence pack covers API docs, MCP tooling, and pricing features, but nothing about isolated test data or sandbox mode. Community evidence even notes self-serve sign-in/testing was removed after a pricing shift, reinforcing the absence of a documented sandbox capability. missing for 10: dedicated sandbox/staging environment, documentation of test workspaces, isolation guarantees for AI testing without touching production data.", "evidenceIds": [ "customer-io-comm-1" ] }, { "productId": "customer-io", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows API docs, OpenAPI spec, and Postman collections exist, but there is no mention of API versioning scheme or a documented deprecation policy anywhere in the pack. Missing for 10: explicit API version numbers/headers, changelog or migration guides, and a stated deprecation/sunset policy for older API versions.", "evidenceIds": [ "customer-io-docs-3", "customer-io-docs-7", "customer-io-probe-2" ] }, { "productId": "customer-io", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Customer.io supports bulk-style actions like triggering a Broadcast to an entire list of recipients at once and triggering automations when profiles join a Segment (docs-4, docs-13, docs-12), which are genuine bulk operations. However there's no explicit evidence of a bulk/batch API endpoint for mass profile updates, imports, or deletions, nor of the MCP/CLI AI tools performing multi-item bulk actions in one command. Missing for 10: documented bulk/batch API endpoints (e.g. batch import, bulk update/delete), evidence of AI assistant executing bulk operations across many items via MCP/CLI, and independent confirmation of bulk operation reliability at scale.", "evidenceIds": [ "customer-io-docs-4", "customer-io-docs-13", "customer-io-docs-12", "customer-io-docs-5", "customer-io-docs-6" ] }, { "productId": "customer-io", "storyId": "automation-rules-engine", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Customer.io's journeys/automations natively trigger actions when profiles perform events or join segments, with rich workflow features like Wait Until delays and branching, and this is fully documented as core product functionality. missing for 10: independent/hands-on verification of automation reliability at scale, and more detail on complex multi-condition rule logic beyond events/segments.", "evidenceIds": [ "customer-io-docs-12", "customer-io-docs-11", "customer-io-docs-18", "customer-io-docs-13", "customer-io-docs-14" ] }, { "productId": "customer-io", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Customer.io's core platform natively supports recurring/triggered workflows (automations triggered by events/segments, chained Wait Until delays, scheduled broadcasts) and docs claim AI can 'build automations' via the CLI/MCP, with MCP able to list active automations. However, there is no concrete evidence of an AI agent actually creating or scheduling a new recurring workflow end-to-end (writes/deletes require separate approval, and only read/list and draft-email examples are shown). missing for 10: a documented example of AI creating/editing a recurring automation via MCP or CLI, independent confirmation of AI-driven automation authoring succeeding.", "evidenceIds": [ "customer-io-docs-1", "customer-io-docs-9", "customer-io-docs-10", "customer-io-docs-12", "customer-io-docs-13", "customer-io-docs-18", "customer-io-probe-3" ] }, { "productId": "customer-io", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of version control, review/approval workflows for automation changes, or rollback of automations. The MCP docs mention read/write approval separation for actions but nothing about versioning or reverting automation states over time.", "evidenceIds": [ "customer-io-docs-9", "customer-io-docs-10" ] }, { "productId": "customer-io", "storyId": "campaign-performance-reporting", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation of campaign/automation reporting dashboards, open/click/unsubscribe/conversion metrics, or time-series analytics views—only references to triggers, segments, and messaging setup. Missing for 10: any mention of a reporting/analytics UI, per-step conversion tracking, or historical metrics API/export.", "evidenceIds": [] }, { "productId": "customer-io", "storyId": "campaign-via-api", "verdict": "partial", "quality": 3, "confidence": "high", "rationale": "Customer.io's own docs state that broadcasts must first be configured in the UI and can then only be triggered via a single API call, directly contradicting the 'entirely through API' requirement; there is no documented endpoint to create or schedule a broadcast/campaign purely programmatically. Missing for 10: a documented API endpoint for creating/scheduling broadcasts or campaigns (not just triggering pre-built ones), and confirmation that campaign scheduling doesn't require dashboard configuration.", "evidenceIds": [ "customer-io-docs-4", "customer-io-docs-13", "customer-io-docs-7" ] }, { "productId": "customer-io", "storyId": "contacts-crud-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm the Pipelines/Track API for creating/updating profiles with custom properties, plus a documented pattern for suppression (sending a 'Suppress Person' event) and deletion, and an OpenAPI spec/Postman collection support production integration. However, there is no explicit evidence of rate limits, batch/bulk endpoints, or production-scale guarantees, and one community report cites bugs with the system as a reason for switching providers. missing for 10: explicit production-scale/rate-limit documentation, independent verification of suppression at scale, and confirmation that the 'delete/suppress via event' pattern is robust in practice.", "evidenceIds": [ "customer-io-docs-3", "customer-io-docs-5", "customer-io-docs-7", "customer-io-docs-8", "customer-io-probe-2", "customer-io-comm-5" ] }, { "productId": "customer-io", "storyId": "create-send-campaign", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs show the core loop: segments for audience selection (docs-11), broadcasts queued to defined recipients at a specified time (docs-13), UI-based broadcast setup with trigger API (docs-4), and Design Studio/drag-and-drop editor for composing messages (docs-16, docs-17). This covers compose, pick audience, and send/schedule as table-stakes. Missing for 10: independent hands-on corroboration of the end-to-end flow and explicit UI screenshots/docs of a scheduling calendar interface (only inferred from 'time you specify').", "evidenceIds": [ "customer-io-docs-11", "customer-io-docs-13", "customer-io-docs-4", "customer-io-docs-16", "customer-io-docs-17" ] }, { "productId": "customer-io", "storyId": "custom-sending-domain", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses domain authentication (SPF/DKIM/DMARC) setup, guided DNS configuration, or verification workflows for sending domains — all evidence concerns AI/MCP integrations, APIs, journeys, and pricing.", "evidenceIds": [] }, { "productId": "customer-io", "storyId": "deliverability-visibility", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions bounce, complaint, or blocklist tracking, deliverability dashboards, or actionable guidance for degraded sending reputation; the docs focus on automations, broadcasts, AI/MCP tooling, and pricing features unrelated to deliverability signals.", "evidenceIds": [] }, { "productId": "customer-io", "storyId": "email-template-editor", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm a drag-and-drop visual editor, code editor, AI-generated global styles, and a Design Studio for drafting emails, showing on-brand visual design tooling. However, there is no explicit evidence of reusable template libraries, personalization token syntax, or cross-email-client rendering/testing guarantees, which are core parts of this story. Missing for 10: reusable template management, personalization token documentation, cross-client rendering testing/preview evidence.", "evidenceIds": [ "customer-io-docs-15", "customer-io-docs-16", "customer-io-docs-17" ] }, { "productId": "customer-io", "storyId": "engagement-data-api-read", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers sending data in (Track/Pipelines API), triggering broadcasts, and AI/MCP automation management, but contains no documentation of an API endpoint for reading a contact's engagement history (messages sent, opened, clicked). Missing for 10: any mention of an Activities/Customer Journey read API, endpoint docs for per-person message history, or examples returning open/click events for a contact.", "evidenceIds": [ "customer-io-docs-3", "customer-io-docs-7" ] }, { "productId": "customer-io", "storyId": "event-triggered-flows", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs clearly describe sending real-time custom events via the Track/Pipelines API (docs-3, docs-7) and triggering automations specifically when a profile performs an event (docs-12), which directly matches the developer story of firing an event to trigger a person-specific automation. Missing for 10: independent/hands-on confirmation that per-person triggering works reliably at scale, and more detail on event payload/attribute matching nuances.", "evidenceIds": [ "customer-io-docs-3", "customer-io-docs-7", "customer-io-docs-12", "customer-io-docs-21" ] }, { "productId": "customer-io", "storyId": "flow-state-inspection", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence describes a per-person view of automation/journey progress or an ops action to pull or re-enter a profile without breaking the flow; docs cover triggering automations, segments, and broadcasts but not individual journey-state visibility or manual re-entry controls.", "evidenceIds": [ "customer-io-docs-12", "customer-io-docs-11" ] }, { "productId": "customer-io", "storyId": "full-list-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack only documents APIs for sending data INTO Customer.io (Track API, Pipelines API, suppression via events) and MCP-based automation reads, but contains no mention of a bulk data export, audience/contact export tool, or open-format download of engagement history and suppression lists. No self-serve export capability is evidenced anywhere in the pack. Missing for 10: any documentation of an export/download feature, open-format (CSV/JSON) audience export, or bulk API read endpoint for full contact/tag/engagement/suppression data.", "evidenceIds": [ "customer-io-docs-3", "customer-io-docs-5", "customer-io-docs-7" ] }, { "productId": "customer-io", "storyId": "guided-platform-migration", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a guided migration tool or import wizard that preserves tags, custom fields, and unsubscribe state from another ESP. Docs discuss general data ingestion via Track/Pipelines API and profile suppression events, but nothing about a dedicated migration/import feature for list portability.", "evidenceIds": [] }, { "productId": "customer-io", "storyId": "metrics-via-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence covers Track/Pipelines APIs for sending data in, broadcast triggering, and MCP for listing automations, but there is no mention of a Reporting/Analytics API or documented endpoints for pulling campaign or flow performance metrics (opens, clicks, conversions, etc.) out to a warehouse or BI tool. Missing for 10: any reporting/metrics API documentation, export/webhook mechanism for performance data, or dashboard-sync examples.", "evidenceIds": [ "customer-io-docs-3", "customer-io-docs-7", "customer-io-docs-9", "customer-io-probe-2" ] }, { "productId": "customer-io", "storyId": "openness-api-parity", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Customer.io explicitly claims 'Your AI can do anything you'd do by clicking or typing' (docs-1) and ships a Pipelines/Track API plus MCP server, but its own docs contradict full parity: broadcasts must be set up in the UI and only triggered via API (docs-4), and segments/journeys are shown as UI-defined constructs (docs-11, docs-12). Independent evidence reinforces this gap—a developer built an open-source alternative specifically due to 'lack of code-based flexibility for defining segments and email flows' (customer-io-comm-3). missing for 10: API-only creation of broadcasts/segments/email flows without prior UI setup, and independent confirmation that MCP/API coverage matches 100% of UI functionality.", "evidenceIds": [ "customer-io-docs-1", "customer-io-docs-4", "customer-io-docs-11", "customer-io-docs-12", "customer-io-comm-3", "customer-io-probe-3" ] }, { "productId": "customer-io", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows APIs for getting data in and out (Track/Pipelines API) but no documented bulk data export tool or open-format full account export/portability feature, nor any 'leave with your data' guarantee. No mention of GDPR-style export or data portability commitments.", "evidenceIds": [] }, { "productId": "customer-io", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Customer.io is a proprietary SaaS messaging platform, not an open-source project; there is no indication of a source-available license, so 'reading source under open license' is a category error for this product type.", "evidenceIds": [] }, { "productId": "customer-io", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Customer.io is a SaaS-only messaging automation platform with no evidence of an on-premise or self-hostable core product; self-hosting is not a plausible axis for this cloud-only category, and evidence confirms only hosted APIs, MCP, and UI features.", "evidenceIds": [] }, { "productId": "customer-io", "storyId": "outbound-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers inbound APIs (Track/Pipelines API), MCP tooling, and broadcast/journey features, but contains no mention of outbound webhooks for delivery/engagement events (sends, opens, clicks, bounces, unsubscribes) or of signed payload verification. missing for 10: documentation of webhook subscription endpoints/UI, event types supported, payload signing/verification mechanism, and any independent confirmation of reliability.", "evidenceIds": [ "customer-io-docs-3", "customer-io-docs-7", "customer-io-docs-13" ] }, { "productId": "customer-io", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions data residency options, regional storage selection, or EU/US hosting choices for Customer.io.", "evidenceIds": [] }, { "productId": "customer-io", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses AI-model training opt-out or data usage controls for Customer.io; all citations concern messaging automation, APIs, and MCP tooling, not AI training data policy.", "evidenceIds": [] }, { "productId": "customer-io", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Customer.io documents explicit mechanisms to delete or suppress profile data via the Track API (Delete Person / Suppress Person events), and its MCP server separates delete operations from reads/writes for explicit approval, giving AI-native users some control over data removal. However, there's no documented retention-policy settings, automated data expiry, bulk/GDPR-scale deletion tooling, or audit trail for deletions. Missing for 10: configurable retention periods, bulk/automated deletion workflows, compliance/audit documentation, independent verification that deletion requests are honored.", "evidenceIds": [ "customer-io-docs-5", "customer-io-docs-10" ] }, { "productId": "customer-io", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry or usage-tracking opt-out settings for AI features or the product itself; the docs focus on API/MCP capabilities, not privacy controls. missing for 10: any mention of telemetry collection, opt-out mechanism, or usage-tracking settings.", "evidenceIds": [] }, { "productId": "customer-io", "storyId": "revenue-attribution", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers messaging setup, broadcasts, segments, APIs, and MCP/AI tooling, but contains no mention of conversion/revenue attribution reporting tied to specific emails or flows beyond opens/clicks. Missing for 10: any documentation of conversion goals, revenue tracking, or attribution reports linking signups/revenue to specific campaigns or flows.", "evidenceIds": [] }, { "productId": "customer-io", "storyId": "scoped-api-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows Customer.io's MCP integration separates reads from writes for AI-action approval (customer-io-docs-10), but there is no evidence of an admin-configurable mechanism to mint scoped or read-only API keys/credentials for external agents or contractors to access analytics data without send capability.", "evidenceIds": [ "customer-io-docs-10" ] }, { "productId": "customer-io", "storyId": "segment-builder", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs describe building segments (e.g., 'Paying Customers', 'Have not logged in recently') from attribute conditions and events, and automations can trigger when profiles join segments, implying live-updating segment membership as data changes. However, tags-based segmentation isn't explicitly documented, and one community post notes frustration with 'lack of code-based flexibility for defining segments,' suggesting some limitations in segment logic sophistication. Missing for 10: explicit tag-criteria documentation, independent confirmation of real-time segment recalculation, and rebuttal to the code-flexibility complaint.", "evidenceIds": [ "customer-io-docs-11", "customer-io-docs-12", "customer-io-docs-7", "customer-io-comm-3" ] }, { "productId": "customer-io", "storyId": "segments-via-api", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs confirm segments exist as a concept (docs-11) and that objects/relationships can be managed via the Track/Pipelines API's /group endpoint (docs-6, docs-7), implying some programmatic control over customer data groupings, but there is no direct evidence of an API to create/update segments, tags, or lists, nor any documented way to read back exact membership. A community report even cites 'lack of code-based flexibility for defining segments' as a driver for building an alternative (comm-3), suggesting real limitations in programmatic segment management. missing for 10: explicit API endpoints for segment/tag/list CRUD, membership read-back examples, and independent confirmation that these work as described.", "evidenceIds": [ "customer-io-docs-6", "customer-io-docs-7", "customer-io-docs-11", "customer-io-comm-3" ] }, { "productId": "customer-io", "storyId": "stack-integrations", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs confirm Customer.io has a Pipelines/Track API that 'most of our integrations are based on' and note users can 'use your preferred tools and sources to transit data through Customer.io,' implying an integrations ecosystem exists, but no evidence names or documents maintained first-party connectors for Stripe, Shopify, Segment, or a product database specifically. missing for 10: explicit documentation of named Stripe/Shopify/Segment/CDP integrations, integration marketplace listing, maintenance/versioning details for these connectors.", "evidenceIds": [ "customer-io-docs-3", "customer-io-docs-21", "customer-io-docs-7" ] }, { "productId": "customer-io", "storyId": "suppression-compliance", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs mention a 'Suppress Person' event and an 'Unsubscribed' segment, implying some suppression capability, but there is no evidence describing automatic bounce or spam-complaint suppression, nor confirmation that suppression is honored uniformly across broadcasts, transactional, and journey sends. Missing for 10: documented bounce/spam-complaint auto-suppression, explicit cross-send-type enforcement, and any compliance (CAN-SPAM/GDPR) framing.", "evidenceIds": [ "customer-io-docs-5", "customer-io-docs-11" ] }, { "productId": "customer-io", "storyId": "templates-as-code", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs mention a 'Code editor' included in the Design Studio for building emails (docs-17) and a Pipelines/Track API for data, but there is no evidence of dedicated template CRUD API endpoints, MJML support, or a documented git/version-control workflow for templates; a community developer even built an OSS alternative citing lack of code-based flexibility for email flows (customer-io-comm-3). missing for 10: template API endpoints with variable substitution docs, MJML/HTML-as-code workflow, version-control integration or CLI template sync.", "evidenceIds": [ "customer-io-docs-17", "customer-io-docs-16", "customer-io-comm-3" ] }, { "productId": "customer-io", "storyId": "transactional-send-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm transactional message use cases (receipts, password resets, shipping updates) and a Track/Pipelines API for sending real-time customer data and events, alongside the same platform's broadcast/marketing capabilities, supporting one-to-one and marketing sends via one API. However, there is no explicit developer-facing example of sending a single transactional message via API call (e.g., a dedicated Transactional API endpoint with code sample) or confirmation of independent/hands-on verification of transactional deliverability. missing for 10: explicit Transactional API code example/endpoint documentation, independent hands-on confirmation of transactional send reliability, clarification on rate/latency guarantees for one-to-one sends.", "evidenceIds": [ "customer-io-docs-14", "customer-io-docs-7", "customer-io-docs-3", "customer-io-docs-13" ] }, { "productId": "customer-io", "storyId": "transactional-separation", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs confirm transactional messaging exists for receipts, confirmations, password resets, etc. (customer-io-docs-14), and separate suppression events can be sent via API (customer-io-docs-5), but there is no explicit documentation of separate deliverability/reputation isolation between transactional and marketing sending, nor confirmation that suppressing a marketing contact does not block transactional receipts. missing for 10: explicit statement that transactional sending reputation/IP pool is isolated from marketing, documented suppression-list scoping (transactional vs marketing), and independent/hands-on confirmation that an unsubscribed contact still receives a transactional receipt.", "evidenceIds": [ "customer-io-docs-14", "customer-io-docs-5", "customer-io-docs-11" ] }, { "productId": "customer-io", "storyId": "visual-automation-builder", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm core lifecycle-automation building blocks: triggering journeys off events/segments (docs-12), chaining multiple delays in a Wait Until block (docs-18), listing active automations (docs-9), and segment-based audience targeting (docs-11). However, the evidence pack never explicitly documents branching logic or exit-rule configuration, the two specific mechanics named in the story, and one community report cites frustration with limited flexibility in defining email flows (customer-io-comm-3). Missing for 10: explicit documentation of conditional branch splits and exit-rule/goal settings within a journey, plus independent hands-on confirmation of multi-step welcome/onboarding/winback builds.", "evidenceIds": [ "customer-io-docs-12", "customer-io-docs-18", "customer-io-docs-9", "customer-io-docs-11", "customer-io-comm-3" ] }, { "productId": "kit", "storyId": "ab-test-campaigns", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Kit's pricing page lists \"A/B test subject lines & content\" as a feature [kit-docs-7], confirming the capability exists, but there's no documentation describing how winners are determined from engagement data (e.g., automatic winner selection, sample size, or evaluation window). Missing for 10: documentation on automatic winner-selection logic/criteria, evidence of engagement-based decisioning, and any hands-on or independent confirmation the feature works as described.", "evidenceIds": [ "kit-docs-7" ] }, { "productId": "kit", "storyId": "agent-audits-program", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Kit has a documented API and an official MCP server that can query live docs and turn natural-language prompts into account actions (e.g., drafting broadcasts), showing programmatic access exists. However, the evidence pack lacks any documentation of endpoints for pulling subscriber/audience lists, automation status, or campaign performance/reporting data — the core of 'report on what's working'. Missing for 10: audience/segment retrieval endpoints, automation status/reporting API, campaign analytics endpoints, and any example of an agent synthesizing a performance report.", "evidenceIds": [ "kit-docs-1", "kit-docs-2", "kit-docs-3", "kit-docs-4", "kit-docs-5", "kit-probe-3" ] }, { "productId": "kit", "storyId": "agent-builds-segment", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Kit's API can target broadcasts to existing segment/tag ids and its MCP server can turn natural language into broadcast actions, but there is no documented endpoint or MCP capability for creating or defining a new segment from a plain-language audience description. Missing for 10: a segment-creation API endpoint, MCP/agent workflow demonstrating natural-language-to-segment translation, and any example of dynamic segment criteria being set programmatically.", "evidenceIds": [ "kit-docs-2", "kit-docs-4", "kit-docs-5", "kit-probe-3" ] }, { "productId": "kit", "storyId": "agent-drives-campaign", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Kit's API and MCP let an agent draft or schedule a broadcast to a segment/tag (kit-docs-1, kit-docs-2, kit-docs-5), and drafts inherently sit unsent until a human acts, which functions as an approval gate. However, there is no explicit documentation of a formal 'queued for human approval' workflow or approval-status field distinct from draft/schedule. Missing for 10: explicit human-approval/review step or status beyond generic draft/schedule, and independent confirmation the MCP flow reliably leaves campaigns unsent pending approval.", "evidenceIds": [ "kit-docs-1", "kit-docs-2", "kit-docs-5", "kit-docs-4" ] }, { "productId": "kit", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Kit provides a live llms.txt file confirmed by probe (HTTP 200 with structured doc links) plus agent-oriented docs including an official MCP server that queries live docs to avoid hallucinated API surfaces. Missing for 10: independent third-party confirmation of an AI agent successfully using llms.txt in practice.", "evidenceIds": [ "kit-probe-1", "kit-docs-4", "kit-docs-5", "kit-probe-3" ] }, { "productId": "kit", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Kit offers subscriber engagement scoring and an MCP integration that lets an agent draft actions like broadcasts from natural-language prompts, which touches on data-driven suggestions, but there is no documented feature that surfaces AI-generated insights or recommendations from a user's own data (e.g., analytics summaries, content suggestions). Missing for 10: dedicated AI insight/analytics generation feature, evidence of proactive suggestions surfaced in-app, independent corroboration of engagement scoring being AI-driven.", "evidenceIds": [ "kit-docs-9", "kit-docs-4", "kit-docs-5", "kit-probe-3" ] }, { "productId": "kit", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Kit ships 'Unlimited Visual Automations' as a core background-automation feature, and its MCP server lets AI agents trigger real account actions (e.g., drafting broadcasts) via natural language, showing some agentic setup capability. However, no evidence shows the MCP or API being used to configure/build visual automation workflows themselves (only broadcast creation is exemplified), and there's no confirmation of persistent, autonomously-triggered background execution driven by an AI agent. Missing for 10: documented API/MCP endpoints for creating or editing automation workflows, evidence of scheduled/triggered autonomous runs initiated by AI, independent confirmation of this working end-to-end.", "evidenceIds": [ "kit-docs-6", "kit-docs-4", "kit-docs-5", "kit-probe-3" ] }, { "productId": "kit", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only documents an MCP server that lets external AI agents (e.g., Claude) query Kit's docs and perform actions via natural language — this is agent-facing infrastructure, not a built-in AI assistant embedded in the Kit product UI itself. No evidence of an in-app assistant for drafting content, automations, or subscriber management.", "evidenceIds": [ "kit-docs-4", "kit-docs-5", "kit-probe-3" ] }, { "productId": "kit", "storyId": "agentic-headless", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Kit exposes a REST API and an MCP server that let third-party tools or agents trigger actions (e.g., drafting broadcasts) programmatically, which could in principle be scripted into a CI pipeline, but there is no documentation of a CLI, headless mode, or explicit CI/automation-pipeline usage. missing for 10: explicit CI/headless execution docs, a CLI or non-interactive mode, and any independent evidence of running Kit unattended in automated pipelines.", "evidenceIds": [ "kit-docs-3", "kit-docs-4", "kit-docs-5", "kit-probe-3" ] }, { "productId": "kit", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Kit is a SaaS email-marketing platform, not an AI agent capable of consuming external MCP servers as tools. The evidence only shows Kit exposing its own MCP server for others (like coding agents) to connect to and use Kit's API, which is the opposite direction from this story's ask.", "evidenceIds": [ "kit-docs-4", "kit-probe-3" ] }, { "productId": "kit", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Kit's developer docs explicitly describe and document an official MCP server ('Kit Developer Docs MCP') that lets agents query live API docs and turn natural-language prompts into real account actions, and a probe confirms this page is live. Missing for 10: independent/hands-on third-party confirmation that the MCP server works as described beyond first-party docs.", "evidenceIds": [ "kit-docs-4", "kit-docs-5", "kit-probe-3" ] }, { "productId": "kit", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Kit ships an official Developer Docs MCP server explicitly designed so natural-language prompts (e.g., 'draft a broadcast to my new subscribers from this week') translate into real account actions, confirmed by both docs and a probe verifying the MCP endpoint exists. Missing for 10: independent/hands-on verification of natural-language command execution beyond the single documented example, and broader coverage of which actions are supported via NL.", "evidenceIds": [ "kit-docs-4", "kit-docs-5", "kit-probe-3" ] }, { "productId": "kit", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Kit's evidence pack covers its REST API, MCP server for AI agents, and marketing features, but nowhere mentions a CLI tool for developers or AI-native workflows. Since Kit is a SaaS platform where an official CLI is a plausible offering, absence of evidence means this axis is unmet rather than inapplicable.", "evidenceIds": [ "kit-docs-3", "kit-docs-4", "kit-probe-1", "kit-probe-2" ] }, { "productId": "kit", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Kit publishes a documented public API (broadcasts, subscribers, segments/tags) with a developer reference and overview explicitly aimed at third-party developers building on the platform, plus an llms.txt for AI-native discovery. Missing for 10: a publicly discoverable OpenAPI/Swagger spec (probe found only 404s) and independent hands-on developer corroboration beyond vendor docs.", "evidenceIds": [ "kit-docs-1", "kit-docs-2", "kit-docs-3", "kit-probe-1", "kit-probe-2" ] }, { "productId": "kit", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Kit has an API and an MCP server for agent integration, but nothing describes issuing scoped or least-privilege API keys/tokens (e.g., granular permission scopes, read-only vs write keys) for agents. missing for 10: any documentation of scoped API key creation, permission/role-based tokens, or least-privilege credential issuance for agent use.", "evidenceIds": [ "kit-docs-3", "kit-docs-4", "kit-docs-5" ] }, { "productId": "kit", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Kit has a REST API with reference docs and an MCP server, but nothing indicates official client SDKs (e.g., Python/JS/Ruby libraries); in fact the OpenAPI spec probe returned 404s, suggesting no machine-generated or officially maintained SDK tooling. missing for 10: evidence of any official SDK/client library, language coverage, or published OpenAPI spec used to generate SDKs.", "evidenceIds": [ "kit-docs-3", "kit-probe-2" ] }, { "productId": "kit", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack of Kit offering webhook subscriptions for events; documentation covers API endpoints, broadcasts, MCP, and pricing but never mentions webhooks. Missing for 10: any webhook documentation, event types, subscription/registration endpoints, or delivery mechanism.", "evidenceIds": [] }, { "productId": "kit", "storyId": "ai-content-drafting", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Kit's API/MCP can create or draft broadcasts programmatically and offers A/B testing of subject lines, but there is no mention of an in-composer AI assistant that drafts or refines subject lines/email copy from a marketer's brief inside the editor itself.", "evidenceIds": [ "kit-docs-1", "kit-docs-7", "kit-docs-5" ] }, { "productId": "kit", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence confirms Kit has API reference docs (create-a-broadcast, overview) but nothing describes an interactive console or runnable/try-it-now examples, and a probe for an OpenAPI/Swagger spec (which typically powers such interactive references) returned 404s across all checked paths.", "evidenceIds": [ "kit-docs-1", "kit-docs-3", "kit-probe-2" ] }, { "productId": "kit", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Kit's developer docs describe an API reference and even an MCP server for live API access, but a direct probe for machine-readable spec endpoints (openapi.json, swagger.json, etc.) returned 404 on all candidates, and no OpenAPI/Swagger download is documented anywhere in the evidence.", "evidenceIds": [ "kit-probe-2", "kit-docs-3", "kit-probe-1" ] }, { "productId": "kit", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a sandbox/test environment, staging account, or test-mode API keys distinct from production data; the docs only describe live API/MCP actions against a real account.", "evidenceIds": [] }, { "productId": "kit", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of API versioning scheme or a documented deprecation policy; the API reference and overview pages mention endpoints but say nothing about version numbers, sunset timelines, or backward-compatibility guarantees. OpenAPI spec probes also 404, further indicating no formal versioned contract is published.", "evidenceIds": [ "kit-docs-3", "kit-probe-2" ] }, { "productId": "kit", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Kit's API/MCP lets users draft and send broadcasts to all or a segment/tag-based subset of subscribers, and the MCP overview shows natural-language prompts triggering such bulk actions, which is a form of bulk operation across many subscriber items. However, there's no evidence of broader bulk operations like batch subscriber import/export, bulk tagging/untagging, or bulk updates via API beyond broadcast targeting. Missing for 10: documented bulk subscriber CRUD/import-export endpoints, evidence of batch processing limits or throughput, and independent confirmation of bulk actions working via the MCP in practice.", "evidenceIds": [ "kit-docs-1", "kit-docs-2", "kit-docs-4", "kit-docs-5", "kit-probe-3" ] }, { "productId": "kit", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Kit's pricing page references 'Unlimited Visual Automations' (kit-docs-6), implying a rule/trigger-based automation builder typical of email platforms, but the evidence pack contains no documentation detailing specific triggers, conditions, or actions, nor any hands-on confirmation of how automations are configured or fire on events. Missing for 10: detailed docs on trigger/event types, action definitions, conditional logic, and independent verification of automation reliability.", "evidenceIds": [ "kit-docs-6" ] }, { "productId": "kit", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Kit supports scheduling one-time broadcasts and offers 'Unlimited Visual Automations' and RSS campaigns (which are inherently recurring), but there's no evidence of an AI-native (API/MCP) mechanism to schedule or manage recurring workflows—broadcast scheduling is single-send and automations are event-triggered rather than time-based recurring jobs. Missing for 10: explicit API/MCP support for recurring/cron-style workflow scheduling, documentation of recurring automation triggers, and independent confirmation these can be set up via AI-native tooling.", "evidenceIds": [ "kit-docs-1", "kit-docs-6", "kit-docs-13", "kit-docs-5" ] }, { "productId": "kit", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence Kit offers version history, review workflows, or rollback for automations/visual automations; docs mention only building automations and MCP-driven actions, not versioning or undo capabilities.", "evidenceIds": [] }, { "productId": "kit", "storyId": "campaign-performance-reporting", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack describes any analytics/reporting feature—no mention of open/click/unsubscribe/conversion tracking per broadcast or automation step; the pack only covers broadcast creation, MCP docs, and pricing feature lists. missing for 10: analytics dashboard docs, per-campaign/per-step metrics, conversion tracking evidence.", "evidenceIds": [] }, { "productId": "kit", "storyId": "campaign-via-api", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Kit's public API docs explicitly support creating/drafting and scheduling broadcasts with targeting by segment/tag, directly enabling campaign creation without the dashboard. Missing for 10: an accessible OpenAPI/swagger spec (probe found 404s), independent developer confirmation of end-to-end scheduling success, and clarity on recurring/RSS campaign API support.", "evidenceIds": [ "kit-docs-1", "kit-docs-2", "kit-docs-3", "kit-probe-2" ] }, { "productId": "kit", "storyId": "contacts-crud-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack only covers broadcasts API, MCP docs, pricing features, and general company history — there is no documentation of contact/subscriber creation, updates, custom field/property management, or suppression via the API, nor any mention of production-scale rate limits or bulk operations.", "evidenceIds": [] }, { "productId": "kit", "storyId": "create-send-campaign", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Kit's API docs confirm drafting/scheduling a broadcast (send/schedule) with audience targeting by segment or tag (kit-docs-1, kit-docs-2), covering the core compose-audience-send/schedule loop, and this is corroborated by the product's broader marketing platform positioning (kit-docs-14, kit-docs-15). missing for 10: independent hands-on user account of composing and sending a real campaign via the UI, and detail on scheduling UI/preview features beyond the API reference.", "evidenceIds": [ "kit-docs-1", "kit-docs-2", "kit-docs-14" ] }, { "productId": "kit", "storyId": "custom-sending-domain", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses domain authentication (SPF/DKIM/DMARC) setup, guided DNS configuration, or verification tooling; the evidence only covers broadcasts, MCP, pricing tiers, and general marketing.", "evidenceIds": [] }, { "productId": "kit", "storyId": "deliverability-visibility", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions bounce, complaint, or blocklist tracking, deliverability metrics, or actionable guidance for degraded sending reputation; the documentation covers broadcasts, pricing features, and MCP/API tooling but nothing about deliverability signal reporting.", "evidenceIds": [] }, { "productId": "kit", "storyId": "email-template-editor", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers broadcast API, MCP integration, and pricing feature bullets (automations, A/B testing, collaborative editing) but contains no mention of a visual email editor, reusable templates, or personalization token rendering across email clients. Since this is a core capability marketers would expect from an email platform, the axis clearly applies, but no supporting evidence exists in the pack.", "evidenceIds": [] }, { "productId": "kit", "storyId": "engagement-data-api-read", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers broadcast creation, MCP server, and pricing features, but nothing shows an API endpoint for retrieving per-subscriber engagement history (sent/opened/clicked events) for a given contact. Subscriber engagement scoring is mentioned only as a pricing feature name, not an API capability.", "evidenceIds": [ "kit-docs-9", "kit-docs-1", "kit-docs-2" ] }, { "productId": "kit", "storyId": "event-triggered-flows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers broadcasts API, MCP docs integration, and pricing features, but contains no mention of a custom-event API or webhook mechanism that lets a developer fire an event to trigger a per-subscriber automation. This is a plausible axis for an email/lifecycle automation platform, but no evidence supports it here.", "evidenceIds": [] }, { "productId": "kit", "storyId": "flow-state-inspection", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only mentions 'Unlimited Visual Automations' as a pricing line item, with no documentation of subscriber-level visibility into automation steps or the ability to pull/re-enter an individual subscriber without disrupting the automation. Missing for 10: any docs on per-subscriber automation position view, manual add/remove-from-automation actions, and confirmation that re-entry doesn't break flow state.", "evidenceIds": [ "kit-docs-6" ] }, { "productId": "kit", "storyId": "full-list-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of a self-serve data export feature (CSV/CSV-like) covering contacts, tags, custom fields, engagement history, or suppression lists; only API endpoints for broadcasts and pricing feature lists are present, none of which address audience export or portability.", "evidenceIds": [] }, { "productId": "kit", "storyId": "guided-platform-migration", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The only relevant evidence is a single marketing line mentioning 'Free migrations' on the pricing page, with no documentation of guided list import, tag/custom-field preservation, or unsubscribe-state handling. missing for 10: any docs on import tooling, evidence tags/custom fields are preserved, evidence unsubscribe status is preserved, guided/step-by-step migration flow.", "evidenceIds": [ "kit-docs-17" ] }, { "productId": "kit", "storyId": "metrics-via-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence only shows API endpoints for creating/scheduling broadcasts and general platform/MCP docs; there is no mention of any API for retrieving campaign or automation ('flow') performance metrics, open/click stats, or analytics exports suitable for warehousing.", "evidenceIds": [ "kit-docs-1", "kit-docs-2", "kit-docs-3", "kit-probe-2" ] }, { "productId": "kit", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Kit's docs show a real API and MCP layer that can create/schedule broadcasts and let agents take 'real account actions,' plus a llms.txt manifest, showing meaningful API parity ambition. But no OpenAPI spec is discoverable (404s across expected paths) and there is no evidence the API covers major UI-only features like landing pages/forms, visual automations, digital product sales, or A/B testing — so full UI-API parity is unconfirmed. Missing for 10: documented API coverage for landing pages/forms/commerce/automations, a public OpenAPI/schema reference, and independent confirmation of parity.", "evidenceIds": [ "kit-docs-1", "kit-docs-3", "kit-docs-4", "kit-docs-5", "kit-probe-1", "kit-probe-2" ] }, { "productId": "kit", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item documents a data export feature or open-format data portability for Kit; only a vague mention of 'Free migrations' (import-side) appears, with no explicit export/download-your-data capability. Data portability is a fair axis for a SaaS platform, but nothing shows Kit actually supports full open-format export.", "evidenceIds": [ "kit-docs-17" ] }, { "productId": "kit", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Kit is a closed-source SaaS email marketing/creator platform; there is no indication its core product source is open at all, and reading proprietary SaaS source code isn't a fair axis for this product category.", "evidenceIds": [] }, { "productId": "kit", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Kit is a SaaS email marketing/creator platform with no self-hosted deployment option evidenced anywhere; self-hosting is not a fair axis for a hosted SaaS product with proprietary pricing tiers and no open-source core.", "evidenceIds": [] }, { "productId": "kit", "storyId": "outbound-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack mentions webhooks, signed payloads, or event subscriptions (sends, opens, clicks, bounces, unsubscribes); the docs cover broadcasts, MCP, and pricing features unrelated to webhook delivery. missing for 10: any documentation of webhook subscription endpoints, event types, or payload signing/verification.", "evidenceIds": [] }, { "productId": "kit", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions data residency, regional storage options, or any data-location controls for Kit.", "evidenceIds": [] }, { "productId": "kit", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses AI-training data opt-out or a privacy policy governing use of user/subscriber data for model training; all evidence concerns email marketing features, MCP/API docs, and business history.", "evidenceIds": [] }, { "productId": "kit", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items address data retention policies, deletion controls, or privacy/data-export mechanisms for AI-native usage; all evidence concerns broadcast features, pricing tiers, and MCP integration.", "evidenceIds": [] }, { "productId": "kit", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items mention telemetry, usage tracking, or opt-out controls for Kit's MCP server, API, or product; the evidence covers unrelated features like broadcasts, pricing, and MCP docs access.", "evidenceIds": [] }, { "productId": "kit", "storyId": "revenue-attribution", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of revenue/signup attribution reporting tied to specific emails or automations — the pack shows broadcast creation, MCP tooling, and general pricing feature bullets (e.g. engagement scoring, A/B testing) but nothing about linking conversions/signups/revenue back to specific emails or flows. Missing for 10: revenue attribution reporting, signup source tracking per email/flow, any analytics dashboard evidence.", "evidenceIds": [ "kit-docs-1", "kit-docs-2", "kit-docs-9", "kit-docs-11" ] }, { "productId": "kit", "storyId": "scoped-api-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of scoped or read-only API key/credential creation in Kit's docs; the evidence pack shows general API access, MCP integration, and pricing features but nothing about granular permission scoping or read-only tokens for third parties/agents.", "evidenceIds": [] }, { "productId": "kit", "storyId": "segment-builder", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Evidence only shows broadcasts can target subscribers by 'segment or tag ids' (kit-docs-2), implying segments exist as a concept, but there's no documentation of building segments from profile attributes or behavioral events, nor confirmation that segments update live as data changes. Missing for 10: segment-builder documentation, behavioral-event triggers, and confirmation of live/dynamic recalculation of membership.", "evidenceIds": [ "kit-docs-2" ] }, { "productId": "kit", "storyId": "segments-via-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows that broadcasts can be targeted using existing segment or tag ids, implying some segmentation exists, but there is no documentation of API endpoints to create/update segments, tags, or lists, nor any evidence of reading back the exact subscriber membership of those groups.", "evidenceIds": [ "kit-docs-2", "kit-docs-3" ] }, { "productId": "kit", "storyId": "stack-integrations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows a generic marketing line about connecting 'creator tools' via apps (kit-docs-15) and a general third-party API note (kit-docs-3); there is no mention of specific maintained first-party integrations with Stripe, Shopify, Segment/CDPs, or product databases anywhere in the pack.", "evidenceIds": [ "kit-docs-15", "kit-docs-3" ] }, { "productId": "kit", "storyId": "suppression-compliance", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items mention unsubscribe suppression, bounce handling, spam complaint suppression, or automatic CAN-SPAM/GDPR compliance across send types; the pack only covers broadcast creation, MCP/API access, and pricing features unrelated to suppression hygiene.", "evidenceIds": [] }, { "productId": "kit", "storyId": "templates-as-code", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of HTML/MJML template files, template-as-code storage, or version-control support for email templates; evidence only covers broadcast creation API, pricing features, and MCP docs tooling.", "evidenceIds": [] }, { "productId": "kit", "storyId": "transactional-send-api", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows Kit's API only supports broadcast/campaign sends to subscriber segments or tags (kit-docs-1, kit-docs-2), with no mention of a transactional API for one-to-one sends like receipts or magic links. The axis is fair to ask of an email-sending platform, but nothing in the pack indicates this capability exists.", "evidenceIds": [ "kit-docs-1", "kit-docs-2" ] }, { "productId": "kit", "storyId": "transactional-separation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Kit is positioned as a creator/newsletter marketing platform (broadcasts, automations, landing pages) with no evidence of a distinct transactional-sending system, templates, or suppression-list separation from marketing sends. Nothing in the evidence pack addresses transactional email, receipts, or reputation isolation.", "evidenceIds": [] }, { "productId": "kit", "storyId": "visual-automation-builder", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Kit's pricing page confirms a visual automation builder (\"Unlimited Visual Automations\") exists, which is consistent with the story's need for lifecycle automations, but the evidence pack contains no documentation detailing branch logic, delay steps, or exit-rule configuration, nor examples of welcome/onboarding/winback templates. Missing for 10: docs on branching logic, delay/wait steps, exit-rule conditions, and concrete welcome/onboarding/winback automation examples.", "evidenceIds": [ "kit-docs-6" ] }, { "productId": "klaviyo", "storyId": "ab-test-campaigns", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "None of the evidence items mention A/B testing, subject line testing, or automatic winner selection based on engagement; only campaign creation via API and general platform features are documented. Missing for 10: any mention of A/B test creation, variant configuration, winning metric selection, or automatic send-to-winner mechanics.", "evidenceIds": [] }, { "productId": "klaviyo", "storyId": "agent-audits-program", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Klaviyo offers a documented REST API (with scoped private keys and OAuth) and a first-party MCP server explicitly built to let AI agents like Claude or Cursor interact with Klaviyo data, which supports programmatic, agent-driven access. However the evidence only shows a campaign-creation example and general API scope docs — there's no direct documentation of pulling audiences, live automation status, or campaign performance/reporting data through the API or MCP server. Missing for 10: explicit endpoints/examples for audience/segment retrieval, automation status, and campaign analytics/reporting accessible via API or MCP, plus independent confirmation of successful end-to-end reporting use.", "evidenceIds": [ "klaviyo-docs-1", "klaviyo-docs-2", "klaviyo-docs-7", "klaviyo-docs-9", "klaviyo-docs-10", "klaviyo-probe-2" ] }, { "productId": "klaviyo", "storyId": "agent-builds-segment", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Klaviyo has a documented MCP server that lets an AI agent act on Klaviyo data via natural language, and the API supports arbitrary schema-free data plus programmatic campaign creation, suggesting an agent could plausibly create segments too. However, no evidence pack item shows a segment-creation endpoint or an MCP example translating a plain-language audience description into an actual segment (the campaign example is the only concrete natural-language-to-action illustration). Missing for 10: explicit segment/API endpoint documentation, a worked example of natural-language-to-segment translation, and independent/hands-on confirmation this works in practice.", "evidenceIds": [ "klaviyo-docs-9", "klaviyo-docs-10", "klaviyo-docs-8", "klaviyo-docs-7", "klaviyo-probe-2" ] }, { "productId": "klaviyo", "storyId": "agent-drives-campaign", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Klaviyo's API and MCP server let an agent draft/create a campaign via natural language (klaviyo-docs-7, klaviyo-docs-9, klaviyo-docs-10), which implies a campaign is created before being sent, but there's no explicit documentation describing a scheduled/queued draft state with a human-approval gate before send. Missing for 10: explicit docs on campaign draft/review states, scheduling API parameters, or an approval workflow between agent creation and send.", "evidenceIds": [ "klaviyo-docs-7", "klaviyo-docs-9", "klaviyo-docs-10" ] }, { "productId": "klaviyo", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe for llms.txt returned 404, and no evidence pack item mentions agent-oriented documentation formats aside from the unrelated MCP server docs, which serve a different purpose (data assistance, not doc discovery for agents).", "evidenceIds": [ "klaviyo-probe-1" ] }, { "productId": "klaviyo", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence only covers Klaviyo's MCP server, which lets external AI agents (Claude, Cursor) query and act on Klaviyo data—this is an external-agent integration, not native in-product AI-generated insights or suggestions surfaced to the user inside Klaviyo's own UI. No evidence pack item describes built-in analytics AI, predictive suggestions, or in-app insight generation. Missing for 10: any documentation of native AI-driven insights/suggestions surfaced within Klaviyo's own dashboard or reporting UI.", "evidenceIds": [ "klaviyo-docs-9", "klaviyo-docs-10", "klaviyo-probe-2" ] }, { "productId": "klaviyo", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Klaviyo's pricing docs confirm the platform supports building automations (flows) that presumably run autonomously once configured, and the MCP server lets an AI agent interact with Klaviyo (e.g., create campaigns) — suggesting some AI-native automation setup is possible. However, the evidence never details how automations trigger/run in the background, whether MCP can create or manage automation flows specifically, or any autonomous execution mechanics beyond a one-line mention. Missing for 10: documentation of flow/automation trigger logic and autonomous runtime behavior, evidence that MCP or another AI-native interface can configure automations (not just campaigns), and independent confirmation of autonomous background execution.", "evidenceIds": [ "klaviyo-docs-11", "klaviyo-docs-9", "klaviyo-docs-10" ] }, { "productId": "klaviyo", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows Klaviyo exposing an MCP server so external AI agents (Claude, Cursor) can act on Klaviyo data — this is the reverse of a built-in assistant inside Klaviyo itself. No documentation of an in-product AI assistant users delegate tasks to directly within the Klaviyo UI.", "evidenceIds": [ "klaviyo-docs-9", "klaviyo-docs-10", "klaviyo-probe-2" ] }, { "productId": "klaviyo", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Klaviyo exposes a full REST API secured by API keys/OAuth, with maintained SDKs (Python, PHP, Ruby, Node) and a sandbox test account for safely automating campaign creation, imports, etc. — enough to script/automate outside the UI, but there is no explicit CI/CD documentation, CLI tool, or headless-runner guidance in the evidence. Missing for 10: dedicated CI/CD integration docs, official CLI, and any explicit statement about running headlessly in automated pipelines.", "evidenceIds": [ "klaviyo-docs-1", "klaviyo-docs-3", "klaviyo-docs-4", "klaviyo-docs-5", "klaviyo-docs-6", "klaviyo-docs-7" ] }, { "productId": "klaviyo", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Klaviyo ships its own MCP server so external AI agents (Claude, Cursor) can connect to it and use Klaviyo's tools — the opposite role from the story, which asks whether Klaviyo itself can plug in and consume other MCP servers' tools. No evidence indicates Klaviyo acts as an MCP client integrating third-party MCP servers.", "evidenceIds": [ "klaviyo-docs-9", "klaviyo-probe-2" ] }, { "productId": "klaviyo", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Klaviyo documents an official MCP server that connects to MCP clients like Claude or Cursor, with example usage shown. Missing for 10: independent/hands-on third-party corroboration and details on scopes/auth specific to the MCP server itself.", "evidenceIds": [ "klaviyo-docs-9", "klaviyo-docs-10", "klaviyo-probe-2" ] }, { "productId": "klaviyo", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Klaviyo ships an official MCP server that connects to AI agents (Claude, Cursor), enabling natural-language commands like \"Create an email campaign promoting our Memorial Day sale\" to operate Klaviyo data and actions. This directly supports natural-language operation, but it's mediated through third-party MCP clients rather than a native NL interface, and there's no independent/hands-on corroboration of reliability. Missing for 10: first-party natural-language UI within Klaviyo itself, independent verification of MCP command success, breadth of supported natural-language operations beyond the single documented example.", "evidenceIds": [ "klaviyo-docs-9", "klaviyo-docs-10", "klaviyo-probe-2" ] }, { "productId": "klaviyo", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows SDKs in Python/PHP/Ruby/Node and an MCP server, but no mention of an official CLI tool anywhere in the docs; llms.txt probe 404s and no CLI reference exists.", "evidenceIds": [ "klaviyo-docs-3", "klaviyo-probe-1" ] }, { "productId": "klaviyo", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Klaviyo publishes a comprehensive public REST API with documented authentication (private API keys, OAuth for partners), scoped permissions, official SDKs in multiple languages, sandbox test accounts, and Postman collections, plus concrete endpoint docs (e.g., create campaign) showing programmatic control. Missing for 10: no evidence of an llms.txt or AI-specific machine-readable API discovery file, and no independent third-party corroboration of API robustness beyond vendor docs.", "evidenceIds": [ "klaviyo-docs-1", "klaviyo-docs-2", "klaviyo-docs-3", "klaviyo-docs-4", "klaviyo-docs-6", "klaviyo-docs-7", "klaviyo-docs-14" ] }, { "productId": "klaviyo", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Klaviyo supports private API keys with scoped permissions and explicitly frames scopes as a way to limit third-party access, which directly supports issuing least-privilege credentials for an agent; OAuth is also recommended for integrator apps. However, there's no documented workflow specifically for provisioning agent-specific credentials, no granular scope list/enumeration shown, and no evidence of key rotation, expiry, or agent-specific audit trails. missing for 10: agent-specific credential provisioning workflow, full enumerated scope list, key rotation/expiry controls, independent confirmation of scope enforcement in practice.", "evidenceIds": [ "klaviyo-docs-1", "klaviyo-docs-2", "klaviyo-docs-14" ] }, { "productId": "klaviyo", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Klaviyo maintains official SDKs in Python, PHP, Ruby, and Node, backed by API key/OAuth authentication, sandboxed test accounts, and Postman collections for development. Missing for 10: independent/third-party corroboration of SDK quality and no explicit mention of typed/async support or SDK versioning cadence.", "evidenceIds": [ "klaviyo-docs-3", "klaviyo-docs-1", "klaviyo-docs-2", "klaviyo-docs-4", "klaviyo-docs-6" ] }, { "productId": "klaviyo", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of webhooks or event subscription mechanisms—only API key auth, SDKs, sandbox testing, and an MCP server for AI agents. Missing for 10: any documentation of a webhooks API, event subscription endpoints, or push-notification mechanism.", "evidenceIds": [] }, { "productId": "klaviyo", "storyId": "ai-content-drafting", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows an MCP server for external AI agents (Claude, Cursor) to create campaigns via API, not a built-in AI assistant inside Klaviyo's own composer for drafting/refining subject lines and copy from a brief. No documentation of an in-app AI content generation feature is present.", "evidenceIds": [ "klaviyo-docs-9", "klaviyo-docs-10", "klaviyo-docs-11" ] }, { "productId": "klaviyo", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Klaviyo provides a Postman collection of pre-built API requests and a sandboxed test account for exploring the API, plus reference pages describing individual endpoints, but this relies on an external tool (Postman) rather than an in-browser interactive 'try it out' console with runnable examples. missing for 10: evidence of a native in-docs interactive API explorer/console, runnable code snippets embedded in the reference pages themselves.", "evidenceIds": [ "klaviyo-docs-6", "klaviyo-docs-4", "klaviyo-docs-5", "klaviyo-docs-7" ] }, { "productId": "klaviyo", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Klaviyo's developer docs describe a full REST API reference, SDKs, and a Postman collection, but no evidence mentions a downloadable OpenAPI/Swagger spec file or machine-readable schema; the llms.txt probe returned 404 as well. missing for 10: explicit OpenAPI/Swagger spec download link, documented schema export, or API definition file.", "evidenceIds": [ "klaviyo-docs-6", "klaviyo-probe-1" ] }, { "productId": "klaviyo", "storyId": "api-sandbox", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Klaviyo explicitly documents a test/sandbox account for testing integrations without affecting production, plus sample data population and Postman collections for exploration. missing for 10: independent/hands-on confirmation of sandbox fidelity, and explicit guidance on using the sandbox specifically with AI agents/MCP workflows.", "evidenceIds": [ "klaviyo-docs-4", "klaviyo-docs-5", "klaviyo-docs-6" ] }, { "productId": "klaviyo", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers authentication, SDKs, sandbox testing, and MCP server support, but contains no mention of API versioning scheme or a documented deprecation policy anywhere in the docs excerpts provided.", "evidenceIds": [] }, { "productId": "klaviyo", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers API authentication, SDKs, sandbox testing, single-campaign creation, and the MCP server, but contains no mention of bulk/batch endpoints, mass import/export operations, or any mechanism for operating on many items at once. Missing for 10: bulk/batch API endpoints, documented rate-limit-aware batch processing, bulk data import/export tooling.", "evidenceIds": [ "klaviyo-docs-7", "klaviyo-docs-8" ] }, { "productId": "klaviyo", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Klaviyo's pricing page mentions users can 'build automations' alongside segmentation and reports, implying event-triggered flow automation exists, but the evidence pack provides no detail on rule definition, trigger types, or conditional logic. missing for 10: documentation of flow/automation trigger configuration, conditional branching, and event-based rule examples.", "evidenceIds": [ "klaviyo-docs-11" ] }, { "productId": "klaviyo", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Klaviyo's automation platform includes 'flows'/automations (klaviyo-docs-11) and campaign creation via API (klaviyo-docs-7), but the evidence never shows an AI-native user scheduling recurring jobs via the API or MCP server—MCP docs only describe one-off campaign creation requests (klaviyo-docs-9, klaviyo-docs-10), not recurring/cron-based automation triggering. Missing for 10: API/MCP documentation of recurring schedule creation, cron-like triggers, or workflow automation exposed to AI agents specifically.", "evidenceIds": [ "klaviyo-docs-11", "klaviyo-docs-7", "klaviyo-docs-9", "klaviyo-docs-10" ] }, { "productId": "klaviyo", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of versioning, review workflows, or rollback capability for Klaviyo automations/flows; docs cover API auth, SDKs, sandbox testing, and MCP server but nothing about automation version history or revert functionality.", "evidenceIds": [] }, { "productId": "klaviyo", "storyId": "campaign-performance-reporting", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no documentation of campaign/automation analytics reporting (opens, clicks, unsubscribes, conversions) or any reporting/analytics API or dashboard feature; it only covers API auth, SDKs, testing, MCP server, and pricing. Missing for 10: any mention of a reporting/analytics dashboard, per-campaign or per-flow-step metrics, or a metrics API endpoint.", "evidenceIds": [] }, { "productId": "klaviyo", "storyId": "campaign-via-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Klaviyo's public API documents a create_campaign endpoint with API key/OAuth auth and SDKs, supporting programmatic campaign creation, but the evidence pack lacks documentation of a scheduling/send endpoint or a full end-to-end example proving a campaign can be scheduled and sent without ever using the dashboard. missing for 10: documented schedule/send-campaign endpoint, evidence of full lifecycle (create+schedule+send) purely via API, independent developer confirmation this works end-to-end.", "evidenceIds": [ "klaviyo-docs-7", "klaviyo-docs-1", "klaviyo-docs-2", "klaviyo-docs-3" ] }, { "productId": "klaviyo", "storyId": "contacts-crud-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm a flexible, schema-less API that accepts arbitrary custom data (klaviyo-docs-8), key-scoped authentication (klaviyo-docs-1, klaviyo-docs-14), and SDKs for production integration (klaviyo-docs-3), plus a sandbox for testing at scale (klaviyo-docs-4, klaviyo-docs-5). However, the pack never cites a specific contacts/profiles create-update-suppress endpoint (only campaign creation is shown), so explicit support for suppression workflows is unproven. Missing for 10: direct documentation of profile create/update endpoints, explicit suppression/unsubscribe API reference, and evidence of rate limits/production-scale throughput guarantees.", "evidenceIds": [ "klaviyo-docs-8", "klaviyo-docs-1", "klaviyo-docs-14", "klaviyo-docs-3", "klaviyo-docs-4", "klaviyo-docs-5" ] }, { "productId": "klaviyo", "storyId": "create-send-campaign", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs show a Create Campaign API, a drag-and-drop email editor, and built-in segmentation/automation, and even AI-assisted campaign creation via the MCP server (e.g., 'Create an email campaign promoting our Memorial Day sale'), covering the compose+audience+send loop. Missing for 10: explicit documentation of the schedule/send UI step and audience-selection mechanics beyond generic 'segmentation' mention.", "evidenceIds": [ "klaviyo-docs-7", "klaviyo-docs-11", "klaviyo-docs-10", "klaviyo-docs-9" ] }, { "productId": "klaviyo", "storyId": "custom-sending-domain", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items address sending-domain authentication (SPF/DKIM/DMARC) or guided DNS setup/verification; all evidence covers API keys, OAuth, SDKs, MCP server, and pricing tiers, which are unrelated to this axis.", "evidenceIds": [] }, { "productId": "klaviyo", "storyId": "deliverability-visibility", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses bounce, complaint, or blocklist deliverability signals, per-send reporting, or actionable guidance for deliverability degradation; the pack only covers API auth, SDKs, MCP server, and pricing. Missing for 10: any documentation of deliverability/bounce/complaint/blocklist metrics, per-send reporting, or guidance features.", "evidenceIds": [] }, { "productId": "klaviyo", "storyId": "email-template-editor", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence confirms Klaviyo has a drag-and-drop email editor and campaign creation API, supporting the visual design part of the story, but there is no mention of reusable template libraries, personalization token syntax, or cross-email-client rendering testing/previews. missing for 10: reusable template library details, personalization token documentation, cross-client rendering/testing evidence.", "evidenceIds": [ "klaviyo-docs-11", "klaviyo-docs-7" ] }, { "productId": "klaviyo", "storyId": "engagement-data-api-read", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items reference an events/metrics API, per-profile activity/timeline endpoint, or any way to retrieve sent/opened/clicked history for a contact — only general API overview, SDKs, campaign creation, and MCP server docs are cited.", "evidenceIds": [] }, { "productId": "klaviyo", "storyId": "event-triggered-flows", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Klaviyo's docs mention flexible, schema-less data ingestion ('arbitrary data') which implies custom event tracking, but the evidence pack lacks any explicit mention of a track-event API or of flows/automations being triggered by specific custom events for an individual profile. Missing for 10: explicit custom event API documentation, explicit flow-trigger-on-event behavior, and confirmation that automations target the specific triggering profile.", "evidenceIds": [ "klaviyo-docs-8", "klaviyo-docs-1" ] }, { "productId": "klaviyo", "storyId": "flow-state-inspection", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence describes flow-level visibility into where a profile sits within an automation, nor any capability to manually pull or re-enter a person mid-flow without breaking it; evidence only covers API auth, SDKs, sandbox testing, campaign creation, and MCP server setup. Missing for 10: automation flow status view per profile, pull/remove from flow controls, re-enter flow functionality, and any documentation of flow integrity safeguards.", "evidenceIds": [] }, { "productId": "klaviyo", "storyId": "full-list-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers API authentication, SDKs, sandbox testing, MCP integration, and pricing tiers, but contains no documentation of a bulk/self-serve export feature covering contacts, custom fields, tags, engagement history, and suppression lists in an open format. Missing for 10: explicit export/download documentation, confirmation of open format (CSV/JSON) output, and evidence suppression list and engagement history are included in any export.", "evidenceIds": [ "klaviyo-docs-8", "klaviyo-docs-12" ] }, { "productId": "klaviyo", "storyId": "guided-platform-migration", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence pack only covers API scopes, SDKs, sandbox testing, MCP server, and pricing tiers—no mention of a guided migration tool, list import wizard, or preservation of tags/custom fields/unsubscribe state from another ESP. Klaviyo docs mention generic 'import or sync your contacts' but nothing about migration-specific data preservation guarantees.", "evidenceIds": [ "klaviyo-docs-12" ] }, { "productId": "klaviyo", "storyId": "metrics-via-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows general API access, authentication, and SDKs, plus a campaign-creation endpoint, but no documentation of any reporting/analytics or metrics endpoints for campaign or flow performance data. Missing for 10: evidence of a Campaigns/Flows reporting or metrics API endpoint, sample response schemas for performance data, or any mention of pulling analytics into a warehouse/BI tool.", "evidenceIds": [ "klaviyo-docs-7", "klaviyo-docs-1", "klaviyo-docs-3" ] }, { "productId": "klaviyo", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Klaviyo's API supports many core objects (campaigns, contacts, arbitrary data) and even an MCP server for agent-driven actions, but the docs describe UI-only features like drag-and-drop email/SMS editors and visual automation builders that aren't shown to have API equivalents. Missing for 10: explicit evidence that visual template/automation building, reporting dashboards, and other UI-exclusive workflows are fully exposed via API, and independent confirmation of full UI/API parity.", "evidenceIds": [ "klaviyo-docs-7", "klaviyo-docs-8", "klaviyo-docs-9", "klaviyo-docs-11" ] }, { "productId": "klaviyo", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Klaviyo has APIs to read data programmatically, but there's no evidence of a bulk data export tool/feature in open formats, nor any documented data-portability or account-export process for users to leave with their data. missing for 10: documented bulk export feature, open-format (e.g., CSV/JSON) full-account export, data portability/exit policy.", "evidenceIds": [] }, { "productId": "klaviyo", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Klaviyo is a closed-source SaaS marketing platform, not an open-source project; source-availability/licensing is not a fair axis for this kind of product.", "evidenceIds": [] }, { "productId": "klaviyo", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Klaviyo is a SaaS marketing platform with no self-hosted/on-premise deployment option; self-hosting is a category error for this cloud-only product.", "evidenceIds": [] }, { "productId": "klaviyo", "storyId": "outbound-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack mentions webhooks, event subscriptions, or signed payloads for delivery/engagement events; the docs cover API auth, SDKs, MCP server, and pricing but nothing about webhook subscriptions or signature verification. missing for 10: webhook subscription API/docs, list of supported event types (sends, opens, clicks, bounces, unsubscribes), payload signing/verification mechanism.", "evidenceIds": [] }, { "productId": "klaviyo", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack mentions data residency, regional storage options, or ability to choose where data is hosted; the docs focus on API scopes, SDKs, and MCP server, not data location controls. missing for 10: any mention of regional data centers, residency options, or data storage location controls.", "evidenceIds": [] }, { "productId": "klaviyo", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses AI-training data usage policies, opt-outs, or any statement about whether customer data is used to train AI/ML models. missing for 10: explicit AI-training data policy, opt-out mechanism, any privacy/DPA statement about model training.", "evidenceIds": [] }, { "productId": "klaviyo", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack items address data retention policies, deletion controls, or export/erasure mechanisms; only API scoping and general auth/testing docs are provided, which don't speak to retention/deletion controls.", "evidenceIds": [] }, { "productId": "klaviyo", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack addresses telemetry/usage-tracking opt-out settings for Klaviyo's own tools (e.g., MCP server or SDKs); the docs focus on API auth, scopes, and testing sandboxes, not telemetry controls.", "evidenceIds": [] }, { "productId": "klaviyo", "storyId": "revenue-attribution", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses revenue/signup attribution to specific emails or flows, conversion tracking, or attribution reporting features; documentation only covers API auth, SDKs, MCP server, and pricing tiers.", "evidenceIds": [] }, { "productId": "klaviyo", "storyId": "scoped-api-keys", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Klaviyo docs confirm private API keys support scopes to limit third-party access (e.g., read-only vs. send permissions), directly matching the ops need to mint restricted credentials for agents/contractors. However, there's no evidence detailing granular scope options (e.g., specific read-only analytics scope vs. campaign-send scope), no UI walkthrough for creating such a key, and no independent/hands-on confirmation of enforcement. missing for 10: granular scope list/UI screenshots, independent verification that scopes actually block send actions, documentation of a dedicated read-only analytics role.", "evidenceIds": [ "klaviyo-docs-1", "klaviyo-docs-14" ] }, { "productId": "klaviyo", "storyId": "segment-builder", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Klaviyo's pricing page confirms segmentation as a core feature alongside automations and reports, but the evidence pack contains no documentation of how segments are built from profile attributes/tags/behavioral events or how they stay dynamically live as data changes. missing for 10: technical docs on segment definition logic, dynamic/live-updating behavior, and support for tags and behavioral event conditions.", "evidenceIds": [ "klaviyo-docs-11" ] }, { "productId": "klaviyo", "storyId": "segments-via-api", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers general API authentication, SDKs, sandbox testing, campaign creation, and MCP server integration, but contains no documentation of specific endpoints for creating/updating segments, tags, or lists, nor for reading back their membership. Missing for 10: any reference to Lists/Segments/Tags API endpoints, membership read endpoints, or examples of programmatic segment management.", "evidenceIds": [ "klaviyo-docs-1", "klaviyo-docs-7", "klaviyo-docs-8" ] }, { "productId": "klaviyo", "storyId": "stack-integrations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack only shows generic API/OAuth/SDK capabilities and a flexible schema, but contains no mention of maintained first-party integrations with Shopify, Stripe, Segment, or a product database — the specific connectors the story asks about.", "evidenceIds": [] }, { "productId": "klaviyo", "storyId": "suppression-compliance", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "None of the evidence items address suppression handling, unsubscribe/bounce/spam-complaint processing, or CAN-SPAM/GDPR compliance mechanics; the pack is limited to API auth, SDKs, MCP server, and pricing. missing for 10: documentation of automatic suppression lists, bounce/complaint handling, cross-channel enforcement across email/SMS sends, and compliance default behavior.", "evidenceIds": [] }, { "productId": "klaviyo", "storyId": "templates-as-code", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows a campaign-creation API and drag-and-drop editors, but nothing about managing email templates as HTML/MJML code, API-managed templates with variable substitution, or version-control workflows for template content.", "evidenceIds": [ "klaviyo-docs-7", "klaviyo-docs-11" ] }, { "productId": "klaviyo", "storyId": "transactional-send-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Klaviyo's API centers on campaigns, flows, SDKs, and MCP integration for marketing sends, but nothing documents a one-to-one transactional email API (e.g., for receipts or magic links) alongside marketing sends.", "evidenceIds": [ "klaviyo-docs-1", "klaviyo-docs-7", "klaviyo-docs-8" ] }, { "productId": "klaviyo", "storyId": "transactional-separation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Klaviyo is fundamentally a marketing/email-marketing platform; the evidence pack shows only marketing campaign creation, API scopes, SDKs, and an MCP server — nothing about a distinct transactional-sending pipeline, dedicated transactional templates, separate suppression logic, or isolated sending reputation that bypasses marketing unsubscribes. No mention of transactional email as a concept at all.", "evidenceIds": [] }, { "productId": "klaviyo", "storyId": "visual-automation-builder", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence only vaguely mentions building 'automations' via drag-and-drop editors, without any detail on branches, delays, exit rules, or lifecycle templates like welcome/onboarding/winback series. Klaviyo is widely known for this capability but the provided evidence pack does not document it in depth. missing for 10: documentation of branching logic, time-delay steps, exit/entry conditions, and specific lifecycle flow templates (welcome, winback, onboarding).", "evidenceIds": [ "klaviyo-docs-11" ] }, { "productId": "loops", "storyId": "ab-test-campaigns", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions A/B testing of subject lines or content, nor any automated winner-selection based on engagement metrics; docs cover campaigns, transactional email, contacts, and API/MCP integration but not split-testing functionality.", "evidenceIds": [] }, { "productId": "loops", "storyId": "agent-audits-program", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Loops exposes API/MCP access to contacts (audiences), events (automation triggers), and campaign content creation, and dashboard analytics on which emails drive upgrades/churn exist, but there's no documented endpoint or MCP capability for pulling campaign performance/results data programmatically — the 'report on what's working' analytics piece is not evidenced as API-accessible. missing for 10: an API/MCP endpoint for campaign performance metrics, evidence of reading analytics/results data programmatically rather than just creating content or contacts.", "evidenceIds": [ "loops-docs-3", "loops-docs-4", "loops-docs-5", "loops-docs-6", "loops-docs-16", "loops-probe-3", "loops-probe-4" ] }, { "productId": "loops", "storyId": "agent-builds-segment", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While Loops exposes API endpoints for contacts, events, and campaign content (loops-docs-3, loops-docs-4, loops-docs-6), and mentions segments only as a dashboard feature that stays dynamic (loops-docs-16), there is no documented endpoint or MCP tool for creating or defining an audience segment programmatically from a natural-language description. No evidence shows an agent can translate plain-language criteria into a real segment via the API or MCP surface.", "evidenceIds": [ "loops-docs-3", "loops-docs-4", "loops-docs-6", "loops-docs-16" ] }, { "productId": "loops", "storyId": "agent-drives-campaign", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Loops explicitly supports agents creating draft campaigns via API that are then reviewed/sent from the dashboard (loops-docs-5, loops-docs-9), and has API/MCP access for contacts, events, and content (loops-docs-3,4,6). However, there's no explicit evidence of segment targeting being part of the drafted campaign creation flow, nor of a 'scheduled/queued' state distinct from 'draft awaiting dashboard review' — the described flow is draft-then-manual-send rather than draft-then-schedule-with-approval-gate. missing for 10: explicit mention of segment-targeted campaign creation via API, a distinct 'scheduled pending approval' state vs plain draft, and independent/hands-on confirmation of this human-approval workflow.", "evidenceIds": [ "loops-docs-5", "loops-docs-9", "loops-docs-3", "loops-docs-4", "loops-docs-6" ] }, { "productId": "loops", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Loops has a live, probe-confirmed llms.txt (loops-probe-1) with dedicated agent-oriented docs at /agents and /agents/api, plus an OpenAPI spec (loops-probe-3) and MCP server for agent connectivity (loops-docs-6, loops-probe-4). Missing for 10: independent hands-on confirmation that an agent successfully consumes llms.txt end-to-end rather than just server availability.", "evidenceIds": [ "loops-probe-1", "loops-docs-9", "loops-docs-6", "loops-probe-3", "loops-probe-4" ] }, { "productId": "loops", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Loops documents analytics/segmentation (loops-docs-16), campaign drafting via API, and an email 'Guardian' check (loops-docs-19), but none of this evidence describes AI-generated insights or suggestions derived from a customer's data (e.g., predictive recommendations, anomaly detection, or AI-surfaced trends). The agent/API/MCP features are about enabling external agents to act on Loops, not the product itself surfacing AI insights.", "evidenceIds": [ "loops-docs-16", "loops-docs-19", "loops-docs-6" ] }, { "productId": "loops", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 6, "confidence": "medium", "rationale": "Loops supports event-triggered workflows (POST /events/send triggers workflows) and lifecycle automations (onboarding, activation, upgrade, winback emails) that are 'set up once' and run autonomously over time, which is the core of background automation for an email/lifecycle platform. Missing for 10: detailed documentation of workflow logic (branching, delays, conditions), and independent/hands-on corroboration that these automations reliably run unattended long-term.", "evidenceIds": [ "loops-docs-3", "loops-docs-15", "loops-docs-16" ] }, { "productId": "loops", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Loops' 'agents' materials describe APIs/MCP/CLI/skills that let external coding agents (Claude, Cursor, etc.) act on Loops on the user's behalf — this is Loops being consumed by an agent, not a built-in AI assistant living inside the Loops product that a user delegates tasks to. No evidence of an in-app assistant (e.g., an AI chat/copilot embedded in the Loops dashboard) exists in the pack.", "evidenceIds": [ "loops-docs-6", "loops-docs-10", "loops-docs-14", "loops-probe-4" ] }, { "productId": "loops", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Loops exposes a REST API with Bearer-token auth, an OpenAPI spec, and a CLI for terminal-based contact/event/email management, which supports scripted/CI usage. However, evidence doesn't explicitly confirm a headless automation workflow (e.g., CI pipeline examples, non-interactive auth for automation, or documented CI recipes) beyond API/CLI existence. Missing for 10: explicit CI/CD examples or automation-pipeline documentation, evidence of non-interactive/service-account auth flow, and independent confirmation of headless use in production automation.", "evidenceIds": [ "loops-docs-1", "loops-docs-3", "loops-docs-4", "loops-docs-7", "loops-probe-3" ] }, { "productId": "loops", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Loops is an email/marketing SaaS platform, not an AI agent — the evidence shows Loops publishing its own MCP server so other agents can call ITS tools (loops-docs-6, loops-probe-4), which is the server role, not Loops acting as a client that plugs in external MCP servers to gain their tools. There is no agentic/assistant capability within Loops that could consume external MCP servers, so this axis is a category error for this product.", "evidenceIds": [ "loops-docs-6", "loops-probe-4" ] }, { "productId": "loops", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Loops documents an official MCP server ('Connect your agent to Loops over MCP. Read and write contacts, events, and content from any client.') with a dedicated docs page, backed by a first-party API and CLI/skills for agent integration. Missing for 10: independent/hands-on verification of the MCP server working end-to-end and more detail on setup/auth for MCP specifically.", "evidenceIds": [ "loops-docs-6", "loops-probe-4", "loops-docs-3", "loops-docs-4" ] }, { "productId": "loops", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Loops provides an MCP server and CLI/skills that let an AI agent operate Loops via natural-language-driven tool calls (create contacts, send events, draft campaigns, run Guardian checks) rather than requiring manual API scripting, which supports agentic/natural-language operation. However, the evidence is all vendor docs describing agent-facing surfaces (MCP, CLI, API) rather than a chat/NL command interface for end-users themselves, and there's no independent/hands-on confirmation of how well natural-language control actually works. Missing for 10: independent verification of natural-language command execution, evidence of an end-user-facing NL/chat interface (not just agent APIs/MCP), and confirmation of reliability/quality of agent-driven actions in practice.", "evidenceIds": [ "loops-docs-6", "loops-docs-10", "loops-docs-19", "loops-probe-4", "loops-docs-3", "loops-docs-4" ] }, { "productId": "loops", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Loops documents an official CLI ('Manage contacts, send events, and test transactional email from your terminal' and 'Install the CLI and skills with one command so your coding agent can use Loops for you'), explicitly positioned for AI-native/agentic workflows. Missing for 10: independent hands-on verification of the CLI's functionality and no detailed command reference in the evidence pack.", "evidenceIds": [ "loops-docs-7", "loops-docs-10" ] }, { "productId": "loops", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Loops provides a documented public REST API with authentication (Bearer token), endpoints for events, contacts, campaigns, and email sending, plus an OpenAPI spec (confirmed live at /openapi.json) and SDKs/CLI for programmatic access. This clearly enables an AI-native user to drive the product entirely via API. Missing for 10: independent third-party developer corroboration of API robustness/versioning in practice beyond founder claims.", "evidenceIds": [ "loops-docs-1", "loops-docs-3", "loops-docs-4", "loops-docs-8", "loops-probe-3", "loops-docs-9" ] }, { "productId": "loops", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows a single Bearer-token API key auth model with no mention of scoped, restricted-permission, or per-agent credentials — only a full-access API key pattern is documented across docs, MCP, and CLI. No evidence of least-privilege or scoped token issuance.", "evidenceIds": [ "loops-docs-1", "loops-docs-6", "loops-probe-3" ] }, { "productId": "loops", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Loops documents official SDKs plus a REST API with Bearer auth, an OpenAPI spec, and a CLI/skills installer for coding agents, indicating strong AI-native SDK support beyond just docs. missing for 10: independent hands-on developer confirmation of SDK reliability/coverage across languages, and no explicit list of supported languages beyond generic 'SDKs' mention.", "evidenceIds": [ "loops-docs-1", "loops-docs-8", "loops-docs-10", "loops-probe-3" ] }, { "productId": "loops", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Loops lets users send events INTO Loops (POST /events/send) and use APIs/MCP/CLI to manage contacts and campaigns, but there is no mention of Loops emitting outbound webhooks that a user's system could subscribe to for platform events (e.g., opens, bounces, unsubscribes). Missing for 10: any documentation of a webhook subscription/callback mechanism, webhook payload schema, or webhook management endpoint.", "evidenceIds": [ "loops-docs-3", "loops-docs-4", "loops-docs-6", "loops-probe-3" ] }, { "productId": "loops", "storyId": "ai-content-drafting", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows API-driven draft campaign creation, agent/MCP integrations, and a text-based editor, but there is no mention of a built-in AI feature inside the composer that drafts or refines subject lines/email copy from a brief — the 'AI drafting' capability described is done by external coding agents via API/MCP, not a native in-app AI assistant.", "evidenceIds": [ "loops-docs-5", "loops-docs-6", "loops-docs-9", "loops-docs-14", "loops-docs-18" ] }, { "productId": "loops", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Loops documents API endpoints (contacts, events, Guardian checks) and exposes a valid openapi.json spec, suggesting a structured API reference exists, but there's no evidence of an interactive 'try it now' console or runnable code examples within the docs themselves. Missing for 10: explicit interactive/try-it API console, runnable code snippets in docs, independent confirmation of the reference UI's interactivity.", "evidenceIds": [ "loops-docs-1", "loops-docs-3", "loops-docs-4", "loops-docs-19", "loops-probe-3" ] }, { "productId": "loops", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A probe confirms a valid OpenAPI spec is served at https://loops.so/openapi.json (contains an 'openapi' key, HTTP 200), and this is complemented by full API reference docs. Missing for 10: independent third-party confirmation that the spec is complete/versioned or widely used by external tooling.", "evidenceIds": [ "loops-probe-3", "loops-docs-1" ] }, { "productId": "loops", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of a sandbox/test environment, test API keys, or staging mode separate from production data; docs cover API auth, SDKs, MCP, and migration guides but nothing about sandboxing. Missing for 10: dedicated sandbox/test-mode environment, test API keys or data isolation guarantees, documentation of a non-production testing workflow.", "evidenceIds": [] }, { "productId": "loops", "storyId": "api-versioning-policy", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "A founder comment mentions the API has versioning ('performant API with versioning for transactional email') and an OpenAPI spec is exposed, but there is no documented deprecation policy, version lifecycle, or migration guidance anywhere in the evidence. Missing for 10: documented deprecation/sunset policy, versioning scheme details, migration guides between API versions.", "evidenceIds": [ "loops-comm-1", "loops-probe-3" ] }, { "productId": "loops", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only single-item API operations (create one contact, send one event, send one transactional email) and no documented bulk/batch endpoints, CSV import, or bulk update/segment operations for AI agents to invoke at scale.", "evidenceIds": [ "loops-docs-3", "loops-docs-4", "loops-docs-2", "loops-probe-3" ] }, { "productId": "loops", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Loops' event API explicitly triggers workflows ('Send an event to trigger workflows') and marketing copy references onboarding/activation/winback flows set up once and running automatically, matching the rule-on-event automation pattern. However, the evidence pack lacks detail on how rules/conditions are authored (branching logic, if/then rules, workflow builder specifics) beyond the event-trigger primitive. Missing for 10: documentation of conditional rule logic/branching in workflows, independent hands-on confirmation of workflow automation depth.", "evidenceIds": [ "loops-docs-3", "loops-docs-15", "loops-docs-4" ] }, { "productId": "loops", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Loops supports event-triggered workflows (POST /events/send) and lifecycle emails (onboarding, winback, etc.) that run automatically once set up, which touches on workflow automation, but there is no evidence of an API or agent-facing capability to schedule recurring jobs/crons directly — workflow triggers are event-based and configured largely via the dashboard. missing for 10: explicit recurring/cron scheduling API, agent-driven workflow creation beyond event triggers, independent confirmation of automation depth.", "evidenceIds": [ "loops-docs-3", "loops-docs-15", "loops-docs-6" ] }, { "productId": "loops", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Loops evidence covers email campaign creation, API/MCP integration, and content authoring, but there is no mention of versioning, reviewing changes, or rolling back automations/workflows — the closest is draft campaigns reviewed before sending, which is not version history or rollback.", "evidenceIds": [] }, { "productId": "loops", "storyId": "campaign-performance-reporting", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers Loops' API, SDKs, CLI, MCP, and migration tooling, plus a generic mention of segmentation revenue impact, but there is no documentation of an analytics dashboard or API reporting opens, clicks, unsubscribes, and conversions per campaign or per automation step over time. Missing for 10: any analytics/reporting endpoint or dashboard docs showing per-campaign/per-step metrics, time-series breakdowns, or conversion attribution.", "evidenceIds": [ "loops-docs-16" ] }, { "productId": "loops", "storyId": "campaign-via-api", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Loops documents a Content API that lets developers create draft campaigns via API, but its own docs explicitly state that review and sending happen from the dashboard ('Create draft campaigns... Review and send from the dashboard'), directly contradicting the 'without touching the dashboard' requirement for sending/scheduling. Missing for 10: documented API endpoints for scheduling/sending a campaign, evidence of full campaign lifecycle (create+schedule+send) without dashboard interaction, and independent confirmation of this workflow.", "evidenceIds": [ "loops-docs-5", "loops-docs-9", "loops-docs-6" ] }, { "productId": "loops", "storyId": "contacts-crud-api", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Evidence confirms a REST API with bearer-token auth and a POST /contacts/create endpoint plus an OpenAPI spec, and dynamic segment properties are mentioned, implying custom contact properties exist. However, there is no explicit documentation of contact update or suppression endpoints, nor any mention of rate limits, bulk operations, or production-scale guarantees. Missing for 10: explicit contacts/update and suppression API endpoints, documented custom-property schema support, and evidence of production-scale throughput/rate limits.", "evidenceIds": [ "loops-docs-1", "loops-docs-4", "loops-docs-16", "loops-probe-3" ] }, { "productId": "loops", "storyId": "create-send-campaign", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs describe composing campaigns via editor or Content API, segmenting/targeting dynamic audiences, and sending from the dashboard, covering the core compose→audience→send loop (loops-docs-5, loops-docs-9, loops-docs-16, loops-docs-17, loops-docs-18). Missing for 10: explicit documentation of a 'schedule for later' feature and independent/hands-on confirmation of the end-to-end campaign flow (community evidence only covers pricing/editor tech, not campaign sending itself).", "evidenceIds": [ "loops-docs-5", "loops-docs-9", "loops-docs-16", "loops-docs-17", "loops-docs-18" ] }, { "productId": "loops", "storyId": "custom-sending-domain", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack describes domain authentication setup (SPF/DKIM/DMARC), DNS record guidance, or verification tooling; one migration doc merely mentions 'verify your domain' as a step without detail. missing for 10: DNS setup guide, SPF/DKIM/DMARC record instructions, verification status UI or API, any deliverability-domain documentation.", "evidenceIds": [ "loops-docs-11" ] }, { "productId": "loops", "storyId": "deliverability-visibility", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack has no mention of bounce, complaint, or blocklist tracking per send, nor any deliverability health dashboard or actionable guidance when deliverability degrades; the closest item ('Guardian checks on an email message') is vague and not tied to bounce/complaint/blocklist metrics. Missing for 10: per-send bounce/complaint/blocklist reporting, deliverability health dashboard, automated alerts or actionable remediation guidance.", "evidenceIds": [ "loops-docs-19" ] }, { "productId": "loops", "storyId": "email-template-editor", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Loops documents an email editor, brand-consistency settings (colors/type/spacing applied automatically), dynamic personalization variables, and reusable content built via templates/Content API (loops-docs-17, loops-docs-18, loops-docs-2, loops-docs-9, loops-docs-14). Community evidence confirms the editor is a customizable, text-based system built on extended MJML (loops-comm-7), which supports templating but is not a classic drag-and-drop visual builder, and no evidence addresses cross-client rendering testing or guarantees. missing for 10: explicit cross-email-client rendering validation, confirmation of a fully visual (non-code) editor experience, independent hands-on review of template reuse in practice.", "evidenceIds": [ "loops-docs-17", "loops-docs-18", "loops-docs-2", "loops-docs-9", "loops-docs-14", "loops-comm-7" ] }, { "productId": "loops", "storyId": "engagement-data-api-read", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents contact creation, event sending, campaign drafting, and transactional email APIs, but nowhere shows an endpoint or documentation for retrieving per-contact engagement history (sends, opens, clicks) via the API. This is a fair capability to expect from an email marketing platform, but no evidence confirms it exists.", "evidenceIds": [ "loops-docs-3", "loops-docs-4", "loops-docs-19" ] }, { "productId": "loops", "storyId": "event-triggered-flows", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Loops' API includes a documented POST /events/send endpoint explicitly for triggering workflows per contact, alongside contact identification via API key/email, matching the story of firing a custom event to trigger a targeted automation. This is corroborated by an OpenAPI spec and SDK/docs references, though there's no independent hands-on report specifically testing event-triggered automation reliability. missing for 10: independent/hands-on verification of event-to-automation triggering, more detail on per-contact targeting semantics.", "evidenceIds": [ "loops-docs-3", "loops-docs-8", "loops-probe-3", "loops-docs-7" ] }, { "productId": "loops", "storyId": "flow-state-inspection", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows Loops can trigger workflows via events and manage contacts via API, but there is no documentation of viewing an individual contact's position within a live automation/workflow or safely pulling/re-entering them without disrupting the flow. Missing for 10: workflow visualization UI, per-contact automation status view, documented pull/re-enter mechanics.", "evidenceIds": [ "loops-docs-3", "loops-docs-4" ] }, { "productId": "loops", "storyId": "full-list-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows API endpoints for creating contacts and sending events, migration guides for moving data INTO Loops from other providers, and CLI/SDK/MCP tooling, but no documented export endpoint, bulk-export feature, or self-serve mechanism to pull contacts, tags, engagement history, or suppression lists OUT of Loops in an open format. Missing for 10: a documented export/download API or dashboard feature, evidence of tags/engagement/suppression data being included in any export, and confirmation of an open file format (CSV/JSON) for full audience export.", "evidenceIds": [ "loops-docs-4", "loops-docs-11", "loops-docs-12", "loops-docs-13" ] }, { "productId": "loops", "storyId": "guided-platform-migration", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Loops publishes migration guides for moving contacts, lists, events, and transactional email from platforms like Brevo and Customer.io with 'before and after code and a cutover checklist,' indicating some list-portability support, but these are developer-facing API migration guides rather than a marketer-friendly guided import wizard. No evidence explicitly confirms preservation of tags, custom fields, or unsubscribe state during import. Missing for 10: explicit UI-based guided import flow for marketers, confirmation that tags/custom fields/unsubscribe status are preserved, and independent evidence of successful migrations.", "evidenceIds": [ "loops-docs-11", "loops-docs-12", "loops-docs-13" ] }, { "productId": "loops", "storyId": "metrics-via-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers API endpoints for creating contacts, sending events, drafting campaign content, and MCP integration, but nothing documents an endpoint for retrieving campaign or flow performance metrics (opens, clicks, conversions, revenue) via API for warehousing or BI use. The analytics mentioned (e.g., 'see which emails lead to upgrades') appear to be dashboard-only features, not exposed via API.", "evidenceIds": [ "loops-docs-3", "loops-docs-4", "loops-docs-5", "loops-docs-16", "loops-probe-3" ] }, { "productId": "loops", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Loops exposes a broad API (contacts, events, transactional email, content/campaign drafting, Guardian checks) plus an OpenAPI spec, CLI, and MCP server, letting AI-native users perform most workflows programmatically (loops-docs-3, loops-docs-4, loops-docs-5, loops-docs-19, loops-probe-3, loops-probe-4). However, campaign creation explicitly notes 'Review and send from the dashboard,' meaning the final send step is UI-only, and analytics/segmentation dashboards have no documented API equivalent. Missing for 10: API parity for sending/publishing campaigns, API access to analytics/segments, independent confirmation of full UI-API parity.", "evidenceIds": [ "loops-docs-3", "loops-docs-4", "loops-docs-5", "loops-docs-6", "loops-docs-19", "loops-probe-3", "loops-probe-4" ] }, { "productId": "loops", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows API endpoints for creating/writing contacts and events, migration guides for moving data INTO Loops from other providers, and an OpenAPI spec, but nothing documents a bulk export feature or open-format data portability for users wanting to leave Loops with their data.", "evidenceIds": [] }, { "productId": "loops", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Loops is a closed-source SaaS email platform; no evidence of an open-source license or public source code repository is provided anywhere in the evidence pack.", "evidenceIds": [] }, { "productId": "loops", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Loops is a hosted SaaS email platform; nothing in the evidence suggests a self-hostable core product, and this is inherently a cloud service (with SES/SES-like MTA infra) rather than software distributed for self-hosting — this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "loops", "storyId": "outbound-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions webhooks for delivery/engagement events (sends, opens, clicks, bounces, unsubscribes) or signed payloads; the docs focus on REST API, MCP, CLI, and events for triggering workflows, not outbound webhook subscriptions. Missing for 10: any webhook subscription endpoint, event types list, and payload signing/verification documentation.", "evidenceIds": [] }, { "productId": "loops", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, regional storage options, or compliance controls letting users choose where data is stored; Loops' documentation focuses on email API, agents, and pricing, not data residency.", "evidenceIds": [] }, { "productId": "loops", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses AI-training data usage, opt-outs, or privacy commitments regarding model training; Loops' materials focus on email/marketing API features, not data-privacy policy toward AI training. Missing for 10: any privacy policy statement, opt-out mechanism, or documentation about AI training data usage.", "evidenceIds": [] }, { "productId": "loops", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items address data retention policies, deletion controls, data export, or GDPR/CCPA-style deletion APIs for contacts/events. The docs focus on sending email, API/SDK usage, and migration, with nothing on how a user or agent controls retention or deletion of stored data. Missing for 10: documented data retention policy, contact/event deletion API or endpoint, data export/erasure workflow, and any privacy compliance certification.", "evidenceIds": [] }, { "productId": "loops", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions telemetry, analytics opt-out, or usage tracking controls for Loops as a product/tool; the evidence focuses on email API/agent features and pricing/UX community feedback. missing for 10: any documentation of telemetry collection or an opt-out mechanism, privacy settings, or data-collection disclosure.", "evidenceIds": [] }, { "productId": "loops", "storyId": "revenue-attribution", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Loops explicitly markets seeing 'which emails lead to upgrades, lower churn, and add revenue' with dynamic segments, suggesting some revenue/conversion attribution beyond opens/clicks, but this is a single marketing-page claim with no detail on methodology, dashboards, or per-flow attribution granularity, and no independent or hands-on corroboration exists in the evidence pack. Missing for 10: detailed docs on how revenue/signup attribution is computed, screenshots or hands-on reports of attribution reporting, and third-party validation.", "evidenceIds": [ "loops-docs-16" ] }, { "productId": "loops", "storyId": "scoped-api-keys", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence shows only a single Bearer-token API key auth model with no mention of scoped, read-only, or per-agent/contractor credentials; nothing indicates permission tiers or restricted-scope keys.", "evidenceIds": [ "loops-docs-1" ] }, { "productId": "loops", "storyId": "segment-builder", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Loops explicitly claims segments are built from profile properties and stay dynamic as data changes (loops-docs-16), and the API supports contacts with properties and events that trigger workflows (loops-docs-3, loops-docs-4), suggesting behavioral/event-based segmentation is possible. However, there's no explicit documentation of tag-based segmentation or a dedicated segment-builder UI walkthrough, and no independent/hands-on confirmation of live segment updates. Missing for 10: explicit tag-attribute mention, segment-builder UI documentation, and independent corroboration of segments staying live.", "evidenceIds": [ "loops-docs-16", "loops-docs-3", "loops-docs-4" ] }, { "productId": "loops", "storyId": "segments-via-api", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Loops' API supports creating/updating contacts and triggering events (loops-docs-4, loops-docs-3), and segments are described as dynamic (loops-docs-16), but no evidence shows an endpoint to read back which contacts belong to a segment, tag, or list. Missing for 10: documented GET/list endpoints for contacts by segment/tag/list, and confirmation that programmatic tag/list membership queries are supported.", "evidenceIds": [ "loops-docs-4", "loops-docs-3", "loops-docs-16", "loops-probe-3" ] }, { "productId": "loops", "storyId": "stack-integrations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows generic API/SDK/events/contacts endpoints and migration guides from competing ESPs (Brevo, Customer.io) — there is no mention of maintained first-party integrations with Stripe, Shopify, Segment, or a product database. Ops users would need to build custom API/webhook connections themselves rather than use a documented native integration.", "evidenceIds": [] }, { "productId": "loops", "storyId": "suppression-compliance", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items describe automatic suppression of unsubscribes, bounces, or spam complaints, nor honoring suppression lists across transactional and marketing sends; the community thread even raises a concern about pricing encouraging unlimited sends without addressing suppression handling. Missing for 10: documentation of automatic unsubscribe/bounce/complaint suppression, cross-send-type enforcement, and CAN-SPAM/GDPR compliance details.", "evidenceIds": [ "loops-comm-5" ] }, { "productId": "loops", "storyId": "templates-as-code", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Loops offers a Content API to author campaigns in code (loops-docs-9), transactional emails with dynamic data variables (loops-docs-2), and an editor built on extended MJML (loops-comm-7), which together enable API-managed templates with variable substitution that a developer could store in their own repo. However, there's no first-party feature for native version-control integration, template diffing, or a documented HTML/MJML file-based workflow — the API/editor split still centers editing in the dashboard. Missing for 10: explicit git/version-control integration or CLI-based template push/pull workflow, and independent confirmation that Content API templates round-trip cleanly with raw MJML/HTML files.", "evidenceIds": [ "loops-docs-9", "loops-docs-2", "loops-comm-7", "loops-docs-5", "loops-docs-14" ] }, { "productId": "loops", "storyId": "transactional-send-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Loops explicitly documents sending transactional emails with dynamic data via API/SDK alongside marketing/lifecycle sends from the same platform, with an OpenAPI spec and events API to trigger workflows, and this is corroborated by community mention of a 'performant API with versioning for transactional email.' Missing for 10: no hands-on developer report of using the transactional send endpoint specifically (e.g. receipts/magic links) beyond docs framing, and no independent deliverability benchmark for transactional use.", "evidenceIds": [ "loops-docs-2", "loops-docs-3", "loops-docs-13", "loops-probe-1", "loops-probe-3", "loops-comm-1" ] }, { "productId": "loops", "storyId": "transactional-separation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs confirm Loops offers distinct transactional email sending (its own API/backend calls) separate from marketing/lifecycle campaigns, and migration guides note transactional vs marketing distinctions, but there is no explicit documentation of separate suppression-list logic or dedicated sending reputation ensuring transactional mail still reaches unsubscribed contacts. Missing for 10: explicit suppression-rule separation, reputation isolation, and confirmation that unsubscribe status doesn't block transactional sends.", "evidenceIds": [ "loops-docs-2", "loops-docs-13", "loops-docs-11", "loops-docs-12" ] }, { "productId": "loops", "storyId": "visual-automation-builder", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Evidence confirms Loops supports lifecycle email use cases like onboarding, activation, upgrade, and winback sequences (loops-docs-15) and event-triggered workflows via API (loops-docs-3), but there is no documentation of branching logic, time-based delays, or exit-rule conditions within a visual automation builder. missing for 10: explicit branch/condition support, delay/wait-step configuration, exit-rule mechanics, and independent confirmation of a multi-step workflow builder.", "evidenceIds": [ "loops-docs-15", "loops-docs-3", "loops-docs-16" ] }, { "productId": "mailchimp", "storyId": "ab-test-campaigns", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Mailchimp's pricing page lists 'Multivariate and A/B Testing' as a feature, confirming the capability exists, but the evidence pack lacks any documentation detailing automatic winner selection based on real engagement metrics (open/click rates) or how variants are configured. missing for 10: first-party docs describing the A/B test setup, winning-criteria selection (opens/clicks/revenue), and automatic winner send-out; independent/hands-on confirmation the feature works as described.", "evidenceIds": [ "mailchimp-docs-9" ] }, { "productId": "mailchimp", "storyId": "agent-audits-program", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Mailchimp's Marketing API (mailchimp-docs-1, mailchimp-docs-4, mailchimp-docs-5) confirms programmatic access to audiences/contacts and campaign sending, implying an API surface an agent could query, but the evidence pack contains no explicit endpoints for automations or campaign performance/reports data, nor any AI-agent/MCP integration proof. Missing for 10: documented reporting/analytics endpoints, automation-status endpoints, and evidence of an agent actually consuming this data end-to-end.", "evidenceIds": [ "mailchimp-docs-1", "mailchimp-docs-4", "mailchimp-docs-5" ] }, { "productId": "mailchimp", "storyId": "agent-builds-segment", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows generic Marketing API capabilities (create contacts, send campaigns) and mentions 'Predictive Segmentation' as a paid feature, but there is no documentation of an API endpoint or agent-facing surface that translates a plain-language audience description into a concrete segment. Missing for 10: any segment-creation API reference, any NL-to-criteria translation mechanism, and any example of an agent invoking this programmatically.", "evidenceIds": [ "mailchimp-docs-4", "mailchimp-docs-10", "mailchimp-docs-1" ] }, { "productId": "mailchimp", "storyId": "agent-drives-campaign", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows only that the Marketing API can create campaigns and send them (with immediate send noted explicitly in mailchimp-docs-5), but there is no evidence of a scheduling/queuing mechanism that lets an agent draft a campaign and leave it pending for human approval before it goes out. No API endpoint, workflow, or documentation describing draft/approval states is present in the evidence pack.", "evidenceIds": [ "mailchimp-docs-5", "mailchimp-docs-1" ] }, { "productId": "mailchimp", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "A direct probe confirms Mailchimp serves an llms.txt file at the expected URL returning HTTP 200 with structured content summarizing the site, which is exactly what this story asks for. However, there's no evidence of broader agent-oriented documentation beyond this single file, nor independent corroboration of its usefulness or completeness. Missing for 10: evidence of dedicated agent/AI-native docs beyond llms.txt, third-party confirmation of quality/usefulness of the file, and any first-party announcement of this feature.", "evidenceIds": [ "mailchimp-probe-1" ] }, { "productId": "mailchimp", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Mailchimp's marketing pages list 'Predictive Segmentation', 'Generative AI features', and 'Actionable insights into audience growth and conversion funnels', suggesting some AI-driven analytics/suggestions exist inside the product, but these are thin pricing-page bullet points with no detailed documentation of how insights are generated or surfaced. Missing for 10: first-party docs explaining the AI insight/suggestion features in depth, independent/hands-on evidence confirming these work as described, and concrete examples of generated insights.", "evidenceIds": [ "mailchimp-docs-9", "mailchimp-docs-10", "mailchimp-docs-11", "mailchimp-docs-12" ] }, { "productId": "mailchimp", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Evidence shows only that RSS campaigns send on a schedule and that the Marketing/Transactional APIs allow event-driven, API-triggered messages, which implies some autonomous background behavior, but there is no direct documentation of Mailchimp's automation/journey-builder feature or AI-native automation setup. Missing for 10: explicit documentation of the Customer Journey/automation builder, evidence of AI-driven or agent-configurable automation setup, and independent confirmation that automations run reliably unattended.", "evidenceIds": [ "mailchimp-docs-5", "mailchimp-docs-1", "mailchimp-docs-2" ] }, { "productId": "mailchimp", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The pricing page lists 'Generative AI features' as an included capability, suggesting some AI assistance exists, but there is no documentation describing a built-in assistant that can be delegated tasks (e.g., autonomously drafting/sending campaigns, managing audiences) rather than just generating content snippets. missing for 10: concrete docs on an AI assistant's task scope, autonomy/delegation workflow, and independent/hands-on confirmation it functions as an agent rather than a content-suggestion tool.", "evidenceIds": [ "mailchimp-docs-11" ] }, { "productId": "mailchimp", "storyId": "agentic-headless", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Mailchimp exposes REST APIs (Marketing API, Transactional/Mandrill API and SMTP) that could in principle be called headlessly from scripts or CI pipelines to create contacts, send campaigns, or send transactional messages (mailchimp-docs-1,2,4,5,6). However there is no first-party guidance on CI/automation workflows, official SDKs, or CLI tooling, and community reports describe accounts being auto-banned or automation silently shut off (e.g., devtools-triggered blocks and Mandrill sending disabled without notice) which undercuts reliable unattended/headless use (mailchimp-comm-4,5,9,11). missing for 10: documented CI/headless automation examples, official CLI/SDK support, evidence of stable unattended operation without account/service suspension.", "evidenceIds": [ "mailchimp-docs-1", "mailchimp-docs-2", "mailchimp-docs-4", "mailchimp-docs-5", "mailchimp-docs-6", "mailchimp-comm-4", "mailchimp-comm-5", "mailchimp-comm-9", "mailchimp-comm-11" ] }, { "productId": "mailchimp", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Mailchimp is a marketing/email SaaS platform, not an AI agent; the story asks about plugging MCP servers into a product so it can use their tools, which applies to agentic/AI-assistant products acting as MCP clients. No evidence shows Mailchimp positioning itself as an MCP client or agent host, so this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "mailchimp", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of an official MCP server for Mailchimp; only Marketing/Transactional REST APIs and an llms.txt file are documented, none of which constitute an MCP server. missing for 10: any mention of MCP protocol support, an official MCP server endpoint, or agent-connection tooling.", "evidenceIds": [] }, { "productId": "mailchimp", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only traditional REST/API endpoints and generic 'Generative AI features' mention, with no natural-language command interface, chat/agent UI, or NL-to-action capability documented. No evidence of an AI assistant that lets users operate Mailchimp via conversational commands.", "evidenceIds": [ "mailchimp-docs-11", "mailchimp-docs-1" ] }, { "productId": "mailchimp", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence pack only shows Mailchimp's REST APIs (Marketing, Transactional, Commerce) and an llms.txt file; there is no mention of an official CLI tool for AI-native workflows. missing for 10: existence of any official CLI, documentation of CLI installation/usage, evidence of AI-native tooling built on a CLI.", "evidenceIds": [ "mailchimp-docs-1", "mailchimp-docs-2", "mailchimp-docs-3" ] }, { "productId": "mailchimp", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Mailchimp exposes well-documented REST APIs (Marketing API, Transactional/Mandrill API, Commerce API) with concrete endpoint-level documentation (contacts, campaigns, SMS, routes) suitable for programmatic/AI-driven control. Missing for 10: independent third-party corroboration of API robustness/rate limits for automated agent use, and no evidence of official SDKs or AI-specific tooling beyond raw REST docs.", "evidenceIds": [ "mailchimp-docs-1", "mailchimp-docs-2", "mailchimp-docs-4", "mailchimp-docs-5", "mailchimp-docs-6", "mailchimp-docs-8" ] }, { "productId": "mailchimp", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of scoped or least-privilege API key/credential issuance for AI agents; Mailchimp docs only describe standard API endpoints for campaigns, contacts, and transactional email with no mention of granular permission scopes or agent-specific credential management.", "evidenceIds": [ "mailchimp-docs-1", "mailchimp-docs-4", "mailchimp-docs-6" ] }, { "productId": "mailchimp", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Mailchimp's Marketing and Transactional REST APIs and endpoint documentation, but nowhere does it mention official client SDKs/libraries for building against these APIs. Missing for 10: any mention of official SDKs in specific languages, SDK repos, or versioned client libraries.", "evidenceIds": [ "mailchimp-docs-1", "mailchimp-docs-2", "mailchimp-docs-4", "mailchimp-docs-6" ] }, { "productId": "mailchimp", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions webhooks or event subscription mechanisms for Mailchimp; only API endpoints for campaigns, contacts, and transactional sending are documented. Absence of evidence for this applicable API capability yields 'none'. missing for 10: webhook documentation, event subscription setup, webhook payload/event types.", "evidenceIds": [] }, { "productId": "mailchimp", "storyId": "ai-content-drafting", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack only mentions 'Generative AI features' as a pricing line item with no description of composer-integrated subject line/copy drafting from a brief, and no docs or hands-on confirmation of this specific workflow. missing for 10: documentation or demo of AI subject-line/copy generation inside the composer, any first-party feature description, independent/hands-on confirmation it works as described.", "evidenceIds": [ "mailchimp-docs-11" ] }, { "productId": "mailchimp", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows standard API reference pages listing endpoints (create contact, send campaign, etc.) with no mention of an interactive console, runnable code samples, or try-it-now sandbox features expected for an AI-native exploratory API reference.", "evidenceIds": [ "mailchimp-docs-4", "mailchimp-docs-5", "mailchimp-docs-6", "mailchimp-docs-7", "mailchimp-docs-8" ] }, { "productId": "mailchimp", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Mailchimp's developer docs describe API endpoints (Marketing, Transactional) but never mentions an OpenAPI/Swagger spec or any downloadable machine-readable schema; the only spec-like artifact found is an llms.txt content summary, not an API spec. Missing for 10: explicit OpenAPI/Swagger file, published schema download link, or documentation referencing machine-readable spec format.", "evidenceIds": [ "mailchimp-docs-1", "mailchimp-docs-4", "mailchimp-probe-1" ] }, { "productId": "mailchimp", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of a sandbox/test environment or test API keys for Mailchimp's API; docs only describe production endpoints (campaigns, contacts, transactional sends) with no mention of a sandbox mode. Missing for 10: any documentation of a sandbox/test environment, test API keys, or dry-run mode that avoids touching production data.", "evidenceIds": [] }, { "productId": "mailchimp", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence confirms Mailchimp has Marketing, Transactional, and Commerce APIs with documented endpoints, but nothing in the pack describes API versioning scheme or a deprecation policy for AI-native reliability; community evidence even shows abrupt, unannounced account/API access shutdowns, undermining trust in predictable API lifecycle.", "evidenceIds": [ "mailchimp-docs-1", "mailchimp-docs-2", "mailchimp-comm-4", "mailchimp-comm-5" ] }, { "productId": "mailchimp", "storyId": "automation-bulk-operations", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Mailchimp's Marketing API allows creating/sending campaigns and contacts (mailchimp-docs-1, mailchimp-docs-4, mailchimp-docs-5), implying some bulk-capable API surface, but there is no documented batch/bulk-specific endpoint in the evidence, and multiple hands-on reports show that performing bulk operations like migrating or importing large contact lists triggered account bans or data loss (mailchimp-comm-3, mailchimp-comm-6), directly contradicting reliable bulk-operation support. missing for 10: explicit batch API documentation, evidence of successful large-scale bulk imports/exports, and independent confirmation that bulk workflows don't trigger account suspension.", "evidenceIds": [ "mailchimp-docs-1", "mailchimp-docs-4", "mailchimp-docs-5", "mailchimp-comm-3", "mailchimp-comm-6" ] }, { "productId": "mailchimp", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Mailchimp’s Transactional API supports event-driven messaging (mailchimp-docs-2) and the Marketing API can trigger campaign sends via API calls, suggesting some rule/event-based automation capability, but no evidence describes a dedicated 'automation rules' or 'journey builder' engine that lets users define custom trigger-action logic. missing for 10: explicit documentation of a rule-builder/automation-journey feature, examples of user-defined trigger conditions, and independent confirmation that such automations work reliably.", "evidenceIds": [ "mailchimp-docs-2", "mailchimp-docs-5", "mailchimp-docs-1" ] }, { "productId": "mailchimp", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The only recurring-schedule evidence is RSS campaigns that send 'according to its schedule' (mailchimp-docs-5); all other campaigns send immediately, and there is no documentation of an API or workflow engine (e.g., Customer Journeys automation) for AI-native users to configure arbitrary recurring jobs. Missing for 10: explicit automation/workflow API docs, recurring-trigger configuration beyond RSS, and any independent confirmation of automation reliability.", "evidenceIds": [ "mailchimp-docs-5", "mailchimp-docs-1" ] }, { "productId": "mailchimp", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of automation versioning, review workflows, or rollback capability in Mailchimp's docs or API; only campaign creation/sending and unrelated API features are documented. missing for 10: version history for automations, diff/review UI, rollback/restore functionality, audit trail of automation changes.", "evidenceIds": [ "mailchimp-docs-5", "mailchimp-docs-9", "mailchimp-docs-10" ] }, { "productId": "mailchimp", "storyId": "campaign-performance-reporting", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The evidence only vaguely gestures at analytics ('actionable insights into audience growth and conversion funnels') without any documentation of per-campaign or per-automation-step reporting on opens, clicks, unsubscribes, or conversions over time. Missing for 10: campaign report API/dashboard details, automation-step-level analytics, time-series breakdowns, and independent corroboration that these metrics are accessible and reliable.", "evidenceIds": [ "mailchimp-docs-12" ] }, { "productId": "mailchimp", "storyId": "campaign-via-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Mailchimp's Marketing API docs confirm creating audience contacts and sending campaigns via API (mailchimp-docs-4, mailchimp-docs-5), showing the core send-via-API capability exists, but the cited evidence only describes immediate sending or RSS-based scheduling, not a general 'schedule campaign' endpoint or full campaign-creation-to-send workflow purely via API. Missing for 10: explicit documented schedule-campaign endpoint, full campaign creation API details, and independent/developer corroboration that the entire flow works without dashboard use.", "evidenceIds": [ "mailchimp-docs-4", "mailchimp-docs-5", "mailchimp-docs-1" ] }, { "productId": "mailchimp", "storyId": "contacts-crud-api", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs confirm a Marketing API contact-creation endpoint ('Create a new omni-channel contact for an audience') and general API access to custom app data, but the pack lacks any explicit documentation of update or suppression (unsubscribe/cleaned) endpoints, custom merge-field schemas, or batch/production-scale operations. Community reports of accounts being banned, contacts wiped, and automated API/browser access blocked (e.g., dev-tools-triggered bans) raise real doubts about reliability of sustained programmatic contact management at scale. Missing for 10: documented update/suppress endpoints, custom property/merge-field API docs, evidence of bulk/batch operations, and independent confirmation of stable production-scale API usage.", "evidenceIds": [ "mailchimp-docs-4", "mailchimp-docs-1", "mailchimp-comm-1", "mailchimp-comm-6", "mailchimp-comm-9", "mailchimp-comm-11" ] }, { "productId": "mailchimp", "storyId": "create-send-campaign", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs show core loop elements: creating audience contacts (docs-4), sending campaigns with scheduling support for RSS campaigns (docs-5), and email design/testing tools (docs-9, docs-14) — this is Mailchimp's flagship product function. Missing for 10: explicit first-party documentation of scheduled (non-RSS) send-later flow and independent hands-on confirmation of a full compose-audience-send cycle; some community reports (mailchimp-comm-12) note deliverability issues on lower tiers that add minor quality risk.", "evidenceIds": [ "mailchimp-docs-4", "mailchimp-docs-5", "mailchimp-docs-9", "mailchimp-docs-14", "mailchimp-comm-12" ] }, { "productId": "mailchimp", "storyId": "custom-sending-domain", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no documentation about domain authentication, SPF/DKIM/DMARC setup, or guided DNS verification flows — only unrelated API docs, pricing feature lists, and community complaints. Missing for 10: any docs or hands-on evidence of domain authentication UI, DNS record generation, or verification status checks.", "evidenceIds": [] }, { "productId": "mailchimp", "storyId": "deliverability-visibility", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation of per-send bounce/complaint/blocklist reporting or actionable deliverability guidance—only generic API descriptions, pricing feature bullets, and community complaints about account bans and poor sender reputation, none of which describe the specific reporting/guidance capability the story asks for.", "evidenceIds": [ "mailchimp-docs-2", "mailchimp-docs-7", "mailchimp-comm-5", "mailchimp-comm-12" ] }, { "productId": "mailchimp", "storyId": "email-template-editor", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Only a generic marketing-site blurb ('Design better emails') hints at a visual editor, with no documentation of reusable templates, personalization/merge tags, or cross-client rendering testing; one community report also describes the drag-and-drop/CKText editor failing to load, undermining reliability. missing for 10: docs on template library, merge-tag personalization, cross-client rendering QA, and independent corroboration of editor quality.", "evidenceIds": [ "mailchimp-docs-14", "mailchimp-comm-10", "mailchimp-comm-11" ] }, { "productId": "mailchimp", "storyId": "engagement-data-api-read", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence shows generic Marketing API endpoints for creating contacts and sending campaigns, but no documentation of an endpoint or object returning per-contact engagement history (sent/opened/clicked activity log) via the API. Mailchimp does have an 'activity' endpoint in reality, but no evidence pack item mentions it, so per the rules absence must be judged as none. missing for 10: any docs referencing a contact activity/engagement history endpoint, example response showing opens/clicks/sends per contact, independent confirmation of this API capability.", "evidenceIds": [ "mailchimp-docs-4", "mailchimp-docs-5" ] }, { "productId": "mailchimp", "storyId": "event-triggered-flows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack references Mailchimp's Marketing API pulling 'custom data' from apps and creating contacts, but nowhere documents a specific Events API or mechanism for firing a custom event that triggers a Customer Journey/automation for a single contact. Without an explicit events endpoint or automation-trigger documentation, there is no evidence this capability is delivered.", "evidenceIds": [ "mailchimp-docs-1", "mailchimp-docs-4" ] }, { "productId": "mailchimp", "storyId": "flow-state-inspection", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence describes Customer Journey Builder subscriber-level visibility, or the ability to locate a specific contact within an automation and pull/re-enter them without disrupting the flow; docs only cover general API/campaign features. Missing for 10: journey/automation visual map showing contact position, per-contact pull/remove/re-entry controls, documentation or hands-on confirmation that this doesn't break the automation state.", "evidenceIds": [] }, { "productId": "mailchimp", "storyId": "full-list-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation of a self-serve export feature covering contacts, custom fields, tags, engagement history, and suppression list in an open format — API docs cover creating contacts and sending campaigns, not exporting. Community reports actually describe cases where audience data could not be self-serve recovered after account issues (mailchimp-comm-1, mailchimp-comm-2), reinforcing the absence of a documented reliable export path. missing for 10: no first-party docs on CSV/data export, no mention of engagement history or suppression list export, no confirmation of open-format guarantee.", "evidenceIds": [ "mailchimp-comm-1", "mailchimp-comm-2", "mailchimp-docs-4" ] }, { "productId": "mailchimp", "storyId": "guided-platform-migration", "verdict": "partial", "quality": 2, "confidence": "low", "rationale": "Mailchimp's marketing site references a 'Switch from another tool' migration feature, but the evidence pack contains no documentation or hands-on detail confirming that tags, custom fields, or unsubscribe state are actually preserved during import. missing for 10: detailed migration/import API or guide docs, evidence of tag/custom-field mapping, evidence of unsubscribe-state preservation, independent confirmation of successful migrations.", "evidenceIds": [ "mailchimp-docs-13" ] }, { "productId": "mailchimp", "storyId": "metrics-via-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Mailchimp has a general Marketing API (contacts, campaign send, transactional email) but contains no documentation of report/analytics endpoints for pulling campaign or automation (flow) performance metrics into an external warehouse or dashboard. Missing for 10: explicit reports/analytics API endpoints, examples of exporting campaign/flow metrics, and any dashboard/warehouse integration evidence.", "evidenceIds": [ "mailchimp-docs-1", "mailchimp-docs-4", "mailchimp-docs-5", "mailchimp-docs-12" ] }, { "productId": "mailchimp", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Mailchimp exposes a fairly complete Marketing API (contacts, campaigns, sending) and a separate Transactional API, showing broad coverage of core UI actions, but there is no first-party claim of full UI/API parity, and community evidence suggests some UI-only features (e.g., the drag-and-drop email builder's advanced editing) are not fully replicable via API/inspector access. missing for 10: explicit vendor statement of full API/UI parity, evidence that advanced design/template features available in the UI builder are fully scriptable via API, independent confirmation of parity across all account/admin settings.", "evidenceIds": [ "mailchimp-docs-1", "mailchimp-docs-4", "mailchimp-docs-5", "mailchimp-docs-2", "mailchimp-comm-11" ] }, { "productId": "mailchimp", "storyId": "openness-full-export", "verdict": "disputed", "quality": 3, "confidence": "medium", "rationale": "Mailchimp's Marketing API allows programmatic access to contacts/campaigns (mailchimp-docs-4), which could support data export, but there's no documented bulk/open-format export-and-leave feature, and community reports describe accounts being banned or deleted with contact lists lost entirely, support falsely claiming data was wiped, and only providing a data dump under public pressure — directly contradicting a smooth 'export all my data and leave' experience. Missing for 10: an official bulk export/data-portability doc, an open standard export format, and confirmation that account closure preserves user access to their data.", "evidenceIds": [ "mailchimp-docs-4", "mailchimp-comm-1", "mailchimp-comm-2", "mailchimp-comm-6" ] }, { "productId": "mailchimp", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Mailchimp is closed-source SaaS; while it references 'Mailchimp Open Commerce' as open source, this is a separate discontinued product, not Mailchimp's own source. No evidence of an open license for Mailchimp's core platform code.", "evidenceIds": [ "mailchimp-docs-3" ] }, { "productId": "mailchimp", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Mailchimp's core email marketing platform is a closed, cloud-only SaaS with no self-hosted deployment option; the only open-source item mentioned (Mailchimp Open Commerce) is a separate commerce stack, not the core marketing product. missing for 10: any self-host package/instructions for the core Marketing/Transactional product, on-prem licensing, or Docker/VM deployment option.", "evidenceIds": [ "mailchimp-docs-1", "mailchimp-docs-2", "mailchimp-docs-3" ] }, { "productId": "mailchimp", "storyId": "outbound-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack includes only general Marketing/Transactional API descriptions and community complaints, with no mention of webhook subscriptions, event types (opens, clicks, bounces, unsubscribes), or signed payload verification. Absence of evidence for this applicable integrations axis means it cannot be credited as delivered.", "evidenceIds": [ "mailchimp-docs-1", "mailchimp-docs-4", "mailchimp-docs-5" ] }, { "productId": "mailchimp", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items mention data residency, region selection, or geographic storage controls for Mailchimp; the docs focus on API features and pricing tiers, not data location controls. Missing for 10: any documentation of region/residency selection, data center choice, or geographic storage controls.", "evidenceIds": [] }, { "productId": "mailchimp", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses AI-training opt-out or data-use controls for Mailchimp; the evidence is limited to product feature descriptions and unrelated account-management/community complaints. Missing for 10: any documentation of an AI-training opt-out setting, privacy policy language on model training, or admin controls to exclude data from AI/ML use.", "evidenceIds": [] }, { "productId": "mailchimp", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No documentation is provided showing Mailchimp gives AI-native users explicit APIs or settings to control data retention/deletion policies; the evidence pack only covers marketing/API features unrelated to retention controls. Community reports actually describe Mailchimp unilaterally deleting accounts and data without user consent or recourse, further indicating the user lacks control over retention/deletion.", "evidenceIds": [ "mailchimp-comm-1", "mailchimp-comm-2", "mailchimp-comm-7" ] }, { "productId": "mailchimp", "storyId": "privacy-telemetry-optout", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "This story concerns AI-native tooling telemetry/usage-tracking opt-out, which applies to AI coding tools/agents, not an email marketing SaaS platform like Mailchimp; the evidence pack contains no such AI-native product surface for this axis to apply to.", "evidenceIds": [] }, { "productId": "mailchimp", "storyId": "revenue-attribution", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack only shows generic marketing/API docs and vague analytics claims (e.g., 'actionable insights into audience growth and conversion funnels', 'predictive segmentation') with no documentation of revenue/signup attribution tied to specific emails or automation flows. No reporting feature, e-commerce revenue-per-campaign dashboard, or flow-level conversion tracking is evidenced.", "evidenceIds": [ "mailchimp-docs-12", "mailchimp-docs-10" ] }, { "productId": "mailchimp", "storyId": "scoped-api-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of API key scoping or read-only/permission-restricted credentials in Mailchimp's docs; the only API key model referenced is a single full-access account key with no mention of granular scopes for analytics-only or contractor access. Missing for 10: any documentation of scoped/read-only API keys, role-based API access, or OAuth scopes limiting an agent to analytics without send permission.", "evidenceIds": [ "mailchimp-docs-1", "mailchimp-docs-4", "mailchimp-docs-5" ] }, { "productId": "mailchimp", "storyId": "segment-builder", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence shows Mailchimp offers 'Predictive Segmentation' and audience growth insights, and the Marketing API supports contact creation and custom data ingestion, which implies segmentation capability, but no direct documentation describes building segments from tags or behavioral events or confirms segments update live as data changes. missing for 10: explicit docs on tag-based segmentation, event-based triggers, and dynamic/live segment updating.", "evidenceIds": [ "mailchimp-docs-10", "mailchimp-docs-4", "mailchimp-docs-12", "mailchimp-docs-1" ] }, { "productId": "mailchimp", "storyId": "segments-via-api", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack only shows generic mentions of Mailchimp's Marketing API and a single endpoint for creating a contact (mailchimp-docs-4), with no documentation of segment, tag, or list CRUD operations or reading back exact membership. Missing for 10: explicit API endpoints for creating/updating segments or tags, listing/reading segment membership, and any developer or hands-on confirmation that programmatic read-back works as described.", "evidenceIds": [ "mailchimp-docs-1", "mailchimp-docs-4" ] }, { "productId": "mailchimp", "storyId": "stack-integrations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack only shows generic Marketing/Transactional APIs and an open-source commerce stack, but contains no mention of maintained first-party integrations with Stripe, Shopify, Segment, or product databases. Missing for 10: explicit Stripe integration docs, Shopify integration docs, Segment/CDP integration docs, and any customer/database sync integration evidence.", "evidenceIds": [ "mailchimp-docs-1", "mailchimp-docs-3" ] }, { "productId": "mailchimp", "storyId": "suppression-compliance", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack documents Mailchimp's suppression list, bounce/complaint handling, or cross-send-type unsubscribe enforcement; docs cited are about API endpoints, pricing tiers, and unrelated features. Community evidence discusses account bans and deliverability reputation issues but does not confirm or deny automatic suppression hygiene, so absence of applicable evidence yields 'none'.", "evidenceIds": [] }, { "productId": "mailchimp", "storyId": "templates-as-code", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation of Mailchimp's Templates API, HTML/MJML template management, merge-tag variable substitution, or any version-control workflow for templates; only general marketing API mentions are present. Missing for 10: Templates API docs, variable/merge-tag substitution details, MJML/HTML support, version-control integration examples.", "evidenceIds": [ "mailchimp-docs-1", "mailchimp-docs-4" ] }, { "productId": "mailchimp", "storyId": "transactional-send-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Mailchimp Transactional (formerly Mandrill) provides API/SMTP-based transactional sending (receipts, event-driven messages) per docs-2, docs-6, docs-7, docs-8, satisfying the core capability, but it is a separate product with its own distinct API endpoint rather than truly unified with the Marketing API, and community reports confirm it was never fully integrated post-acquisition (mailchimp-comm-8) and has suffered abrupt, unexplained shutdowns of transactional sending (mailchimp-comm-4, mailchimp-comm-5). missing for 10: evidence of a single unified API/dashboard combining marketing and transactional sends, and confirmation of reliable uptime for transactional messages without account suspension risk.", "evidenceIds": [ "mailchimp-docs-2", "mailchimp-docs-6", "mailchimp-docs-7", "mailchimp-docs-8", "mailchimp-comm-8", "mailchimp-comm-4", "mailchimp-comm-5" ] }, { "productId": "mailchimp", "storyId": "transactional-separation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Mailchimp Transactional (Mandrill) is documented as a distinct product with its own API/SMTP, dedicated IPs, and routing separate from the Marketing API, which supports the general premise of separated transactional infrastructure. However, there is no direct evidence that transactional sends bypass marketing suppression/unsubscribe lists or that templates are kept fully separate, and community reports (e.g., Mandrill sends being silently shut off after a single complaint, poor integration between the two products) suggest the separation is imperfect and reputation is fragile rather than cleanly isolated. Missing for 10: explicit documentation of suppression-list independence for transactional sends, confirmation that unsubscribed marketing contacts still receive transactional receipts, and evidence of fully separate templates/reputation management.", "evidenceIds": [ "mailchimp-docs-2", "mailchimp-docs-6", "mailchimp-docs-7", "mailchimp-docs-8", "mailchimp-comm-4", "mailchimp-comm-5", "mailchimp-comm-8" ] }, { "productId": "mailchimp", "storyId": "visual-automation-builder", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation of Mailchimp's journey/automation builder, branching logic, delays, or exit rules — only generic API references (campaigns, transactional email, contacts) and pricing-page feature bullets (A/B testing, predictive segmentation, AI) that don't address multi-step lifecycle automations.", "evidenceIds": [] } ]