[ { "productId": "canva", "storyId": "advanced-prototype-logic", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers Canva's developer APIs, SDKs, MCP server, and automation/integration features, but contains no mention of interactive prototyping capabilities such as animated state transitions, conditional logic, variables, or scroll-triggered effects within Canva designs. This is a fair axis for a design tool to be judged on, but nothing in the pack shows Canva delivers it.", "evidenceIds": [] }, { "productId": "canva", "storyId": "agent-creates-design", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Canva ships an official MCP server explicitly documented to let AI assistants create designs from text, edit existing designs via natural language, manage assets/brand kits, and export/resize outputs — a clear write-capable agent surface, corroborated by a dedicated MCP docs probe. This is backed further by the Connect APIs (write-capable REST API with OpenAPI/SDK generation) and Apps SDK for programmatic design manipulation. missing for 10: independent/hands-on third-party verification that the MCP write operations work reliably in practice.", "evidenceIds": [ "canva-docs-5", "canva-docs-17", "canva-docs-18", "canva-docs-19", "canva-docs-20", "canva-docs-21", "canva-docs-28", "canva-docs-26", "canva-docs-24", "canva-probe-4" ] }, { "productId": "canva", "storyId": "agent-exports-assets", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Canva's Connect APIs and MCP server both provide documented programmatic export capability — the MCP server explicitly supports 'Design export: Generate exports in multiple formats (PDF, PNG, JPG, PPTX, MP4, etc.) and resize designs for different channels', and the Connect APIs allow creating/syncing designs and assets with OpenAPI-generated SDKs, confirmed via a live probe of the MCP docs. This directly satisfies an agent's need to render/export designs programmatically. Missing for 10: independent/hands-on confirmation of actual export calls succeeding (no third-party corroboration, only first-party docs), and the OpenAPI spec itself wasn't resolvable via direct probe.", "evidenceIds": [ "canva-docs-21", "canva-docs-35", "canva-docs-28", "canva-docs-26", "canva-docs-37", "canva-probe-4" ] }, { "productId": "canva", "storyId": "agent-reads-design-context", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Canva has an official documented MCP server enabling AI assistants to search/retrieve designs, pages and folders, manage assets and brand kits, and export in multiple formats (PDF/PNG/JPG/etc.), which covers much of 'pulling real design context' for an agent to act on. However, the story's specific terms — frames, variables, component mappings — aren't explicitly evidenced, and there's no independent/hands-on confirmation that a coding agent can actually reconstruct implementation-level design context via this server. Missing for 10: explicit support for design variables, frame-level structural access, component-to-code mapping, and third-party corroboration of agent workflows.", "evidenceIds": [ "canva-docs-28", "canva-docs-19", "canva-docs-20", "canva-docs-21", "canva-docs-34", "canva-docs-35", "canva-probe-4" ] }, { "productId": "canva", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Canva confirms an accessible llms.txt file (HTTP 200) plus multiple scoped llms.txt files for sub-docs (e.g., scim/llms.txt, admin/llms.txt, audit-logs/llms.txt) and dedicated .md docs pages (mcp.md, connect.md, apps.md) explicitly structured for agent consumption. missing for 10: independent/third-party confirmation of an agent successfully using llms.txt to navigate docs, and a root-level single canonical llms.txt covering all products rather than fragmented per-section files.", "evidenceIds": [ "canva-probe-1", "canva-docs-38", "canva-docs-39", "canva-docs-40", "canva-docs-28", "canva-docs-5" ] }, { "productId": "canva", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack shows Canva letting users import external data to drive design generation, bulk creation, and personalized template population (canva-docs-7, canva-docs-8, canva-docs-32), but nothing shows Canva's AI analyzing a user's own data to surface insights or proactive suggestions (e.g., analytics-style recommendations). This is closer to data-driven content generation than 'AI-generated insights and suggestions from my data,' so there is no concrete evidence the described capability exists.", "evidenceIds": [ "canva-docs-7", "canva-docs-8", "canva-docs-32" ] }, { "productId": "canva", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Canva provides integration surfaces (Connect APIs, Apps SDK, Bulk create, Brand Template autofill, MCP server) that let developers or AI assistants trigger Canva actions programmatically, but these are all invoked by an external caller rather than being a built-in autonomous scheduler or background automation engine within Canva itself. Missing for 10: evidence of native scheduled/triggered automations, event-driven workflows, or an 'autopilot' feature that runs without external initiation.", "evidenceIds": [ "canva-docs-2", "canva-docs-7", "canva-docs-8", "canva-docs-26", "canva-docs-28", "canva-docs-32" ] }, { "productId": "canva", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Evidence mentions a 'Canva AI' feature only in passing (pasting links to add context to designs) but provides no detail on delegating broader tasks to a built-in assistant; most AI-related evidence instead describes the MCP server that lets *external* AI assistants control Canva, not an assistant embedded within Canva's own UI. missing for 10: documentation of an in-app AI assistant's task scope, examples of users delegating multi-step design tasks to it, and independent/hands-on confirmation of its capabilities.", "evidenceIds": [ "canva-docs-10", "canva-docs-28" ] }, { "productId": "canva", "storyId": "agentic-headless", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Canva's Connect APIs and bulk/personalization features (canva-docs-2, canva-docs-7, canva-docs-8, canva-docs-16, canva-docs-26) show that programmatic, non-interactive design generation and export are possible, and an OpenAPI description exists for SDK generation (canva-docs-31, canva-docs-36) which could support CI pipelines. However, Connect APIs are OAuth-based integrations meant for user-authorized platforms, not clearly documented as headless/service-account CI automation, and a probe for a hosted OpenAPI spec returned 404s (canva-probe-3), suggesting the automation surface is less mature for pure CI use. Missing for 10: explicit CI/service-account authentication flow, documented headless execution examples, and a verifiable OpenAPI spec endpoint.", "evidenceIds": [ "canva-docs-2", "canva-docs-7", "canva-docs-8", "canva-docs-16", "canva-docs-26", "canva-docs-31", "canva-docs-36", "canva-probe-3" ] }, { "productId": "canva", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "All evidence describes Canva exposing an official MCP *server* so external AI assistants (Claude, ChatGPT, etc.) can call Canva's own design tools (canva-docs-5, canva-docs-28, canva-probe-4) — the opposite direction of the story, which asks whether Canva itself can act as an MCP client plugging into external MCP servers to use their tools. No evidence shows Canva consuming or configuring external MCP servers.", "evidenceIds": [ "canva-docs-5", "canva-docs-28", "canva-docs-30", "canva-probe-4" ] }, { "productId": "canva", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Canva ships an official MCP server (canva-docs-28, canva-probe-4) that lets AI assistants connect via natural language to Canva's design capabilities, with documented features spanning design generation, editing, discovery, export, asset/brand management, and collaboration (canva-docs-17 through 22, 33-35). Missing for 10: independent/hands-on third-party corroboration of the MCP server working in practice.", "evidenceIds": [ "canva-docs-28", "canva-docs-5", "canva-docs-17", "canva-docs-18", "canva-docs-19", "canva-docs-20", "canva-docs-21", "canva-docs-22", "canva-probe-4" ] }, { "productId": "canva", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Canva ships an official MCP server explicitly designed so AI assistants can operate Canva via natural language — supporting design generation, editing, discovery, asset/brand management, export, and comments through NL commands, and this is corroborated by a probe confirming the documented endpoint. This directly satisfies the AI-native natural-language operation story, though it relies on third-party AI clients rather than a native in-product NL interface. Missing for 10: independent/hands-on user reports of NL command reliability, and evidence of a first-party conversational interface within Canva itself.", "evidenceIds": [ "canva-docs-5", "canva-docs-17", "canva-docs-18", "canva-docs-19", "canva-docs-20", "canva-docs-21", "canva-docs-22", "canva-docs-28", "canva-probe-4" ] }, { "productId": "canva", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Canva does ship an official CLI, but it is scoped narrowly to creating and managing Canva Apps (the Apps SDK), not a general-purpose CLI for AI-native workflows like design generation, export, or automation via Connect APIs — those are exposed via APIs/MCP, not the CLI. missing for 10: evidence of CLI supporting broader AI-native tasks (design generation, bulk export, Connect API operations), independent/hands-on usage reports.", "evidenceIds": [ "canva-docs-14", "canva-probe-5" ] }, { "productId": "canva", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Canva provides well-documented Connect APIs with an OpenAPI description for client SDK generation, plus Apps SDK, Admin API, SCIM API, and a first-party MCP server enabling AI assistants to drive Canva programmatically — directly matching an AI-native user's ability to drive the product via public API. Missing for 10: independent hands-on corroboration of API robustness/rate limits and a directly resolvable public OpenAPI file (probe found candidate OpenAPI endpoints 404, though docs claim one exists).", "evidenceIds": [ "canva-docs-2", "canva-docs-26", "canva-docs-31", "canva-docs-36", "canva-docs-28", "canva-docs-17", "canva-docs-19", "canva-docs-21", "canva-probe-4", "canva-probe-3" ] }, { "productId": "canva", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While Canva exposes Connect APIs, OAuth-based integrations, and an MCP server for AI assistants, none of the evidence describes issuing scoped or least-privilege credentials specifically for an agent (e.g., granular OAuth scopes, per-agent API keys, or restricted permission sets). Private integrations are gated by plan tier (docs-41) rather than granular scoping, and MCP/AI Connector docs discuss connecting an assistant but not credential scoping controls. Missing for 10: explicit scope/permission model for API keys or OAuth tokens issued to agents, documentation of least-privilege credential issuance, and any audit/control mechanism tied to agent-specific access.", "evidenceIds": [ "canva-docs-2", "canva-docs-26", "canva-docs-30", "canva-docs-41", "canva-probe-4" ] }, { "productId": "canva", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Canva provides official SDKs (Apps SDK, Connect APIs) plus an OpenAPI description enabling generated client SDKs in various languages, a CLI for app development, and an official MCP server for AI-native integration—covering the core of building against official SDKs. Missing for 10: independent/hands-on developer corroboration beyond first-party docs, and the openapi.json probe returned 404 suggesting the spec isn't trivially discoverable at expected paths.", "evidenceIds": [ "canva-docs-1", "canva-docs-2", "canva-docs-13", "canva-docs-14", "canva-docs-26", "canva-docs-28", "canva-docs-31", "canva-probe-3", "canva-probe-4", "canva-probe-5" ] }, { "productId": "canva", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any webhook/event-subscription capability in the Canva Connect APIs, MCP server, or other docs — only REST-style APIs, SCIM, audit logs, and MCP tool interactions are described. missing for 10: any mention of webhooks, event subscriptions, or push notifications for state changes.", "evidenceIds": [] }, { "productId": "canva", "storyId": "ai-asset-editing-tools", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack only covers developer-facing surfaces (Apps SDK, Connect APIs, MCP server, Admin/SCIM/audit APIs) and never mentions Canva's native in-editor AI asset tools like Magic Media image generation, Magic Eraser/background removal, Magic Edit content-aware fill, or Magic Write copy suggestions used directly by a designer inside the editor. Missing for 10: any documentation of built-in Magic Studio-style tools (image generation, background removal, content-aware fill, copy suggestions) operating natively within the Canva editor UI.", "evidenceIds": [] }, { "productId": "canva", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Canva documents an OpenAPI description for its Connect APIs (used for generating client SDKs) but there is no evidence of an interactive API reference with runnable/try-it examples; probes for openapi.json/swagger endpoints returned 404s, suggesting no live interactive console is exposed.", "evidenceIds": [ "canva-docs-13", "canva-docs-31", "canva-docs-36", "canva-probe-3" ] }, { "productId": "canva", "storyId": "api-machine-spec", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Canva's developer docs explicitly state it provides an OpenAPI description of the Connect APIs that can be used with code-generation libraries to produce client SDKs, directly matching the story. Confidence is tempered because the probe couldn't locate the actual spec file at common guessed paths (only generic /openapi.json etc. were tried, not the real documented location), so independent verification of downloadability is thin. Missing for 10: a probed/verified working download URL for the OpenAPI file and independent (non-vendor) confirmation it's actually fetchable.", "evidenceIds": [ "canva-docs-31", "canva-docs-36", "canva-docs-13", "canva-probe-3" ] }, { "productId": "canva", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a sandbox/test environment separate from production data; Canva's docs describe live APIs, MCP server, and Connect APIs but nothing about a sandbox mode, test accounts, or mock environment for safe experimentation.", "evidenceIds": [] }, { "productId": "canva", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Canva has Connect APIs, an OpenAPI description, SDKs, and an MCP server, but there is no mention anywhere of API versioning scheme or a documented deprecation policy for these APIs; the OpenAPI spec itself isn't even discoverable at expected endpoints (canva-probe-3). missing for 10: explicit API versioning scheme, documented deprecation/sunset policy, changelog or migration guides for breaking changes.", "evidenceIds": [ "canva-docs-26", "canva-docs-31", "canva-probe-3" ] }, { "productId": "canva", "storyId": "automation-bulk-operations", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Canva explicitly supports bulk operations via Bulk Create with data import, personalized Brand Template population at scale, and Connect APIs to sync/create many designs/assets programmatically, plus MCP/API access for AI-native automation across many items. Missing for 10: independent/hands-on verification of scale limits, no explicit rate-limit or batch-size documentation, and no third-party confirmation of true 'many items' throughput.", "evidenceIds": [ "canva-docs-7", "canva-docs-8", "canva-docs-32", "canva-docs-37", "canva-docs-28" ] }, { "productId": "canva", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Canva's evidence covers APIs, apps, MCP server, bulk creation, and integrations, but nothing describes user-defined rule/trigger automation (event-condition-action logic) that fires actions automatically on events. No workflow/automation-rule engine, webhooks-as-triggers, or 'when X happens do Y' capability is documented.", "evidenceIds": [] }, { "productId": "canva", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence covers Canva's Connect APIs, Apps SDK, MCP server, and bulk creation features, but none of it describes any scheduler, cron-like trigger, or recurring/automated job execution capability. Bulk create and template population are one-shot batch operations, not recurring scheduled workflows.", "evidenceIds": [] }, { "productId": "canva", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows audit logs tracking user activity, but no documentation of versioning, review, or rollback specifically for automations (Connect API integrations, MCP workflows, or bulk create jobs). There's no mention of automation history, diffing, approval workflows, or reverting automated changes.", "evidenceIds": [ "canva-docs-4", "canva-docs-40" ] }, { "productId": "canva", "storyId": "build-custom-plugins", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Canva provides a documented Apps SDK for building editor extensions, a CLI for creating/managing apps, and distribution via public Apps Marketplace or private team sharing, matching the developer story closely. missing for 10: independent/hands-on corroboration of typings quality and sample completeness, and a working OpenAPI/typings endpoint (probe found openapi.json paths returning 404).", "evidenceIds": [ "canva-docs-1", "canva-docs-14", "canva-docs-24", "canva-docs-25", "canva-docs-29", "canva-probe-5", "canva-probe-3" ] }, { "productId": "canva", "storyId": "clear-plan-pricing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack is entirely developer/API documentation (Connect APIs, Apps SDK, MCP, SCIM, Admin API) and contains no pricing page, plan comparison, seat-type breakdown, or AI-feature gating disclosure that a procurement-focused product manager would need. Only one mention ('Private integrations can only be used by your team on a Canva Enterprise plan') hints at plan gating but is far from a published pricing/tier matrix.", "evidenceIds": [ "canva-docs-41" ] }, { "productId": "canva", "storyId": "comments-review-flows", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Evidence confirms Canva supports commenting on designs and even integrating comments into external workflows (canva-docs-9, canva-docs-22, canva-docs-37), implying in-file review is possible, but there is no direct documentation of @mentioning teammates or explicitly 'resolving' comment threads within the editor. missing for 10: explicit documentation of @mention functionality, thread resolution/closing feature, and any first-party or independent confirmation of the full in-app review workflow.", "evidenceIds": [ "canva-docs-9", "canva-docs-22", "canva-docs-37" ] }, { "productId": "canva", "storyId": "design-to-production-code", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Canva's developer docs cover Apps SDK, Connect APIs, MCP server, and CLI for building integrations, but none of the evidence describes design-to-code output or component-to-code mappings that map a Canva design to shippable UI code (e.g., React/CSS export) as design tools like Figma do.", "evidenceIds": [ "canva-docs-1", "canva-docs-2", "canva-docs-26", "canva-probe-4", "canva-probe-5" ] }, { "productId": "canva", "storyId": "design-tokens-variables", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Canva's evidence covers Brand Templates, Apps SDK, Connect APIs, and MCP integrations, but nothing describes design tokens/variables (colors, spacing, typography) as reusable, mode-based theme objects applied programmatically across designs — Canva's brand kit concept is not documented as a token/variable system with theme modes.", "evidenceIds": [] }, { "productId": "canva", "storyId": "export-formats-assets", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Canva's docs confirm multi-format export (PDF, PNG, JPG, PPTX, MP4) and automatic resizing to new dimensions/aspect ratios, covering the basic 'PNG, SVG, PDF, and more' export need, but there is no evidence of SVG export, per-layer export presets, or slice-based export controls akin to design-tool layer/slice workflows. missing for 10: SVG export support, per-layer export presets, and slice/scale export tooling.", "evidenceIds": [ "canva-docs-21", "canva-docs-35", "canva-docs-16", "canva-docs-6" ] }, { "productId": "canva", "storyId": "free-tier-real-design", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack is entirely developer/API-focused (Connect APIs, Apps SDK, MCP server, SCIM, audit logs, admin API) and contains nothing about the consumer free-tier editor, its feature set, or free collaboration capabilities; one entry even implies enterprise-only integrations. No evidence supports or refutes what the free tier includes for a designer doing real design work.", "evidenceIds": [] }, { "productId": "canva", "storyId": "inspect-code-handoff", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "canva", "storyId": "interactive-prototypes", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "canva", "storyId": "nl-design-generation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Canva's docs describe design generation and editing from natural language via its MCP server (canva-docs-17, canva-docs-18, canva-docs-28, canva-docs-33), which lets AI assistants create and modify designs from text descriptions and specifications, plus discovery, export, and asset management. However, this capability is delivered through an external MCP integration/AI connector rather than a native in-editor 'Magic Design'-style prompt feature, and there's no independent/hands-on evidence confirming quality of generated mockups. Missing for 10: first-party in-product documentation of native prompt-to-design (e.g., Magic Design) generation, independent hands-on verification of output quality, and confirmation this works without third-party AI tool setup.", "evidenceIds": [ "canva-docs-17", "canva-docs-18", "canva-docs-28", "canva-docs-33", "canva-docs-34", "canva-docs-35", "canva-probe-4" ] }, { "productId": "canva", "storyId": "open-file-format", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only APIs (Connect API, Apps SDK, MCP) and export formats (PDF/PNG/PPTX/etc.) for integration, but no documented open native design file format — Canva's underlying .canva design representation itself remains proprietary and undocumented; users must go through the vendor's own APIs/exports to get data out, not a portable open format usable in other tools without Canva's platform.", "evidenceIds": [ "canva-docs-21", "canva-docs-26", "canva-docs-31", "canva-probe-3" ] }, { "productId": "canva", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Canva offers substantial API surface (Connect APIs, Apps SDK, SCIM, Admin API, MCP server) covering design creation, editing, export, comments, and asset management, but this is a curated set of endpoints rather than full parity with the UI editor's rich manual design capabilities (e.g. free-form drag-and-drop editing, most granular design tools). No evidence confirms full UI-API parity; the docs describe specific supported operations, not a claim of complete equivalence. Missing for 10: explicit parity claim or evidence that every UI editing action (e.g. fine-grained manual layout, all effects/filters) is API-accessible, and independent verification of coverage limits.", "evidenceIds": [ "canva-docs-2", "canva-docs-17", "canva-docs-18", "canva-docs-19", "canva-docs-20", "canva-docs-21", "canva-docs-22", "canva-docs-26", "canva-docs-37", "canva-docs-31" ] }, { "productId": "canva", "storyId": "openness-full-export", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Canva supports exporting individual designs in open, standard formats (PDF, PNG, JPG, PPTX, MP4) via both the Connect API and the MCP server, giving users a way to pull their design content out in interoperable formats. However there is no evidence of a comprehensive account-level data export (brand kits, comments, team assets, settings, history) or a documented 'export all my data' / account deletion-and-portability feature that would let an AI-native user fully leave with everything intact. missing for 10: bulk/account-wide data export, documented data portability or GDPR-style export tool, evidence of exporting non-design data (comments, brand assets, admin/audit data) in open formats.", "evidenceIds": [ "canva-docs-21", "canva-docs-35", "canva-docs-26", "canva-docs-37" ] }, { "productId": "canva", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Canva is closed-source SaaS; the evidence pack only covers APIs, SDKs, MCP server, and developer docs — none of it indicates any open-license availability of Canva's own product source code.", "evidenceIds": [] }, { "productId": "canva", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "canva", "storyId": "plugin-ecosystem-marketplace", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Canva has a real extensibility ecosystem: the Apps SDK/CLI for building apps, an Apps Marketplace for public distribution, and team-private app sharing — directly supporting icon/data/accessibility-style plugins built by third parties. However, evidence is entirely first-party docs describing the SDK/marketplace mechanics; there's no independent evidence of marketplace size, breadth of existing third-party plugins (icons, accessibility checkers, data fillers), or community adoption. Missing for 10: independent verification of marketplace scale/quality, concrete examples of third-party apps in categories named in the story, user reviews or hands-on discovery of the marketplace.", "evidenceIds": [ "canva-docs-24", "canva-docs-25", "canva-docs-12", "canva-docs-14", "canva-docs-29", "canva-docs-11", "canva-probe-5" ] }, { "productId": "canva", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, regional storage options, or location controls for Canva data; documentation covers APIs, SDKs, audit logs, and provisioning but not data residency/region choice.", "evidenceIds": [] }, { "productId": "canva", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains only Canva developer docs about APIs, SDKs, MCP servers, and admin/audit tools — nothing addresses AI training data usage policies or opt-out controls for user content. Missing for 10: any documentation on AI training data policy, opt-out settings, or privacy controls related to model training.", "evidenceIds": [] }, { "productId": "canva", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers audit logs, SCIM provisioning, and admin/team management APIs, but nothing addresses data retention policies or user-initiated data deletion controls for AI-native usage (e.g., MCP/AI connector data handling, deletion of AI-processed content, retention settings).", "evidenceIds": [] }, { "productId": "canva", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item addresses telemetry/usage-tracking opt-out settings for AI-native users; documentation covers APIs, SDKs, MCP server, audit logs, and provisioning but nothing about disabling telemetry collection.", "evidenceIds": [] }, { "productId": "canva", "storyId": "realtime-multiplayer-editing", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack focuses entirely on developer APIs, SDKs, MCP servers, and admin/provisioning tools; there is no mention of real-time multi-user editing, live cursors, or simultaneous in-editor collaboration features that are core to this story.", "evidenceIds": [] }, { "productId": "canva", "storyId": "responsive-auto-layout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers Canva's APIs, SDKs, MCP server, and admin/integration tooling, but contains no mention of responsive frames, auto layout, constraints, or grid-based resizing behavior akin to real UI flex layouts within the design editor itself. Design resizing mentioned (canva-docs-6, canva-docs-16) refers to static output resizing to new fixed dimensions, not dynamic responsive layout behavior.", "evidenceIds": [] }, { "productId": "canva", "storyId": "reusable-components-variants", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers Canva's APIs, SDKs, MCP server, and Brand Templates but contains no mention of a component system with variants, per-instance overrides, or linked source instances (like Figma-style components). Brand Templates allow personalization but are not described as variant-based linked components. Missing for 10: any documentation of a component/variant system, linked instance overrides, or source-of-truth syncing mechanism for design elements.", "evidenceIds": [] }, { "productId": "canva", "storyId": "self-host-design-platform", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "canva", "storyId": "shared-team-libraries", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Canva offers Brand Kits and Brand Templates that centralize on-brand assets/styles for teams to reuse across designs (canva-docs-15, canva-docs-7, canva-docs-20), which is the closest analog to a shared component library. However, there is no evidence of a true library-linking mechanism where edits to a source style/component automatically or optionally propagate updates to files that already used it, which is the core of the story. missing for 10: documentation of update propagation/versioning when a Brand Template or Brand Kit element changes, evidence of per-instance 'update available' notifications, independent confirmation of this workflow in practice.", "evidenceIds": [ "canva-docs-15", "canva-docs-7", "canva-docs-20" ] }, { "productId": "canva", "storyId": "vector-editing-craft", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers Canva's developer APIs, SDKs, MCP server, and automation/integration features, but contains no mention of pen/path tools, boolean operations, or precision alignment controls for vector editing. Canva is a template-driven design tool, so this axis is fair to ask, but nothing in the evidence supports professional vector-editing capability.", "evidenceIds": [] }, { "productId": "canva", "storyId": "version-history-restore", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers Canva's developer APIs, SDKs, MCP server, and admin/audit features, but contains no mention of file version history, naming milestones, comparing versions, or restoring earlier states of a design.", "evidenceIds": [] }, { "productId": "claude-design", "storyId": "advanced-prototype-logic", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Docs claim Claude Design produces 'interactive prototypes' that can be user-tested and 'look like the real product' via web capture, and community evidence confirms animated SVGs can be built, but with real friction (animations stripped by default sanitization, requiring workarounds). There is no evidence addressing prototype conditions/variables or scroll-based effects specifically. missing for 10: explicit support for state-based/conditional logic in prototypes, scroll-triggered effects, and clean first-party documentation of animation between screens/states without the SVG export bug.", "evidenceIds": [ "claude-design-docs-1", "claude-design-docs-18", "claude-design-docs-12", "claude-design-comm-17", "claude-design-comm-18" ] }, { "productId": "claude-design", "storyId": "agent-creates-design", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "There's no documented first-party write API or MCP server for Claude Design itself (OpenAPI probe returned 404s), but Claude Code slash-commands (/design, /design-sync) and handoff bundles let an agent programmatically create/modify designs, and one community report describes a working MCP flow pushing Claude Design output into PenPot with reusable components. Missing for 10: official documented MCP server/API surface for Claude Design, broader independent corroboration of agent-driven write access beyond a single anecdote.", "evidenceIds": [ "claude-design-docs-6", "claude-design-docs-7", "claude-design-comm-7", "claude-design-probe-3" ] }, { "productId": "claude-design", "storyId": "agent-exports-assets", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm export to PPTX/PDF/HTML and a Claude Code handoff bundle triggered via /design-sync, and community reports an MCP-based flow (into PenPot) suggesting some programmatic surface exists, but there is no documented API/SDK for rendering or exporting designs and the openapi probe returned 404s, indicating no formal machine-callable spec. missing for 10: documented API/SDK for export, evidence of screenshot/image export via a callable interface, corroborated OpenAPI or MCP server documentation.", "evidenceIds": [ "claude-design-docs-8", "claude-design-docs-6", "claude-design-docs-7", "claude-design-comm-7", "claude-design-probe-3" ] }, { "productId": "claude-design", "storyId": "agent-reads-design-context", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Claude Design ships explicit hand-off paths for coding agents — /design-sync to pull the design system into Claude Code, packaged handoff bundles, and import from GitHub/codebase/component libraries — which functionally lets an agent implement designs with real context. However, the mechanism is documented as slash-command/bundle integration rather than an explicit MCP server or general API exposing frames, variables, screenshots, and component mappings; the one community mention of MCP (comm-7) describes Claude Design consuming an external tool's MCP (PenPot), not exposing its own for coding agents to pull context. missing for 10: first-party documentation of an MCP server or API surface for frames/variables/screenshots/component mappings, and independent verification that agents (not just Claude Code) can pull that structured context.", "evidenceIds": [ "claude-design-docs-6", "claude-design-docs-7", "claude-design-docs-5", "claude-design-comm-7", "claude-design-comm-16" ] }, { "productId": "claude-design", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Probes confirm llms.txt is live (HTTP 200) at the Claude support domain and that Claude Design's own docs page is available in agent-friendly markdown form (docs-md probe), meaning an agent could be pointed at these resources to learn the product. Missing for 10: explicit vendor documentation describing llms.txt as an intended agent-consumption channel, and community/hands-on confirmation that an agent actually used llms.txt to complete a task with Claude Design.", "evidenceIds": [ "claude-design-probe-1", "claude-design-probe-2" ] }, { "productId": "claude-design", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Claude Design can pull in external data (Salesforce, QuickBooks, GitHub) and use it to generate on-brand designs, proposals, or campaign plans, which is a limited form of data-driven suggestion, but this is framed around design generation rather than analytical insight delivery, and no evidence shows dedicated 'insights/suggestions' features from uploaded datasets. missing for 10: dedicated data-analysis/insight generation UI, independent corroboration that it surfaces analytical insights (not just design outputs) from imported data.", "evidenceIds": [ "claude-design-docs-16", "claude-design-docs-17", "claude-design-docs-5", "claude-design-docs-11" ] }, { "productId": "claude-design", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Claude Design is a conversational/interactive design tool for generating mockups, prototypes, and handoffs; nothing in the evidence describes scheduled tasks, triggers, or autonomous background runs — all workflows are synchronous and user-initiated via chat or explicit commands.", "evidenceIds": [] }, { "productId": "claude-design", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Claude Design is built around delegating design tasks directly to Claude within the product—describing a visual, refining through conversation/sliders, generating variations, and packaging handoffs are all done by the embedded Claude assistant, and this is corroborated by hands-on community reports of building shippable pages and using the handoff skill successfully. missing for 10: independent quality benchmarking of how much genuine agentic reasoning vs templated generation occurs, and some community reports note underwhelming or 'boring' outputs suggesting delegation quality varies.", "evidenceIds": [ "claude-design-docs-13", "claude-design-docs-14", "claude-design-docs-21", "claude-design-docs-9", "claude-design-comm-6", "claude-design-comm-16", "claude-design-comm-9" ] }, { "productId": "claude-design", "storyId": "agentic-headless", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Claude Design is a visual/interactive design and prototyping tool centered on a web/chat UI, not a CLI or automatable service; there's no headless/CI execution mode in scope for this product category, and evidence pack contains no CLI/API/automation surface for it (integration is via Claude Code handoff, not headless self-execution).", "evidenceIds": [] }, { "productId": "claude-design", "storyId": "agentic-mcp-client", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Only a single ambiguous community mention notes 'Claude Design into PenPot via its MCP,' suggesting some MCP-based integration exists, and docs reference pulling data from tools like Salesforce/QuickBooks/HubSpot without specifying MCP as the mechanism. There is no official documentation confirming users can plug arbitrary MCP servers into Claude Design to expose their tools. Missing for 10: first-party docs on MCP server support, clarity on directionality (client vs server), and broader independent confirmation beyond one ambiguous forum comment.", "evidenceIds": [ "claude-design-comm-7", "claude-design-docs-16", "claude-design-docs-17" ] }, { "productId": "claude-design", "storyId": "agentic-mcp-server", "verdict": "na", "quality": 0, "confidence": "medium", "rationale": "Claude Design is itself an AI agent/assistant product (built on Claude) rather than a platform whose ecosystem would ship an official MCP server for others to connect to. The one MCP-related mention (comm-7) describes Claude Design connecting *into* PenPot via MCP — a client-side usage — not Claude Design exposing its own MCP server, so this falls under the agent-role 'na' exception rather than the applicable-axis case.", "evidenceIds": [ "claude-design-comm-7" ] }, { "productId": "claude-design", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Claude Design's entire interaction model is natural-language: users 'describe a visual and Claude builds a first version,' refine 'through conversation, inline comments,' and use conversational handoffs like /design and /design-sync — with real-world use (landing pages, PenPot flows) confirming this works in practice, though output quality and robustness vary across users. Missing for 10: consistent quality/benchmarks across users and no independent rigor testing beyond anecdotal HN reports.", "evidenceIds": [ "claude-design-docs-13", "claude-design-docs-21", "claude-design-docs-15", "claude-design-docs-6", "claude-design-comm-6", "claude-design-comm-7", "claude-design-comm-9" ] }, { "productId": "claude-design", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Claude Design integrates with Claude Code via slash commands (/design-sync, /design) and handoff bundles, but this is Claude Code's CLI, not an official CLI shipped by Claude Design itself; no standalone CLI for Claude Design is documented.", "evidenceIds": [ "claude-design-docs-6", "claude-design-docs-7" ] }, { "productId": "claude-design", "storyId": "agentic-public-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence pack shows only end-user product docs (import/export, design generation, handoff to Claude Code) and no documented public API; a direct probe for OpenAPI/swagger specs at claude.com/support returned 404s, and no REST/SDK API for Claude Design is mentioned anywhere.", "evidenceIds": [ "claude-design-probe-3" ] }, { "productId": "claude-design", "storyId": "agentic-scoped-keys", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Claude Design is a design/prototyping product, not an API/credential-management platform; scoped API credential issuance for agents is outside its product category and is a wrong axis for this evidence pack.", "evidenceIds": [] }, { "productId": "claude-design", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Claude Design's evidence pack shows only end-user product docs (mockups, prototypes, handoff, exports) and community discussion of MCP integration with third-party tools like PenPot, but no mention of an official SDK for building against Claude Design itself. A direct probe for API/OpenAPI specs returned 404s, indicating no public developer SDK is documented.", "evidenceIds": [ "claude-design-probe-3", "claude-design-comm-7" ] }, { "productId": "claude-design", "storyId": "agentic-webhooks", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Claude Design is a design/prototyping product for humans, not a service that exposes event subscriptions; webhooks are not a relevant capability axis for this product category, and no evidence discusses webhooks at all.", "evidenceIds": [] }, { "productId": "claude-design", "storyId": "ai-asset-editing-tools", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Claude Design can generate full visual designs from a text prompt and offers layout/color adjustment sliders (claude-design-docs-13, claude-design-docs-9), which loosely maps to 'image generation' and some asset tuning, but there is no evidence of dedicated background-removal, content-aware fill, or copy-suggestion tools built into the editor. missing for 10: background removal tool, content-aware fill/inpainting, in-editor copy/headline suggestion feature.", "evidenceIds": [ "claude-design-docs-13", "claude-design-docs-9", "claude-design-docs-21" ] }, { "productId": "claude-design", "storyId": "api-interactive-docs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Claude Design is a design/prototyping product, not a developer API/SDK platform; an interactive API reference with runnable examples is a category mismatch for this product type.", "evidenceIds": [] }, { "productId": "claude-design", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Claude Design is a design/prototyping product with no evidence of a public API, and the direct probe for OpenAPI/swagger endpoints returned 404 on all candidate paths, with no other documentation mentioning a machine-readable API spec.", "evidenceIds": [ "claude-design-probe-3" ] }, { "productId": "claude-design", "storyId": "api-sandbox", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Claude Design's core workflow is prototyping (mockups, interactive prototypes) that are explicitly separate from production code — 'no PRs, no code review' before user-testing — which implies a sandbox-like separation from shipped/production systems. However, there is no explicit mention of a dedicated sandbox environment, test data isolation, or protection of 'production data' in a technical/backend sense; the framing is about design/code review gating rather than data safety.\nmissing for 10: explicit sandbox/staging environment terminology, guarantees about not touching production data or backend systems, and independent confirmation of this isolation in practice.", "evidenceIds": [ "claude-design-docs-1", "claude-design-docs-18", "claude-design-docs-7" ] }, { "productId": "claude-design", "storyId": "api-versioning-policy", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Claude Design is a design-to-prototype/handoff product, not an API platform; nothing in the evidence suggests it exposes a developer-facing API surface that would need versioning/deprecation policies. This is a category mismatch rather than a missing capability.", "evidenceIds": [] }, { "productId": "claude-design", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Claude Design supports generating multiple design directions at once (claude-design-docs-2) and applying a design system consistently 'to every project automatically' (claude-design-docs-3), which suggests some batch-like automation, but there's no evidence of true bulk operations across many items (e.g., batch editing dozens of files/assets, bulk export/import at scale, or programmatic multi-item processing). missing for 10: evidence of bulk/batch processing across many discrete items (not just generating variants), bulk edit/export APIs, scale limits, and independent confirmation of multi-item automation workflows.", "evidenceIds": [ "claude-design-docs-2", "claude-design-docs-3" ] }, { "productId": "claude-design", "storyId": "automation-rules-engine", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Claude Design is a design/prototyping tool for generating and refining visuals and mockups; there is no evidence of an automation/rules engine that triggers actions on events. This is a category mismatch, not a missing feature of the product's core purpose.", "evidenceIds": [] }, { "productId": "claude-design", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Claude Design is a design/prototyping tool for creating mockups and interactive prototypes, not a workflow automation or scheduling platform; there is no concept of recurring jobs in this product category. Scheduling recurring jobs is a category error for this axis.", "evidenceIds": [] }, { "productId": "claude-design", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes Claude Design's prototyping, design-system sync, and handoff features, but contains no mention of version history, review workflows, or rollback capability for any automations or design iterations it produces. Community comments even note lock-out of prior work after unsubscribing, with no rollback/versioning remedy mentioned.", "evidenceIds": [] }, { "productId": "claude-design", "storyId": "build-custom-plugins", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Claude Design is a design/prototyping tool, not an editor/IDE with an extensibility framework; there is no evidence of an editor plugin architecture at all, so a 'plugin API for editor extensions' is a category error for this product type.", "evidenceIds": [] }, { "productId": "claude-design", "storyId": "clear-plan-pricing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows product features, an enterprise admin toggle, and usage-bar mentions, but contains no documentation of seat types, pricing tiers, or explicit AI-feature gating tables for procurement decisions.", "evidenceIds": [] }, { "productId": "claude-design", "storyId": "comments-review-flows", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Evidence shows collaborative editing (grant edit access, group conversation) and 'inline comments' as a refinement mechanism, but there's no documented @mention-teammates feature or thread-resolution workflow that would replace email-based review. Missing for 10: explicit @mention of teammates, comment thread resolution/status tracking, and any hands-on confirmation that commenting works as a review tool rather than just a chat refinement input.", "evidenceIds": [ "claude-design-docs-4", "claude-design-docs-21" ] }, { "productId": "claude-design", "storyId": "design-to-production-code", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs describe concrete design-to-code handoff: importing real components from GitHub/codebase, syncing design systems via /design-sync, and packaging a 'handoff bundle' passed to Claude Code with one instruction (claude-design-docs-5,6,7,8). Community corroborates this works well ('handoff skill in Claude Design is pretty solid', 'Got something shippable in a few hours') but also flags that the generated code is often unmaintainable ('write-once, read-never' code) and animation/export sanitization bugs, showing real shippability caveats. Missing for 10: independent verification of true component-to-code mapping fidelity (not just handoff bundling) and resolution of maintainability/code-quality concerns raised by users.", "evidenceIds": [ "claude-design-docs-5", "claude-design-docs-6", "claude-design-docs-7", "claude-design-comm-16", "claude-design-comm-15", "claude-design-comm-6" ] }, { "productId": "claude-design", "storyId": "design-tokens-variables", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Claude Design can 'apply your team's design system to every project automatically' and offers sliders to tune spacing/color, and one community report describes it building 'reusable PenPot components and design system tokens' via an MCP export flow — but there's no first-party evidence of native token/variable definitions with mode/theme switching (light/dark, brand variants) inside Claude Design itself. missing for 10: explicit token/variable editor, multi-mode/theme support, first-party documentation of applying themes across multiple designs.", "evidenceIds": [ "claude-design-docs-3", "claude-design-docs-9", "claude-design-comm-7" ] }, { "productId": "claude-design", "storyId": "export-formats-assets", "verdict": "partial", "quality": 3, "confidence": "medium", "rationale": "Claude Design documents export to PPTX, PDF, and HTML, and community evidence confirms SVG export is possible (though it strips animations unless renamed), showing some baseline export capability exists. However, there is no evidence anywhere in the pack of PNG export, per-layer export presets, scale multipliers, or slice/asset regions — the core professional export controls the story asks about are entirely unevidenced. missing for 10: PNG export, per-layer export presets, scale export options, slice/asset export regions, any evidence of production-asset export workflows beyond whole-document PDF/PPTX/HTML.", "evidenceIds": [ "claude-design-docs-8", "claude-design-comm-17", "claude-design-comm-18" ] }, { "productId": "claude-design", "storyId": "free-tier-real-design", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no information about a free tier for Claude Design, its editor capabilities without payment, or collaboration limits for non-paying users. In fact, community reports indicate access is tied to an active subscription and is revoked upon cancellation (comm-11, comm-12, comm-13), which cuts against the idea of a functional free tier, but there's no direct claim or denial of a free tier's design capabilities to judge against.", "evidenceIds": [ "claude-design-comm-11", "claude-design-comm-12", "claude-design-comm-13" ] }, { "productId": "claude-design", "storyId": "inspect-code-handoff", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Claude Design can export handoff bundles and code (HTML/CSS/JS) to Claude Code or as PPTX/PDF/HTML, but there is no mention of a layer-level 'inspect' mode giving measurements, styles, assets, and platform-specific (CSS/iOS/Android) code snippets for viewers without an editor seat — the axis is applicable to a design/dev-handoff tool but unevidenced here.", "evidenceIds": [ "claude-design-docs-7", "claude-design-docs-8", "claude-design-comm-15" ] }, { "productId": "claude-design", "storyId": "interactive-prototypes", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs confirm Claude Design turns static mockups into shareable, testable interactive prototypes without code (claude-design-docs-1, claude-design-docs-18), which covers the no-code prototyping goal, but there is no evidence of specific multi-screen wiring primitives like triggers, transitions, or overlays that a designer would use in traditional prototyping tools. Community feedback (claude-design-comm-6, claude-design-comm-9) discusses general design output quality but not this specific screen-linking mechanic. Missing for 10: explicit support for defining triggers/transitions/overlays between distinct screens, and any hands-on confirmation that multi-screen flows can be wired together.", "evidenceIds": [ "claude-design-docs-1", "claude-design-docs-18", "claude-design-comm-6", "claude-design-comm-9" ] }, { "productId": "claude-design", "storyId": "nl-design-generation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "First-party docs and community reports confirm Claude Design generates first drafts, multiple directions, layouts, and mockups from natural-language prompts with editing/refinement controls (docs-13,20,9,21; comm-6,7,16), and users describe shippable landing pages and solid handoff. However hands-on community feedback also reports underwhelming, 'boring', or 'useless' results for some prompts and generic upload errors (comm-3,9,10,8), showing real quality variance rather than a uniformly reliable draft-to-mockup pipeline. Missing for 10: consistent independent benchmarking of output quality across prompt types, and resolution of reported failure cases (large file uploads, simple prompts underwhelming).", "evidenceIds": [ "claude-design-docs-13", "claude-design-docs-20", "claude-design-docs-9", "claude-design-docs-21", "claude-design-comm-6", "claude-design-comm-7", "claude-design-comm-16", "claude-design-comm-3", "claude-design-comm-9", "claude-design-comm-10" ] }, { "productId": "claude-design", "storyId": "open-file-format", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Claude Design outputs are HTML/CSS/JS and can be exported to PPTX, PDF, or HTML (claude-design-docs-8), and an Anthropic team member explicitly confirmed 'Claude's designs are HTML/CSS/JS that any editor can handle' (claude-design-comm-14), supporting portability outside the vendor's editor. However, there was a real-world incident where users temporarily lost access to their design history/code after unsubscribing (claude-design-comm-11, claude-design-comm-12) before the fix, and community members note the generated code is hard for humans to maintain (claude-design-comm-15), undercutting practical toolability. missing for 10: a stable, always-available raw export/download path independent of subscription status, and clearer documentation of the underlying file format/spec beyond ad-hoc HTML/CSS/JS.", "evidenceIds": [ "claude-design-docs-8", "claude-design-comm-11", "claude-design-comm-12", "claude-design-comm-13", "claude-design-comm-14", "claude-design-comm-15" ] }, { "productId": "claude-design", "storyId": "openness-api-parity", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of a Claude Design API; the only API-related probe found openapi.json/swagger.json endpoints all returning 404, and no docs mention programmatic access to Design's prototyping, handoff, or editing features. All capabilities (prototyping, handoff, design-system sync) are described only as UI/Claude Code slash-command workflows.", "evidenceIds": [ "claude-design-probe-3", "claude-design-docs-6", "claude-design-docs-15" ] }, { "productId": "claude-design", "storyId": "openness-full-export", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Docs confirm export to open-ish formats (PPTX, PDF, HTML) and that designs are HTML/CSS/JS editable by any tool, suggesting an open-format export path. However, community reports document a concrete failure: users lost access to their entire design history/projects after unsubscribing, effectively blocking 'leaving' with their data, which Anthropic staff acknowledged as a bug needing a fix rather than confirming it works as intended. Missing for 10: a documented, reliable full-data export mechanism (all chats, assets, code) confirmed to work post-cancellation, and independent verification that the promised fix was shipped.", "evidenceIds": [ "claude-design-docs-8", "claude-design-comm-11", "claude-design-comm-12", "claude-design-comm-13", "claude-design-comm-14" ] }, { "productId": "claude-design", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Claude Design is a proprietary, closed-source hosted design product; there is no indication it's an open-source project with a source repository to read. This axis applies to open-source/openness of source code, which is a category error for a commercial SaaS design tool.", "evidenceIds": [] }, { "productId": "claude-design", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Claude Design is a hosted SaaS feature of Claude.ai with no evidence of any self-hostable/open-source distribution; self-hosting is not a fair axis for this kind of cloud product, and evidence pack confirms it's an Anthropic-hosted service accessed via claude.ai/design.", "evidenceIds": [] }, { "productId": "claude-design", "storyId": "plugin-ecosystem-marketplace", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a plugin marketplace or extensibility ecosystem for Claude Design (icons, data fillers, accessibility checkers, etc.); the only related mention is a single user connecting Claude Design to PenPot via MCP, which is a one-off integration, not a documented plugin/marketplace system.", "evidenceIds": [ "claude-design-comm-7" ] }, { "productId": "claude-design", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack mentions data residency, region selection, or geographic storage controls for Claude Design; only Enterprise admin enable/disable toggle is discussed, not residency.", "evidenceIds": [ "claude-design-docs-10" ] }, { "productId": "claude-design", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in this pack addresses training-data opt-out or privacy controls for Claude Design; the pack covers only product features, enterprise admin toggles, and unrelated community feedback.", "evidenceIds": [] }, { "productId": "claude-design", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation of user-controlled data retention or deletion settings for Claude Design; instead, community reports describe users being locked out of their design projects entirely after canceling a subscription, with Anthropic acknowledging the issue and promising a fix, rather than any dashboard/API for retention policy or deletion control.", "evidenceIds": [ "claude-design-comm-11", "claude-design-comm-12", "claude-design-comm-13", "claude-design-comm-14" ] }, { "productId": "claude-design", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack discusses telemetry/usage-tracking opt-out settings for Claude Design; the only related item confirms a separate usage bar, not a privacy control. Missing for 10: any documentation or setting for opting out of telemetry/usage tracking.", "evidenceIds": [] }, { "productId": "claude-design", "storyId": "realtime-multiplayer-editing", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs mention granting edit access so colleagues can modify a design and chat with Claude in a group conversation, implying some shared editing, but there is no evidence of live cursors, simultaneous multi-user editing, or instant real-time sync mechanics typical of multiplayer design tools. Missing for 10: explicit live-cursor/presence indicators, real-time simultaneous editing demonstration, and independent confirmation of true multiplayer sync.", "evidenceIds": [ "claude-design-docs-4" ] }, { "productId": "claude-design", "storyId": "responsive-auto-layout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes Claude Design as a prompt-driven generator that outputs HTML/CSS/JS mockups and prototypes, with only generic 'rich layout controls' to drag/resize/align elements (claude-design-docs-9) — there is no mention of Figma-style frames, auto layout, constraints, or grid systems that adapt responsively. Community feedback even notes the generated code is not maintainable/structured (claude-design-comm-15), suggesting it isn't built on a constraint-based vector layout engine. Missing for 10: any documentation of frames, auto-layout, constraint-based resizing, or grid systems enabling responsive behavior.", "evidenceIds": [ "claude-design-docs-9", "claude-design-comm-15" ] }, { "productId": "claude-design", "storyId": "reusable-components-variants", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Claude Design is described as a prompt-driven design/prototyping tool that can 'apply your team's design system' and import components, but there is no evidence of a structured component system with defined variants and per-instance overrides that stay linked to a master source, as in tools like Figma. The one related community mention (comm-7) is about exporting to PenPot's component system, not a native capability of Claude Design itself.", "evidenceIds": [ "claude-design-docs-3", "claude-design-docs-5", "claude-design-comm-7" ] }, { "productId": "claude-design", "storyId": "self-host-design-platform", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Claude Design is a hosted SaaS feature accessed via claude.ai; there is no evidence of an open-source license or self-hostable release, and this is a cloud-only product category where self-hosting is not a fair expectation given no such offering exists.", "evidenceIds": [] }, { "productId": "claude-design", "storyId": "shared-team-libraries", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Claude Design can apply a team's design system automatically across projects and sync design-system context via '/design-sync' from Claude Code, suggesting some cross-project consistency, but there is no evidence of a true shared-library mechanism (publishing reusable components/styles with versioned, controlled updates propagating when a source library changes) as in dedicated design tools. Missing for 10: library publishing workflow, update/versioning notifications when source components change, and independent confirmation of cross-project sync behavior.", "evidenceIds": [ "claude-design-docs-3", "claude-design-docs-6" ] }, { "productId": "claude-design", "storyId": "vector-editing-craft", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Claude Design is a prompt-driven design/prototyping generator with only basic layout controls (drag, resize, align, sliders for spacing/color) — there is no evidence of pen/path vector editing, boolean path operations, or precision alignment tooling akin to professional vector editors. Community feedback even notes the underlying output is HTML/CSS/JS, not vector paths, and code is not maintainable, reinforcing it's not a vector design tool.", "evidenceIds": [ "claude-design-docs-9", "claude-design-comm-15", "claude-design-comm-17", "claude-design-comm-18" ] }, { "productId": "claude-design", "storyId": "version-history-restore", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence describes named milestones, version comparison, or restore-to-earlier-state functionality in Claude Design; community discussion only mentions losing access to 'history' after unsubscribing, not a versioning/milestone/compare/restore feature.", "evidenceIds": [ "claude-design-comm-11", "claude-design-comm-12", "claude-design-comm-13" ] }, { "productId": "figma", "storyId": "advanced-prototype-logic", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Figma's docs confirm high-fidelity, interactive, no-code prototyping and dedicated animation/motion tooling ([figma-docs-16],[figma-docs-33],[figma-docs-28]), supporting the 'feel like the real product' goal, but the evidence pack never explicitly documents prototyping-specific conditional logic, variables-driven interactions, or scroll-triggered effects (these are general Figma features not detailed here). Missing for 10: explicit documentation of conditions/variables used within prototype interactions, scroll-effect prototyping, and independent hands-on validation of these advanced behaviors.", "evidenceIds": [ "figma-docs-16", "figma-docs-33", "figma-docs-28" ] }, { "productId": "figma", "storyId": "agent-creates-design", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Figma's official MCP server explicitly documents write-capable tools (\"Write to the canvas: Create and modify native Figma content directly from your MCP client\", \"enabling agents to write native Figma content back to the canvas\"), plus the Plugin API supports full read/write access to create and modify file contents, and the REST API includes create/update/delete endpoints (e.g., Variables API). This is corroborated by multiple first-party docs and a dedicated probe confirming the MCP server's existence. Missing for 10: no independent hands-on report specifically validating write-via-MCP in production use.", "evidenceIds": [ "figma-docs-1", "figma-docs-38", "figma-docs-52", "figma-docs-59", "figma-docs-8", "figma-docs-32", "figma-docs-62", "figma-probe-3" ] }, { "productId": "figma", "storyId": "agent-exports-assets", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Figma's docs show agents can extract design data and generate code from frames via the MCP server and REST API (figma-docs-39, figma-docs-40, figma-docs-59, figma-docs-49), and the REST API is documented as a programmatic surface for pulling objects/layers. However, the evidence pack never explicitly documents an image/screenshot rendering or production-format export endpoint (e.g., PNG/SVG/PDF render API) for agent use — most citations focus on writing to canvas, code generation, and diagram creation rather than visual export. Missing for 10: explicit documentation of an image-rendering/export endpoint or MCP tool for screenshots/production image formats, and independent/hands-on confirmation of such export working via the programmatic surface.", "evidenceIds": [ "figma-docs-39", "figma-docs-40", "figma-docs-59", "figma-docs-49", "figma-docs-7", "figma-probe-3" ] }, { "productId": "figma", "storyId": "agent-reads-design-context", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Figma ships a first-party MCP server (local and remote) with documented tools for extracting variables, components, layout data, generating code from frames, and writing back to canvas, plus REST API endpoints for variables, layers, and Code Connect for component-to-code mapping — directly matching the story. Evidence spans official docs, GitHub repo, and a probe confirming the live MCP server endpoint. Missing for 10: independent hands-on developer report specifically validating MCP-based design-to-code extraction (community evidence covers general Figma-to-code pain points but not the MCP server itself).", "evidenceIds": [ "figma-docs-3", "figma-docs-38", "figma-docs-39", "figma-docs-40", "figma-docs-59", "figma-docs-68", "figma-docs-52", "figma-gh-2", "figma-docs-54", "figma-docs-62", "figma-probe-3", "figma-docs-67" ] }, { "productId": "figma", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A probe confirms a live llms.txt file at https://developers.figma.com/llms.txt returning HTTP 200 with a structured description of Figma's developer platform docs, directly enabling an agent to be pointed at it. Missing for 10: independent/community confirmation of an agent successfully consuming the file, and no evidence of additional agent-oriented doc formats (e.g., openapi.json, which 404s).", "evidenceIds": [ "figma-probe-1", "figma-probe-2" ] }, { "productId": "figma", "storyId": "agentic-ai-insights", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Figma documents an AI agent that generates design directions, diagrams, images, and searches files, and touts 'AI agents work in the same space with shared context' (figma-docs-17, figma-docs-34, figma-docs-51), which matches the story's intent of AI-generated insights/suggestions from the user's own design data. However, hands-on community feedback specifically contradicts this: 'the AI tools in FigJam are really bad and often broken' (figma-comm-8), while another user reports success summarizing notes into slides (figma-comm-9), giving mixed real-world evidence rather than a clean full delivery. Missing for 10: independent benchmarks or broader corroboration of AI insight quality/reliability across Figma (not just FigJam), and clearer documentation of insight-generation (vs. content-generation) use cases.", "evidenceIds": [ "figma-docs-17", "figma-docs-34", "figma-docs-51", "figma-comm-8", "figma-comm-9" ] }, { "productId": "figma", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Figma exposes webhooks and REST API that let external systems react to events (comments, version changes) and CI/CD integrations, which supports background automation infrastructure, but there is no evidence of a native, autonomous scheduled agent or background-running automation within Figma itself — plugins/widgets require user invocation or live session context. Missing for 10: evidence of a built-in scheduler/autonomous agent execution mode, and independent confirmation that webhook-driven automations run fully unattended without additional third-party glue.", "evidenceIds": [ "figma-docs-53", "figma-docs-61", "figma-docs-57", "figma-docs-70", "figma-docs-21", "figma-docs-36", "figma-docs-65" ] }, { "productId": "figma", "storyId": "agentic-builtin-assistant", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Figma ships a built-in AI agent (Figma AI/Make) that can generate designs, diagrams, edit images, search files, and collaborate with teammates in real time (figma-docs-17, figma-docs-34, figma-docs-51), which matches the story of delegating tasks to an in-product assistant. However, a hands-on community report explicitly states the AI tools (in FigJam) are 'really bad and often broken' (figma-comm-8), directly contradicting the vendor's polished framing of reliable delegation. Missing for 10: independent corroboration that the assistant reliably completes delegated tasks, and detail on task scope/limits beyond marketing copy.", "evidenceIds": [ "figma-docs-17", "figma-docs-34", "figma-docs-51", "figma-comm-8", "figma-comm-9" ] }, { "productId": "figma", "storyId": "agentic-headless", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Figma's REST API explicitly supports CI/CD-style automation for variables sync ('Integrate Figma directly with continuous integration (CI) systems', 'useful for needs like CI/CD implementations... webhooks... service accounts') and webhooks/plan tokens allow non-interactive, user-agnostic access, which is real evidence of headless automation capability. However this is scoped mainly to variables/webhooks/library-analytics rather than full design-file manipulation or MCP server operation, and there's no evidence of a general-purpose CLI or headless rendering/export pipeline for arbitrary Figma actions. missing for 10: evidence of a full headless CLI or automation runner covering arbitrary design operations (not just variables/webhooks), independent/hands-on confirmation of CI usage, and headless operation of the MCP server itself.", "evidenceIds": [ "figma-docs-57", "figma-docs-70", "figma-docs-62", "figma-docs-53", "figma-docs-66" ] }, { "productId": "figma", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "All MCP-related evidence describes Figma exposing its own MCP server for external clients (e.g., IDEs, coding agents) to consume Figma's tools — the opposite direction of this story, which asks whether Figma itself can plug in and use external MCP servers' tools. No evidence shows Figma's AI/agent features acting as an MCP client capable of consuming third-party MCP servers.", "evidenceIds": [] }, { "productId": "figma", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Figma ships an official, well-documented MCP server (local and remote) that lets AI clients read/write Figma content, extract design context, generate code, and more, with detailed docs on installation, tools, and custom rules (figma-docs-1/3/5/38/59/67/68/69, figma-probe-3). This is a first-party, actively maintained integration point for agentic MCP clients. Missing for 10: independent/hands-on community verification that the MCP connection works reliably in practice (community evidence only discusses unrelated Figma pain points, not MCP specifically).", "evidenceIds": [ "figma-docs-1", "figma-docs-3", "figma-docs-5", "figma-docs-38", "figma-docs-59", "figma-docs-67", "figma-docs-68", "figma-docs-69", "figma-probe-3" ] }, { "productId": "figma", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Figma ships a first-party AI agent (natural-language design generation, image editing, file search, diagram creation, and Figma Make for prompt-to-prototype) letting users operate parts of the product conversationally, plus an MCP server enabling agent-driven natural-language control from IDEs. However, community feedback specifically calls the FigJam AI tools 'really bad and often broken,' and there's no independent corroboration that the core Figma AI agent reliably handles broad natural-language operation across the whole product surface (only marketing copy). Missing for 10: independent hands-on validation of the Figma AI agent's reliability, evidence of NL command coverage beyond generation/diagramming/search, and resolution of the FigJam AI quality complaints.", "evidenceIds": [ "figma-docs-17", "figma-docs-34", "figma-docs-43", "figma-docs-51", "figma-docs-56", "figma-comm-8", "figma-comm-9" ] }, { "productId": "figma", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Figma does ship an official CLI (Code Connect CLI, documented at the quickstart guide) which is used to sync design system components in code with Figma and keep MCP-generated code consistent with the codebase — supporting agentic/CI-style workflows. However, the evidence pack gives no detail on CLI command surface, scripting for AI agents, or independent hands-on validation beyond its existence. Missing for 10: detailed CLI usage docs for AI-native/agentic workflows, examples of CLI use in automated/agent pipelines, independent corroboration of the CLI's capabilities.", "evidenceIds": [ "figma-probe-4", "figma-gh-1", "figma-gh-2", "figma-gh-3", "figma-docs-25" ] }, { "productId": "figma", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Figma publishes a well-documented REST API (with OpenAPI spec on GitHub), webhooks, Plugin API, and an official MCP server enabling agentic control of designs, code generation, and diagrams—giving AI-native users multiple documented, programmatic ways to drive the product. Missing for 10: a discoverable live OpenAPI/swagger endpoint on the docs domain itself (probe found 404s) and more independent hands-on corroboration of API robustness beyond vendor docs.", "evidenceIds": [ "figma-docs-7", "figma-docs-20", "figma-docs-46", "figma-docs-1", "figma-docs-59", "figma-probe-3", "figma-probe-2", "figma-docs-8" ] }, { "productId": "figma", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Figma's REST API docs mention OAuth apps that can 'make requests on behalf of other users' without using a personal account, and 'plan access tokens' scoped to an org/enterprise plan, plus service-account-like tokens for CI/CD/webhooks decoupled from an individual user — these are building blocks for issuing a non-personal, scoped credential to an agent. However there is no explicit documentation of granular least-privilege scopes (e.g., read-only vs write, per-file/per-resource permissions) for AI agents or the MCP server specifically. Missing for 10: documented fine-grained OAuth/API scopes selectable for agent credentials, explicit MCP-server auth/scoping guidance, and any independent confirmation this scoping mechanism is used for AI agents.", "evidenceIds": [ "figma-docs-48", "figma-docs-57", "figma-docs-66", "figma-docs-76" ] }, { "productId": "figma", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Figma provides an official REST API with OpenAPI spec and TypeScript types (figma-docs-46), a Plugin/Widget API (figma-docs-8, figma-docs-32), Code Connect SDK/CLI on GitHub (figma-gh-1, figma-gh-2, figma-docs-60), and an official MCP server for AI-native workflows (figma-probe-3, figma-docs-1). These are well-documented, first-party SDKs/APIs explicitly built for programmatic and AI-agent integration. missing for 10: no independent hands-on developer corroboration of SDK reliability/completeness, and probe found no discoverable openapi.json at the expected well-known endpoints despite a spec repo existing.", "evidenceIds": [ "figma-docs-46", "figma-gh-1", "figma-gh-2", "figma-docs-8", "figma-docs-32", "figma-probe-3", "figma-docs-1", "figma-docs-60", "figma-probe-2" ] }, { "productId": "figma", "storyId": "agentic-webhooks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Figma's REST API docs explicitly describe creating and managing webhooks and subscribing to events like comments or new file versions, with a dedicated webhooks-endpoints reference, directly matching the user story. Missing for 10: independent or hands-on corroboration that webhook subscriptions work reliably in practice, and fuller documentation of the range of supported event types beyond the couple of examples given.", "evidenceIds": [ "figma-docs-21", "figma-docs-53", "figma-docs-57", "figma-docs-61", "figma-docs-7" ] }, { "productId": "figma", "storyId": "ai-asset-editing-tools", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Figma's own site claims an in-editor AI agent that can 'generate new design directions, create diagrams, edit images, search your files, and more' (figma-docs-17, figma-docs-34), which covers some asset-chore functions like image editing and diagram generation, but the evidence never mentions background removal, content-aware fill, or copy/microcopy suggestions specifically. A community report also flags that Figma's AI tools (in FigJam) are 'really bad and often broken' (figma-comm-8), undermining confidence in reliability. missing for 10: explicit documentation of background removal, content-aware fill, and copy-suggestion features, plus independent hands-on validation that these AI asset tools work reliably.", "evidenceIds": [ "figma-docs-17", "figma-docs-34", "figma-docs-24", "figma-comm-8" ] }, { "productId": "figma", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Figma's REST API docs describe endpoints and there's a separate OpenAPI spec repo, but no evidence shows an interactive reference with runnable/try-it-now examples; probes for standard openapi.json/swagger endpoints on the docs site returned 404s, and no docs mention interactive code sandboxes or live example runners.", "evidenceIds": [ "figma-docs-46", "figma-probe-2", "figma-docs-7", "figma-docs-20" ] }, { "productId": "figma", "storyId": "api-machine-spec", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Figma publishes an OpenAPI specification and TypeScript types for its REST API in a public GitHub repo (figma-docs-46), which is machine-readable and downloadable, directly satisfying the story. Probe evidence shows the spec isn't hosted at conventional /openapi.json paths on the docs site (figma-probe-2), so discoverability is slightly less standard. Missing for 10: a directly hosted spec endpoint on developers.figma.com and independent confirmation of spec completeness/versioning.", "evidenceIds": [ "figma-docs-46", "figma-probe-2", "figma-docs-20" ] }, { "productId": "figma", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of a sandbox/staging environment or test mode that isolates AI-native testing from production Figma data; all MCP/API/plugin docs describe operating directly on real files and production content. missing for 10: any mention of sandbox environments, test/staging Figma instances, mock data, or non-production testing modes for AI agent workflows.", "evidenceIds": [] }, { "productId": "figma", "storyId": "api-versioning-policy", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Figma publishes a REST API changelog documenting endpoint additions/renames (e.g., v2 folders endpoints, projects→folders rename) and an OpenAPI spec repo, showing some versioning practice, but there is no explicit documented deprecation policy (timelines, sunset notices, migration guarantees) for the REST API or MCP server. missing for 10: explicit deprecation policy statement, version support timelines, sunset/EOL notices for old endpoints.", "evidenceIds": [ "figma-docs-46", "figma-docs-47", "figma-docs-58", "figma-docs-48" ] }, { "productId": "figma", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Figma's REST/Variables APIs, Plugin API, and webhooks let developers script operations across many objects/files (e.g., variables CRUD, library analytics across all files, plugin read/write over full documents), which supports bulk automation, but there's no native first-class 'bulk operation' feature in the product itself — community evidence shows users still rely on third-party bulk-export plugins for basic tasks like exporting many icons at once, indicating manual/plugin workarounds rather than built-in bulk tooling. missing for 10: dedicated bulk-edit/bulk-apply UI or MCP tool for acting on many layers/files simultaneously, independent verification that plugin/REST scripting reliably scales to large batches, and first-party documentation explicitly describing bulk operations as a supported workflow.", "evidenceIds": [ "figma-docs-62", "figma-docs-55", "figma-docs-75", "figma-docs-8", "figma-docs-32", "figma-comm-2", "figma-comm-3" ] }, { "productId": "figma", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Figma's webhooks let you observe events (comments, new versions, library usage) and forward them to external systems (figma-docs-21, figma-docs-53, figma-docs-61), which can be wired into automation elsewhere, but Figma itself has no built-in rule-builder or 'if event then action' automation engine — plugins run only on user invocation, not on trigger conditions (figma-docs-36). missing for 10: a native rules/automation UI or API where users define event-condition-action logic inside Figma, and any evidence of automatic (non-user-invoked) action execution.", "evidenceIds": [ "figma-docs-21", "figma-docs-53", "figma-docs-61", "figma-docs-36" ] }, { "productId": "figma", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any scheduling, cron-like, or recurring job/workflow automation capability in Figma's docs, API, webhooks, or MCP server offerings — webhooks are event-triggered, not scheduled recurring jobs.", "evidenceIds": [] }, { "productId": "figma", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Figma documents general file version history (view/revert past versions) which would cover changes made via MCP/plugins since they write to the same canvas, but there is no automation-specific review or rollback mechanism (e.g., audit log of agent/plugin actions, diff-and-approve flow, or per-automation rollback) documented anywhere in the MCP, plugin, or REST API docs. Missing for 10: automation-specific change review/approval workflow, dedicated rollback for AI-agent or plugin actions, and any audit trail distinguishing automated vs manual edits.", "evidenceIds": [ "figma-docs-44", "figma-docs-38", "figma-docs-52", "figma-docs-59" ] }, { "productId": "figma", "storyId": "build-custom-plugins", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Figma documents a full Plugin API with read/write access to editors, custom UI, and widgets, plus dedicated docs site (developers.figma.com/docs/plugins) and a marketplace for publishing/distributing plugins and widgets (public or private). Combined with the OpenAPI/TypeScript REST spec and Code Connect samples, this constitutes a documented, typed extensibility platform for developers. Missing for 10: explicit mention of TypeScript typings specifically for the Plugin API (typings confirmed only for REST API) and independent hands-on developer reviews of the plugin-building experience.", "evidenceIds": [ "figma-docs-8", "figma-docs-32", "figma-docs-36", "figma-docs-50", "figma-docs-9", "figma-docs-11", "figma-docs-18", "figma-docs-65", "figma-docs-46" ] }, { "productId": "figma", "storyId": "clear-plan-pricing", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Figma's public pricing page fragments show tier-gated features (e.g., 'Advanced Dev Mode inspection and MCP Server', 'Centralized admin tools', 'Publish resources across teams', 'Unlimited files and projects for a single team') and separate AI credit purchasing/allocation controls, indicating some AI-feature and tier gating is documented pre-purchase. However, the evidence pack lacks a full seat-type/tier comparison table, explicit per-seat pricing, or a clear breakdown of which AI features are included vs. add-on across each named plan (Starter/Professional/Organization/Enterprise), so the story is only partially supported. Missing for 10: explicit seat-type definitions, full tier-by-tier feature/price matrix, and clear enumeration of which AI capabilities are gated at each tier.", "evidenceIds": [ "figma-docs-12", "figma-docs-13", "figma-docs-23", "figma-docs-24", "figma-docs-29", "figma-docs-37", "figma-docs-45" ] }, { "productId": "figma", "storyId": "comments-review-flows", "verdict": "full", "quality": 6, "confidence": "medium", "rationale": "Figma's core design docs confirm in-file commenting and multiplayer feedback (\"share feedback with comments... in one shared space\") and webhooks reference collaborators commenting on files, matching the core of the story, but the pack never explicitly documents @mention or thread-resolution UI. Missing for 10: explicit documentation of @mentioning teammates and resolving comment threads, plus independent hands-on confirmation beyond marketing copy.", "evidenceIds": [ "figma-docs-14", "figma-docs-53" ] }, { "productId": "figma", "storyId": "design-to-production-code", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Figma ships strong first-party dev-handoff tooling: Code Connect maps design system components directly to real production code snippets in Dev Mode (figma-gh-1, figma-gh-2, figma-docs-60), and the MCP server can generate code from selected frames, extract variables/component context, and even build into production (figma-docs-2, figma-docs-39, figma-docs-25, figma-docs-35). However, community evidence shows real friction shipping that code as-is — devs complain about receiving absolute-position CSS instead of usable layout code and needing manual/plugin workarounds for asset export (figma-comm-1, figma-comm-2, figma-comm-3), and advanced Dev Mode/MCP access is gated behind a paid tier (figma-comm-4, figma-docs-23). Missing for 10: independent hands-on validation that MCP/Code-Connect output is shippable without manual cleanup at scale, and confirmation that core code-mapping isn't paywalled behind Dev Mode license.", "evidenceIds": [ "figma-gh-1", "figma-gh-2", "figma-docs-60", "figma-docs-39", "figma-docs-25", "figma-docs-35", "figma-comm-1", "figma-comm-2", "figma-comm-3", "figma-comm-4", "figma-docs-23" ] }, { "productId": "figma", "storyId": "design-tokens-variables", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Figma has native Variables with modes/themes for color, spacing, and typography, a dedicated Variables REST API for querying/creating/updating/deleting tokens and syncing design-system source of truth, and MCP tooling (get_variable_defs) that extracts variables and styles across selections; docs explicitly recommend using Figma variables for tokens across spacing, color, radius, and typography. Missing for 10: independent hands-on validation of multi-mode theming workflows and more detail on cross-file/library token publishing at scale.", "evidenceIds": [ "figma-docs-54", "figma-docs-62", "figma-docs-68", "figma-docs-73", "figma-docs-40" ] }, { "productId": "figma", "storyId": "export-formats-assets", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Community evidence confirms Figma supports exporting assets like SVGs (e.g., 'Copy as SVG', bulk export plugins) but only as a workaround with friction, not a documented first-party export panel with per-layer presets, scales, and slices. missing for 10: first-party docs describing the native Export panel, per-layer export presets, multiple-scale exports (1x/2x/3x), PDF export, and slice tool; independent confirmation these work smoothly without third-party plugins.", "evidenceIds": [ "figma-comm-2", "figma-comm-3" ] }, { "productId": "figma", "storyId": "free-tier-real-design", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack describes general Figma design/collaboration/prototyping capabilities (figma-docs-14, figma-docs-15, figma-docs-16) but nothing ties these specifically to the free tier vs. paid seats, and no pricing page excerpt enumerates what the free plan includes. One community note (figma-comm-4) even flags a trend of moving features (Dev Mode) behind paid licenses, but this doesn't confirm or deny free-tier usability for core design work either way.", "evidenceIds": [] }, { "productId": "figma", "storyId": "inspect-code-handoff", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Figma's Dev Mode (figma-docs-64, figma-docs-23) and MCP tools like get_variable_defs (figma-docs-68) support extracting styles, variables, and code snippets, and Code Connect (figma-gh-1/3) provides true-to-production code snippets, with community confirmation of CSS export workflows (figma-comm-1/2/3). However, the evidence never explicitly confirms iOS/Android snippet generation, precise measurement tooling, or that Dev Mode access requires only a lower-cost seat rather than a full editor seat—figma-docs-23 even ties 'Advanced Dev Mode inspection' to a paid plan tier. missing for 10: explicit iOS/Android code snippet evidence, explicit seat-tier confirmation (dev seat vs editor seat), measurement/spec inspection detail beyond CSS.", "evidenceIds": [ "figma-docs-64", "figma-docs-23", "figma-docs-68", "figma-gh-1", "figma-gh-3", "figma-comm-1", "figma-comm-2", "figma-comm-4" ] }, { "productId": "figma", "storyId": "interactive-prototypes", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Figma's docs explicitly market native, no-code prototyping tools ('high-fidelity, no-code, interactive prototypes'), which per general product knowledge and the docs pages cover triggers, transitions, and overlays as core interaction-design features, corroborated by community praise for Figma's rapid feature growth in design/prototyping workflows. missing for 10: no explicit line-item mention of 'triggers/transitions/overlays' terminology or independent hands-on walkthrough of building a specific prototype flow.", "evidenceIds": [ "figma-docs-16", "figma-docs-33", "figma-docs-15", "figma-comm-6" ] }, { "productId": "figma", "storyId": "nl-design-generation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Figma's own docs describe a genuine prompt-to-design capability: the 'Figma agent' can 'generate new design directions, create diagrams, edit images' from prompts, users can 'riff with the agent to refine' designs, and 'Figma Make' turns ideas into working prototypes (figma-docs-17, figma-docs-34, figma-docs-51, figma-docs-56, figma-docs-43). However, this is all vendor-sourced documentation with no independent hands-on report confirming quality of first drafts/mockups, and one community comment specifically calls Figma's AI tools 'really bad and often broken' (figma-comm-8), tempering confidence in real-world reliability. Missing for 10: independent/hands-on verification of output quality for full mockups, and resolution of the community report of broken AI features.", "evidenceIds": [ "figma-docs-17", "figma-docs-34", "figma-docs-51", "figma-docs-56", "figma-docs-43", "figma-comm-8" ] }, { "productId": "figma", "storyId": "open-file-format", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Figma's native .fig file format itself is not published as an open or documented spec; the evidence pack offers only programmatic access surfaces (REST API, Plugin API, MCP server) rather than an open file-format specification, and community evidence explicitly states design work is 'locked up in a proprietary format' with no offline/local file portability without constant export.", "evidenceIds": [ "figma-comm-7", "figma-docs-7", "figma-docs-20" ] }, { "productId": "figma", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Figma exposes several APIs (REST API for reading objects/variables/webhooks, Plugin API for read/write of file contents, MCP server for writing to canvas and generating code/diagrams) that cover a large slice of UI functionality, but no evidence shows full parity — e.g. REST API is largely read/webhook-oriented, Plugin API requires running inside the editor UI rather than headless automation, and there's no documented API for things like commenting, team/project management, or prototyping interactions equivalent to the UI. missing for 10: comprehensive REST/plugin coverage of comments, prototyping settings and team administration, evidence of headless parity without plugin UI context, and independent confirmation that API-driven workflows match UI capability 1:1.", "evidenceIds": [ "figma-docs-7", "figma-docs-8", "figma-docs-38", "figma-docs-59", "figma-docs-62", "figma-docs-32", "figma-probe-3" ] }, { "productId": "figma", "storyId": "openness-full-export", "verdict": "disputed", "quality": 3, "confidence": "medium", "rationale": "Figma's REST API and Plugin API let you extract objects, layers, properties, and variables (figma-docs-7, figma-docs-26, figma-docs-49, figma-docs-62), suggesting some data portability, but this falls short of a genuine 'export all your data in open formats and leave' capability — designs remain in Figma's proprietary .fig format with no full-fidelity open-format export. A hands-on community report explicitly contradicts openness: 'your full fidelity work is locked up in a proprietary format' and you can't keep a fully up-to-date local copy without constant manual export (figma-comm-7). Missing for 10: an actual bulk/full-fidelity open-format export or documented data-portability/exit path, and no contradiction from independent sources.", "evidenceIds": [ "figma-docs-7", "figma-docs-26", "figma-docs-49", "figma-docs-62", "figma-comm-7" ] }, { "productId": "figma", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Figma is a closed, proprietary commercial product; while some peripheral tools (Code Connect, REST API spec) are open-sourced on GitHub, there is no evidence the core Figma product/application source is available under an open license. Community feedback even notes work is 'locked up in a proprietary format,' reinforcing that the core product is closed-source.", "evidenceIds": [ "figma-comm-7", "figma-gh-1", "figma-gh-2", "figma-docs-46" ] }, { "productId": "figma", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Figma is a cloud SaaS product with no evidence of any self-hosted deployment option for the core design tool; evidence only covers cloud APIs, plugins, MCP server, and community complaints about proprietary lock-in (figma-comm-7), which actually underscores the lack of self-hosting.", "evidenceIds": [ "figma-comm-7" ] }, { "productId": "figma", "storyId": "plugin-ecosystem-marketplace", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Figma's Plugin API and Widget API docs explicitly support a broad plugin ecosystem for reading/writing files, building custom UI, and automating workflows 'from documentation to communication' (figma-docs-8, figma-docs-18, figma-docs-36, figma-docs-65), and independent hands-on community evidence confirms designers actually use third-party plugins (e.g., bulk SVG export/autonaming) in real workflows (figma-comm-3). Private plugin support for internal tooling is also documented (figma-docs-9). Missing for 10: explicit mention of a named 'Community/marketplace' browsing experience, plugin counts/scale, or concrete examples of accessibility-checker/data-filler plugins specifically.", "evidenceIds": [ "figma-docs-8", "figma-docs-9", "figma-docs-11", "figma-docs-18", "figma-docs-36", "figma-docs-65", "figma-comm-3" ] }, { "productId": "figma", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data residency, regional storage options, or data localization controls for Figma; all evidence concerns design tools, MCP server capabilities, plugins, and collaboration features. Missing for 10: any mention of region selection, data residency commitments, or storage location controls.", "evidenceIds": [] }, { "productId": "figma", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items address AI training data opt-out or data usage controls for AI model training; nothing about privacy settings, data processing agreements, or opt-out toggles is mentioned. This axis applies to Figma (a data-handling SaaS platform where AI training opt-out is a plausible policy), but no supporting evidence exists.", "evidenceIds": [] }, { "productId": "figma", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items address data retention policies, deletion controls, or data lifecycle management for AI features; the pack covers MCP tooling, plugins, REST API, and collaboration features but nothing about retention/deletion settings for AI-native users. missing for 10: data retention policy documentation, deletion controls/APIs, AI data handling/opt-out settings, any admin controls for purging AI-processed data.", "evidenceIds": [] }, { "productId": "figma", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items address telemetry/usage-tracking opt-out settings or privacy controls of that kind; the evidence covers MCP server features, plugins, REST API, and admin/SCIM tools but nothing about disabling analytics or tracking for AI-native workflows.", "evidenceIds": [] }, { "productId": "figma", "storyId": "realtime-multiplayer-editing", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Figma's docs directly describe multiplayer files with shared editing, cursor chat, and real-time collaboration ('Work together in multiplayer files... talk to your teammates with cursor chat in one shared space'; 'together in real-time and in one place'), matching the core story. Missing for 10: independent/hands-on verification of live-cursor sync at scale and no detail on conflict resolution or performance limits under heavy concurrent editing.", "evidenceIds": [ "figma-docs-14", "figma-docs-27", "figma-docs-63" ] }, { "productId": "figma", "storyId": "responsive-auto-layout", "verdict": "partial", "quality": 2, "confidence": "low", "rationale": "Evidence pack lacks any explicit mention of Figma's auto layout, constraints, or grid systems; the closest is a vague marketing line about 'best-in-class layout tools' which doesn't name the specific responsive-design mechanics the story describes. missing for 10: explicit docs on auto layout resizing behavior, constraint pinning, layout grids, and any hands-on corroboration of responsive frame behavior.", "evidenceIds": [ "figma-docs-15" ] }, { "productId": "figma", "storyId": "reusable-components-variants", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The evidence pack only points to a help-center article titled 'Guide to components in Figma' without any quoted content describing variants, per-instance overrides, or override-linking behavior; no other citation addresses this specific design-systems capability. Missing for 10: explicit documentation or examples of variant properties, instance override behavior, and how overrides remain linked to (and reset from) the main component.", "evidenceIds": [ "figma-docs-72", "figma-docs-30" ] }, { "productId": "figma", "storyId": "self-host-design-platform", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Figma is a proprietary, cloud-hosted SaaS platform; there is no evidence of an open-source license or self-hosting option for the full design platform—evidence even notes proprietary file format lock-in and cloud dependency. Community comments confirm files are locked in a proprietary format with no local self-hosted alternative.", "evidenceIds": [ "figma-comm-7" ] }, { "productId": "figma", "storyId": "shared-team-libraries", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm centralized library/asset publishing across teams and departments (figma-docs-37, figma-docs-12) and a components guide (figma-docs-72), plus library usage analytics across files (figma-docs-55, figma-docs-75), but there is no explicit documentation in the pack of the update-notification/accept-update workflow when a library source component changes. missing for 10: explicit doc on component update propagation/notification workflow, independent/hands-on confirmation of controlled update behavior.", "evidenceIds": [ "figma-docs-37", "figma-docs-12", "figma-docs-72", "figma-docs-55", "figma-docs-75", "figma-docs-45" ] }, { "productId": "figma", "storyId": "vector-editing-craft", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Figma's marketing copy claims 'best-in-class layout tools, type tools' and pixel-precise control (figma-docs-15, figma-docs-63), which implies strong vector design capability, but the evidence pack contains no specific documentation of pen/path tools, boolean operations, or precise alignment guides — these are core to the story but unevidenced here. Community comments focus on SVG export and Dev Mode licensing, not vector editing quality itself. Missing for 10: explicit docs/screenshots of pen tool and path editing, boolean operation (union/subtract/intersect) documentation, alignment/snapping tool documentation, and independent hands-on corroboration of vector editing precision.", "evidenceIds": [ "figma-docs-15", "figma-docs-63", "figma-docs-28" ] }, { "productId": "figma", "storyId": "version-history-restore", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Only one citation confirms version history viewing exists (figma-docs-44), but the evidence pack contains no detail on naming milestones, comparing versions, or restoring earlier states. missing for 10: documentation on naming/labeling versions, comparing diffs between versions, restoring a file to an earlier state, and any independent/hands-on corroboration of these capabilities.", "evidenceIds": [ "figma-docs-44" ] }, { "productId": "framer", "storyId": "advanced-prototype-logic", "verdict": "disputed", "quality": 6, "confidence": "medium", "rationale": "Framer's docs and community evidence confirm rich prototyping behavior — animated menus, carousels, code overrides, data-driven components (docs-45, docs-32, comm-4, comm-5) — but hands-on reports document concrete failures that undercut the 'feels like the real product' promise: animations distort during size/geometry changes, performance is inconsistent (60fps then chugging) on the same device, and users found Framer 'falls through' vs alternatives like react-spring when chaining multiple interactive animated nodes (comm-6, comm-7, comm-9). Missing for 10: independent verification that these animation glitches are resolved, and evidence of robust variable/conditional-logic support beyond CMS data binding.", "evidenceIds": [ "framer-docs-45", "framer-docs-32", "framer-docs-4", "framer-comm-4", "framer-comm-5", "framer-comm-6", "framer-comm-7", "framer-comm-9" ] }, { "productId": "framer", "storyId": "agent-creates-design", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Framer documents a write-capable external agent connection (Claude Code, Cursor, Codex) that can create/update/delete CMS items and modify canvas/components, backed by branching for safe review, plus a plugin surface and Server API for programmatic site automation. Missing for 10: independent/hands-on verification of the external-agent write capability and no public OpenAPI/formal API schema was found by the probe.", "evidenceIds": [ "framer-docs-18", "framer-docs-24", "framer-docs-25", "framer-docs-34", "framer-docs-35", "framer-docs-36", "framer-docs-17", "framer-docs-23", "framer-docs-30" ] }, { "productId": "framer", "storyId": "agent-exports-assets", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers AI agents editing designs, CMS, and publishing via a Server API and external-agent connections, but nothing documents a programmatic render/export surface for screenshots, images, or production-format exports (no export/render API endpoints, and probes for OpenAPI specs all 404).", "evidenceIds": [ "framer-docs-5", "framer-docs-17", "framer-docs-40", "framer-probe-3" ] }, { "productId": "framer", "storyId": "agent-reads-design-context", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Framer explicitly markets a native 'External Agent' connection (explicitly positioned as replacing a separate MCP server) that gives coding agents like Claude Code, Cursor, and Codex direct access to canvas, components, CMS, and project context, including read/write CMS operations and branch-based change tracking. This closely matches the story's intent of pulling design context to implement changes, but the specific mention of 'screenshots' and 'variables' from the story isn't explicitly evidenced, and all evidence is first-party docs with no independent/hands-on corroboration of the external agent workflow. Missing for 10: explicit support for screenshots/variables extraction, independent developer confirmation of the external-agent integration working as described.", "evidenceIds": [ "framer-docs-34", "framer-docs-17", "framer-docs-30", "framer-docs-39", "framer-docs-18", "framer-docs-24", "framer-docs-35", "framer-docs-25", "framer-docs-36" ] }, { "productId": "framer", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A direct probe confirms Framer serves a live llms.txt file (HTTP 200) with structured content describing the product and linking to migration/agent docs, and Framer's own docs explicitly reference this file's content, giving concrete first-party evidence an AI agent can be pointed at it. missing for 10: independent/third-party confirmation that agents successfully consume this llms.txt in practice, and broader agent-oriented doc coverage beyond this single file.", "evidenceIds": [ "framer-probe-1", "framer-docs-28", "framer-docs-41" ] }, { "productId": "framer", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Framer's AI agent can restructure and suggest layout changes based on generic 'conversion best practices' (framer-docs-31/38) and manages/organizes CMS data (framer-docs-8/46), which is adjacent to data-driven suggestions, but there is no evidence of the agent analyzing a user's actual site/traffic data to surface analytical insights (e.g., performance, SEO, or visitor-behavior insights) — it is generative/design assistance rather than data-insight generation. Missing for 10: evidence of analytics-based insight generation, data pattern analysis, or dashboards summarizing user's own data trends.", "evidenceIds": [ "framer-docs-31", "framer-docs-38", "framer-docs-8", "framer-docs-46", "framer-docs-18" ] }, { "productId": "framer", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Framer offers a Server API to 'automate publishing' and lets external agents (Claude Code, Cursor, Codex) connect to read/write CMS and canvas content, which could be wired into autonomous workflows, but all documented agent interactions (framer-docs-6/7/26) are conversational/on-demand rather than scheduled or self-running background automations. There's no evidence of triggers, schedules, or persistent background jobs — only API/connection primitives a developer would need to build such automation themselves.\nmissing for 10: evidence of scheduled/triggered background jobs, autonomous run history or logs, native workflow/automation builder distinct from on-demand AI agent editing.", "evidenceIds": [ "framer-docs-5", "framer-docs-17", "framer-docs-18", "framer-docs-30", "framer-docs-34" ] }, { "productId": "framer", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Framer ships multiple built-in AI agents (design agent, CMS agent, AI website generator) that users can delegate tasks to directly inside the product via natural-language prompts (e.g. 'make responsive', 'switch to dark mode', CMS setup) as shown in framer-docs-6,7,8,26,42,43,44,45,46. Missing for 10: independent/hands-on user reports specifically validating the built-in AI assistant's reliability and quality (community evidence available only covers older, unrelated Framer features like animations).", "evidenceIds": [ "framer-docs-6", "framer-docs-7", "framer-docs-8", "framer-docs-26", "framer-docs-42", "framer-docs-43", "framer-docs-44", "framer-docs-45", "framer-docs-46" ] }, { "productId": "framer", "storyId": "agentic-headless", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Framer advertises a Server API to 'automate publishing' and integrate with external products/agents, which implies some programmatic/headless control, but there is no documented CLI, headless mode, or CI-specific workflow, and probes show no OpenAPI/API spec is publicly discoverable (404s). Missing for 10: concrete CI/headless setup docs, API reference/spec, and evidence of automated pipeline usage beyond marketing blurbs.", "evidenceIds": [ "framer-docs-5", "framer-docs-40", "framer-docs-17", "framer-probe-3" ] }, { "productId": "framer", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Framer explicitly states it does not use or require MCP: 'Framer doesn't require a separate MCP server... without manual MCP setup' (framer-docs-34). Its actual capability runs in the opposite direction — external agents like Claude Code/Cursor connect INTO Framer to use Framer's canvas/CMS as their tool, not Framer consuming external MCP servers' tools. No evidence exists of Framer acting as an MCP client that plugs in third-party MCP servers.", "evidenceIds": [ "framer-docs-34", "framer-docs-39", "framer-docs-30" ] }, { "productId": "framer", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Framer explicitly states it does not offer an MCP server, instead providing a proprietary 'External Agent connection' as an alternative to MCP (framer-docs-34). Since the story specifically asks for an official MCP server, and Framer's own docs assert this is unnecessary and unsupported by design, there is no evidence of an actual MCP server implementation.", "evidenceIds": [ "framer-docs-34" ] }, { "productId": "framer", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Framer's docs show extensive natural-language operation: the built-in AI agent creates pages/sections/copy from prompts, restyles sites ('switch to dark mode', 'use Fraunces/Inter'), reorganizes layouts, and makes sites responsive purely from described intent, plus a CMS agent and external-agent integration (Claude Code/Cursor/Codex) that reads/writes CMS and canvas content via natural language, with branch-based safety for changes. missing for 10: independent/hands-on third-party corroboration of the AI/CMS agent actually working as described (community evidence only covers older non-AI Framer features).", "evidenceIds": [ "framer-docs-6", "framer-docs-7", "framer-docs-8", "framer-docs-19", "framer-docs-20", "framer-docs-27", "framer-docs-31", "framer-docs-37", "framer-docs-43", "framer-docs-44", "framer-docs-45", "framer-docs-34", "framer-docs-18", "framer-docs-25" ] }, { "productId": "framer", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Framer offers a Server API, plugins, and 'External Agent' connections for tools like Claude Code/Cursor/Codex, but nowhere does it mention an official Framer CLI tool for AI-native workflows.", "evidenceIds": [] }, { "productId": "framer", "storyId": "agentic-public-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Framer documents a Server API for automation/publishing and CMS API access, and explicitly supports external AI agents (Claude Code, Cursor, Codex) reading/writing CMS collections via a documented connection flow (framer-docs-5,17,30,39,40,18,24,34). However, probes could not locate any formal API reference/spec (openapi.json/swagger 404s) or detailed endpoint documentation, and there's no independent/community corroboration of developers actually using the Server API. Missing for 10: a discoverable formal API reference/spec, authentication/rate-limit details, and independent developer testimonials confirming API usability.", "evidenceIds": [ "framer-docs-5", "framer-docs-17", "framer-docs-30", "framer-docs-39", "framer-docs-18", "framer-docs-24", "framer-docs-34", "framer-probe-3" ] }, { "productId": "framer", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Framer documents a Server API and an 'External Agent' connection that lets tools like Claude Code or Cursor read/write CMS data and canvas content, with branching as a safety net for undoing changes — but nothing describes issuing scoped, least-privilege, or permission-limited API credentials/tokens for agents. Missing for 10: any mention of API key scopes, granular permission levels, token expiration/rotation, or role-based credential issuance for agent access.", "evidenceIds": [ "framer-docs-17", "framer-docs-18", "framer-docs-25", "framer-docs-34", "framer-docs-35", "framer-docs-36" ] }, { "productId": "framer", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Framer documents a Server API, plugin SDK, code components/overrides, and an external agent connection for building integrations against official interfaces, but there's no discoverable OpenAPI spec (probe found 404s across candidate paths) and no independent/hands-on developer corroboration of building against these SDKs. missing for 10: publicly discoverable API reference/OpenAPI spec, independent developer corroboration, SDK versioning/language support details.", "evidenceIds": [ "framer-docs-5", "framer-docs-17", "framer-docs-23", "framer-docs-32", "framer-docs-34", "framer-probe-3" ] }, { "productId": "framer", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers Framer's Server API, external agent connections, and plugins, but there is no mention of webhooks or event subscription mechanisms anywhere in the docs or probes; OpenAPI probe also returned 404s with no webhook spec found.", "evidenceIds": [] }, { "productId": "framer", "storyId": "ai-asset-editing-tools", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Framer's AI agent can generate pages, sections, copy, and visuals (framer-docs-6, framer-docs-26, framer-docs-42) and adjust layout/typography, which touches copy suggestions and some visual generation, but there is no evidence of dedicated asset-chore tools like image generation, background removal, or content-aware fill inside the editor. missing for 10: dedicated image generation tool, background removal feature, content-aware fill capability, hands-on/independent verification of these specific asset tools.", "evidenceIds": [ "framer-docs-6", "framer-docs-26", "framer-docs-42", "framer-docs-7" ] }, { "productId": "framer", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Framer documents a Server API, Plugins API, and Code Components, but nothing shows an interactive API reference with runnable/try-it examples; probe evidence explicitly found no OpenAPI/swagger spec (404s at all candidate paths), suggesting no such interactive reference exists.", "evidenceIds": [ "framer-docs-5", "framer-docs-17", "framer-probe-3" ] }, { "productId": "framer", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Framer offers a Server API and llms.txt, but the probe explicitly found no OpenAPI/swagger spec at any standard location, and no docs mention a downloadable machine-readable API spec.", "evidenceIds": [ "framer-probe-3", "framer-docs-5", "framer-docs-17" ] }, { "productId": "framer", "storyId": "api-sandbox", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Framer offers staging environments on a framer.app domain and automatic branching so AI agent changes happen in isolation from the live/main project, which functions as a sandbox against production data (framer-docs-10, framer-docs-11, framer-docs-25, framer-docs-36). However, this is framed around publishing workflow rather than an explicit 'sandbox for AI testing' concept, and there's no independent/hands-on verification of how well this isolation holds up in practice. missing for 10: independent corroboration of branch/staging isolation, explicit documentation framing this as an AI-testing sandbox rather than a publishing safety net, and detail on whether CMS data changes by agents are also fully isolated from production before merge.", "evidenceIds": [ "framer-docs-10", "framer-docs-11", "framer-docs-25", "framer-docs-36", "framer-docs-47" ] }, { "productId": "framer", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of API versioning scheme or a documented deprecation policy; probes for OpenAPI specs returned 404s and the Server/Fetch API docs mention no versioning or deprecation terms.", "evidenceIds": [ "framer-docs-5", "framer-docs-40", "framer-probe-3" ] }, { "productId": "framer", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Framer docs show AI-agent driven bulk operations such as site-wide typography/color/theme changes across all pages (framer-docs-19,27,37), CSV bulk import of CMS collections with row-level linking (framer-docs-9,21), and external agents able to create/update/delete many CMS items via API (framer-docs-18,24,35). This supports bulk operations across many items in CMS/design contexts, but there's no documentation of scale limits, batch API endpoints, or performance at large volumes. Missing for 10: explicit bulk/batch API documentation with scale guarantees, independent verification of bulk operations succeeding at scale, and rate-limit/performance details.", "evidenceIds": [ "framer-docs-9", "framer-docs-19", "framer-docs-21", "framer-docs-27", "framer-docs-37", "framer-docs-18", "framer-docs-24", "framer-docs-35" ] }, { "productId": "framer", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Framer's API, plugins, and AI agents that can be manually invoked to modify a site, and an API to 'automate publishing,' but nothing describes a rule engine or event-trigger system where a user defines 'when X happens, do Y' automation. missing for 10: any event-trigger/rule definition mechanism, conditional logic, or workflow automation builder within Framer.", "evidenceIds": [ "framer-docs-5", "framer-docs-17", "framer-docs-40" ] }, { "productId": "framer", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Framer's evidence covers AI agents editing sites, CMS, publishing, plugins, and APIs, but there is no mention of scheduling recurring jobs, cron-like automation, or workflow triggers of any kind. Automate publishing via API is a manual/programmatic action, not a recurring scheduled job.", "evidenceIds": [] }, { "productId": "framer", "storyId": "automation-versioned-workflows", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Framer documents automatic version history with instant rollback, plus branching that isolates every external-agent change so it can be reviewed, undone, or merged — directly matching version/review/rollback for automations run by AI agents. Missing for 10: independent/hands-on confirmation of rollback and branch-merge workflows in practice, and detail on granular diff/review UI beyond marketing docs.", "evidenceIds": [ "framer-docs-12", "framer-docs-22", "framer-docs-25", "framer-docs-36", "framer-docs-11", "framer-docs-10" ] }, { "productId": "framer", "storyId": "build-custom-plugins", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Framer documents a Plugins API for building editor extensions (small apps that interact with the Editor/CMS, insert layers/images/code components) with a dedicated developer docs section, plus adjacent APIs like Code Components and Server API. However, the evidence pack doesn't show explicit TypeScript typings references or concrete code samples/SDK repo, and there's no distribution/marketplace mechanism described. missing for 10: explicit typings documentation, concrete code samples/starter templates, and a distribution/marketplace mechanism for sharing plugins.", "evidenceIds": [ "framer-docs-2", "framer-docs-23", "framer-docs-29", "framer-docs-33", "framer-docs-3", "framer-docs-32" ] }, { "productId": "framer", "storyId": "clear-plan-pricing", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no pricing page, plan tiers, seat types, or AI-feature gating details — only one mention that a static-file hosting feature is limited to Pro/Enterprise plans, which does not constitute published plan/seat/AI-gating clarity. No pricing comparison table or plan documentation is present anywhere in the pack.", "evidenceIds": [] }, { "productId": "framer", "storyId": "comments-review-flows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers branching, version history, AI agents, CMS, and plugins, but nothing about in-file commenting, @mentions of teammates, or resolving comment threads for design review. Missing for 10: any mention of a commenting feature, mention/notification system, or thread resolution workflow.", "evidenceIds": [] }, { "productId": "framer", "storyId": "design-to-production-code", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Framer supports genuine dev-facing code integration: Code Components (custom React components with visual controls), Code Overrides (HOCs), a Server API, and native external-agent access letting Claude Code/Cursor/Codex read/write the live project and CMS with branch-based review before merge. Community evidence corroborates that Framer's React-based model lets designers hand off working code to developers. However this is not a classic 'export design to code' pipeline (no Figma-style code export or OpenAPI-documented flow was found), and independent verification is thin/dated.\nmissing for 10: independent hands-on proof of shipping exported production code, more recent community validation of the external-agent/dev handoff workflow.", "evidenceIds": [ "framer-docs-3", "framer-docs-32", "framer-docs-34", "framer-docs-39", "framer-docs-17", "framer-docs-25", "framer-comm-4" ] }, { "productId": "framer", "storyId": "design-tokens-variables", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Framer's AI agent can restyle colors/typography/light-dark mode on request (framer-docs-27, framer-docs-37, framer-docs-19), but there is no documentation of an actual design-token or variables system with reusable modes/themes applied consistently across designs, akin to Figma variables. Missing for 10: token/variable definition, mode-based theme switching infrastructure, cross-file/theme propagation evidence.", "evidenceIds": [ "framer-docs-27", "framer-docs-37", "framer-docs-19" ] }, { "productId": "framer", "storyId": "export-formats-assets", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Framer is a website-builder/publishing tool, not a design/prototyping tool with layer-based asset export; the evidence pack contains no mention of PNG/SVG/PDF export, per-layer presets, scales, or slices — these are Figma/Sketch-style export features absent from Framer's docs.", "evidenceIds": [] }, { "productId": "framer", "storyId": "free-tier-real-design", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no mention of Framer's free tier, its editor limitations, or free-tier collaboration features; all docs focus on AI agents, CMS, publishing, and plugins. No pricing or free-plan details are cited anywhere in the pack, so there is no basis to confirm the free tier supports real design work or collaboration.", "evidenceIds": [] }, { "productId": "framer", "storyId": "inspect-code-handoff", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "framer", "storyId": "interactive-prototypes", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence shows Framer supports animations/transitions (community: 'great for more advanced prototyping needs w/ animations/transitions', 'declarative approach to creating nice animations') and AI-driven interactive elements like 'sticky navbars, animated menus, interactive carousels' handled via prompts, but there is no explicit documentation of screen-to-screen click triggers or overlay linking (the core Figma-style prototyping flow), and community reports flag animation performance/rendering glitches that undercut confidence. missing for 10: explicit docs on multi-screen click triggers, overlay/modal linking, and independent confirmation of a smooth no-code prototyping flow.", "evidenceIds": [ "framer-comm-2", "framer-comm-6", "framer-comm-7", "framer-docs-45" ] }, { "productId": "framer", "storyId": "nl-design-generation", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Framer's docs describe a native AI agent that generates editable pages, sections, copy, and visuals from prompts, with examples of full mockup edits (typography, grid layouts, brand colors, dark mode, responsive breakpoints, image-based starting points), and lets designers keep refining in canvas after generation. This directly matches the story of prompt-driven first drafts/layouts/mockups, though evidence is entirely vendor-sourced with no independent hands-on verification of AI-generation quality — missing for 10: independent/hands-on confirmation that generated designs are production-quality, and any documented limitations of the AI design agent.", "evidenceIds": [ "framer-docs-6", "framer-docs-7", "framer-docs-26", "framer-docs-42", "framer-docs-43", "framer-docs-44", "framer-docs-45", "framer-docs-19", "framer-docs-20", "framer-docs-27", "framer-docs-37" ] }, { "productId": "framer", "storyId": "open-file-format", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Framer's APIs, plugins, CMS, and AI agent integrations, but contains no mention of an open or documented native file format that would let designs be opened, edited, or version-controlled by external tools outside Framer's own editor/publishing pipeline. Version history, branching, and staging are all Framer-hosted features, not a portable file format.", "evidenceIds": [] }, { "productId": "framer", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Framer exposes a Server API, external-agent connections (Claude Code/Cursor/Codex) with read/write CMS access, branching for review, and a plugin system that covers many editor actions (framer-docs-5,17,18,24,34,35,36) — a real, substantive API surface for automation. But there's no evidence of full UI/API parity: no OpenAPI/spec discoverable (framer-probe-3 shows 404s across candidate spec paths), and the docs describe specific supported actions (CMS CRUD, canvas edits via agent, publishing) rather than a claim that literally everything achievable in the UI (e.g. fine-grained styling, animation editing, plugin marketplace management) is scriptable via API. Missing for 10: a documented comprehensive API/OpenAPI reference proving parity, explicit vendor claim of 'anything in UI = anything via API', and independent/hands-on verification of that parity.", "evidenceIds": [ "framer-docs-5", "framer-docs-17", "framer-docs-18", "framer-docs-24", "framer-docs-34", "framer-docs-35", "framer-docs-36", "framer-probe-3" ] }, { "productId": "framer", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Framer offers a Server API, CMS API, and CSV import for content, but nothing indicates the ability to export a full site (design, code, CMS data) into open, portable formats for migration away from Framer; only 'migrate to Framer' flows are documented, not exit/export paths.", "evidenceIds": [ "framer-docs-9", "framer-docs-21", "framer-docs-5", "framer-docs-40" ] }, { "productId": "framer", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Framer is closed-source proprietary SaaS; no evidence pack item mentions an open-license source availability, and this is a mainstream commercial product with no indication of open-sourcing its codebase.", "evidenceIds": [] }, { "productId": "framer", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "framer", "storyId": "plugin-ecosystem-marketplace", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Framer's docs confirm a plugin system where developers build 'small apps' that can interact with the editor and CMS (inserting images, layers, code components, modifying canvas), which is the technical foundation for a plugin ecosystem. However, the evidence never mentions a marketplace, community size, or specific plugin categories like icon packs, data fillers, or accessibility checkers as the story asks. Missing for 10: evidence of an actual marketplace/directory, plugin discovery UX, and named example plugins matching icons/data/accessibility use cases.", "evidenceIds": [ "framer-docs-2", "framer-docs-23", "framer-docs-29", "framer-docs-33" ] }, { "productId": "framer", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses data residency, region selection, or data storage location controls for Framer; all evidence concerns AI agents, CMS, plugins, and publishing features unrelated to data residency.", "evidenceIds": [] }, { "productId": "framer", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data usage for AI training, opt-out controls, or any privacy policy statement regarding training data; all evidence concerns AI agent features, plugins, and site publishing.", "evidenceIds": [] }, { "productId": "framer", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers Framer's AI agent, plugins, API, CMS, and version control/branching features, but contains no mention of data retention policies, data deletion controls, or privacy/compliance settings for AI-native workflows. Missing for 10: any documentation on data retention periods, user-initiated data deletion, or privacy controls specific to AI agent interactions.", "evidenceIds": [] }, { "productId": "framer", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry, usage tracking, or an opt-out mechanism for Framer's AI features or platform; all documentation covers unrelated capabilities (agents, CMS, plugins, publishing).", "evidenceIds": [] }, { "productId": "framer", "storyId": "realtime-multiplayer-editing", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack includes docs on branching, version history, and staging (framer-docs-11, framer-docs-47) but contains no mention of real-time multiplayer editing, live cursors, or instant sync between simultaneous editors — the actual claim in the story. Missing for 10: any documentation of live cursors, simultaneous multi-user canvas editing, or real-time presence indicators.", "evidenceIds": [] }, { "productId": "framer", "storyId": "responsive-auto-layout", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Framer's docs describe breakpoint-based responsive adaptation ('adapts your layouts across breakpoints, adjusting stacks, type sizes, spacing, and image crops') and general layout refinement (breakpoints, spacing) via its AI agent and canvas, which implies auto-layout/stack-style responsive behavior. However, there's no explicit documentation of manual constraints or grid systems, and no independent/hands-on corroboration of responsive resizing quality. missing for 10: explicit docs on constraints/grid systems, independent verification of resize fidelity across breakpoints, non-AI-agent manual workflow evidence.", "evidenceIds": [ "framer-docs-43", "framer-docs-42" ] }, { "productId": "framer", "storyId": "reusable-components-variants", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Evidence only touches tangentially on component reuse — Code Components with visual controls and Code Overrides (HOCs to modify Layer/Component properties) hint at reusable, overridable components, but nothing explicitly describes variants or per-instance overrides that stay linked to a master component/source as in a design-system workflow. Missing for 10: explicit documentation of component 'variants', proof that per-instance overrides remain linked/syncable to the source component, and any hands-on/community confirmation of this specific workflow.", "evidenceIds": [ "framer-docs-3", "framer-docs-32" ] }, { "productId": "framer", "storyId": "self-host-design-platform", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "framer", "storyId": "shared-team-libraries", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Framer's evidence covers code components, plugins, branching, and CMS collections within a single project, but there is no documentation of a shared/team library mechanism that publishes components or styles across separate files or projects with controlled update propagation (the Figma-style library model). Missing for 10: any mention of a cross-project component/style library, library publishing workflow, or update-notification/accept-update mechanism.", "evidenceIds": [ "framer-docs-3", "framer-docs-32", "framer-docs-23" ] }, { "productId": "framer", "storyId": "vector-editing-craft", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack is entirely about Framer's AI agents, CMS, publishing, plugins, and developer API — nothing addresses classic vector tools like pen/path editing, boolean operations, or precise alignment guides. No documentation or community evidence confirms these professional vector-editing capabilities.", "evidenceIds": [] }, { "productId": "framer", "storyId": "version-history-restore", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Framer's docs confirm automatic version history with one-click restore of any prior version (framer-docs-12, framer-docs-22) and branching to isolate changes (framer-docs-11, framer-docs-47), but there is no evidence of naming/labeling milestones or a side-by-side compare view between versions. Missing for 10: named milestone/checkpoint labeling, version comparison/diff view, independent hands-on confirmation of restore workflow.", "evidenceIds": [ "framer-docs-12", "framer-docs-22", "framer-docs-11", "framer-docs-47" ] }, { "productId": "penpot", "storyId": "advanced-prototype-logic", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows basic board-to-board click interactions (penpot-docs-12, penpot-docs-14); nothing in the pack mentions animation/transition effects between states, conditional logic, variables, or scroll-triggered effects in prototypes. This is an applicable axis for a prototyping tool, but no evidence supports these advanced behaviors.", "evidenceIds": [ "penpot-docs-12", "penpot-docs-14" ] }, { "productId": "penpot", "storyId": "agent-creates-design", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Penpot has a documented MCP server explicitly for connecting AI agents/LLMs to design files, plus a plugin API and integrations API with access to objects/properties, suggesting agents can both read and modify designs programmatically. However, MCP docs emphasize agents 'reading and interacting with' design files for high-fidelity artifacts, and no independent/hands-on evidence confirms write operations via MCP or API actually work reliably, and openapi/spec probes 404'd. Missing for 10: concrete documented write/create examples via MCP or API, independent verification of agent-driven design edits, and a working OpenAPI spec.", "evidenceIds": [ "penpot-docs-1", "penpot-docs-3", "penpot-docs-19", "penpot-docs-27", "penpot-docs-31", "penpot-docs-32", "penpot-docs-39", "penpot-probe-2", "penpot-probe-3" ] }, { "productId": "penpot", "storyId": "agent-exports-assets", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Penpot documents an MCP server and plugin/integrations API that give agents structured access to design objects and mentions 'external rendering' as a use case (penpot-docs-3, penpot-docs-32), and design data can be exported to open formats like SVG/CSS/HTML (penpot-docs-21). However, there is no concrete documented endpoint or workflow for producing screenshots/images/production export files programmatically, no OpenAPI spec was found (404s in penpot-probe-2), and one community report describes limited export options (single-layer only) in hands-on use (penpot-comm-5). missing for 10: an explicit documented render/export API or endpoint (PNG/PDF/JPG) usable by an agent, a working OpenAPI/schema reference, and independent confirmation that MCP/plugin API can actually produce exported artifacts.", "evidenceIds": [ "penpot-docs-3", "penpot-docs-32", "penpot-docs-21", "penpot-probe-2", "penpot-comm-5" ] }, { "productId": "penpot", "storyId": "agent-reads-design-context", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Penpot documents an official MCP server that lets any AI agent connect to design files, described as exposing design as code so agents can 'read and interact with your design files' and access objects/properties for external rendering, plus separate docs on components, design tokens, and design-system alignment (penpot-docs-1,18,27,31,32,3,33,11). However, no evidence explicitly confirms screenshot export or an explicit frame/variable/component mapping schema via the MCP itself, and there is no independent/hands-on report validating the MCP server's real-world behavior (only first-party docs, plus unrelated community comments about general app performance). Missing for 10: independent verification of MCP server functioning as described, explicit mention of screenshot/visual context retrieval, and concrete component-mapping API examples.", "evidenceIds": [ "penpot-docs-1", "penpot-docs-18", "penpot-docs-27", "penpot-docs-31", "penpot-docs-32", "penpot-docs-3", "penpot-docs-11", "penpot-docs-33", "penpot-probe-3" ] }, { "productId": "penpot", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Penpot publishes an actual llms.txt at penpot.app/llms.txt with structured content (design tokens, plugin marketplace stats, Figma comparison) as cited in penpot-docs-16 and penpot-docs-24, plus extensive agent-oriented docs for its MCP server (penpot-docs-1, 18, 19, 27, 31, 32, 39) explicitly aimed at AI agents. missing for 10: an independent probe directly confirming penpot.app/llms.txt resolves (the probe only tested the unrelated help.penpot.app subdomain and got 404), and no third-party corroboration that agents successfully consume this file.", "evidenceIds": [ "penpot-docs-16", "penpot-docs-24", "penpot-docs-1", "penpot-docs-31", "penpot-docs-32", "penpot-probe-3", "penpot-probe-1" ] }, { "productId": "penpot", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Penpot exposes design data via its MCP server, plugin API, and webhooks so external AI agents can read structured design data and generate outputs (e.g., 'AI agents can easily read and interact with your design files creating high-fidelity artifacts') and the plugin marketplace lists AI tools, but there is no evidence of a built-in, first-party AI feature that proactively surfaces insights or suggestions from a user's own data inside the product itself. Missing for 10: a native in-app AI insight/suggestion feature (vs. only enabling third-party agents via MCP/plugins), and any hands-on/community confirmation that such insights actually work in practice.", "evidenceIds": [ "penpot-docs-1", "penpot-docs-32", "penpot-docs-40", "penpot-docs-24", "penpot-docs-3" ] }, { "productId": "penpot", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Penpot provides building blocks for automation—an MCP server for AI agents, outbound webhooks, and a plugin API—that could underlie background automations, but all documented use cases (prompting an agent, plugin actions) are interactive/triggered rather than autonomous, unattended background processes. Missing for 10: explicit support for scheduled/triggered autonomous agent runs, evidence of agents operating without a live user prompt, and any independent confirmation that MCP/webhook flows work unattended in production.", "evidenceIds": [ "penpot-docs-1", "penpot-docs-4", "penpot-docs-27", "penpot-docs-31", "penpot-docs-40" ] }, { "productId": "penpot", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Penpot's AI evidence is entirely about an MCP server that lets external AI agents/LLMs connect to Penpot's data (penpot-docs-1, penpot-docs-18, penpot-docs-27, penpot-docs-31, penpot-docs-39), not about a built-in AI assistant embedded in the product itself that users can delegate tasks to. No evidence describes an in-app AI chat/copilot feature within the Penpot UI.", "evidenceIds": [ "penpot-docs-1", "penpot-docs-18", "penpot-docs-27", "penpot-docs-31", "penpot-docs-39", "penpot-probe-3" ] }, { "productId": "penpot", "storyId": "agentic-headless", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Penpot ships self-host/Docker/Kubernetes deployment, access-token API auth, outbound webhooks, and a self-hostable MCP server, all of which could underpin automated/CI-driven workflows, but there is no explicit documentation of a headless mode, CLI, or CI pipeline usage. missing for 10: explicit headless/CI-mode documentation, CLI tooling, independent evidence of running Penpot in a CI pipeline.", "evidenceIds": [ "penpot-docs-7", "penpot-docs-8", "penpot-docs-6", "penpot-docs-4", "penpot-docs-19", "penpot-docs-30" ] }, { "productId": "penpot", "storyId": "agentic-mcp-client", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Penpot's docs mention connecting to another product's MCP server ('Migrate your designs into Penpot by connecting Figma's and Penpot's MCPs') implying Penpot can act as an MCP client to import tool-provided data, but the vast majority of MCP evidence describes Penpot instead exposing its OWN MCP server for agents to connect to (the reverse relationship), not Penpot consuming arbitrary external MCP servers' tools. Missing for 10: documentation of a general-purpose MCP client/connector UI in Penpot, a list of supported external MCP servers beyond Figma, and any hands-on/community confirmation that this client-side integration works.", "evidenceIds": [ "penpot-docs-15", "penpot-docs-25", "penpot-docs-18" ] }, { "productId": "penpot", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Penpot documents an official MCP server (agnostic, self-hostable, connects to any AI agent/LLM) with dedicated marketing/docs pages and a probe confirming the page exists. Missing for 10: independent/hands-on third-party confirmation that the MCP server works reliably in practice, and technical setup/API reference details beyond marketing copy.", "evidenceIds": [ "penpot-docs-1", "penpot-docs-18", "penpot-docs-19", "penpot-docs-27", "penpot-docs-31", "penpot-docs-32", "penpot-docs-39", "penpot-probe-3" ] }, { "productId": "penpot", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Penpot ships an official MCP server explicitly designed so any AI agent/LLM can 'prompt away' to read and manipulate design files expressed as code, directly enabling natural-language operation, backed by plugin API and access tokens for automation. Missing for 10: independent/hands-on evidence of the MCP server actually working end-to-end in practice (community comments only cover general UI/performance, not the AI/MCP workflow specifically).", "evidenceIds": [ "penpot-docs-1", "penpot-docs-31", "penpot-docs-32", "penpot-docs-39", "penpot-docs-3", "penpot-probe-3" ] }, { "productId": "penpot", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers Penpot's MCP server, plugin API, webhooks, and REST access tokens, but there is no mention of an official CLI tool for interacting with Penpot as an AI-native workflow interface. The MCP server is a distinct integration mechanism, not a CLI.", "evidenceIds": [] }, { "productId": "penpot", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Penpot documents a public API with access tokens for authentication, outbound webhooks, an open plugin API, and a dedicated MCP server explicitly designed to let any AI agent/LLM read and manipulate design files programmatically. This directly matches the ai-native 'documented public API' story, with self-hosting options adding control. Missing for 10: no discoverable OpenAPI/swagger spec (probes returned 404s) and no independent/hands-on verification of the API's completeness beyond vendor docs.", "evidenceIds": [ "penpot-docs-1", "penpot-docs-3", "penpot-docs-6", "penpot-docs-19", "penpot-docs-27", "penpot-docs-31", "penpot-docs-32", "penpot-probe-2", "penpot-probe-3" ] }, { "productId": "penpot", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Penpot offers access tokens as an alternative login method for API access (penpot-docs-6) and an MCP server for agent connectivity, but there is no evidence of scoped or least-privilege credential issuance (e.g., permission-limited tokens, role-based scopes, or granular API key restrictions) for agents. Missing for 10: scoped/permission-limited token creation, documentation of least-privilege API key options, agent-specific credential management.", "evidenceIds": [ "penpot-docs-6", "penpot-docs-3" ] }, { "productId": "penpot", "storyId": "agentic-sdks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Penpot ships an official, self-hostable MCP server plus an open plugin API, REST API with access-token auth, and webhooks, explicitly marketed for AI agents to build against (penpot-docs-1,3,5,6,19,27,31,32,39). This gives AI-native users concrete official SDK/API surfaces to integrate with, though there's no independent hands-on validation of the MCP server and the OpenAPI spec could not be located (probe 404s), missing for 10: independent developer confirmation of MCP/plugin SDK reliability, discoverable machine-readable OpenAPI/llms.txt spec.", "evidenceIds": [ "penpot-docs-1", "penpot-docs-3", "penpot-docs-5", "penpot-docs-6", "penpot-docs-19", "penpot-docs-27", "penpot-docs-31", "penpot-docs-32", "penpot-docs-39", "penpot-probe-2", "penpot-probe-3" ] }, { "productId": "penpot", "storyId": "agentic-webhooks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Penpot's docs explicitly describe outbound webhooks that notify external apps of events like file creation/update or comments, directly matching the 'subscribe to events via webhooks' story. Evidence is vendor-documentation only with no independent/hands-on confirmation of webhook delivery or a full list of supported event types. Missing for 10: independent corroboration of webhook triggering/reliability, complete event-type catalogue, and details on webhook subscription management (payload schema, retries, security).", "evidenceIds": [ "penpot-docs-4", "penpot-docs-20", "penpot-docs-3" ] }, { "productId": "penpot", "storyId": "ai-asset-editing-tools", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Penpot's AI evidence is entirely about an MCP server for connecting external AI agents/LLMs to structured design data, plus a generic plugin marketplace mention of 'AI tools' — there is no evidence of built-in editor features like image generation, background removal, content-aware fill, or copy suggestions happening inside Penpot itself. The axis is fair for a design tool (competitors ship such features), but nothing in the pack shows Penpot delivering it natively.", "evidenceIds": [ "penpot-docs-1", "penpot-docs-2", "penpot-docs-24", "penpot-docs-31", "penpot-docs-32", "penpot-docs-40" ] }, { "productId": "penpot", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence describes integrations-api, plugin system, and MCP server, but there is no interactive API reference with runnable examples; probes explicitly found no OpenAPI/swagger spec or llms.txt documentation endpoint (404s), and no mention of interactive docs/sandbox exists.", "evidenceIds": [ "penpot-probe-1", "penpot-probe-2", "penpot-docs-3" ] }, { "productId": "penpot", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "While Penpot documents a general integrations API, access tokens, webhooks, and an MCP server, there is no evidence of a downloadable OpenAPI/Swagger spec; direct probes for openapi.json/swagger.json/llms.txt all returned 404s, indicating no machine-readable API spec is published.", "evidenceIds": [ "penpot-probe-1", "penpot-probe-2", "penpot-docs-3", "penpot-docs-6" ] }, { "productId": "penpot", "storyId": "api-sandbox", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Penpot has no dedicated 'sandbox' or 'test environment' feature, but its self-hosting capability (docker-compose/Kubernetes deployment, MCP server deployable in your own environment) means an AI-native user could spin up an isolated instance separate from production to test agent workflows. This is an indirect workaround rather than a purpose-built sandbox/staging mode. Missing for 10: an explicit sandbox/staging environment feature, documented separation of test vs production data within the same account, and any first-party guidance on safely testing AI agents against non-production data.", "evidenceIds": [ "penpot-docs-7", "penpot-docs-19", "penpot-docs-30", "penpot-docs-38", "penpot-docs-28" ] }, { "productId": "penpot", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence documents an API, plugin system, and MCP server, but there is no mention of API versioning or a documented deprecation policy anywhere in the docs; probes for OpenAPI/llms.txt spec files return 404, and no changelog or versioning policy is cited. Missing for 10: any documented API version scheme, deprecation notices, or changelog governing breaking changes.", "evidenceIds": [ "penpot-docs-3", "penpot-docs-6", "penpot-probe-1", "penpot-probe-2" ] }, { "productId": "penpot", "storyId": "automation-bulk-operations", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Penpot's API/plugin/MCP layer gives extensive access to objects and properties and lets agents 'automate tedious tasks' across files (penpot-docs-2, penpot-docs-3, penpot-docs-32), which implies scriptable bulk edits, but no documentation specifically demonstrates a bulk-operation workflow (e.g., batch renaming/restyling hundreds of layers). Independent hands-on reports directly contradict smooth handling of many items: performance 'starts dropping rapidly once you approach around 1000 layers' versus design systems that run 10-40k layers (penpot-comm-19), and users report crashes/lag with just a few pages of content (penpot-comm-10, penpot-comm-16). Missing for 10: a documented bulk-edit API/plugin example, benchmarks showing many-item operations succeed at scale, and resolution of the reported performance degradation.", "evidenceIds": [ "penpot-docs-2", "penpot-docs-3", "penpot-docs-32", "penpot-comm-19", "penpot-comm-10", "penpot-comm-16" ] }, { "productId": "penpot", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Penpot's outbound webhooks fire on events (file created/updated, comment added) which is a basic event-trigger mechanism, and the MCP/plugin API lets external agents react to design changes, but there is no evidence of a rules engine or conditional logic layer where users define custom trigger conditions and automated actions within Penpot itself. missing for 10: rule-definition UI/config (conditions, filters, action chains), documented automation recipes, evidence of in-app if/then automation beyond simple webhook event notification.", "evidenceIds": [ "penpot-docs-4", "penpot-docs-20", "penpot-docs-3", "penpot-docs-5" ] }, { "productId": "penpot", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Penpot's automation surface (MCP server, plugins, webhooks, API/access tokens) supports event-driven and on-demand integrations, but no evidence anywhere in the pack describes scheduling, cron-like recurring jobs, or timed workflow execution.", "evidenceIds": [ "penpot-docs-4", "penpot-docs-18", "penpot-docs-27", "penpot-docs-40" ] }, { "productId": "penpot", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Penpot's evidence covers MCP/AI agent integration, webhooks, and plugins for automation, but there is no evidence of any mechanism to version, review, or roll back these automations themselves (e.g., automation history, audit trail, or undo for agent-driven actions). Component 'restore main' (docs-23) is a design-object rollback, not an automation/workflow rollback feature.", "evidenceIds": [] }, { "productId": "penpot", "storyId": "build-custom-plugins", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm a dedicated plugin API/system ('Our plugin system brings the flexibility that devs need to create and publish their own custom extensions', 'Use community plugins or build your own') and a marketplace with 113+ community plugins, showing developers can and do build/distribute extensions. However, no evidence pack item shows actual typings, SDK reference docs, or code samples/tutorials for the plugin API itself. Missing for 10: documented TypeScript typings, published API reference/sample plugin code, and independent developer confirmation of using the plugin API to build an extension.", "evidenceIds": [ "penpot-docs-5", "penpot-docs-24", "penpot-docs-37" ] }, { "productId": "penpot", "storyId": "clear-plan-pricing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack includes only scattered pricing-page fragments (unlimited teams, no file limits, community plugins) but no structured breakdown of seat types, plan tiers, or which AI features are gated behind specific plans. No pricing table, tier comparison, or AI-feature-gating disclosure is documented anywhere in the pack.", "evidenceIds": [ "penpot-docs-9", "penpot-docs-10", "penpot-docs-29", "penpot-docs-37" ] }, { "productId": "penpot", "storyId": "comments-review-flows", "verdict": "partial", "quality": 2, "confidence": "low", "rationale": "The only evidence touching commenting is a passing mention that webhooks fire when 'a comment has been added,' which confirms Penpot has some in-file commenting feature but gives no detail on @mentions or resolving threads. missing for 10: documentation or hands-on evidence of @mention functionality, thread resolution UI, and any community confirmation that review workflows actually replace email in practice.", "evidenceIds": [ "penpot-docs-4", "penpot-docs-20" ] }, { "productId": "penpot", "storyId": "design-to-production-code", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Penpot documents code-based design representation and an 'Inspect' feature exporting CSS/HTML/SVG/JSON that matches the design (penpot-docs-21/36), plus an MCP server letting AI agents read structured design files to generate implementation artifacts (penpot-docs-32, penpot-docs-40). However, this is inspection/AI-assisted code generation rather than a documented direct component-to-framework code mapping (e.g., to React/Vue components), and no independent/hands-on evidence confirms developers actually shipping this code in production. missing for 10: first-party or community evidence of production-ready component-to-code bindings (framework-specific), and independent verification that generated/inspected code is usable as-is in shipped products.", "evidenceIds": [ "penpot-docs-21", "penpot-docs-36", "penpot-docs-32", "penpot-docs-40", "penpot-gh-1" ] }, { "productId": "penpot", "storyId": "design-tokens-variables", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Penpot explicitly advertises 'Design Tokens' and being a 'native design tokens' tool (penpot-docs-33, penpot-docs-13, penpot-docs-16), which directly supports token management for design-system alignment, but the evidence is only marketing tagline text with no detail on how modes/themes, multiple token sets, or per-property (color/spacing/typography) switching actually work. Components with linked instances (penpot-docs-11, penpot-docs-22) support consistency but aren't tokens/modes specifically, and no community/hands-on evidence corroborates the tokens feature's real-world robustness. Missing for 10: detailed documentation of token modes/themes switching, per-property (color/spacing/typography) token examples, and independent/hands-on confirmation that the tokens feature works as described.", "evidenceIds": [ "penpot-docs-33", "penpot-docs-13", "penpot-docs-16", "penpot-docs-11", "penpot-docs-22" ] }, { "productId": "penpot", "storyId": "export-formats-assets", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no first-party documentation describing PNG/SVG/PDF export options, per-layer export presets, scale factors, or slices — the closest data point is a community comment noting an inability to export a whole file (only a single layer) in past testing, which is negative rather than supportive. No docs, help-center pages, or hands-on confirmation of this specific export feature set are present.", "evidenceIds": [ "penpot-comm-5" ] }, { "productId": "penpot", "storyId": "free-tier-real-design", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Penpot's free tier documented as unlimited files and unlimited teams, and the core editor supports components, prototyping, and design tokens without gating behind a paid seat, which supports real collaborative design work. However, community reports raise concerns about performance/stability at scale (crashes, lag with many layers/pages) that could limit usability for complex real work, and there's no explicit pricing-page breakdown confirming free tier includes full collaboration features like comments/permissions. missing for 10: explicit free-tier feature/seat-limit breakdown from pricing page, independent confirmation that free tier collaboration (comments, real-time multiplayer) is unrestricted, and resolution of community-reported performance/stability issues affecting real design work.", "evidenceIds": [ "penpot-docs-9", "penpot-docs-10", "penpot-docs-29", "penpot-docs-37", "penpot-docs-11", "penpot-docs-12", "penpot-comm-10", "penpot-comm-15", "penpot-comm-19" ] }, { "productId": "penpot", "storyId": "inspect-code-handoff", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Penpot's docs confirm an inspect feature giving code that matches design (CSS, HTML, SVG, JSON), which covers measurements/styles/code snippets for web, but there is no evidence of iOS/Android-specific code snippet generation, and no explicit documentation confirming that viewers/inspectors can access this without a paid editor seat. missing for 10: iOS/Android code snippet support, explicit seat-free viewer/inspect access confirmation, asset-export detail in inspect mode.", "evidenceIds": [ "penpot-docs-21", "penpot-docs-36", "penpot-docs-9", "penpot-docs-10" ] }, { "productId": "penpot", "storyId": "interactive-prototypes", "verdict": "full", "quality": 6, "confidence": "medium", "rationale": "Docs confirm board-to-board prototyping (\"connect boards, which can act as screens\") and marketing copy references \"interactive prototypes\"; a community user corroborates that \"the prototype feature is working great.\" However, the evidence pack never details specific trigger types, transition/animation options, or overlay behavior, so the depth of the interaction model is unverified. Missing for 10: detailed documentation of trigger/transition/overlay options, hands-on walkthrough of prototyping fidelity, and independent reviews specifically praising or critiquing interaction complexity.", "evidenceIds": [ "penpot-docs-12", "penpot-docs-14", "penpot-comm-3" ] }, { "productId": "penpot", "storyId": "nl-design-generation", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Penpot's docs describe an official MCP server that lets external AI agents/LLMs read and interact with Penpot files to 'generate editable design structures' and 'create high-fidelity artifacts' from prompts (penpot-docs-31, penpot-docs-32, penpot-docs-40), which supports the story's intent of prompt-driven design generation. However, this is achieved only via connecting a third-party AI agent through MCP rather than a native in-app 'type a prompt, get a mockup' feature, and there is no independent/hands-on evidence confirming actual generated layouts or mockups working in practice. Missing for 10: first-party demo/screenshots of actual generated designs, independent hands-on reports validating the MCP-to-mockup pipeline, and a native in-app generation UI not dependent on external agent setup.", "evidenceIds": [ "penpot-docs-31", "penpot-docs-32", "penpot-docs-40", "penpot-docs-27", "penpot-probe-3" ] }, { "productId": "penpot", "storyId": "open-file-format", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Penpot explicitly documents an open file format built on open standards (SVG, CSS, HTML, JSON) and is open-source itself, explicitly contrasting with Figma's proprietary format, plus REST/webhook APIs and plugin system for external tooling. Missing for 10: independent third-party corroboration that .penpot files are actually opened/edited by external tools, and no dedicated file-format spec page (llms.txt/openapi probes 404).", "evidenceIds": [ "penpot-docs-21", "penpot-docs-36", "penpot-docs-16", "penpot-docs-3", "penpot-gh-1" ] }, { "productId": "penpot", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Penpot documents extensive API/plugin/MCP access (objects and properties, webhooks, access tokens, plugin system) suggesting broad programmatic control, but there's no explicit claim or evidence of full UI/API parity, and probes for an OpenAPI/swagger spec returned 404s, indicating the API surface isn't fully discoverable or documented as complete. missing for 10: explicit statement of full UI-API feature parity, a public OpenAPI/schema reference, and independent (community/hands-on) confirmation that all UI actions are scriptable via API.", "evidenceIds": [ "penpot-docs-3", "penpot-docs-5", "penpot-docs-6", "penpot-docs-24", "penpot-probe-1", "penpot-probe-2", "penpot-probe-3" ] }, { "productId": "penpot", "storyId": "openness-full-export", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Penpot's docs promise open formats and no lock-in — the file format is open and design data can be inspected/exported as CSS, HTML, SVG and JSON, and self-hosting gives full data control (penpot-docs-21, penpot-docs-28, penpot-docs-38). However, a hands-on community report directly contradicts full-data export, stating the user 'had no way to export the diagram as one file like PDF or JPG. I could only export a layer' (penpot-comm-5), suggesting the practical export experience falls short of the 'export all my data and leave' promise. Missing for 10: independent confirmation of a full-project/all-files bulk export, verification that the open file format round-trips completely outside Penpot, and resolution of the community-reported export limitation.", "evidenceIds": [ "penpot-docs-21", "penpot-docs-28", "penpot-docs-38", "penpot-comm-5" ] }, { "productId": "penpot", "storyId": "openness-open-license", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Penpot is well known as open-source (GitHub repo referenced) and docs emphasize open standards, open file format, and self-hosting, which implies source availability, but the evidence pack lacks an explicit statement of the license type or direct pointer to license terms. missing for 10: explicit license name/terms (e.g. MPL-2.0) cited in docs, confirmation that full source (not just plugin API) is published under that license.", "evidenceIds": [ "penpot-gh-1", "penpot-docs-21", "penpot-docs-36", "penpot-docs-38", "penpot-docs-16" ] }, { "productId": "penpot", "storyId": "openness-self-host", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Penpot provides extensive first-party self-hosting documentation (Docker Compose, Kubernetes/Helm, on-prem, private cloud, 3-minute setup) and explicitly extends this to AI tooling by letting users deploy the MCP server in their own environment. Community reports corroborate real-world self-hosting via Docker/Portainer working, though some users report performance/stability issues at scale, which is a caveat rather than a failure of the self-hosting capability itself. Missing for 10: no first-party performance benchmarks or hardening guidance addressing the community-reported instability at scale.", "evidenceIds": [ "penpot-docs-7", "penpot-docs-8", "penpot-docs-19", "penpot-docs-28", "penpot-docs-30", "penpot-docs-38", "penpot-comm-9", "penpot-comm-12", "penpot-comm-17" ] }, { "productId": "penpot", "storyId": "plugin-ecosystem-marketplace", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm a plugin system and marketplace with 113+ plugins spanning accessibility, code, AI, and design-system tools, plus first-party docs on building custom extensions (penpot-docs-24, penpot-docs-5, penpot-docs-37). However, community feedback suggests the plugin ecosystem is still much smaller/younger than competitors' (penpot-comm-20 notes needing a 'bigger community' and missing pre-built element libraries), and there's no independent hands-on verification of marketplace breadth or quality. Missing for 10: independent reviews of specific popular plugins, marketplace UI/discovery evidence, and confirmation the 113+ figure reflects an actively curated, high-quality ecosystem rather than a small/early one.", "evidenceIds": [ "penpot-docs-24", "penpot-docs-5", "penpot-docs-37", "penpot-comm-20" ] }, { "productId": "penpot", "storyId": "privacy-data-residency", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Penpot's self-hosting model directly delivers data residency choice: users can run Penpot in their own private cloud, on-premise, or any environment, giving full control over where data lives, and this extends to the MCP server which can also be self-deployed rather than routed through third-party clouds. Missing for 10: no first-party managed 'choose a region' feature for the SaaS-hosted offering, and no independent verification of self-hosting data-control claims beyond docs.", "evidenceIds": [ "penpot-docs-7", "penpot-docs-28", "penpot-docs-38", "penpot-docs-19", "penpot-docs-30" ] }, { "productId": "penpot", "storyId": "privacy-no-training", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Penpot's self-host docs explicitly state 'Keep your data private. Penpot will never make business out of your data. Choose where and how to host it,' which directly addresses preventing data from being used for AI training by keeping it entirely under user control. However, this is a general privacy claim tied to self-hosting rather than an explicit AI-training opt-out policy or contractual clause, and there's no independent/third-party corroboration of this specific promise. missing for 10: explicit AI-training-specific policy/ToS language, independent verification of the no-training claim, and clarity on cloud-hosted (non-self-hosted) instance data usage.", "evidenceIds": [ "penpot-docs-28", "penpot-docs-7", "penpot-docs-38" ] }, { "productId": "penpot", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Penpot's self-hosting and 'keep your data private... decide where and how to host it' messaging (penpot-docs-28, penpot-docs-38) implies data control, and self-hosting inherently lets an org manage retention/deletion at the infrastructure level. However, there is no explicit documentation of data retention policies, file/version deletion mechanisms, or GDPR-style erasure APIs for AI-processed data. Missing for 10: explicit retention/deletion API or policy docs, confirmation of how MCP-accessed data is retained/purged, independent verification of deletion behavior.", "evidenceIds": [ "penpot-docs-28", "penpot-docs-38", "penpot-docs-7", "penpot-docs-19" ] }, { "productId": "penpot", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows self-hosting and data-privacy marketing claims (e.g., 'Keep your data private,' 'never make business out of your data') but never mentions a specific telemetry/usage-tracking opt-out setting or configuration flag. Missing for 10: explicit documentation of a telemetry toggle or opt-out mechanism, confirmation of what usage data is collected, and any independent verification that self-hosted instances disable tracking by default.", "evidenceIds": [ "penpot-docs-28", "penpot-docs-38", "penpot-docs-7" ] }, { "productId": "penpot", "storyId": "realtime-multiplayer-editing", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Vendor marketing implies real-time team collaboration (\"Collaborate freely with unlimited teams\") but the evidence pack contains no explicit documentation of live cursors or instant sync mechanics, and multiple hands-on reports describe multiplayer editing as crash-prone or unstable when several users work simultaneously on a file. Missing for 10: first-party docs describing live cursor presence/conflict resolution, latency/sync architecture details, and independent confirmation that simultaneous multi-user editing performs reliably at scale.", "evidenceIds": [ "penpot-docs-9", "penpot-docs-29", "penpot-comm-10", "penpot-comm-11", "penpot-comm-15" ] }, { "productId": "penpot", "storyId": "responsive-auto-layout", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Penpot's marketing docs confirm CSS Grid and Flex layout support for responsive/rules-based interfaces (penpot-docs-14), but there is no detailed documentation of auto-layout resizing behavior, constraints on resize, or grid systems working the way native UI frameworks flex, and no independent/hands-on confirmation of these specific behaviors. Community evidence instead focuses on general performance/stability issues rather than confirming responsive layout fidelity. missing for 10: detailed docs on constraints-based resizing, auto-layout resize rules, hands-on validation that grids/flex truly mimic real UI responsiveness.", "evidenceIds": [ "penpot-docs-14", "penpot-docs-21" ] }, { "productId": "penpot", "storyId": "reusable-components-variants", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm reusable components with main/copy linkage that propagates updates (docs-11,22,23,35) and mention 'variants' as an extension (docs-41), but per-instance override mechanics are never detailed, and hands-on community feedback explicitly flags that variants/states handling is 'not on Figma's level yet' and components can feel 'difficult to handle' (penpot-comm-1, penpot-comm-9). missing for 10: documented per-instance override workflow, detailed variant creation/management docs, independent confirmation that variant/override linkage works robustly at scale.", "evidenceIds": [ "penpot-docs-11", "penpot-docs-22", "penpot-docs-23", "penpot-docs-35", "penpot-docs-41", "penpot-comm-1", "penpot-comm-9" ] }, { "productId": "penpot", "storyId": "self-host-design-platform", "verdict": "disputed", "quality": 6, "confidence": "medium", "rationale": "Docs strongly document self-hosting (docker compose, Kubernetes/Helm/OpenShift/Rancher, private cloud/on-prem deployment, 'set up in 3 minutes', open-source positioning vs Figma) with no artificial file/team limits, which is exactly what this story asks for. However, hands-on community reports contradict the ease-of-self-hosting claim: one user says self-hosted instance 'crash after a few minutes and not properly show the canvases. So a no for me' and others report it being 'unstable, very crash prone... huge memory hog' under modest load, even as another reports self-hosting 'works fine so far.' Missing for 10: explicit confirmation of the open-source license terms/scope, independent security/production-readiness audit of self-hosted deployments, and resolution of the stability complaints reported by self-hosters.", "evidenceIds": [ "penpot-docs-7", "penpot-docs-8", "penpot-docs-26", "penpot-docs-30", "penpot-docs-38", "penpot-comm-9", "penpot-comm-11", "penpot-comm-10", "penpot-comm-12" ] }, { "productId": "penpot", "storyId": "shared-team-libraries", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm components as reusable linked elements where main-component edits propagate to copies (penpot-docs-22, penpot-docs-23, penpot-docs-35) and design-system features like tokens and variants (penpot-docs-33, penpot-docs-41), but there is no explicit documentation of publishing a library across separate files/projects or how updates propagate cross-project, and community feedback notes design-system/component sophistication lags Figma (penpot-comm-1, penpot-comm-9). missing for 10: explicit shared/library publishing workflow across files and projects, documentation of controlled update/accept-changes flow at library level, independent confirmation of cross-project sync reliability.", "evidenceIds": [ "penpot-docs-22", "penpot-docs-23", "penpot-docs-35", "penpot-docs-33", "penpot-docs-41", "penpot-comm-1", "penpot-comm-9" ] }, { "productId": "penpot", "storyId": "vector-editing-craft", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers AI/MCP integration, self-hosting, plugins, components, prototyping, and design tokens, but contains no documentation or hands-on mention of pen/path editing tools, boolean path operations, or precise alignment/snapping features. Community comments only compare general UI polish and performance to Figma without confirming these specific vector-editing capabilities.", "evidenceIds": [] }, { "productId": "penpot", "storyId": "version-history-restore", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions version history, milestone naming, diffing/comparison, or restore-to-earlier-state functionality; only component/plugin/self-host/AI features are documented. missing for 10: any mention of file version history, snapshot naming, diff/compare tools, or restore capability.", "evidenceIds": [] }, { "productId": "rive", "storyId": "advanced-prototype-logic", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Rive's docs explicitly describe building interactive experiences responding to user input, screen size, scrolling, and application state (rive-docs-4), state machines/timelines/keyframes (rive-docs-3), and real-time data binding so animations update live (rive-docs-5, rive-docs-15, rive-docs-22, rive-docs-24) — directly matching the story's states/conditions/variables/scroll-effect prototyping needs. Community evidence corroborates real-world adoption for production-grade interactive UI/UX work (rive-comm-4, rive-comm-7), and Rive's own positioning ('no mockups, no prototypes... the real thing', rive-docs-16/23) reflects this design intent. Missing for 10: independent hands-on verification specifically of scroll-triggered effects and variable/condition-driven prototypes beyond vendor docs.", "evidenceIds": [ "rive-docs-3", "rive-docs-4", "rive-docs-5", "rive-docs-15", "rive-docs-22", "rive-docs-24", "rive-docs-16", "rive-docs-23", "rive-comm-4", "rive-comm-7" ] }, { "productId": "rive", "storyId": "agent-creates-design", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Rive documents an MCP integration letting AI tools create/modify complex View Models, State Machines, Layouts, and Shapes directly in the Editor, plus RML (a text-based file format) explicitly designed so coding agents can build/modify Rive files programmatically, and a CLI for agent-driven generation. This goes beyond read-only access into genuine write-capable agent tooling. Missing for 10: independent/hands-on verification of the MCP tools working in practice and more detail on the write-scope limits of the MCP server.", "evidenceIds": [ "rive-docs-10", "rive-docs-19", "rive-docs-27", "rive-docs-25", "rive-docs-30", "rive-docs-26", "rive-probe-4" ] }, { "productId": "rive", "storyId": "agent-exports-assets", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Rive's runtimes expose a documented programmatic Renderer interface (rive-gh-1/2/3) and RML lets code/agents construct .riv content (rive-docs-13/20/25), and export docs describe exporting to 'websites, mobile apps, games, videos' (rive-docs-7/8) — suggesting a programmatic render/export surface. However, hands-on community feedback directly contradicts image/video export capability: users report 'no way to export movie files' and ask 'can I export video assets with this!?' with no answer (rive-comm-3, rive-comm-10), indicating the actual exportable surface is limited to the .riv runtime format rather than screenshots/images/production video as the story requires. Missing for 10: documented API/CLI for agent-triggered screenshot or image/video export, and confirmation this gap has been resolved.", "evidenceIds": [ "rive-gh-1", "rive-gh-2", "rive-docs-7", "rive-docs-8", "rive-docs-13", "rive-docs-20", "rive-comm-3", "rive-comm-10" ] }, { "productId": "rive", "storyId": "agent-reads-design-context", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Rive documents an official MCP server connecting the Editor to AI tools, letting agents create/modify View Models (variable-like data), State Machines, Layouts, and Components (rive-docs-19, rive-docs-27, rive-docs-10, rive-docs-11). This covers some 'design context' elements (component mappings, variables via view models) but is framed as agents building/editing Rive content rather than pulling existing frames or screenshots for implementation elsewhere. Missing for 10: explicit screenshot/frame extraction capability, evidence of pulling existing design context (vs. generating new), and independent/hands-on confirmation of the MCP workflow.", "evidenceIds": [ "rive-docs-19", "rive-docs-27", "rive-docs-10", "rive-docs-11", "rive-docs-15", "rive-probe-4" ] }, { "productId": "rive", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Rive's docs are served in agent-friendly markdown (docs/*.md return 200) and explicitly point agents to a documentation index at https://rive.app/docs/llms.txt for full-site context, satisfying the 'point an agent at llms.txt or agent-oriented docs' story even though the conventional root-level /llms.txt path 404s. Missing for 10: direct confirmation that /docs/llms.txt itself loads and contains a complete index, and no independent (non-vendor) corroboration of an agent successfully using it.", "evidenceIds": [ "rive-probe-1", "rive-probe-2", "rive-docs-1" ] }, { "productId": "rive", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Rive is a vector animation/design and runtime tool; its AI agent and MCP tools assist with generating code, layouts, and animations, not with producing analytical insights or suggestions from a user's business/usage data. This 'data insights' framing is a category mismatch for a design/animation product, so the axis does not apply.", "evidenceIds": [] }, { "productId": "rive", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Rive's docs describe an AI agent, MCP integration, and CLI that generate animations/code when invoked by a user, but there is no evidence of scheduled, unattended, or continuously running background automations — all AI features described are user-triggered, session-based interactions within the Editor.", "evidenceIds": [ "rive-docs-9", "rive-docs-19", "rive-docs-26", "rive-docs-27" ] }, { "productId": "rive", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Rive documents a built-in AI agent inside the Editor that writes code, generates layouts, data models, view models, state machines, and animation, letting users delegate repetitive/complex tasks to it directly within the product. This is first-party documented with specific task examples, though there's no independent/hands-on corroboration of its real-world effectiveness. Missing for 10: independent verification of the AI agent's actual capability/reliability, more detail on scope/limits of delegation.", "evidenceIds": [ "rive-docs-9", "rive-docs-10", "rive-docs-18", "rive-docs-27", "rive-docs-26" ] }, { "productId": "rive", "storyId": "agentic-headless", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Rive exposes open-source runtimes as libraries (rive-gh-1, rive-docs-21) and mentions a CLI for agent-driven generation and RML for text-based/coding-agent workflows (rive-docs-26, rive-docs-25, rive-docs-30), suggesting some scriptable/automatable path outside the interactive editor. However, there is no explicit documentation of headless rendering, CI pipeline integration, or automated test/build usage of the CLI or runtimes. Missing for 10: explicit headless/CI usage docs, CLI reference/flags, examples of automated pipeline integration, independent confirmation of non-interactive operation.", "evidenceIds": [ "rive-docs-26", "rive-docs-25", "rive-docs-30", "rive-gh-1", "rive-docs-21" ] }, { "productId": "rive", "storyId": "agentic-mcp-client", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Rive documents a first-party MCP integration letting users connect the Rive Editor to AI tools via MCP so AI can perform tasks like creating View Models, State Machines, Layouts, and Shapes, confirmed by both docs and a dedicated probe. Missing for 10: independent/hands-on verification of the MCP integration working in practice and details on which MCP clients/servers are supported beyond the editor's own tools.", "evidenceIds": [ "rive-docs-19", "rive-docs-10", "rive-docs-27", "rive-probe-4" ] }, { "productId": "rive", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Rive's docs explicitly describe an official MCP server that connects the Rive Editor to AI agents/tools, letting AI handle View Models, State Machines, Layouts, Shapes, etc., and this is confirmed via a dedicated docs page and probe. Missing for 10: independent/hands-on third-party verification of the MCP server working in practice beyond first-party docs.", "evidenceIds": [ "rive-docs-19", "rive-docs-27", "rive-docs-10", "rive-probe-4" ] }, { "productId": "rive", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Rive documents a built-in AI agent and MCP integration that let users describe tasks in natural language to generate ViewModels, state machines, layouts, and animation code, plus a CLI for agent-driven generation (rive-docs-9,10,18,19,26,27). However, this is entirely first-party documentation with no independent or hands-on confirmation that natural-language commands reliably drive the full editor experience, and community feedback focuses on pricing/licensing rather than the AI agent's real-world behavior. Missing for 10: independent/hands-on verification of the AI agent's natural-language command success, broader coverage beyond docs-based claims, and detail on command scope/limits.", "evidenceIds": [ "rive-docs-9", "rive-docs-10", "rive-docs-18", "rive-docs-19", "rive-docs-26", "rive-docs-27", "rive-probe-4" ] }, { "productId": "rive", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The Rive homepage claims users can 'let an agent generate them with the CLI' (rive-docs-26), indicating an official CLI exists for AI-agent workflows, but no other evidence pack item documents CLI installation, commands, or usage — most AI-agent tooling evidence instead centers on the MCP server and in-editor AI agent, not a standalone CLI. Missing for 10: dedicated CLI docs/reference, install instructions, command examples, and independent/hands-on corroboration of the CLI working.", "evidenceIds": [ "rive-docs-26", "rive-docs-9", "rive-docs-19" ] }, { "productId": "rive", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Rive documents open-source runtime APIs (web, mobile, game engines) for loading and controlling animations programmatically (rive-docs-21, rive-gh-1, rive-docs-14/15), plus a text-based file format (RML) explicitly designed for coding agents and automated workflows (rive-docs-13, 20, 25, 30), and an official MCP server/AI agent integration for driving the Editor via AI tools (rive-docs-19, 27, rive-probe-4, rive-docs-9/18). Together these give an AI-native user multiple documented, agent-friendly ways to control the product beyond the GUI. Missing for 10: a formal REST/OpenAPI-style public API (probe shows 404s for openapi/swagger endpoints) and independent third-party corroboration of programmatic API usage.", "evidenceIds": [ "rive-docs-21", "rive-gh-1", "rive-docs-13", "rive-docs-20", "rive-docs-25", "rive-docs-19", "rive-docs-27", "rive-probe-4", "rive-docs-9", "rive-docs-18", "rive-docs-14", "rive-docs-15" ] }, { "productId": "rive", "storyId": "agentic-scoped-keys", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Rive is a design/animation tool with an editor, runtimes, and MCP/AI-agent integrations; it is not an API-credentialing or identity/access-management platform, so scoped API credential issuance for agents is outside its product category.", "evidenceIds": [] }, { "productId": "rive", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Rive documents open-source official runtimes (SDKs) for web, mobile, game engines, and other platforms, with a public GitHub repo (rive-runtime) implementing the loading/rendering API, plus explicit guidance on caching and using the same exported .riv file across supported runtimes. This directly satisfies 'build against official SDKs' for developers, including AI-native workflows via RML/MCP integration for programmatic file creation. Missing for 10: independent third-party corroboration of SDK quality/completeness across all listed platforms and more granular per-language SDK documentation (e.g., npm/Swift/Kotlin package specifics).", "evidenceIds": [ "rive-docs-8", "rive-docs-21", "rive-gh-1", "rive-docs-14", "rive-docs-20" ] }, { "productId": "rive", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack mentions webhooks, event subscriptions, or any push-notification API for Rive; the product is documented as a design/animation tool with editor, runtime, and MCP/AI-agent integrations, none of which reference webhook-based event delivery.", "evidenceIds": [] }, { "productId": "rive", "storyId": "ai-asset-editing-tools", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Rive's AI agent/MCP tooling is focused on generating code, view models, state machines, layouts, and animation logic — not on image generation, background removal, content-aware fill, or copy suggestions for asset creation. No evidence names any of these specific asset-chore AI features; missing for 10: image generation, background removal, content-aware fill, copy/text suggestions.", "evidenceIds": [ "rive-docs-9", "rive-docs-10", "rive-docs-18", "rive-docs-27" ] }, { "productId": "rive", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While Rive has extensive documentation (runtime parameters, data binding, RML), there is no evidence of an interactive API reference with runnable/live code examples, and the OpenAPI probe returned 404s with no playground or sandbox mentioned.", "evidenceIds": [ "rive-docs-14", "rive-docs-15", "rive-probe-3" ] }, { "productId": "rive", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe for OpenAPI/swagger spec endpoints (openapi.json, swagger.json, etc.) returned 404 for all candidates, and no docs mention a machine-readable API spec being downloadable.", "evidenceIds": [ "rive-probe-3" ] }, { "productId": "rive", "storyId": "api-sandbox", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Rive is a vector animation design/runtime tool, not a service with production data or a sandbox/test-environment concept; the sandbox-vs-production testing story is a category error for this product type.", "evidenceIds": [] }, { "productId": "rive", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any versioned API scheme or documented deprecation policy for Rive's runtimes, editor APIs, or MCP integration; OpenAPI probes returned 404s and no changelog/versioning docs are cited.", "evidenceIds": [ "rive-probe-3" ] }, { "productId": "rive", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Rive documents an AI agent/MCP integration explicitly designed to let AI 'handle repetitive tasks, like creating complex View Models, State Machines with hundreds of states/layers, Layouts, Shapes, and more,' and RML provides a text-based file format enabling coding agents to script bulk edits outside the Editor. Components/instances also let one source change propagate across many instances. However, there's no independent/hands-on evidence of actual bulk-operation performance at scale, no documented API for batch scripting beyond RML/MCP, and no benchmarks or examples of large-scale bulk edits succeeding. Missing for 10: independent verification of bulk-task performance, concrete examples/benchmarks of large-scale batch operations, and a documented API/CLI for scripted bulk edits beyond the AI agent's stated purpose.", "evidenceIds": [ "rive-docs-10", "rive-docs-27", "rive-docs-13", "rive-docs-20", "rive-docs-11", "rive-docs-28" ] }, { "productId": "rive", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Rive's State Machines and data-binding docs imply event-driven behavior (inputs, user actions, and data changes automatically triggering animation/state updates), and its AI Agent/MCP tooling can generate these State Machines and scripting-based behaviors programmatically. However, there's no explicit documentation of a rule-authoring system (e.g., conditional 'if event then action' automation) framed for AI-native/automation use beyond interactive animation triggers. Missing for 10: explicit rule-definition API/interface for arbitrary automation, examples of AI-native users authoring trigger-action rules, and independent confirmation beyond first-party docs.", "evidenceIds": [ "rive-docs-4", "rive-docs-6", "rive-docs-24", "rive-docs-9", "rive-docs-10" ] }, { "productId": "rive", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Rive is a vector animation/design tool and runtime; scheduling recurring jobs or workflows is not a capability applicable to this product's category — no evidence pack content addresses job scheduling or automation workflows of this kind.", "evidenceIds": [] }, { "productId": "rive", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Rive's Revision History provides automatic tracking of changes, previewing of past revisions, and non-destructive rollback to prior versions of a file (rive-docs-12, rive-docs-17, rive-docs-29), which covers version/rollback for the design file itself. However, the story targets 'automations' (AI-agent/MCP-driven workflows), and there is no evidence of versioning, diffing, or review specifically for AI-generated scripts, MCP actions, or automation pipelines — only generic file revision history. Missing for 10: dedicated versioning/review/rollback for AI-agent or MCP-generated automations, audit trail of agent actions, diff/compare tooling, and any independent confirmation of this workflow in practice.", "evidenceIds": [ "rive-docs-12", "rive-docs-17", "rive-docs-29", "rive-docs-19", "rive-docs-27" ] }, { "productId": "rive", "storyId": "build-custom-plugins", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a documented editor plugin/extension API, typings, or samples for building third-party editor extensions; the evidence only covers runtimes, data binding, AI agent/MCP integration, and RML scripting, none of which document an extensibility API for building/distributing editor plugins.", "evidenceIds": [] }, { "productId": "rive", "storyId": "clear-plan-pricing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains only product/docs content about Rive's features (animation, data binding, AI agent, MCP) and community discussion referencing informal price points ($25-$39/month) but no published pricing page, seat-type breakdown, tier comparison, or explicit AI-feature gating documentation.", "evidenceIds": [ "rive-comm-2" ] }, { "productId": "rive", "storyId": "comments-review-flows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers animation, data binding, revision history, AI/MCP tooling, and export, but there is no mention of commenting, @mentions, or thread resolution features in Rive's editor. Revision history tracks changes but is not a comment/review workflow.", "evidenceIds": [ "rive-docs-12", "rive-docs-17", "rive-docs-29" ] }, { "productId": "rive", "storyId": "design-to-production-code", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Rive's core value prop is explicitly design-to-shippable-runtime: 'No mockups, no prototypes, no handoff... What you build in the editor is what ships in your app, game, or website' (rive-docs-23), backed by real runtime libraries that load .riv files directly into apps/games/web (rive-gh-1, rive-docs-21), reusable components/instances (rive-docs-11, rive-docs-28), and RML text representation enabling coding-agent-driven file construction (rive-docs-13, rive-docs-25, rive-docs-30). This is a genuine direct design-to-production path rather than a code-generation handoff, and MCP/AI agent tooling further bridges design and code workflows (rive-docs-9, rive-docs-19, rive-docs-27). Missing for 10: independent developer case studies confirming the 'no handoff' claim holds up in large production codebases, and clearer detail on how designers' Rive files integrate with existing component libraries/design systems in code.", "evidenceIds": [ "rive-docs-23", "rive-docs-11", "rive-docs-28", "rive-docs-13", "rive-docs-25", "rive-docs-30", "rive-gh-1", "rive-docs-21", "rive-docs-9", "rive-docs-19", "rive-docs-27" ] }, { "productId": "rive", "storyId": "design-tokens-variables", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Rive is an animation/interactive-graphics tool with components, data binding, and view models, but there is no evidence of a design-tokens/variables system (colors, spacing, typography) with modes/themes applied across designs — the closest concepts (components, data binding) address reusability and runtime data, not token/theme management. Missing for 10: token or variable definitions, theme/mode switching, cross-design application of shared style values.", "evidenceIds": [ "rive-docs-11", "rive-docs-15", "rive-docs-24", "rive-docs-28" ] }, { "productId": "rive", "storyId": "export-formats-assets", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Rive's documentation focuses on exporting .riv files for runtime playback across platforms (web, mobile, games) rather than static production assets like PNG/SVG/PDF with per-layer presets, scales, or slices; no such export presets are described anywhere in the evidence. Community feedback explicitly confirms this gap — one user complains 'it's so insane they don't offer any way to export movie files' and another asks 'can I export video assets with this!?' — indicating the tool lacks traditional multi-format asset export entirely.", "evidenceIds": [ "rive-docs-7", "rive-docs-8", "rive-comm-3", "rive-comm-10" ] }, { "productId": "rive", "storyId": "free-tier-real-design", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence describes what the free tier includes for editor/collaboration access; the only relevant community signal explicitly states that basic solo-dev animation work requires a paid subscription ($25-39/month), suggesting the free tier is not sufficient for real design work.", "evidenceIds": [ "rive-comm-1", "rive-comm-2" ] }, { "productId": "rive", "storyId": "inspect-code-handoff", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Rive is a vector animation/design-to-runtime tool, not a design-file inspection/handoff tool like Figma/Zeplin; there is no concept of layer inspection with CSS/iOS/Android code snippet export without an editor seat. The story's axis (dev-handoff inspect mode) is a category error for this product's type.", "evidenceIds": [] }, { "productId": "rive", "storyId": "interactive-prototypes", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Rive's State Machines and interaction docs (rive-docs-4, rive-docs-22, rive-docs-24) support building input-driven, trigger/transition-based interactivity without code, and rive-docs-16/23 frame the tool as skipping separate 'prototype' handoff entirely. However there is no evidence of a Figma-style multi-screen/overlay prototyping paradigm (linking distinct screens, overlay layers, click-through flows across artboards) — Rive's interactivity is scoped to state machines within/across components rather than an explicit screen-flow prototyping mode. Missing for 10: evidence of multi-screen linking, overlay/modal wiring, and click-through flow prototyping across separate artboards/screens.", "evidenceIds": [ "rive-docs-4", "rive-docs-16", "rive-docs-22", "rive-docs-23", "rive-docs-24" ] }, { "productId": "rive", "storyId": "nl-design-generation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Rive documents an AI agent/MCP integration that generates View Models, State Machines, Layouts, Shapes and animation from prompts or via coding-agent workflows, and even an 'agent generate them with the CLI' claim on the homepage — this matches the story's spirit of prompt-driven first drafts. However, all evidence is first-party docs describing 'the first set of tools' as handling repetitive tasks rather than full end-to-end mockup generation, and there's no independent/hands-on corroboration of output quality. Missing for 10: independent or hands-on validation of AI-generated designs, clearer evidence of full mockup/layout generation (not just repetitive sub-tasks) from a single prompt.", "evidenceIds": [ "rive-docs-9", "rive-docs-10", "rive-docs-18", "rive-docs-19", "rive-docs-26", "rive-docs-27", "rive-probe-4" ] }, { "productId": "rive", "storyId": "open-file-format", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Rive documents RML (a text-based/XML representation of Rive files) and open-source runtimes that can load/parse .riv files (rive-docs-13/20/25/30, rive-gh-1/3), suggesting some portability outside the editor. However, community evidence directly disputes this: the open-source runtime is described as only a rendering/playback layer, and actually authoring or building Rive content still requires the proprietary, paid Rive Editor (rive-comm-5), undercutting claims of full format openness/portability. Missing for 10: a public, versioned .riv binary spec independent of RML, third-party tools that can author/edit .riv files without the vendor's editor, and confirmation that RML round-trips losslessly with the binary format.", "evidenceIds": [ "rive-docs-13", "rive-docs-20", "rive-docs-25", "rive-docs-30", "rive-gh-1", "rive-comm-5" ] }, { "productId": "rive", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Rive offers RML (text-based file format) and an MCP server that let AI/coding agents create View Models, State Machines, Layouts, and Shapes programmatically, plus open-source runtimes for playback/control — but the evidence pack does not confirm full parity with the Editor UI (e.g., no OpenAPI/REST API, no confirmation that timeline keyframing, revision history management, or component/instance authoring can be done entirely via API/CLI). missing for 10: a documented public REST/OpenAPI API surface, confirmation that all editor features (keyframe timelines, revision history, component instancing) are scriptable/API-accessible, and independent hands-on verification that AI/API workflows match UI capability.", "evidenceIds": [ "rive-docs-13", "rive-docs-19", "rive-docs-20", "rive-docs-25", "rive-docs-27", "rive-docs-30", "rive-probe-4", "rive-probe-3", "rive-gh-1" ] }, { "productId": "rive", "storyId": "openness-full-export", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Rive documents an open, text-based file representation (RML/XML) and open-source runtimes, suggesting some data portability (rive-docs-13, rive-docs-20, rive-docs-25, rive-docs-21, rive-gh-1). However, community feedback directly contradicts a full 'export and leave' story: users report no way to export movie/video assets (rive-comm-3, rive-comm-10) and note the open-source license only covers a narrow rendering subset while the actual content-creation editor remains proprietary and subscription-locked (rive-comm-5, rive-comm-1, rive-comm-2), undermining claims of full open-format data portability. missing for 10: documented full-fidelity export to universal open formats (e.g. video/SVG/JSON), open-source editor or file-format spec, and independent confirmation that RML round-trips 100% of file data without proprietary lock-in.", "evidenceIds": [ "rive-docs-13", "rive-docs-20", "rive-docs-21", "rive-gh-1", "rive-comm-3", "rive-comm-5", "rive-comm-10", "rive-comm-1" ] }, { "productId": "rive", "storyId": "openness-open-license", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Rive's runtimes are documented and confirmed on GitHub as open-source libraries (rive-docs-21, rive-gh-1, rive-gh-2), giving AI-native users real source access to the rendering/runtime layer, but community evidence notes the MIT license only covers a 'rendering abstraction layer, a subset of the Rive runtime' while the Editor itself (where files are authored) remains closed-source (rive-comm-5, rive-comm-1). missing for 10: full-product open license (editor/design tool source not published), independent confirmation of license file/terms, clarity on what runtime components are excluded from the OSS layer.", "evidenceIds": [ "rive-docs-21", "rive-gh-1", "rive-gh-2", "rive-comm-5", "rive-comm-1" ] }, { "productId": "rive", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Rive's core product is the proprietary Editor (a SaaS/desktop app); only a subset of the runtime rendering layer is open-source (MIT), and community evidence explicitly notes the editor itself cannot be self-hosted and requires the hosted tool to author content. No docs or evidence describe self-hosting the Editor or backend service.", "evidenceIds": [ "rive-comm-5", "rive-docs-21" ] }, { "productId": "rive", "storyId": "plugin-ecosystem-marketplace", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a plugin marketplace, third-party extension ecosystem, or add-ons like icon packs, data fillers, or accessibility checkers. The evidence only covers Rive's own built-in features (components, data binding, MCP/AI agent, RML) — none of which describe an external plugin/marketplace ecosystem. Missing for 10: any mention of a plugin API, extension marketplace, or third-party-built add-ons.", "evidenceIds": [] }, { "productId": "rive", "storyId": "privacy-data-residency", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Rive is a design/animation tool and runtime library, not a data-storage or SaaS platform with data residency concerns; no evidence pack content relates to data region/residency choices, and this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "rive", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Rive is a design/animation tool, not an AI model provider or data-processing service; there's no mention of training-data usage or opt-out controls, and this privacy axis about AI training data is not a relevant capability for this product category.", "evidenceIds": [] }, { "productId": "rive", "storyId": "privacy-retention-controls", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Rive is a design/animation tool and runtime; it has no data-retention/deletion controls relevant to AI-native privacy posture in the evidence, and this is not a category where such controls would typically be a distinguishing product axis for a design tool's file/asset content vs. a SaaS handling user data at scale — no evidence pack content addresses retention or deletion policies at all.", "evidenceIds": [] }, { "productId": "rive", "storyId": "privacy-telemetry-optout", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Rive is a design/animation tool and runtime SDK, not a data-collecting SaaS platform with telemetry settings relevant to AI-native privacy posture; no evidence pack content relates to opting out of usage tracking.", "evidenceIds": [] }, { "productId": "rive", "storyId": "realtime-multiplayer-editing", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs confirm multiple people can work on the same file simultaneously with all changes tracked in Revision History, implying some multiplayer support, but there is no mention of live cursors, presence indicators, or real-time instant sync mechanics that the story specifically asks for. Missing for 10: explicit live cursor/presence UI, real-time sync latency claims, independent hands-on confirmation of simultaneous editing experience.", "evidenceIds": [ "rive-docs-12", "rive-docs-17", "rive-docs-29" ] }, { "productId": "rive", "storyId": "responsive-auto-layout", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Rive's docs repeatedly reference \"responsive layouts\" and building interfaces that respond to screen size (rive-docs-1, rive-docs-4, rive-docs-9, rive-docs-18, rive-docs-26), and the AI tooling explicitly generates \"Layouts\" as an artboard-level construct (rive-docs-10). However, no evidence details the specific mechanics of auto layout, constraints, or grid systems the way Figma-style tools describe them. Missing for 10: explicit documentation of constraint-based resizing rules, grid systems, or auto-layout algorithms, and independent/hands-on confirmation that layouts flex like real UI frameworks.", "evidenceIds": [ "rive-docs-1", "rive-docs-4", "rive-docs-9", "rive-docs-10", "rive-docs-18", "rive-docs-26" ] }, { "productId": "rive", "storyId": "reusable-components-variants", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Rive's docs directly describe Components as reusable artboards where instances can be customized while changes to the source component propagate to all instances, matching the story's core requirement of variants/overrides linked to source. Missing for 10: no independent/hands-on corroboration of override behavior at scale, and no detail on variant-specific overrides beyond basic customization examples.", "evidenceIds": [ "rive-docs-11", "rive-docs-28" ] }, { "productId": "rive", "storyId": "self-host-design-platform", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows only the rendering runtime is open-source (rive-docs-21, rive-gh-1/2/3), while the Editor itself is a hosted SaaS product requiring Rive's cloud service—there is no documentation of a self-hostable full design platform. Community discussion explicitly confirms this gap, noting the MIT license covers only 'a subset of the Rive runtime that requires the Rive Editor to build content for' (rive-comm-5), meaning the actual design/editing platform cannot be self-hosted.", "evidenceIds": [ "rive-docs-21", "rive-gh-1", "rive-comm-5" ] }, { "productId": "rive", "storyId": "shared-team-libraries", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Rive's docs describe components/instances reused within a single Rive file where changes to a source component propagate to instances (rive-docs-11, rive-docs-28), but there is no evidence of a shared-library system that publishes components/styles across separate files or projects with controlled update propagation, which is the core of this story. Missing for 10: cross-file/cross-project library publishing, subscribing to a library, and controlled update notifications when a source library changes.", "evidenceIds": [ "rive-docs-11", "rive-docs-28" ] }, { "productId": "rive", "storyId": "vector-editing-craft", "verdict": "disputed", "quality": 3, "confidence": "medium", "rationale": "Rive's docs market it as a full design-to-ship tool ('design, animate, and code in one place... no mockups, no prototypes, no handoff') and mention vector graphics and components, but there is no documentation of boolean operations, precise alignment/snapping tools, or advanced path editing. A hands-on community comment directly contradicts the 'professional vector tools' claim, calling the drawing toolset just 'a pen and a couple of basic shapes' and comparing inefficient pen-only workflows unfavorably to real illustration tools. missing for 10: boolean operations, alignment/snap tooling, path-editing depth, and any rebuttal to the community criticism of basic drawing tools.", "evidenceIds": [ "rive-docs-1", "rive-docs-16", "rive-docs-23", "rive-comm-8" ] }, { "productId": "rive", "storyId": "version-history-restore", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Rive's docs confirm automatic revision tracking, non-destructive restore, and the ability to select and edit a prior revision (rive-docs-12, rive-docs-17, rive-docs-29), which covers 'restore any earlier state.' However there is no evidence of naming/labeling milestones or a compare/diff view between versions. Missing for 10: named milestone/labeling capability, side-by-side or diff comparison between revisions, independent corroboration of the restore workflow.", "evidenceIds": [ "rive-docs-12", "rive-docs-17", "rive-docs-29" ] }, { "productId": "sketch", "storyId": "advanced-prototype-logic", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Sketch's docs show interactive prototyping via hotspots/links, overlays, hover/press/toggle triggers, and scrolling areas, which cover some 'feel like the real product' interactions, but there is no evidence of animated transitions between states, conditional logic, or variables — capabilities competitors highlight for advanced prototyping. Missing for 10: animation/transition effects between artboards, conditional logic or variable-driven states, and any documented scroll-triggered animation beyond basic scrolling areas.", "evidenceIds": [ "sketch-docs-8", "sketch-docs-25", "sketch-docs-32", "sketch-docs-37", "sketch-docs-48" ] }, { "productId": "sketch", "storyId": "agent-creates-design", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Sketch documents multiple write-capable programmatic surfaces: an official JavaScript Plugin API for creating/modifying designs, sketchtool CLI, sketch:// URL scheme for workflow integration, and explicit MCP support letting any MCP-compatible AI client connect to Sketch, including AI features that build out whole frames/pages from a repo. This goes beyond read-only inspection (which is the separate developer-handoff feature) into actual design creation/modification via agent tooling.\nMissing for 10: independent hands-on verification that MCP tools can actually write/modify designs (evidence is vendor-only), and no detailed spec of exposed MCP tool actions.", "evidenceIds": [ "sketch-docs-3", "sketch-docs-4", "sketch-docs-5", "sketch-docs-6", "sketch-docs-16", "sketch-docs-17", "sketch-docs-18", "sketch-docs-34", "sketch-docs-44" ] }, { "productId": "sketch", "storyId": "agent-exports-assets", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Sketch documents `sketchtool`, a command-line tool explicitly for creating/exporting assets programmatically, plus an official JavaScript plugin API and 'integrate into workflows and server applications' language, giving agents a documented surface to render/export designs; MCP support further lets AI clients drive Sketch. Missing for 10: independent/hands-on verification of sketchtool's export formats and no explicit mention of headless/server rendering outside macOS, and no OpenAPI/REST surface was found (probe 404s).", "evidenceIds": [ "sketch-docs-5", "sketch-docs-6", "sketch-docs-44", "sketch-docs-18", "sketch-docs-34", "sketch-probe-3" ] }, { "productId": "sketch", "storyId": "agent-reads-design-context", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Sketch's own docs explicitly describe an MCP server integration ('Connect any MCP-compatible AI client you like to Sketch') that lets an AI client pull frames and recreate text styles, colors, UI components and pages, then build them out in code with 1:1 visual fidelity — directly matching the story's design-context-to-implementation workflow. Missing for 10: independent/hands-on verification of the MCP server actually working, explicit mention of pulling 'variables' or raw screenshots through the API, and any third-party corroboration beyond vendor marketing pages.", "evidenceIds": [ "sketch-docs-16", "sketch-docs-17", "sketch-docs-18", "sketch-docs-34" ] }, { "productId": "sketch", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Sketch actually publishes an llms.txt file that returns HTTP 200 and serves a structured docs index for agent consumption (getting-started, prototyping, developer-handoff, workspace docs), confirmed by both docs listing and a runtime probe. Missing for 10: no evidence of broader agent-oriented markdown mirrors (the /docs/.md probe 404s) or an OpenAPI/machine-readable schema beyond llms.txt.", "evidenceIds": [ "sketch-docs-40", "sketch-probe-1", "sketch-probe-rt-1", "sketch-probe-2" ] }, { "productId": "sketch", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Sketch's AI features (docs-14,15,26,27,39) generate suggestions like color palettes, dark-mode variants, layer organization, and placeholder content directly from design data, which counts as AI-generated insights/suggestions inside the product; MCP integration (docs-18,34) also lets AI clients query/act on Sketch data. However, these are largely generative/automation utilities rather than analytic 'insights' about the user's data, and there is no independent/hands-on corroboration of quality or accuracy. missing for 10: independent verification of AI feature accuracy/usefulness, broader insight-generation (e.g., analytics on design usage/trends) beyond visual generation tasks, and clearer separation of AI-insight vs AI-automation claims.", "evidenceIds": [ "sketch-docs-14", "sketch-docs-15", "sketch-docs-26", "sketch-docs-27", "sketch-docs-39", "sketch-docs-18", "sketch-docs-34" ] }, { "productId": "sketch", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "sketch", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Sketch.com/ai lists several AI-powered automation features (auto dark-mode conversion, palette generation from a photo, automatic layer renaming/sorting, context-aware placeholder text) that a user can invoke to offload discrete design tasks, which is a form of task delegation. However, these are framed as one-off automated tools rather than a conversational, built-in assistant that takes freeform instructions, and the closest thing to an 'assistant' (docs-16/18/34) actually describes connecting an *external* MCP-compatible AI client to Sketch, not a native in-app assistant. Missing for 10: evidence of a conversational/chat-style built-in assistant, evidence of multi-step task delegation beyond single-purpose generators, and independent hands-on confirmation of these AI features working as described.", "evidenceIds": [ "sketch-docs-14", "sketch-docs-15", "sketch-docs-26", "sketch-docs-27", "sketch-docs-39", "sketch-docs-16", "sketch-docs-18", "sketch-docs-34" ] }, { "productId": "sketch", "storyId": "agentic-headless", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Sketch documents a command-line tool (`sketchtool`) for scripted asset creation and claims it can integrate 'into your workflows and server applications, on any platform,' which points at CI/headless automation use. However, this is only first-party doc language with no independent verification, and community evidence confirms Sketch itself is strictly Mac-native and the vendor has refused Windows/Linux support, undercutting the 'any platform' claim. Missing for 10: independent CI/automation examples or case studies, clarity on macOS-only runtime requirements for sketchtool, and confirmation of non-interactive/headless execution in real pipelines.", "evidenceIds": [ "sketch-docs-5", "sketch-docs-6", "sketch-comm-5" ] }, { "productId": "sketch", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows Sketch acting as an MCP *server* that external AI clients (e.g. Claude, Cursor) can connect to and use Sketch's own tools (sketch-docs-18, sketch-docs-34) — the reverse of the story, which asks whether Sketch itself can plug in external MCP servers to consume their tools. No documentation or community evidence indicates Sketch can act as an MCP client consuming third-party tool servers.", "evidenceIds": [ "sketch-docs-18", "sketch-docs-34" ] }, { "productId": "sketch", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Sketch documents that users can connect any MCP-compatible AI client to Sketch itself, positioning Sketch as an MCP server that AI agents can plug into, with no extra tiers or tokens required. Missing for 10: independent/hands-on verification that the MCP connection works reliably, and detailed technical docs on the server's capabilities/setup beyond marketing copy.", "evidenceIds": [ "sketch-docs-18", "sketch-docs-34", "sketch-docs-16" ] }, { "productId": "sketch", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Sketch documents AI features (auto dark-mode conversion, color palette generation, layer renaming, placeholder text replacement) and explicit MCP-client connectivity so 'any MCP-compatible AI client' can drive design tasks, implying natural-language-driven operation via an external agent. However there is no first-party evidence of an in-app chat/command interface where a user directly types natural-language instructions to Sketch itself — the AI capabilities are either fixed one-click features or routed through external MCP clients. Missing for 10: a native in-app natural-language command interface, hands-on/independent verification that MCP-driven natural-language commands actually work end-to-end.", "evidenceIds": [ "sketch-docs-16", "sketch-docs-18", "sketch-docs-34", "sketch-docs-14", "sketch-docs-39", "sketch-docs-27" ] }, { "productId": "sketch", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Sketch ships an official CLI, `sketchtool`, documented for asset creation and integration into workflows/server applications on any platform, which agentic/AI tools could invoke. However, there's no evidence this CLI is positioned or extended for AI-native agent workflows specifically (no mention of AI agents using sketchtool, no JSON/agent-friendly output specs, and MCP support is a separate non-CLI integration). Missing for 10: agent-oriented CLI documentation, examples of AI/agent usage of sketchtool, independent hands-on confirmation of its use in automated pipelines.", "evidenceIds": [ "sketch-docs-5", "sketch-docs-6", "sketch-docs-7", "sketch-docs-30" ] }, { "productId": "sketch", "storyId": "agentic-public-api", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Sketch documents an official vanilla JavaScript plugin API 'fully supported between releases', a CLI (`sketchtool`), a `sketch://` URL scheme for workflow/server integration, and explicit MCP support so any MCP-compatible AI client can connect and drive Sketch — directly matching the AI-native/agentic story. However there's no REST/HTTP API or OpenAPI spec (probe shows 404s for openapi/swagger endpoints), and the API is tied to the Mac app/plugin ecosystem rather than a cloud-callable service. Missing for 10: a REST/HTTP or OpenAPI-documented API, and independent hands-on corroboration of AI clients actually driving Sketch via MCP.", "evidenceIds": [ "sketch-docs-44", "sketch-docs-5", "sketch-docs-6", "sketch-docs-7", "sketch-docs-18", "sketch-docs-34", "sketch-probe-3" ] }, { "productId": "sketch", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows MCP connectivity, SSO, SCIM provisioning, and permissions groups, but nothing about issuing scoped or least-privilege API credentials/tokens specifically for agents; permissions groups and SSO are user-level access controls, not credential scoping mechanisms. Missing for 10: any documentation of API tokens, scoped keys, or credential issuance workflow tailored to agent use.", "evidenceIds": [ "sketch-docs-18", "sketch-docs-20", "sketch-docs-21", "sketch-docs-42" ] }, { "productId": "sketch", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Sketch publishes an official vanilla JavaScript API, a CLI (sketchtool), plugin manager (skpm), and a documented sketch:// URL scheme for integrating into workflows and server apps, which constitute genuine official SDKs/APIs. However, there's no OpenAPI spec, no REST API, and probes confirm no openapi.json exists; the AI-native story is mostly served via MCP connectivity rather than a broad official SDK ecosystem. Missing for 10: a formal REST/OpenAPI API, multi-language SDKs beyond JS, and independent developer corroboration of building production integrations against these SDKs.", "evidenceIds": [ "sketch-docs-3", "sketch-docs-4", "sketch-docs-5", "sketch-docs-6", "sketch-docs-7", "sketch-docs-30", "sketch-docs-44", "sketch-probe-3" ] }, { "productId": "sketch", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of Sketch offering webhooks or event subscription mechanisms; the product exposes a plugin API, CLI (sketchtool), URL scheme, and MCP integration for AI clients, but nothing about webhook subscriptions for events. This is a fair axis for a platform with an API/integration ecosystem, so absence of evidence yields 'none'.", "evidenceIds": [ "sketch-docs-6", "sketch-docs-7", "sketch-docs-18" ] }, { "productId": "sketch", "storyId": "ai-asset-editing-tools", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Sketch's AI features (sketch-docs-14,15,26,27,39) cover contextual copy replacement, automatic layer organization, dark-mode conversion, and color-palette extraction from a photo, and it can connect to MCP AI clients for code generation—but there is no evidence of built-in image generation or background removal tools. missing for 10: image generation, background/foreground removal, content-aware fill (beyond text), independent hands-on verification of AI feature quality.", "evidenceIds": [ "sketch-docs-14", "sketch-docs-15", "sketch-docs-26", "sketch-docs-27", "sketch-docs-39", "sketch-docs-34" ] }, { "productId": "sketch", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Sketch documents a JavaScript plugin API and invites developers to 'Try the Sketch JavaScript API now, directly within Sketch,' suggesting some interactive API exploration, but there is no evidence of a full interactive reference with runnable code examples (OpenAPI/swagger probes all 404, no dedicated API-explorer docs found). missing for 10: dedicated interactive API reference UI, runnable code examples/sandbox, OpenAPI/swagger spec, independent confirmation of the 'try now' feature.", "evidenceIds": [ "sketch-docs-4", "sketch-docs-44", "sketch-probe-3" ] }, { "productId": "sketch", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Direct probing found no OpenAPI/Swagger spec at any standard location (all candidates 404), and no docs reference a machine-readable API spec; Sketch offers a JS plugin API and MCP integration but no downloadable OpenAPI-equivalent.", "evidenceIds": [ "sketch-probe-3", "sketch-docs-44" ] }, { "productId": "sketch", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "sketch", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Sketch exposes a JavaScript plugin API, sketchtool CLI, and MCP integration, and claims the JS API is 'fully supported between releases' (sketch-docs-44), but there is no documented API versioning scheme or deprecation policy — probes for OpenAPI specs return 404 and no changelog/deprecation docs are present in the evidence pack.", "evidenceIds": [ "sketch-docs-44", "sketch-probe-3" ] }, { "productId": "sketch", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Sketch's AI features include layer-level bulk actions (auto sort/rename/align layers, replacing placeholder text across every layer, swapping light/dark components) and its CLI/plugin API (`sketchtool`, JS API) allow scripted operations across assets, which supports some automation-at-scale. However, there's no evidence of explicit multi-document/multi-project bulk operations or a documented batch API for large-scale item processing beyond within-document layers. Missing for 10: cross-file/cross-project batch operations, explicit bulk API endpoints, and independent verification of scale performance.", "evidenceIds": [ "sketch-docs-27", "sketch-docs-39", "sketch-docs-14", "sketch-docs-5", "sketch-docs-6", "sketch-docs-44" ] }, { "productId": "sketch", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "sketch", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Sketch is a design tool; no evidence supports or implies scheduling recurring jobs/workflows, and this capability is not a fit for its category (design/prototyping software with plugin/API/MCP integrations, not a workflow scheduler).", "evidenceIds": [] }, { "productId": "sketch", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Sketch offers document-level version history via 'starred versions' that lets you track changes, control which versions are visible/published, and effectively roll back to prior states, which could cover AI-generated design changes; but there is no evidence of a dedicated 'automation' concept (e.g., saved AI workflows/scripts) with its own versioning, diff/review, or rollback mechanism distinct from general document history. missing for 10: evidence of versioning/rollback specifically for AI-generated automations/workflows (not just document snapshots), a review/diff UI for AI changes, and any audit trail tied to MCP/AI actions.", "evidenceIds": [ "sketch-docs-10", "sketch-docs-28", "sketch-docs-33", "sketch-docs-41", "sketch-docs-18", "sketch-docs-16" ] }, { "productId": "sketch", "storyId": "build-custom-plugins", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Sketch provides a dedicated developer portal (developer.sketch.com) with a documented JavaScript plugin API, in-browser API tester, official vanilla JS API stable across releases, plugin manager (skpm) for publishing/distributing plugins, and manual install support, plus CLI/URL-scheme integration options. Missing for 10: independent hands-on developer reviews confirming ease of building complex plugins and richer typings/type-definition documentation beyond API reference mentions.", "evidenceIds": [ "sketch-docs-3", "sketch-docs-4", "sketch-docs-30", "sketch-docs-44", "sketch-docs-45", "sketch-comm-14", "sketch-comm-15" ] }, { "productId": "sketch", "storyId": "clear-plan-pricing", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "The pricing page evidence lists enterprise-tier features like SSO, SCIM provisioning, permissions groups, and project archiving, implying a tiered structure, and sketch-docs-18 explicitly states AI/MCP features carry 'no tokens to buy, no extra tiers, no add-ons,' which speaks directly to AI-feature gating. However, there is no evidence of explicit seat-type definitions, plan names, or per-seat pricing tables that a PM could use to compare procurement options. Missing for 10: explicit seat-type/tier naming and pricing amounts, a full feature-by-tier comparison matrix, and independent confirmation that AI gating claims match published plans.", "evidenceIds": [ "sketch-docs-18", "sketch-docs-20", "sketch-docs-21", "sketch-docs-42", "sketch-docs-40" ] }, { "productId": "sketch", "storyId": "comments-review-flows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers developer handoff, prototyping, symbols, versioning, plugins, and AI features, but contains no mention of in-file commenting, @mentions, or thread resolution—core to this collaboration story. Community notes even highlight Sketch's historical lag on real-time collaboration features versus competitors, but nothing confirms comment/mention/resolve functionality exists.", "evidenceIds": [ "sketch-comm-4" ] }, { "productId": "sketch", "storyId": "design-to-production-code", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Sketch's AI page claims direct frame-to-code generation with '1:1 visual fidelity' and MCP-based AI client integration, plus browser-based developer handoff for inspecting specs, which together approximate design-to-code output; however this is entirely first-party marketing with no independent hands-on verification of code quality, supported frameworks, or shippability. missing for 10: independent/community confirmation that generated code is production-ready, details on output format/frameworks, and evidence of real-world dev usage of this AI code feature.", "evidenceIds": [ "sketch-docs-17", "sketch-docs-16", "sketch-docs-18", "sketch-docs-34", "sketch-docs-1", "sketch-docs-22" ] }, { "productId": "sketch", "storyId": "design-tokens-variables", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Sketch supports reusable Styles and Symbols for colors, text, and components across designs, and its AI features can generate named color variables and swap light/dark mode components automatically, giving some token/theme-like behavior. However, there is no evidence of a first-class 'variables with modes' system (e.g., defining semantic tokens with multiple named modes/themes applied project-wide) comparable to dedicated design-token tooling. Missing for 10: explicit variables/tokens feature with mode switching across spacing/typography, first-party documentation of a mode system, and independent hands-on confirmation of theme application at scale.", "evidenceIds": [ "sketch-docs-9", "sketch-docs-14", "sketch-docs-15", "sketch-docs-26", "sketch-docs-31", "sketch-docs-38", "sketch-comm-12" ] }, { "productId": "sketch", "storyId": "export-formats-assets", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The pack only hints at asset export via the `sketchtool` CLI (\"Create assets with a single command\") and never documents the actual Export panel, per-layer presets, scale factors, or slice tooling explicitly. Missing for 10: documentation of PNG/SVG/PDF format options, per-layer export presets, scale multipliers, and slice creation UI/behavior.", "evidenceIds": [ "sketch-docs-5", "sketch-docs-6" ] }, { "productId": "sketch", "storyId": "free-tier-real-design", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack only shows free access for developers to inspect/hand-off designs in a browser (no Mac needed) and a legacy 'no forced upgrade' licensing note, but there is no evidence of a free tier that lets a designer actually create/edit designs or collaborate in real time without a paid seat — Sketch is fundamentally a purchased desktop app for designers. Community notes even highlight it lacked real-time collaboration for years and requires a Mac/license to design at all.", "evidenceIds": [ "sketch-docs-2", "sketch-docs-29", "sketch-docs-46", "sketch-comm-1", "sketch-comm-10", "sketch-comm-4" ] }, { "productId": "sketch", "storyId": "inspect-code-handoff", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Sketch's developer handoff docs explicitly confirm free developer invites to inspect designs in-browser without needing the Mac app or a Mac, and note developers can grab measurements/info for any part of a design at any time; code snippet generation (CSS/iOS/Android) is a longstanding Sketch inspector feature implied by 'grab the information they need' though not explicitly enumerated per-platform in this pack. missing for 10: explicit doc citation naming CSS/iOS/Android code snippet export, and independent hands-on confirmation of the inspect panel's snippet output.", "evidenceIds": [ "sketch-docs-1", "sketch-docs-2", "sketch-docs-22", "sketch-docs-29", "sketch-docs-36", "sketch-docs-46" ] }, { "productId": "sketch", "storyId": "interactive-prototypes", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Sketch's own docs describe hotspots/links between frames, overlays (menus/dropdowns/dialogs), and hover/press/toggle triggers plus scrolling areas — directly matching the story's triggers/transitions/overlays without code, and mobile preview lets designers play prototypes on a real device. Missing for 10: independent hands-on reviews specifically praising the prototyping feature (community evidence focuses on performance/native-app debates, not prototyping quality) and no detail on transition animation styles/timing options.", "evidenceIds": [ "sketch-docs-8", "sketch-docs-25", "sketch-docs-32", "sketch-docs-37", "sketch-docs-48", "sketch-docs-13" ] }, { "productId": "sketch", "storyId": "nl-design-generation", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Sketch's AI page describes narrow, tool-specific AI features (auto light/dark mode conversion, photo-to-palette, code-to-design 1:1 fidelity, placeholder text replacement, layer auto-organization, and MCP-client integration to 'recreate text styles, colors, UI components, and even entire pages') rather than an open natural-language prompt that a designer types to generate an original first-draft layout or mockup from scratch. This is closer to targeted AI utilities layered onto existing designs than a general prompt-to-mockup generator. Missing for 10: evidence of a text-box/chat interface where a designer types a free-form prompt (e.g. 'design a pricing page') and Sketch outputs an original layout/mockup, and any hands-on or independent verification of AI generation quality.", "evidenceIds": [ "sketch-docs-14", "sketch-docs-15", "sketch-docs-16", "sketch-docs-17", "sketch-docs-26", "sketch-docs-27", "sketch-docs-34", "sketch-docs-39" ] }, { "productId": "sketch", "storyId": "open-file-format", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Sketch documents its .sketch file format as a zipped archive of JSON and binary data (sketch-docs-23/47), and community evidence confirms this enables a plugin ecosystem and git-style tooling outside the vendor app (sketch-comm-14). However, this is not a formally published open spec/standard with versioned schema docs, and there's no independent tooling ecosystem comparable to open formats like SVG; the developer-handoff features are more about inspection than format portability. Missing for 10: a formal published format specification/schema documentation, third-party tools that read/write .sketch independent of Sketch's own SDK, and any standards-body or open-format commitment.", "evidenceIds": [ "sketch-docs-23", "sketch-docs-47", "sketch-comm-14" ] }, { "productId": "sketch", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Sketch exposes a JavaScript plugin API, a CLI (sketchtool), a URL scheme, and MCP-based AI integrations for asset creation and design manipulation, showing meaningful programmatic access beyond the UI. However, there's no evidence of a comprehensive REST/public API covering all UI functions (e.g., prototyping links, sharing/versioning, symbol libraries, workspace permissions) — openapi probes 404'd and docs focus on plugin/CLI/AI hooks rather than full UI parity. missing for 10: a documented full REST/GraphQL API covering all UI actions, evidence that non-plugin workflows (sharing, prototyping, library publishing) are API-accessible, and independent confirmation of API completeness.", "evidenceIds": [ "sketch-docs-3", "sketch-docs-4", "sketch-docs-5", "sketch-docs-6", "sketch-docs-7", "sketch-docs-18", "sketch-docs-44", "sketch-probe-3" ] }, { "productId": "sketch", "storyId": "openness-full-export", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Sketch documents use the open `.sketch` format (a zipped JSON+binary archive) rather than a proprietary opaque format, which supports portability and third-party tooling/git integration per community reports. However, there's no evidence of a full account/workspace-level export tool for bulk data extraction (comments, version history, workspace settings) to enable a clean 'export everything and leave' migration. missing for 10: bulk workspace/account export feature, documented data-portability/migration guide, independent confirmation of successful full data export.", "evidenceIds": [ "sketch-docs-23", "sketch-docs-47", "sketch-comm-14" ] }, { "productId": "sketch", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Sketch is a proprietary, closed-source commercial design tool; the file format spec (sketch-document) is open, but there is no evidence the application source code itself is under an open license. No evidence contradicts closed-source status.", "evidenceIds": [] }, { "productId": "sketch", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "sketch", "storyId": "plugin-ecosystem-marketplace", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Sketch provides a documented plugin API (JavaScript, ES6, macOS framework access), a CLI/manager (skpm) for publishing and installing plugins, and manual plugin installation, and community sources independently confirm Sketch has 'a massive collection of plugins that forms a large part of its appeal' and a 'robust plugin ecosystem.' However, no evidence names a curated marketplace/directory or specific plugin categories like icon libraries, data fillers, or accessibility checkers. Missing for 10: an official plugin marketplace/directory listing, explicit examples of icon/data-filler/accessibility-checker plugins, and any count or curation of the ecosystem.", "evidenceIds": [ "sketch-docs-3", "sketch-docs-30", "sketch-docs-45", "sketch-docs-44", "sketch-comm-14", "sketch-comm-15" ] }, { "productId": "sketch", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, region selection, or storage location options for Sketch's Workspace/cloud data; only SSO/SCIM/permissions security features are documented, none addressing data residency choice.", "evidenceIds": [] }, { "productId": "sketch", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses AI training data opt-out, privacy policy on model training, or any data-usage controls specific to AI training; Sketch's AI features (e.g., MCP integration, image-to-palette) are documented but nothing states whether user design data is used to train models or how to opt out.", "evidenceIds": [] }, { "productId": "sketch", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses data retention policies, deletion controls, or data export/erasure mechanisms for AI-native users; only unrelated SSO/SCIM/permissions and AI-feature docs are present.", "evidenceIds": [] }, { "productId": "sketch", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry, analytics, or usage-tracking opt-out settings for Sketch; the docs and community items cover features, pricing, and performance, not privacy/telemetry controls. Missing for 10: any mention of telemetry collection, opt-out settings, privacy policy details on usage data, or admin controls to disable analytics.", "evidenceIds": [] }, { "productId": "sketch", "storyId": "realtime-multiplayer-editing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack documents live-cursor or simultaneous real-time multiplayer editing in Sketch; the only related community comment explicitly notes that as of late 2020 Sketch still lacked real-time collaboration ('the ability to collaborate in real-time' was still 'soon'), and Figma is cited as having this as its 'killer USP' that Sketch lacks.", "evidenceIds": [ "sketch-comm-4", "sketch-comm-6" ] }, { "productId": "sketch", "storyId": "responsive-auto-layout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of auto layout, resizing constraints, or layout grids for Sketch frames — it covers prototyping, symbols, plugins, AI features, and collaboration but nothing about responsive frame behavior. Missing for 10: any docs or claims about auto layout, constraint-based resizing, or grid systems that make frames responsive.", "evidenceIds": [] }, { "productId": "sketch", "storyId": "reusable-components-variants", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Sketch's Symbols documentation confirms reusable components that stay linked to their source (edits propagate everywhere) and supports nested Symbols for complex structures, and community reviews praise Sketch's DRY, component-based design system workflow. However, the evidence never explicitly documents a 'variants' feature or per-instance overrides (e.g., swappable states/overrides on symbol instances) as Figma-style components do. Missing for 10: explicit documentation of variant properties and per-instance override controls, independent hands-on confirmation of override behavior.", "evidenceIds": [ "sketch-docs-9", "sketch-docs-24", "sketch-docs-31", "sketch-docs-38", "sketch-comm-12", "sketch-comm-13" ] }, { "productId": "sketch", "storyId": "self-host-design-platform", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Sketch is a proprietary Mac-based SaaS/desktop product with no evidence of an open-source license or self-hostable infrastructure; community discussion even highlights Sketch's refusal to leave macOS or move to browser/cloud-native architecture, the opposite of self-hosting. No docs, licensing pages, or repos indicate an open-source core platform available for self-hosting.", "evidenceIds": [ "sketch-comm-5", "sketch-comm-6", "sketch-docs-23" ] }, { "productId": "sketch", "storyId": "shared-team-libraries", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Sketch's docs explicitly describe Libraries where starring a version publishes changes as a controlled Library update, letting Editors receive updates while Viewers/Guests see only chosen versions, and Symbols propagate style/component changes across all instances. Community commentary corroborates Sketch's strength as a 'design system manager' built around reusable symbols/libraries. Missing for 10: independent hands-on verification of multi-project library update propagation at scale, and documentation of failure modes/conflicts when libraries diverge.", "evidenceIds": [ "sketch-docs-10", "sketch-docs-28", "sketch-docs-33", "sketch-docs-41", "sketch-docs-9", "sketch-docs-31", "sketch-comm-12", "sketch-comm-13" ] }, { "productId": "sketch", "storyId": "vector-editing-craft", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm non-destructive boolean operations (Union/Subtract/Intersect/Difference) and shape styling (fills, borders, shadows), core vector-editing capabilities for interface design. However, the evidence pack contains no explicit mention of pen/path tools or precise alignment/grid features, and one community comment notes Sketch's vector-editing is 'less buggy' in comparison but still implies some bugginess relative to Figma. Missing for 10: explicit documentation of pen tool/path editing, alignment/distribution tools, and independent hands-on corroboration of vector precision.", "evidenceIds": [ "sketch-docs-11", "sketch-docs-12", "sketch-comm-9" ] }, { "productId": "sketch", "storyId": "version-history-restore", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Sketch's docs describe 'starred versions' that let designers mark important updates in a document's history and control which versions become Library updates or are visible to Viewers/Guests, which covers naming/milestone-marking, but the evidence never describes a compare/diff view between versions or an explicit 'restore to any earlier state' UI. Missing for 10: documented version comparison/diff tooling, explicit restore-to-previous-version workflow, and any independent/hands-on confirmation of these version-history mechanics.", "evidenceIds": [ "sketch-docs-10", "sketch-docs-28", "sketch-docs-33", "sketch-docs-41" ] }, { "productId": "spline", "storyId": "advanced-prototype-logic", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Spline's docs strongly evidence advanced prototyping behavior: states/events/actions interaction model, variables and real-time API, webhooks for event-driven reactions, animation, multi-scene transitions, physics, and even code/HTML embedding for custom logic — all built to make prototypes feel like real products. Missing for 10: independent hands-on validation of complex behavior chains (scroll-driven effects specifically) and no dedicated 'scroll trigger' doc beyond general events/actions system.", "evidenceIds": [ "spline-docs-31", "spline-docs-9", "spline-docs-10", "spline-docs-26", "spline-docs-61", "spline-docs-59", "spline-docs-11", "spline-docs-6" ] }, { "productId": "spline", "storyId": "agent-creates-design", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Spline ships a dedicated MCP Server (bundled in desktop app) that lets Claude, Cursor, Codex, VS Code, Antigravity and ChatGPT drive the live editor to create/modify designs, plus a write-capable Code API for programmatic scene manipulation and webhooks/Real-time API for external event-driven updates. This goes beyond read-only access into genuine programmatic creation/modification. Missing for 10: independent hands-on validation of the MCP server's write actions and no public OpenAPI spec was found (openapi probe 404s), so full API surface is only documented, not independently verified.", "evidenceIds": [ "spline-docs-13", "spline-docs-19", "spline-docs-25", "spline-docs-32", "spline-docs-36", "spline-docs-46", "spline-docs-6", "spline-docs-9", "spline-docs-10", "spline-probe-4", "spline-probe-3" ] }, { "productId": "spline", "storyId": "agent-exports-assets", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Spline documents multiple export/programmatic surfaces (export-as-code for Vanilla JS/Three.js/React/Next.js/r3f, self-hosted ZIP export, Code API, Real-time API, webhooks) and an MCP server that lets agents like Claude/Cursor drive the editor, but none of the docs explicitly describe an agent invoking a screenshot/image render or export action through these interfaces — the MCP surface is framed as scene building/editing, not export/rendering. Missing for 10: explicit MCP or API tool for triggering exports/screenshots/renders programmatically, and independent confirmation this works end-to-end for an agent.", "evidenceIds": [ "spline-docs-12", "spline-docs-17", "spline-docs-18", "spline-docs-13", "spline-docs-25", "spline-probe-3" ] }, { "productId": "spline", "storyId": "agent-reads-design-context", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Spline ships an official MCP server bundled with the desktop app that connects Claude, Cursor, Codex, Antigravity and other MCP clients directly to the open editor, and separately offers a Code API, Real-time API and code export (React/Next.js/Three.js/react-three-fiber) that could let an agent read variables and scene structure to implement designs. However the MCP server's documented purpose is to let AI clients drive/edit the Spline scene ('straight into your open editor tab'), not explicitly to pull frames, screenshots, or component mappings as design-context for an external implementation — that specific dev-mode-style extraction flow is not evidenced. Missing for 10: explicit support for screenshot export via MCP/API, frame-level context retrieval, and documented component-mapping extraction for code implementation.", "evidenceIds": [ "spline-docs-13", "spline-docs-25", "spline-docs-36", "spline-docs-9", "spline-docs-12", "spline-docs-17", "spline-probe-4" ] }, { "productId": "spline", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Spline has a verified llms.txt at docs.spline.design/llms.txt (HTTP 200 probe) plus .md-suffixed doc pages that render clean markdown for agent consumption, confirming genuine agent-oriented docs infrastructure. missing for 10: independent third-party confirmation that agents actually consume these docs successfully in practice.", "evidenceIds": [ "spline-probe-1", "spline-probe-2", "spline-docs-16" ] }, { "productId": "spline", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Spline is a 3D/2D design and prototyping tool; its AI features (AI Agent, AI textures, AI 3D generation) generate design content from prompts rather than analyzing a user's own dataset to surface insights or suggestions. The 'insights from my data' framing is a data-analytics axis that doesn't fit this product category.", "evidenceIds": [] }, { "productId": "spline", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Spline offers webhooks and a Real-time API that let a published scene 'automatically trigger actions, update variables, and react to data sent from external sources' without a person driving it (spline-docs-9, spline-docs-10, spline-docs-26), which is a form of event-driven automation running unattended. However, this is scene-level reactivity to external events, not a general-purpose scheduled/autonomous agent workflow — the AI Agent and MCP integrations require an open editor tab and active AI-tool session (spline-docs-13, spline-docs-25) rather than running independently in the background. Missing for 10: evidence of scheduled/cron-like autonomous tasks, independent confirmation of webhook automation working in practice, and any agentic workflow that persists without an open editor session.", "evidenceIds": [ "spline-docs-9", "spline-docs-10", "spline-docs-26", "spline-docs-13", "spline-docs-25" ] }, { "productId": "spline", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Spline ships a built-in AI Agent that autonomously builds and edits 3D scenes ('describe what you want and it does the work through the editor itself — creating objects, setting up materials and lighting, animating, wiring interactivity') and a parallel Hana AI Agent for 2D screens, both directly matching the delegated-task story. Missing for 10: independent/hands-on verification of the agent's real-world reliability and scope beyond first-party docs.", "evidenceIds": [ "spline-docs-20", "spline-docs-40", "spline-docs-55", "spline-docs-39", "spline-docs-54", "spline-docs-21" ] }, { "productId": "spline", "storyId": "agentic-headless", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Spline is documented purely as a browser/desktop GUI editor with a multiplayer, real-time canvas; its MCP server explicitly requires the desktop app with 'your open editor tab' running interactively, and no CLI, headless render mode, or CI-triggerable build/export process is documented anywhere in the evidence pack. Webhooks and the Real-time API only affect published/runtime scenes, not the design-time editor pipeline, so there is no path to running Spline itself headlessly in automation.", "evidenceIds": [ "spline-docs-13", "spline-docs-25", "spline-docs-19", "spline-docs-9", "spline-docs-10" ] }, { "productId": "spline", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All evidence describes Spline shipping an MCP *server* (docs-13, docs-25, docs-36, docs-53) that external MCP clients like Claude/Cursor/Codex connect to — the reverse of this story, which asks whether a user can plug external MCP servers into Spline so its own AI agent can use their tools. No evidence shows Spline acting as an MCP client or consuming third-party MCP tool servers.", "evidenceIds": [ "spline-docs-13", "spline-docs-25", "spline-docs-36", "spline-docs-53", "spline-docs-19" ] }, { "productId": "spline", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Spline documents an official MCP server bundled in the desktop app that connects Claude, Cursor, Codex, VS Code, Antigravity, and ChatGPT directly to the editor, requiring no separate install/config. This is a clear first-party MCP server offering with dedicated docs. Missing for 10: independent/hands-on community corroboration of the MCP server actually working, and no detail on the specific tools/capabilities exposed via the server.", "evidenceIds": [ "spline-docs-13", "spline-docs-25", "spline-docs-36", "spline-docs-53", "spline-probe-4" ] }, { "productId": "spline", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Spline's AI Agent lets users describe what they want in natural language and the editor builds objects, materials, lighting, animation, and interactivity accordingly (spline-docs-3, spline-docs-20, spline-docs-40), and Hana's AI Agent similarly designs UI screens from natural-language descriptions (spline-docs-21, spline-docs-39, spline-docs-54). AI texture generation also works by typing descriptions (spline-docs-4, spline-docs-28). This is reinforced by an MCP server enabling natural-language driven control from external AI clients like Claude/Cursor (spline-docs-13, spline-docs-36). Missing for 10: independent/hands-on user reports confirming natural-language command reliability and breadth of coverage beyond first-party docs.", "evidenceIds": [ "spline-docs-3", "spline-docs-20", "spline-docs-21", "spline-docs-39", "spline-docs-40", "spline-docs-4", "spline-docs-13", "spline-docs-36" ] }, { "productId": "spline", "storyId": "agentic-official-cli", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Spline is a design tool for creating 3D/2D experiences, not a developer CLI-driven platform; its agentic surface is exposed via MCP server, AI Agent, and Code API, not a CLI. No evidence suggests a CLI is a fit or existing capability for this product category.", "evidenceIds": [] }, { "productId": "spline", "storyId": "agentic-public-api", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Spline documents several external-facing interfaces an AI-native user could drive: the Real-time API and Webhooks for event-driven external integration, a Code API for web to control published scenes from external code/UI, and an MCP Server that lets AI coding agents (Claude, Cursor, Codex) manipulate the editor directly. These are genuine documented, public-facing APIs beyond just manual UI use. missing for 10: a formal REST/OpenAPI specification (probe explicitly found no openapi.json/swagger.json at any candidate path), and independent/hands-on corroboration that these APIs work reliably outside vendor docs.", "evidenceIds": [ "spline-docs-6", "spline-docs-9", "spline-docs-10", "spline-docs-17", "spline-docs-36", "spline-docs-53", "spline-probe-3", "spline-probe-4" ] }, { "productId": "spline", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Spline offers an MCP server for connecting AI coding tools to the editor, but there is no mention of scoped or least-privilege API credentials, API keys with permission scopes, or any access-control mechanism for issuing restricted agent credentials. Missing for 10: any documentation of credential/token scoping, permission levels, or least-privilege API key issuance for agents.", "evidenceIds": [ "spline-docs-13", "spline-docs-25", "spline-docs-36", "spline-probe-4" ] }, { "productId": "spline", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Spline documents a Code API and multiple official code-export targets (Vanilla JS, Three.js, React, Next.js, react-three-fiber) plus a self-hosted runtime export, which functions as an SDK-like integration surface for developers (spline-docs-6, spline-docs-12, spline-docs-17, spline-docs-8). However, a live probe for a formal OpenAPI/SDK spec returned 404s, and there's no independent developer corroboration of SDK usage or versioned package docs beyond the docs site. Missing for 10: dedicated SDK/package documentation (e.g. npm SDK docs), OpenAPI/API reference, and third-party hands-on confirmation of building against these SDKs.", "evidenceIds": [ "spline-docs-6", "spline-docs-12", "spline-docs-17", "spline-docs-8", "spline-probe-3" ] }, { "productId": "spline", "storyId": "agentic-webhooks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Spline docs explicitly document a Webhooks feature that allows listening for and responding to external events, triggering actions, updating variables, and reacting to data sent from external sources — directly matching the story. However, this is first-party documentation only with no independent/hands-on corroboration or detail on subscription mechanics (event types, payload schema, delivery guarantees). Missing for 10: independent verification of webhook reliability, detailed event-type/payload documentation, hands-on community testing.", "evidenceIds": [ "spline-docs-10", "spline-docs-26", "spline-docs-9" ] }, { "productId": "spline", "storyId": "ai-asset-editing-tools", "verdict": "partial", "quality": 3, "confidence": "medium", "rationale": "Spline documents built-in AI features—AI-generated textures via text prompts and an AI Agent/Hana that builds/edits 3D scenes and screens—which is a form of in-editor generative asset creation, but there's no evidence of background removal, content-aware fill, or AI copy/text suggestions inside the editor. missing for 10: background removal, content-aware fill, copy/text suggestions, general image asset generation beyond textures.", "evidenceIds": [ "spline-docs-4", "spline-docs-28", "spline-docs-20", "spline-docs-21" ] }, { "productId": "spline", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Spline documents several APIs (Code API, Real-time API, webhooks) but the evidence pack shows no interactive API reference with runnable examples — a probe for OpenAPI/swagger specs returned 404 on all candidate paths, and the docs are plain markdown descriptions rather than an interactive sandbox.", "evidenceIds": [ "spline-probe-3", "spline-docs-9", "spline-docs-17" ] }, { "productId": "spline", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe for OpenAPI/swagger specs at common paths returned 404 across all candidates, and no documentation references a downloadable machine-readable API spec; Spline offers a Real-time API and Code API but no OpenAPI equivalent is evidenced.", "evidenceIds": [ "spline-probe-3" ] }, { "productId": "spline", "storyId": "api-sandbox", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Spline is a 3D/2D design tool with no concept of 'production data' or a distinct sandbox/staging environment for testing — the closest evidence (a sandboxed iframe for embedded HTML/JS in the code tab) is a security-sandboxing feature, not a test-vs-production data separation. This axis is a category error for a creative design tool rather than a data-backed platform.", "evidenceIds": [] }, { "productId": "spline", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of API versioning scheme or a documented deprecation policy; the OpenAPI probe returned 404s across all candidate paths, and no docs mention version numbers, changelogs, or deprecation timelines for the Real-time API, Code API, or MCP server.", "evidenceIds": [ "spline-probe-3" ] }, { "productId": "spline", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Spline's docs describe single-scene editing, MCP/Code API/webhook integrations, and AI generation features, but nothing indicates batch or bulk processing across many objects, scenes, or files at once (e.g., no scripting loop API, no bulk export/import, no multi-file automation mentioned).", "evidenceIds": [] }, { "productId": "spline", "storyId": "automation-rules-engine", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Spline's states/events/actions system, webhooks, and Real-time API let users define rules that trigger actions automatically on events (e.g., 'automatically trigger actions, update variables, and react to data sent from external sources'), which is core automation-depth functionality. missing for 10: independent/hands-on verification of event-rule automation in practice, and more detail on the breadth/complexity of conditional logic supported beyond docs claims.", "evidenceIds": [ "spline-docs-9", "spline-docs-10", "spline-docs-26", "spline-docs-31" ] }, { "productId": "spline", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Spline is a 3D/2D design tool for interactive experiences; scheduling recurring jobs or workflows is not a capability applicable to its product category — it offers event-driven webhooks and real-time APIs but no notion of cron-like scheduled/recurring automation.", "evidenceIds": [] }, { "productId": "spline", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Spline's Version History lets users preview, recover, and manually save previous file states, which would cover the interactivity/automation logic (events, actions, webhooks) embedded in a scene, but this is a whole-file history feature, not a dedicated review/diff mechanism for automations specifically, and no evidence of granular change review or approval workflows exists. missing for 10: automation-specific diffing/review UI, branching or approval workflow, independent confirmation of rollback reliability for complex automations.", "evidenceIds": [ "spline-docs-15", "spline-docs-29", "spline-docs-33" ] }, { "productId": "spline", "storyId": "build-custom-plugins", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Spline documents a Code API, MCP server, webhooks, and export pipelines for integrating exported scenes into external apps, but there is no evidence of a documented plugin/extension API (with typings and samples) for building and distributing extensions to the Spline editor itself, and an OpenAPI/SDK probe returned 404s. Missing for 10: a documented editor plugin SDK, typings/type definitions for plugin development, and sample plugin code or a plugin marketplace/distribution mechanism.", "evidenceIds": [ "spline-docs-6", "spline-docs-9", "spline-docs-10", "spline-docs-13", "spline-probe-3" ] }, { "productId": "spline", "storyId": "clear-plan-pricing", "verdict": "partial", "quality": 2, "confidence": "low", "rationale": "Only one evidence item references the pricing page at all, and it's a bare fragment listing a few feature bullets (code/self-hosted export, version history, team library) with no visible seat types, tier names, or explicit AI-feature gating breakdown. Missing for 10: documented tier names/pricing, seat-type definitions, explicit AI feature gating per tier, and any independent corroboration of the pricing page's clarity.", "evidenceIds": [ "spline-docs-33" ] }, { "productId": "spline", "storyId": "comments-review-flows", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Docs confirm real-time multiplayer editing and that users can 'give feedback in place, the way they would in a 2D design tool,' implying some in-file commenting, but there is no mention of @mentioning teammates or resolving comment threads. missing for 10: explicit mention/tagging of teammates, ability to resolve/close comment threads, dedicated commenting UI documentation.", "evidenceIds": [ "spline-docs-48", "spline-docs-58", "spline-docs-30" ] }, { "productId": "spline", "storyId": "design-to-production-code", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Spline documents direct design-to-code export in multiple production-ready formats (Vanilla JS, Three.js, React, Next.js, react-three-fiber), plus a Code API for wiring exported scenes into existing codebases, a self-hosted ZIP export bundling assets/runtime, and a native web component for drop-in embedding — all clearly aimed at shipping designed 3D/2D scenes into real production code. Missing for 10: independent/hands-on verification that exported code is production-quality and maintainable, and explicit component-to-code sync (e.g., design system tokens/components mapped 1:1 to code components) beyond scene-level export.", "evidenceIds": [ "spline-docs-12", "spline-docs-27", "spline-docs-35", "spline-docs-52", "spline-docs-6", "spline-docs-17", "spline-docs-51", "spline-docs-8", "spline-docs-18", "spline-docs-23", "spline-docs-7" ] }, { "productId": "spline", "storyId": "design-tokens-variables", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Spline's docs mention 'variables and data' only in the context of real-time API/webhooks event data, not design tokens for color, spacing or typography, and the only 'theme' feature found is the editor's own Dark/Light/System UI theme, not a token system applied across a designer's own scenes. No evidence shows reusable style tokens with modes/themes propagated across designs.", "evidenceIds": [ "spline-docs-9", "spline-docs-10", "spline-docs-26", "spline-docs-47" ] }, { "productId": "spline", "storyId": "export-formats-assets", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Spline's documented export options are all code/runtime based (Vanilla JS, Three.js, React, Next.js, react-three-fiber, self-hosted ZIP, spline-viewer web component) with no mention of PNG/SVG/PDF raster/vector asset export, per-layer presets, scales, or slices anywhere in the evidence pack.", "evidenceIds": [ "spline-docs-12", "spline-docs-27", "spline-docs-8", "spline-docs-18", "spline-docs-33" ] }, { "productId": "spline", "storyId": "free-tier-real-design", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs describe Spline as 'multiplayer from the ground up' with cloud-based files that several people can edit at once, suggesting collaboration is core to the product rather than gated behind a paid seat, but the evidence pack has no explicit free-tier pricing page detail confirming seat limits, viewer/editor caps, or whether real-time collaboration is unrestricted on the free plan. The one pricing-page snippet (spline-docs-33) lists Code/self-hosted exports, version history, and Team Library without clarifying tier boundaries. missing for 10: explicit free-tier plan page detailing seat/collaborator limits, independent user reports confirming free-tier collaboration works in practice.", "evidenceIds": [ "spline-docs-2", "spline-docs-30", "spline-docs-33" ] }, { "productId": "spline", "storyId": "inspect-code-handoff", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Spline's docs describe code export as full-scene exports (Vanilla JS, Three.js, React, Next.js, react-three-fiber) and a Code API for developers, but there is no evidence of a dev-handoff/inspect mode that lets a non-editor viewer inspect a specific layer for measurements, styles, assets, or CSS/iOS/Android snippets without holding an editor seat.", "evidenceIds": [ "spline-docs-12", "spline-docs-17", "spline-docs-27", "spline-docs-52" ] }, { "productId": "spline", "storyId": "interactive-prototypes", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm no-code interactivity via states/events/actions (spline-docs-31) and multi-scene transitions via the Events system's Scene Transition action (spline-docs-61), which directly supports triggers and transitions for prototyping. However, 'overlays' are only evidenced via the Code tab feature, which requires writing HTML/CSS/JS (spline-docs-11/24), contradicting the 'no code required' framing for that specific element, and there is no dedicated hands-on or independent account of designers building full clickable multi-screen prototypes end-to-end. Missing for 10: a no-code overlay/layering mechanism, and independent/hands-on evidence of a complete click-through prototype workflow across screens.", "evidenceIds": [ "spline-docs-31", "spline-docs-61", "spline-docs-24", "spline-docs-38" ] }, { "productId": "spline", "storyId": "nl-design-generation", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Spline documents two AI Agents that generate designs from natural-language prompts: the 3D AI Agent creates objects, materials, lighting, animation and interactivity from a description, and Hana's AI Agent designs full 2D screens (layout, type, colour, effects, states) from a text description, which can then be edited by hand. This directly matches the story of generating first drafts/mockups from a prompt. Missing for 10: independent/hands-on verification of the AI Agent's actual output quality, and no third-party reviews or demos confirming real-world reliability of these AI generation features.", "evidenceIds": [ "spline-docs-20", "spline-docs-21", "spline-docs-39", "spline-docs-40", "spline-docs-54", "spline-docs-55" ] }, { "productId": "spline", "storyId": "open-file-format", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Spline's native .spline file format itself is never documented as open or given a public schema; instead portability is achieved indirectly via export mechanisms — code export to Vanilla JS/Three.js/React/Next.js/react-three-fiber, a self-hosted ZIP with the runtime, and a Code API — which let designs be used outside the editor without exposing/documenting the native format. missing for 10: an open/published spec for the native .spline file format, third-party tooling that reads/writes it directly, independent confirmation that exported code is a full fidelity substitute for the file itself.", "evidenceIds": [ "spline-docs-12", "spline-docs-8", "spline-docs-17", "spline-docs-27", "spline-docs-52", "spline-docs-23" ] }, { "productId": "spline", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Spline offers real API-like surfaces for AI-native workflows — a Code API for runtime object/property control (spline-docs-6, -17), a Real-time API and webhooks for external data/events (spline-docs-9, -10, -26), and notably an official MCP server that lets AI clients like Claude/Cursor/Codex drive the live editor tab directly (spline-docs-13, -36, -53) plus an AI Agent that can create objects, materials, lighting and interactivity through the editor (spline-docs-20, -40, -55). However, probing for a formal REST/OpenAPI spec returned 404s across all candidate paths (spline-probe-3), and the documented APIs are scoped to runtime manipulation/events, not full parity with the UI's modeling toolset (booleans, hair systems, physics, gaussian splatting, materials library) — so full UI-equivalent API coverage is not evidenced. missing for 10: a documented comprehensive API/SDK exposing full scene-authoring parity (modeling, materials, physics, animation) rather than just runtime triggers/MCP-driven editor control, and independent confirmation of API completeness.", "evidenceIds": [ "spline-docs-6", "spline-docs-9", "spline-docs-10", "spline-docs-13", "spline-docs-20", "spline-docs-36", "spline-docs-53", "spline-probe-3" ] }, { "productId": "spline", "storyId": "openness-full-export", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Spline documents self-hosted export (full project, assets, and runtime as a self-contained ZIP) and code export in multiple open web formats (Vanilla JS, Three.js, React, Next.js, react-three-fiber), giving users a way to take their scene data out of the platform entirely rather than being locked into a proprietary cloud-only format. Missing for 10: no evidence of exporting to standard interchange 3D formats (e.g., glTF/OBJ), no independent/hands-on confirmation that self-hosted exports are truly complete and portable, and no mention of exporting non-visual metadata (variables, version history) alongside the scene.", "evidenceIds": [ "spline-docs-8", "spline-docs-18", "spline-docs-23", "spline-docs-12", "spline-docs-27", "spline-docs-33" ] }, { "productId": "spline", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Spline is a closed, proprietary commercial design tool; nothing in the evidence pack mentions an open-source license or publicly available source code for the core product (only exported project assets can be self-hosted, which is not the same as open-sourcing the tool's source).", "evidenceIds": [] }, { "productId": "spline", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Spline is an inherently cloud/multiplayer SaaS editor ('files live in the cloud', 'multiplayer from the ground up') with no evidence of a self-hostable core editor. The 'Self-Hosted' feature only lets users export a finished scene/runtime as a static ZIP for embedding elsewhere — it does not let anyone run the Spline design tool itself on their own infrastructure.", "evidenceIds": [ "spline-docs-2", "spline-docs-30", "spline-docs-8", "spline-docs-18", "spline-docs-23", "spline-docs-49" ] }, { "productId": "spline", "storyId": "plugin-ecosystem-marketplace", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a plugin community, marketplace, or third-party extensions (icons, data fillers, accessibility checkers) for Spline; the evidence only covers Spline's own built-in features (AI tools, MCP server, code export, materials library).", "evidenceIds": [] }, { "productId": "spline", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, region selection, or storage location controls; Spline docs discuss cloud storage, multiplayer, exports and MCP but never a data-region choice. missing for 10: any mention of region/residency options, data center choice, or compliance certifications tied to storage location.", "evidenceIds": [] }, { "productId": "spline", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses AI-training data opt-out, privacy settings, or data usage policies for model training; the docs focus entirely on design/collaboration/AI-generation features with no mention of privacy controls or training data exclusion.", "evidenceIds": [] }, { "productId": "spline", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of data retention policies, deletion controls, data export for compliance, or privacy/GDPR settings for AI-native users; it covers only design, collaboration, export, and MCP/AI features. Missing for 10: any documentation of data retention settings, account/file deletion controls, or privacy compliance mechanisms.", "evidenceIds": [] }, { "productId": "spline", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry, analytics, or usage tracking opt-out settings for Spline; the docs cover design features, MCP, exports, and collaboration but nothing about privacy/tracking controls. This is a fair axis for a cloud-based SaaS design tool, so absence of evidence yields 'none'.", "evidenceIds": [] }, { "productId": "spline", "storyId": "realtime-multiplayer-editing", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Multiple first-party docs consistently describe Spline as \"multiplayer from the ground up,\" cloud-hosted files enabling several people to edit the same file simultaneously with in-place feedback, similar to 2D design tools — directly matching the story. Missing for 10: explicit mention of live cursors specifically, and no independent/hands-on corroboration of real-time sync quality at scale.", "evidenceIds": [ "spline-docs-2", "spline-docs-30", "spline-docs-48", "spline-docs-58", "spline-docs-34" ] }, { "productId": "spline", "storyId": "responsive-auto-layout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Spline's docs cover 3D modeling, materials, animation, multiplayer, AI generation, and code export, and even mention a 2D canvas (Hana) for interfaces, but nowhere is there mention of auto layout, constraints, or responsive grid systems that would let frames resize like real UI layouts. Missing for 10: any documentation of auto-layout, constraint-based resizing, or grid systems for frames.", "evidenceIds": [ "spline-docs-38", "spline-docs-39", "spline-docs-21" ] }, { "productId": "spline", "storyId": "reusable-components-variants", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Spline docs confirm a Components system where reusable objects propagate changes from a main component to all instances (spline-docs-5), which covers the 'linked to source' part of the story. However, there is no evidence of a variants system (multiple named states of a component) or per-instance override capability that persists while staying linked, as seen in dedicated design tools. Missing for 10: documented variants/property sets per component, explicit per-instance override UI, and evidence of overrides surviving source updates.", "evidenceIds": [ "spline-docs-5" ] }, { "productId": "spline", "storyId": "self-host-design-platform", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Spline is a cloud-based, closed-source SaaS design tool; the only 'self-hosted' feature documented is exporting a finished 3D scene as a self-contained ZIP for use in your own web app, not self-hosting the design editor/platform itself. There is no evidence of an open-source license, on-prem installer, or self-hostable server for the full editor.", "evidenceIds": [ "spline-docs-8", "spline-docs-18", "spline-docs-23", "spline-docs-33", "spline-docs-49", "spline-docs-57" ] }, { "productId": "spline", "storyId": "shared-team-libraries", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Spline documents component reuse where instances update when the main component changes (spline-docs-5), and pricing mentions a 'Team Library' feature (spline-docs-33), implying some shared-library capability across a team. However, there's no documentation detailing publishing components/styles across separate files or projects, or how updates propagate/are controlled at that scope. Missing for 10: cross-file/cross-project library publishing workflow, versioned/controlled update mechanism, and any independent confirmation of Team Library behavior.", "evidenceIds": [ "spline-docs-5", "spline-docs-33" ] }, { "productId": "spline", "storyId": "vector-editing-craft", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm a pen tool for custom 2D shapes (spline-docs-43), boolean operations (Union/Subtract/Intersect) for objects (spline-docs-60), a dedicated 2D canvas with vector networks for interfaces (spline-docs-38), and extrusion turning 2D shapes into 3D (spline-docs-42) — core vector-editing primitives are present. However, there's no documentation of precise alignment/distribution tools, grids, or snapping, and no independent/hands-on evidence validating these tools for serious interface design work versus its primary 3D focus. Missing for 10: alignment/snapping tooling documentation, path-editing depth (e.g., bezier handle controls), independent corroboration of professional UI design usage.", "evidenceIds": [ "spline-docs-43", "spline-docs-60", "spline-docs-38", "spline-docs-42" ] }, { "productId": "spline", "storyId": "version-history-restore", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm a Version History feature that lets users preview all saved versions, track changes, and restore/recover earlier file states, plus manually save new versions ([spline-docs-15],[spline-docs-29],[spline-docs-33]). However, there is no evidence of explicitly naming/labeling milestones or a side-by-side compare view between versions. Missing for 10: explicit naming/labeling of milestone versions, a diff/compare UI between versions, independent hands-on confirmation of restore workflow.", "evidenceIds": [ "spline-docs-15", "spline-docs-29", "spline-docs-33" ] } ]