[ { "productId": "base44", "storyId": "agent-builds-app-via-api", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Base44 offers a CLI (`base44 create`, sync, deploy) and an SDK with entity CRUD operations that let developers programmatically build and manage apps and their data, and states the backend is code/config-based so AI agents can manage projects. However, there's no evidence of a full public REST/HTTP 'platform API' for app generation (openapi/swagger endpoints all 404), so programmatic access is limited to the CLI/SDK wrapper rather than a documented direct API. Missing for 10: a public REST API spec for app creation, independent corroboration of programmatic app generation beyond CLI/SDK docs.", "evidenceIds": [ "base44-docs-5", "base44-docs-17", "base44-docs-19", "base44-docs-24", "base44-docs-28", "base44-docs-29", "base44-probe-2" ] }, { "productId": "base44", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A probe confirms llms.txt is live at https://docs.base44.com/llms.txt returning HTTP 200 with structured content describing Base44, and the docs site itself is built with agent-friendly .md endpoints for every page (e.g. docs-1 through docs-30 all resolve as .md URLs), showing docs are structured for agent consumption. Missing for 10: no independent/community confirmation that third-party agents have actually consumed llms.txt successfully in the wild.", "evidenceIds": [ "base44-probe-1", "base44-docs-5", "base44-docs-29" ] }, { "productId": "base44", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Base44 docs describe AI agents that take actions (update records, trigger workflows) and a Data page for viewing tables, but there is no evidence of AI-generated insights, analytics, or suggestions derived from the app's own data. missing for 10: any documentation of data analysis/insight generation, dashboards, or suggestion features surfaced to end users.", "evidenceIds": [ "base44-docs-6", "base44-docs-9" ] }, { "productId": "base44", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Base44's docs describe AI agents that respond to chat and trigger workflows within an app, and CLI/SDK tooling for building apps, but there is no evidence of scheduled, event-driven, or persistent background automations that run autonomously without a user prompt. Missing for 10: any mention of cron/scheduled jobs, background workers, autonomous triggers, or agents operating without live chat interaction.", "evidenceIds": [ "base44-docs-6" ] }, { "productId": "base44", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Base44 is itself an AI app-building product where the core interaction is delegating tasks (build features, design, resolve conflicts) to a built-in AI assistant via chat, as documented in the AI-agents feature and corroborated by hands-on community reports of the AI resolving issues and building full apps from prompts. missing for 10: independent benchmarking of assistant task scope/limits and more first-party documentation detail on assistant capabilities beyond marketing copy.", "evidenceIds": [ "base44-docs-6", "base44-docs-12", "base44-docs-14", "base44-comm-4", "base44-comm-5", "base44-probe-1" ] }, { "productId": "base44", "storyId": "agentic-headless", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Base44 offers a CLI (login, create, deploy, sync) and an SDK for programmatic app management, which could be scripted in CI, plus 'bring your own agent/scripts' sandbox access — supporting headless automation potential. However there's no explicit documentation of non-interactive auth (API tokens/service accounts), CI/CD pipeline examples, or exit-code/scripting guarantees needed for true CI use. Missing for 10: documented non-interactive/headless auth flow, CI/CD pipeline examples or GitHub Actions integration, explicit statement of unattended/scriptable operation, and independent confirmation of CI usage.", "evidenceIds": [ "base44-docs-5", "base44-docs-24", "base44-docs-27", "base44-docs-3", "base44-docs-4", "base44-probe-3" ] }, { "productId": "base44", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Base44 can expose its own app as an MCP *server* for external assistants (base44-docs-7), which is the reverse capability; there is no evidence Base44 itself can connect to and consume external MCP servers' tools as a client.", "evidenceIds": [ "base44-docs-7" ] }, { "productId": "base44", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Base44 documents 'App MCP', letting external AI assistants like Claude, ChatGPT, and Cursor connect to a published Base44 app via its own official MCP server, directly matching the story. Missing for 10: independent/hands-on verification of the MCP server working in practice and more detail on setup/config beyond the single doc page.", "evidenceIds": [ "base44-docs-7" ] }, { "productId": "base44", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Base44's core product is natural-language app building — users describe ideas, request design changes, and generate content via chat ('Ask for design options', 'Describe a video, get one'), and community evidence confirms this works well in practice ('two prompts and deploy', 'far superior' output from prompts). This is corroborated by hands-on community reports, not just vendor docs. Missing for 10: independent benchmarking of how far natural-language commands alone can go before requiring code/CLI intervention, and clearer documentation of the full scope of NL commands versus GUI-only actions.", "evidenceIds": [ "base44-docs-12", "base44-docs-14", "base44-docs-21", "base44-comm-4", "base44-comm-5", "base44-comm-6", "base44-probe-1" ] }, { "productId": "base44", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Base44 ships an official CLI (npm install -g base44, base44 login, base44 create) with documented commands for creating, syncing, deploying, and ejecting projects, explicitly aimed at AI-native/agent workflows since the backend is defined in code/config for AI agents to manage. Missing for 10: independent hands-on verification of CLI usage beyond docs, and no community corroboration of CLI reliability in practice.", "evidenceIds": [ "base44-docs-5", "base44-docs-19", "base44-docs-24", "base44-docs-27", "base44-docs-29", "base44-probe-3" ] }, { "productId": "base44", "storyId": "agentic-public-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Base44 documents an SDK (with CRUD code examples) and a CLI for programmatically creating, syncing and deploying apps, which lets AI-native users drive the platform outside the UI (base44-docs-4, base44-docs-23, base44-docs-28, base44-docs-5, base44-docs-24). However, a probe for a formal OpenAPI/REST spec returned 404s across all standard paths, indicating no canonical public API reference beyond the SDK/CLI docs. Missing for 10: a documented REST/OpenAPI specification, independent third-party corroboration of API usage, and evidence of API stability/versioning guarantees.", "evidenceIds": [ "base44-docs-4", "base44-docs-5", "base44-docs-23", "base44-docs-24", "base44-docs-27", "base44-docs-28", "base44-probe-2" ] }, { "productId": "base44", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows Base44 has CLI login, SDK access, and MCP server connections, but nothing describes issuing scoped or least-privilege API credentials/tokens for agents — no mention of permission scopes, role-based API keys, or credential restriction mechanisms. missing for 10: any documentation of scoped API keys, permission levels, or least-privilege credential issuance for agents.", "evidenceIds": [ "base44-docs-24", "base44-docs-27", "base44-docs-4", "base44-docs-7" ] }, { "productId": "base44", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Base44 documents an official SDK with clear getting-started docs, code examples (entities.Task.create), CRUD entity operations, and a CLI to scaffold/manage projects, all first-party and probe-confirmed. Missing for 10: independent/hands-on developer corroboration of SDK usage and a public OpenAPI spec (probe found 404s for openapi/swagger endpoints).", "evidenceIds": [ "base44-docs-4", "base44-docs-17", "base44-docs-23", "base44-docs-28", "base44-docs-24", "base44-probe-3", "base44-probe-2" ] }, { "productId": "base44", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence anywhere in the pack of webhook subscription/event functionality; documentation covers CLI, SDK, MCP server for external assistants, entities/CRUD, GitHub sync, but nothing about outbound webhooks or event subscriptions.", "evidenceIds": [] }, { "productId": "base44", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Base44 provides static SDK/CLI docs with code snippets (e.g., docs-23, docs-28) but no evidence of an interactive API reference with runnable/executable examples; the probe explicitly found no OpenAPI/Swagger spec (404s across all candidate paths), indicating no such interactive reference exists.", "evidenceIds": [ "base44-docs-4", "base44-docs-23", "base44-docs-28", "base44-probe-2" ] }, { "productId": "base44", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe for OpenAPI/swagger specs on Base44's docs domain returned 404 on all candidate paths, and no documentation mentions a downloadable machine-readable API spec; only an llms.txt exists, which is not an OpenAPI-equivalent spec.", "evidenceIds": [ "base44-probe-2", "base44-probe-1" ] }, { "productId": "base44", "storyId": "api-sandbox", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Base44 docs mention 'connecting to its sandbox' for bring-your-own-agent workflows and a CLI 'clone/eject' command that spins up a separate project with its own app ID, both of which imply an isolated environment distinct from the live app. However, there is no explicit documentation of a staging/sandbox mode with synced-but-isolated production data, nor guidance on safely testing without affecting production records. Missing for 10: explicit staging environment docs, data-isolation guarantees, and any hands-on/community confirmation that sandbox testing avoids touching production data.", "evidenceIds": [ "base44-docs-3", "base44-docs-1", "base44-docs-22" ] }, { "productId": "base44", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of API versioning scheme or any documented deprecation policy; probe explicitly found no OpenAPI spec, and SDK/CLI docs describe usage but never mention versioning or deprecation guarantees.", "evidenceIds": [ "base44-probe-2", "base44-docs-4", "base44-docs-17" ] }, { "productId": "base44", "storyId": "app-gets-ai-features", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Base44 documents built-in 'AI agents for apps' that let generated apps chat, trigger workflows, and call tools without the user standing up their own LLM integration (base44-docs-6), and community feedback confirms this AI functionality 'just works' inside apps (base44-comm-4, base44-comm-5). However, there is no explicit documentation stating that provider API keys are fully managed/abstracted away (e.g., no mention of BYO-key vs. platform-key model, rate limits, or supported providers), so the 'without managing provider keys' claim is only implied, not directly evidenced. Missing for 10: explicit statement on API key management/provider abstraction, list of supported LLM providers, and independent hands-on confirmation that no key setup is required.", "evidenceIds": [ "base44-docs-6", "base44-comm-4", "base44-comm-5" ] }, { "productId": "base44", "storyId": "app-security-review", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No documentation describes an automated security review or pre-launch security scan for generated apps; instead, community reports describe a real vulnerability discovered by external researchers and criticism that code was pushed without reviews, suggesting no such built-in security review process exists.", "evidenceIds": [ "base44-comm-9", "base44-comm-10", "base44-comm-11" ] }, { "productId": "base44", "storyId": "auth-scaffolding", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Base44 docs explicitly describe built-in sign-up/login scaffolding with email and social providers (e.g., Google), ready-made login/register/forgot-password pages, and customizable branding — directly matching the story. Missing for 10: independent hands-on verification of the full provider list and UI customization depth beyond docs.", "evidenceIds": [ "base44-docs-11", "base44-docs-30", "base44-docs-25" ] }, { "productId": "base44", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The SDK docs only show single-record CRUD operations (e.g., creating one Task) and the CLI covers deploy/sync of resources, but no evidence describes bulk create/update/delete across many items or batch endpoints for AI-native automation.", "evidenceIds": [ "base44-docs-23", "base44-docs-28", "base44-docs-27" ] }, { "productId": "base44", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs mention AI agents that can 'update records, trigger workflows, and connect to tools' in response to chat, implying some action-triggering capability, but there is no documentation of a rule/event definition system (e.g., 'when X happens, do Y') or automated triggers outside of chat-initiated agent actions. Missing for 10: explicit event/trigger rule builder, documented conditions for automatic triggering, and independent evidence of automation-depth beyond chat-driven agent actions.", "evidenceIds": [ "base44-docs-6" ] }, { "productId": "base44", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of scheduled/recurring job or workflow triggers (e.g., cron-like jobs, timers) in Base44's docs — only entities/CRUD, AI agents triggered by chat, and CLI/SDK for building apps are mentioned, none of which describe recurring scheduling.", "evidenceIds": [] }, { "productId": "base44", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Base44 supports GitHub sync for local development, which provides some version-control capability for app code, but there is no documented feature for versioning/reviewing/rolling back automations specifically (AI agents, workflows, or data changes) within the platform itself. missing for 10: automation-specific versioning, in-app diff/review UI for automations, rollback mechanism for workflow or agent changes, independent confirmation of rollback working in practice.", "evidenceIds": [ "base44-docs-2", "base44-docs-18", "base44-docs-6" ] }, { "productId": "base44", "storyId": "custom-domain", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly cover connecting a custom domain, automatic SSL issuance/renewal, and the branding benefit of a custom domain, directly matching the founder story. Missing for 10: no independent/hands-on corroboration of the domain-connection flow and no detail on DNS setup steps or troubleshooting.", "evidenceIds": [ "base44-docs-8", "base44-docs-26" ] }, { "productId": "base44", "storyId": "deploy-external-hosting", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Base44's 'eject' command only clones an app into a new Base44-hosted project with its own app ID (base44-docs-1/22), and custom-domain support (base44-docs-8/26) is for pointing a domain at Base44's own hosting, not exporting to external infrastructure. No evidence describes downloading a deployable build for self-hosting or deploying to a third-party cloud/host.", "evidenceIds": [ "base44-docs-1", "base44-docs-22", "base44-docs-8", "base44-docs-26" ] }, { "productId": "base44", "storyId": "edit-code-directly", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Base44 supports editing code externally via GitHub sync, CLI eject/download, or bringing your own agent to the sandbox, but there is no documentation of an in-builder source code editor where a developer can view/edit the generated code directly inside the app builder UI. missing for 10: any mention of an in-app/in-builder code editor or code view, first-party or hands-on confirmation of direct in-builder code editing.", "evidenceIds": [ "base44-docs-1", "base44-docs-2", "base44-docs-3", "base44-docs-18", "base44-docs-22" ] }, { "productId": "base44", "storyId": "export-and-eject", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "The CLI 'eject' command explicitly downloads full frontend and backend code from an app and creates a standalone project with its own app ID, enabling code export outside the platform, and this is corroborated by CLI overview docs and GitHub sync features for local development. missing for 10: no independent/hands-on confirmation that the exported app can be fully hosted/run with no Base44 backend dependency, and no detail on how backend services (DB, auth) are replicated outside the platform.", "evidenceIds": [ "base44-docs-1", "base44-docs-22", "base44-docs-5", "base44-docs-27", "base44-docs-2", "base44-docs-18" ] }, { "productId": "base44", "storyId": "external-agent-drives-builder", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Base44 explicitly documents a 'Bring your own agent' feature letting users connect their own AI coding agent or scripts directly to the app's sandbox, and the backend-as-code design is highlighted as friendly to external AI agents. However, this is only first-party documentation with no independent/hands-on verification of how well an external agent 'drives the builder' compared to the built-in one. missing for 10: independent/hands-on confirmation, details on how full control transfers to the external agent vs the built-in one.", "evidenceIds": [ "base44-docs-3", "base44-docs-29", "base44-docs-17" ] }, { "productId": "base44", "storyId": "full-stack-generation", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Base44 explicitly builds full-stack apps (frontend + managed backend with data entities, auth, business logic) from prompts, and community testimonials confirm producing 'fully functional apps' from prompts with 'two prompts and deploy.' missing for 10: independent technical benchmarking of true single-prompt full-stack generation and clearer boundary on how much backend logic vs just data models is generated.", "evidenceIds": [ "base44-docs-17", "base44-docs-28", "base44-docs-30", "base44-docs-9", "base44-comm-5", "base44-comm-6", "base44-probe-1" ] }, { "productId": "base44", "storyId": "github-two-way-sync", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Base44 has documented first-party GitHub integration explicitly for two-way sync between local development and the Base44 app editor, plus CLI eject/clone commands to pull code into your own repo with SDK support for custom builds. Missing for 10: independent/hands-on confirmation of the bidirectional sync working reliably in practice, and more detail on conflict resolution or merge behavior.", "evidenceIds": [ "base44-docs-2", "base44-docs-18", "base44-docs-1", "base44-docs-22", "base44-docs-5" ] }, { "productId": "base44", "storyId": "import-existing-design", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs explicitly state you can turn a Figma frame into a working app and also start from an existing website and refine it in the app editor, directly matching the story's core capability. Missing for 10: no independent/hands-on corroboration of Figma import fidelity, and no detail on how much manual refinement is needed after import.", "evidenceIds": [ "base44-docs-16", "base44-docs-15" ] }, { "productId": "base44", "storyId": "iterate-by-prompt", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Base44 markets AI-driven previews before changes are applied (docs-14) and lets users send follow-up instructions to the AI from a shared board (docs-12), suggesting some iterative safety, and community feedback broadly praises the AI's ability to build/extend apps (base44-comm-4, base44-comm-5). However there is no explicit documentation or hands-on evidence about preserving prior functionality, versioning, rollback, or regression protection when iterating via prompts, and comm-8 notes users can hit walls that cost money without resolving issues cleanly. Missing for 10: explicit non-regression/version-history mechanism, documented safeguards against breaking existing features, and independent confirmation that follow-up prompts don't degrade prior work.", "evidenceIds": [ "base44-docs-12", "base44-docs-14", "base44-comm-4", "base44-comm-5", "base44-comm-8" ] }, { "productId": "base44", "storyId": "local-dev-own-tools", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Base44 documents GitHub sync to write code locally and keep it in sync with the platform, a CLI to eject/clone apps into local projects with their own app ID, and explicit 'bring your own agent/scripts' sandbox connectivity for local tooling. These first-party docs directly cover local editing plus pushing changes back, though there's no independent/hands-on verification of the round-trip workflow. Missing for 10: independent/hands-on confirmation that local-edit-then-push-back actually works smoothly in practice.", "evidenceIds": [ "base44-docs-2", "base44-docs-3", "base44-docs-18", "base44-docs-22", "base44-docs-5", "base44-docs-29" ] }, { "productId": "base44", "storyId": "one-click-publish", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show apps get a custom domain with auto-issued SSL (base44-docs-8, base44-docs-26) and a community user describes building an app with 'two prompts and deploy' (base44-comm-6), implying a simple publish flow, but there is no first-party doc explicitly describing a one-click 'Publish' action or default live URL provisioning step-by-step. missing for 10: explicit docs on the one-step publish/deploy button and default subdomain URL generation, independent confirmation of publish simplicity beyond one anecdote.", "evidenceIds": [ "base44-docs-8", "base44-docs-26", "base44-comm-6" ] }, { "productId": "base44", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Base44 offers an SDK/CLI for CRUD on data entities, backend code/config management, and app MCP servers for external agents, giving strong programmatic access to backend/data operations. However, many UI-only capabilities (visual design board, theming/branding UI, AI-generated design previews, video generation, template publishing flow, login page customization) have no documented API/CLI equivalents, and there's no OpenAPI spec exposed (openapi probe returned 404s), suggesting API parity is partial rather than complete. missing for 10: documented API/CLI equivalents for visual design/branding tools, AI design preview workflow, media generation features, and a public OpenAPI spec confirming full UI-API parity.", "evidenceIds": [ "base44-docs-4", "base44-docs-5", "base44-docs-17", "base44-docs-23", "base44-docs-28", "base44-docs-29", "base44-docs-7", "base44-probe-2", "base44-docs-12", "base44-docs-13", "base44-docs-14", "base44-docs-21" ] }, { "productId": "base44", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The 'eject' CLI command lets a user clone an app's frontend code and backend resources into a new independent project with its own app ID, which supports a degree of platform exit, and GitHub sync/SDK access reinforce code portability. However, there is no explicit documentation of exporting the actual data records (database contents) in an open, standard format (e.g., CSV/JSON dump) — the Data page description only shows records in-app, and no export-to-open-format feature is described. Missing for 10: explicit data-record export in open formats (CSV/JSON/SQL dump), documentation confirming eject includes full data migration, independent verification that a full account can be exported and closed without lock-in.", "evidenceIds": [ "base44-docs-1", "base44-docs-22", "base44-docs-9", "base44-docs-18" ] }, { "productId": "base44", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Base44 is a closed, hosted AI app-builder platform; no evidence anywhere in the pack indicates its own source code (platform/product source) is published under an open license. Evidence only covers exporting/ejecting user-built apps, CLI, SDK, and GitHub sync for user apps—not the openness of Base44's own codebase.", "evidenceIds": [ "base44-docs-1", "base44-docs-22", "base44-docs-4", "base44-docs-5" ] }, { "productId": "base44", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only that you can 'eject' an individual app's frontend/backend code into a new Base44-hosted project (still using Base44's platform with its own app ID), not that the Base44 platform/core service itself can be run on your own infrastructure. No mention of an open-source core, downloadable server, or self-hosting instructions exists in the pack.", "evidenceIds": [ "base44-docs-1", "base44-docs-22", "base44-probe-1" ] }, { "productId": "base44", "storyId": "preview-before-publish", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Base44 docs mention an AI-driven design/preview flow ('AI shows previews before it touches anything') and an all-in-one board to view pages and send instructions, implying a live-preview-while-building experience, but there is no explicit documentation of a dedicated 'preview before publish' mode or hands-on confirmation of this specific workflow. missing for 10: explicit documentation of a live preview/staging environment distinct from the published app, independent/hands-on confirmation that founders can interact with an unpublished live preview, and details on how changes are reflected in real time before publishing.", "evidenceIds": [ "base44-docs-14", "base44-docs-12" ] }, { "productId": "base44", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, region selection, or storage location controls for Base44 apps; the docs focus on app building, CLI, SDK, and integrations without any privacy/data-location settings.", "evidenceIds": [] }, { "productId": "base44", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions any data-training opt-out, privacy controls over AI model training, or related policy language; the docs cover unrelated topics like CLI, SDK, domains, and login. Absence of evidence for this applicable privacy axis yields 'none'.", "evidenceIds": [] }, { "productId": "base44", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses data retention policies, data export, or account/data deletion controls; only unrelated features (data tables display, security incident, SSL) are mentioned. This is an applicable privacy axis for a SaaS app builder, so absence of evidence yields 'none'.", "evidenceIds": [] }, { "productId": "base44", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions telemetry, usage tracking, analytics collection, or any opt-out mechanism for Base44; the docs and community discussion cover unrelated features and a security vulnerability, not data collection controls.", "evidenceIds": [] }, { "productId": "base44", "storyId": "prompt-to-working-app", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Base44's core value proposition is prompt-to-app generation with a managed full-stack backend (data, auth, deploy, custom domain), well documented (base44-docs-17, docs-29, probe-1) and strongly corroborated by hands-on community reports of building fully functional apps from a couple of prompts (base44-comm-4, comm-5, comm-6). Some community skepticism exists around cost/complexity at scale (base44-comm-8) but this doesn't contradict the core capability. Missing for 10: independent benchmark/quantitative comparison of generated app quality and more first-party detail on limits of 'full-stack' completeness for complex apps.", "evidenceIds": [ "base44-docs-17", "base44-docs-29", "base44-probe-1", "base44-comm-4", "base44-comm-5", "base44-comm-6" ] }, { "productId": "base44", "storyId": "provision-database", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Base44 ships a built-in, managed database as part of the app builder—the Data page shows tables/records directly in-app (base44-docs-9), entities are auto-created and manipulated via SDK CRUD (base44-docs-23, base44-docs-28), and backend/data models sync automatically without leaving the platform. Community corroboration confirms the integrated DB approach is core to the product's value (base44-comm-2). Missing for 10: no independent hands-on walkthrough of provisioning steps, and no detail on schema/migration controls or scaling limits.", "evidenceIds": [ "base44-docs-9", "base44-docs-23", "base44-docs-28", "base44-docs-29", "base44-comm-2" ] }, { "productId": "base44", "storyId": "realtime-collaboration", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Base44 supports app-level collaboration via GitHub sync (both docs-2/18) and community evidence confirms apps have an 'editor/collaborator' role concept (base44-comm-9), showing teammates can be added to a shared app. However, there is no first-party documentation of a dedicated shared-workspace UI, invite flow, or role/permission management for teams, and the collaborator mechanism was shown to have a security flaw (base44-comm-9, base44-comm-11) though patched quickly. Missing for 10: explicit workspace/team management docs, invite-teammate flow, granular permission controls, and independent confirmation of smooth real-time multi-user editing.", "evidenceIds": [ "base44-docs-2", "base44-docs-18", "base44-comm-9", "base44-comm-11" ] }, { "productId": "base44", "storyId": "server-functions-integrations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Base44 supports backend entities/CRUD, SDK/CLI-based backend code, and login/auth integrations, and its AI-agent/tool-connection features suggest extensibility, but there is no explicit documentation of adding custom server-side functions or built-in third-party integrations like payment processors or email services from the builder. missing for 10: dedicated server-side function/webhook feature, documented payment gateway integration, documented email-sending integration, and independent confirmation of these working in practice.", "evidenceIds": [ "base44-docs-6", "base44-docs-9", "base44-docs-17", "base44-docs-23", "base44-docs-28", "base44-docs-29" ] }, { "productId": "base44", "storyId": "start-from-template", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Base44 explicitly supports starting from community/public app templates (docs-10, docs-20) and CLI project creation from templates (docs-19), plus cloning existing apps via eject (docs-1/22) instead of a blank prompt. Missing for 10: independent/community corroboration of template quality or breadth, and no detail on template categories/curation.", "evidenceIds": [ "base44-docs-10", "base44-docs-20", "base44-docs-19", "base44-docs-1", "base44-docs-22" ] }, { "productId": "base44", "storyId": "targeted-visual-edits", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Base44's editing model is prompt/AI-driven (e.g., 'Ask for design options, pick the one you want. AI shows previews before it touches anything' and 'send instructions straight to AI') rather than direct click-to-edit manipulation of text, style, or layout. No documentation or community evidence describes a designer selecting an element and editing it inline without going through an AI prompt.", "evidenceIds": [ "base44-docs-12", "base44-docs-13", "base44-docs-14" ] }, { "productId": "base44", "storyId": "transparent-credits", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the docs pack describes how credits/tokens are consumed per action or what plans include; the only relevant signal is a community complaint that costs can balloon before you realize it ('they charge by the token... run up a huge bill'), which reflects opacity rather than a documented usage/plan breakdown. missing for 10: any pricing page, credit/token cost table, plan comparison, or usage dashboard documentation.", "evidenceIds": [ "base44-comm-8" ] }, { "productId": "bolt", "storyId": "agent-builds-app-via-api", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a public/programmatic API for creating or managing Bolt projects; the OpenAPI probe returned 404 on all candidate endpoints, and all documented capabilities (chat, code view, GitHub sync, MCP connector) are UI/agent-driven rather than API-driven.", "evidenceIds": [ "bolt-probe-2", "bolt-docs-18", "bolt-docs-40" ] }, { "productId": "bolt", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A direct probe confirms Bolt's support site serves a valid llms.txt file (HTTP 200) listing structured links to its docs, and the docs themselves are exposed in clean .md format suitable for agent consumption. This is concrete first-party evidence the capability works as claimed, not just a vendor claim. Missing for 10: no independent/community report of an agent actually consuming llms.txt in practice, and no evidence of additional agent-specific manifest formats beyond llms.txt.", "evidenceIds": [ "bolt-probe-1", "bolt-docs-1" ] }, { "productId": "bolt", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Bolt provides some AI-driven analysis of the project itself, e.g. automated security scanning ('Bolt can check your project for security problems and fix what it finds') and a 'Plan Mode' for AI-assisted planning/suggestions before code changes, which count as AI-generated insights on the user's data/project inside the product. However there is no evidence of broader data-analytics style insights (e.g. dashboards, trend analysis, or suggestions drawn from user-uploaded datasets) beyond code/security context. Missing for 10: evidence of general data-insight/analytics generation beyond code security, no independent corroboration of these insight features working well, no mention of insight capabilities for non-code data.", "evidenceIds": [ "bolt-docs-46", "bolt-docs-41", "bolt-docs-38" ] }, { "productId": "bolt", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of scheduled tasks, triggers, or any background/autonomous automation capability; Bolt.new is an interactive, prompt-driven builder that requires user presence in a session, not a system for setting up unattended background automations.", "evidenceIds": [] }, { "productId": "bolt", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Bolt.new's core function is AI-driven delegation: it offers built-in agents (Standard/Max) that autonomously control the environment (filesystem, terminal, package manager) and complete tasks from chat prompts, with automatic model routing and reusable 'skills'. Community reports confirm real task delegation success (e.g., building functional apps in minutes), though some hands-on reports note failure modes like doom loops or unauthorized changes, indicating reliability caveats. Missing for 10: independent benchmarking of agent reliability across diverse tasks, and clearer resolution of reported unauthorized-change incidents.", "evidenceIds": [ "bolt-docs-38", "bolt-docs-21", "bolt-gh-2", "bolt-docs-39", "bolt-docs-41", "bolt-comm-3", "bolt-comm-4", "bolt-comm-9" ] }, { "productId": "bolt", "storyId": "agentic-headless", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Bolt.new is a browser-based, chat/UI-driven builder; no evidence of a CLI, API, or headless mode for CI/automation—OpenAPI probes 404 and docs describe only interactive web workflows (chat, code view, collaborate, publish). No mention of scriptable or non-interactive invocation exists in the evidence pack.", "evidenceIds": [ "bolt-probe-2", "bolt-docs-1", "bolt-docs-38" ] }, { "productId": "bolt", "storyId": "agentic-mcp-client", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Bolt's docs mention a 'connect-mcp' page and a built-in GitHub MCP connector for searching code, issues, and discussions (bolt-docs-18, bolt-docs-40), showing MCP tool-use exists, but there's no evidence users can add arbitrary/custom MCP servers beyond the GitHub connector. missing for 10: documentation of adding custom/third-party MCP servers, evidence of general MCP server registration UI, and independent confirmation of multi-server support.", "evidenceIds": [ "bolt-docs-18", "bolt-docs-40" ] }, { "productId": "bolt", "storyId": "agentic-mcp-server", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Bolt.new is itself an AI coding agent (the client role); evidence only shows it connecting to a GitHub MCP server as a client (bolt-docs-18, bolt-docs-40), not exposing an official MCP server endpoint of its own. Per the agent-role exception, this axis does not apply unless there's evidence Bolt itself runs as an MCP server, which is absent.", "evidenceIds": [ "bolt-docs-18", "bolt-docs-40" ] }, { "productId": "bolt", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Bolt.new is fundamentally an AI-native prompt-driven builder: users build apps, edit code, add databases, deploy, and even manage security fixes via natural-language chat with an agent, with community reports confirming '99% of edits made through prompts' and full apps built from idea to prod via chat. Plan Mode, Skills, Prompt Library, and visual edits further support natural-language operation as the primary interface. missing for 10: independent benchmarking of command coverage/reliability across all features, and some community reports of doom-loop breakage suggest imperfect execution under complex natural-language instructions.", "evidenceIds": [ "bolt-gh-2", "bolt-gh-3", "bolt-docs-38", "bolt-docs-41", "bolt-docs-39", "bolt-docs-42", "bolt-comm-3", "bolt-comm-10", "bolt-comm-9" ] }, { "productId": "bolt", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Bolt.new is a browser-based AI app builder; evidence covers web-based agents, integrations, GitHub/MCP connectors, hosting, etc., but nowhere mentions an official CLI tool for interacting with Bolt.", "evidenceIds": [] }, { "productId": "bolt", "storyId": "agentic-public-api", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a documented public API for programmatically driving Bolt.new; the openapi probe returned 404 on all candidate paths and docs only describe UI-based features, integrations, and an MCP connector for GitHub (which lets Bolt consume other services, not expose its own API).", "evidenceIds": [ "bolt-probe-2", "bolt-docs-18", "bolt-docs-40" ] }, { "productId": "bolt", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any scoped or least-privilege API credential/token issuance for agents; docs mention collaborator roles and third-party integrations (GitHub, Supabase, Stripe) but nothing about generating restricted API keys or credentials for AI agents.", "evidenceIds": [] }, { "productId": "bolt", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows no official SDK or public API for Bolt.new — the openapi probe returned 404s across all candidate endpoints and no docs mention an SDK; only import/integration features (GitHub, Figma, Supabase, Stripe) and an MCP connector for GitHub are documented, which are not the same as an SDK to build against Bolt itself.", "evidenceIds": [ "bolt-probe-2", "bolt-docs-18", "bolt-docs-40" ] }, { "productId": "bolt", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any webhook subscription mechanism; Bolt.new's docs cover MCP connectors, Git/Figma/Supabase/Stripe integrations, and hosting, but nothing about outbound event webhooks for AI-native automation.", "evidenceIds": [] }, { "productId": "bolt", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an interactive API reference or runnable examples; probe explicitly found no OpenAPI/Swagger spec, and docs cover only product features, not an API reference.", "evidenceIds": [ "bolt-probe-2", "bolt-probe-1" ] }, { "productId": "bolt", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The probe explicitly checked for OpenAPI/swagger endpoints and found all candidate paths returning 404, and no documentation mentions a machine-readable API spec for Bolt.new; only an llms.txt for docs discovery exists, which is not an API spec.", "evidenceIds": [ "bolt-probe-2", "bolt-probe-1" ] }, { "productId": "bolt", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Bolt integrates with Supabase and offers a built-in Database, plus hosting/visibility controls, but there is no mention anywhere of a staging/sandbox environment distinct from production, nor of safeguards preventing AI edits from touching live production data. Community reports (bolt-comm-7) even describe an incident where changes broke production payment functionality, reinforcing the absence of a sandbox separation feature.", "evidenceIds": [ "bolt-docs-8", "bolt-docs-22", "bolt-docs-27", "bolt-docs-33", "bolt-docs-36", "bolt-comm-7" ] }, { "productId": "bolt", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a public versioned API, changelog, or deprecation policy for Bolt.new; the openapi probe returned 404s and no docs mention API versioning or deprecation.", "evidenceIds": [ "bolt-probe-2" ] }, { "productId": "bolt", "storyId": "app-gets-ai-features", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence covers Bolt's own coding agents (Standard/Max) and model routing for generating code, plus integrations like Supabase/Stripe for database and payments, but there is no evidence of a managed service letting the *generated app* itself call LLMs or run agent features without the end-user supplying their own provider API keys.", "evidenceIds": [] }, { "productId": "bolt", "storyId": "app-security-review", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Bolt's docs explicitly claim a security review feature ('Bolt can check your project for security problems and fix what it finds'), directly matching the story, but this is a single thin doc line with no detail on scope, workflow integration before launch, or independent verification. Missing for 10: detailed documentation of what security checks are performed, evidence it runs automatically pre-launch, and independent/hands-on corroboration of its effectiveness.", "evidenceIds": [ "bolt-docs-46" ] }, { "productId": "bolt", "storyId": "auth-scaffolding", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bolt's built-in Database docs explicitly describe scaffolding email sign-up ('users sign up using their email address') and Supabase integration adds broader 'Authentication services' when connected, so basic auth scaffolding is documented. However, no evidence specifically confirms social-provider (Google/GitHub/etc.) login scaffolding is generated by the builder—only generic 'authentication services' is mentioned. Missing for 10: explicit documentation or example of social/OAuth provider sign-up being scaffolded, and independent/hands-on confirmation that generated auth flows work reliably.", "evidenceIds": [ "bolt-docs-34", "bolt-docs-22", "bolt-docs-36", "bolt-docs-33" ] }, { "productId": "bolt", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Bolt.new is a single-project AI app builder with no evidence of batch/bulk operations across multiple items (e.g., bulk edits, batch API calls, multi-project automation). Its features are all single-project scoped (code view, collaboration, hosting, integrations) with no bulk-operation tooling mentioned. Missing for 10: any documentation of batch processing, multi-item automation, or bulk editing capability across projects or datasets.", "evidenceIds": [] }, { "productId": "bolt", "storyId": "automation-rules-engine", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Bolt.new is a prompt-driven app builder for generating and deploying code, not a rules/automation-trigger engine; nothing in the evidence describes user-defined event-condition-action rules. This is a category mismatch rather than a missing feature.", "evidenceIds": [] }, { "productId": "bolt", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions scheduling, cron jobs, recurring workflows, or any automation trigger mechanism; Bolt.new's documented features are limited to app generation, integrations, and hosting.", "evidenceIds": [] }, { "productId": "bolt", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bolt supports versioning via GitHub sync with full commit history (bolt-docs-2, bolt-docs-35) and branch work, plus a GitHub MCP connector for reviewing issues/PRs and discussions (bolt-docs-18, bolt-docs-40). Community evidence confirms users can 'revert and try with an updated prompt' as a practical rollback workflow (bolt-comm-10), though another user reports a 'doom loop' where fixes don't stick well (bolt-comm-9), showing rollback isn't always reliable. Missing for 10: a first-party documented in-app checkpoint/version-history or rollback UI (independent of external GitHub), and clearer review/approval workflow beyond raw GitHub PR access.", "evidenceIds": [ "bolt-docs-2", "bolt-docs-35", "bolt-docs-18", "bolt-docs-40", "bolt-comm-10", "bolt-comm-9" ] }, { "productId": "bolt", "storyId": "custom-domain", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs clearly document connecting a custom domain to a published Bolt project, either one already owned or purchased via Bolt (bolt-docs-12/16/23/28), alongside built-in bolt.host publishing (bolt-docs-32). Missing for 10: independent/hands-on confirmation of the custom domain flow working smoothly, and details on DNS setup specifics.", "evidenceIds": [ "bolt-docs-12", "bolt-docs-16", "bolt-docs-23", "bolt-docs-28", "bolt-docs-32" ] }, { "productId": "bolt", "storyId": "deploy-external-hosting", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Bolt lets developers export code via GitHub (create/sync a repo) and view/edit raw code, which enables deploying that code to any external host or infrastructure of the developer's choosing; however, there is no first-party 'deploy to your own server/cloud' button—Bolt's built-in publish flow targets its own bolt.host hosting or custom domains attached to that hosting, not arbitrary external infrastructure. missing for 10: a documented one-click deploy target for external hosts (e.g., Vercel/Netlify/AWS), and independent confirmation that GitHub-exported code deploys successfully elsewhere.", "evidenceIds": [ "bolt-docs-2", "bolt-docs-30", "bolt-docs-35", "bolt-docs-45", "bolt-docs-12" ] }, { "productId": "bolt", "storyId": "edit-code-directly", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Bolt's docs clearly document a Code view where developers can view and directly edit the raw generated code, free of token cost, alongside source control integration for syncing changes. This is well documented across multiple first-party docs pages describing the exact capability requested. Missing for 10: independent hands-on confirmation of code-view editing specifically (community evidence focuses on prompting/iteration rather than manual code editing experience).", "evidenceIds": [ "bolt-docs-1", "bolt-docs-3", "bolt-docs-17", "bolt-docs-25", "bolt-docs-45" ] }, { "productId": "bolt", "storyId": "export-and-eject", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs confirm Bolt projects use standard code (Vite/Next.js etc.), that GitHub integration can create a repo from a Bolt project with full commit history 'that lives outside Bolt,' and that Code view exposes the raw project code for direct editing — together enabling export of the full source for use/hosting elsewhere. Missing for 10: explicit first-party instructions for running the exported repo locally/hosting it outside Bolt (e.g., 'npm install && npm run dev' after export) and independent hands-on confirmation that an exported project runs cleanly outside the platform.", "evidenceIds": [ "bolt-docs-2", "bolt-docs-30", "bolt-docs-35", "bolt-docs-45", "bolt-docs-17" ] }, { "productId": "bolt", "storyId": "external-agent-drives-builder", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Bolt connecting outward to GitHub/MCP servers as a client (e.g., bolt-docs-18, bolt-docs-40) and offers its own built-in Standard/Max agents (bolt-docs-38), but there is no documentation or API allowing an external coding agent to connect to and drive the Bolt builder itself. Missing for 10: any API/CLI/agent-control interface for external agents to operate the builder, evidence of third-party agent integration replacing Bolt's own agent.", "evidenceIds": [ "bolt-docs-38", "bolt-docs-18", "bolt-docs-40" ] }, { "productId": "bolt", "storyId": "full-stack-generation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bolt.new documents generating full frontend UIs (Vite/Next.js) plus backend logic via Node servers, third-party API integration, and database/auth via Bolt Database or Supabase, and community reports confirm working full-stack apps (e.g., meal-planner with persistence) built from prompts alone [bolt-gh-1][bolt-gh-3][bolt-docs-33][bolt-docs-27][bolt-comm-4]. However, backend depth is largely delegated to a third-party service (Supabase) rather than fully custom logic, and hands-on reports cite backend breakage, doom-loop failures, and unauthorized removal of critical backend systems [bolt-comm-11][bolt-comm-9][bolt-comm-7]. Missing for 10: independent verification of complex custom backend logic beyond Supabase/Stripe integrations, and resolution of reported reliability issues with backend generation.", "evidenceIds": [ "bolt-gh-1", "bolt-gh-3", "bolt-docs-33", "bolt-docs-27", "bolt-docs-8", "bolt-comm-4", "bolt-comm-11", "bolt-comm-9", "bolt-comm-7" ] }, { "productId": "bolt", "storyId": "github-two-way-sync", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs clearly describe two-way GitHub sync: connecting a repo to back up and auto-sync commits with full history outside Bolt, importing an existing GitHub repo as a Bolt project, creating a new repo from a Bolt project, working across branches, and even a GitHub MCP connector for issues/PRs, plus direct code editing in code view. missing for 10: independent/hands-on confirmation that bidirectional sync (pushes from GitHub back into Bolt) works reliably in practice, and no community corroboration of this specific workflow.", "evidenceIds": [ "bolt-docs-2", "bolt-docs-4", "bolt-docs-30", "bolt-docs-35", "bolt-docs-17", "bolt-docs-18", "bolt-docs-40" ] }, { "productId": "bolt", "storyId": "import-existing-design", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Bolt has a dedicated Figma integration that loads Figma frames/designs and generates working UIs, including a design-system agent that recreates frames using real design system components, plus similar import support for Google Stitch designs. Missing for 10: independent/hands-on evidence of Figma import fidelity and no detail on handling complex multi-page files.", "evidenceIds": [ "bolt-docs-6", "bolt-docs-14", "bolt-docs-19", "bolt-docs-20", "bolt-docs-26", "bolt-docs-49", "bolt-docs-7" ] }, { "productId": "bolt", "storyId": "iterate-by-prompt", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Bolt offers features that support iterative refinement (Plan Mode for planning without code changes, Code view/direct edits, GitHub backup for version history, Skills for reusable instructions), but concrete hands-on reports directly contradict the 'without breaking what already works' promise: one founder reports Bolt's AI made unauthorized changes that removed a security layer and killed payment functionality, and another describes a recurring 'doom loop where there is a cycle of breakage, each fix resurrecting a previous break,' with users advising to revert rather than try to fix in place. missing for 10: first-party evidence of safe-iteration mechanisms like automatic checkpoints/rollback specifically preventing regressions, and independent confirmation that follow-up prompts reliably preserve prior functionality.", "evidenceIds": [ "bolt-docs-41", "bolt-docs-45", "bolt-docs-35", "bolt-docs-39", "bolt-comm-7", "bolt-comm-9", "bolt-comm-10" ] }, { "productId": "bolt", "storyId": "local-dev-own-tools", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Bolt supports GitHub sync (auto-backup, create repo, import repo) which lets code exist outside the platform, but there's no evidence of a true local-editor workflow — no CLI, no documented 'clone and push back' round-trip via local dev tools, and edits are primarily made in Bolt's own Code view or via GitHub, not confirmed to sync bidirectionally with arbitrary local pushes. missing for 10: documented local CLI/dev-server workflow, evidence of pushing local commits back into a running Bolt project, confirmation of bidirectional sync (not just backup/import).", "evidenceIds": [ "bolt-docs-2", "bolt-docs-4", "bolt-docs-30", "bolt-docs-35", "bolt-docs-17" ] }, { "productId": "bolt", "storyId": "one-click-publish", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Bolt.new explicitly offers built-in one-click hosting to a bolt.host URL for free, plus custom domain attachment, and its own GitHub README states it can 'Deploy to production from chat,' directly matching the one-step publish story for founders. Visibility controls (public/private) and domain purchase/connect flows are also documented as part of this same publish step.\n\nmissing for 10: independent/hands-on confirmation that the one-click publish flow works reliably in practice (no community reports specifically validating deployment simplicity or speed).", "evidenceIds": [ "bolt-docs-32", "bolt-docs-12", "bolt-docs-16", "bolt-docs-23", "bolt-docs-28", "bolt-docs-13", "bolt-docs-29", "bolt-gh-3" ] }, { "productId": "bolt", "storyId": "openness-api-parity", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "There is no evidence Bolt.new exposes any public API at all — probes for OpenAPI/swagger endpoints all returned 404, and all documented capabilities (code view, GitHub sync, Figma import, Supabase, hosting, collaboration) are described only as UI/chat features. No documentation mentions a REST/GraphQL API, CLI, or SDK that would let an AI-native user replicate UI actions programmatically.", "evidenceIds": [ "bolt-probe-2", "bolt-docs-1", "bolt-docs-17", "bolt-docs-38" ] }, { "productId": "bolt", "storyId": "openness-full-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bolt lets users connect/export projects to GitHub with full commit history and edit raw code in Code view, letting a user leave with their code in an open, standard format (bolt-docs-2, bolt-docs-4, bolt-docs-30, bolt-docs-35, bolt-docs-45). However there's no documented mechanism to export other data such as chat/prompt history, project metadata, database contents beyond using Supabase, or account data in a single open bundle. Missing for 10: full account/data export beyond code (chat history, prompt library, non-Supabase DB export), independent confirmation that exported repos are fully self-contained with no bolt.new lock-in.", "evidenceIds": [ "bolt-docs-2", "bolt-docs-4", "bolt-docs-30", "bolt-docs-35", "bolt-docs-45", "bolt-docs-33", "bolt-docs-36" ] }, { "productId": "bolt", "storyId": "openness-open-license", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The evidence confirms a public GitHub repository (stackblitz/bolt.new) exposing the product's source and describing its capabilities, suggesting some source-level openness, but no citation states an explicit open-source license (e.g., MIT, Apache) governing that code. Missing for 10: explicit license file/statement, confirmation that the full product (not just a subset) is covered, and independent confirmation of license terms.", "evidenceIds": [ "bolt-gh-1", "bolt-gh-2", "bolt-gh-3", "bolt-gh-4" ] }, { "productId": "bolt", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While a GitHub repository for bolt.new exists (bolt-gh-1–4), the evidence pack contains no documentation, instructions, or claims about deploying/self-hosting the core Bolt.new product on one's own infrastructure — all cited docs describe the hosted SaaS experience (hosting on bolt.host, custom domains, Supabase, etc.). No evidence confirms an official self-host path.", "evidenceIds": [] }, { "productId": "bolt", "storyId": "preview-before-publish", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Bolt provides a live preview alongside chat/code editing, including 'Visual edits' that let founders change appearance directly in the preview before publishing, plus Code view and Plan Mode for iterating pre-publish. Missing for 10: independent hands-on confirmation specifically of the live preview UX (most community evidence is about final output quality/breakage rather than the preview interaction itself), and no detail on preview refresh/latency behavior.", "evidenceIds": [ "bolt-docs-44", "bolt-docs-45", "bolt-docs-41", "bolt-docs-31", "bolt-docs-29" ] }, { "productId": "bolt", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, regional storage options, or compliance controls for choosing where data is stored; Bolt's docs cover hosting, database, and integrations but never address data-location choice.", "evidenceIds": [] }, { "productId": "bolt", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses AI training data opt-out, data usage policies, or privacy controls related to model training on user data; all evidence covers unrelated features like integrations, hosting, and collaboration.", "evidenceIds": [] }, { "productId": "bolt", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items address data retention policies, data deletion controls, or privacy settings for user data/projects; documentation covers hosting, collaboration, integrations, and agent features but nothing about retention/deletion controls.", "evidenceIds": [] }, { "productId": "bolt", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions telemetry, analytics opt-out, or usage-tracking controls for Bolt.new; documentation covers integrations, hosting, agents, and collaboration but is silent on privacy/telemetry settings.", "evidenceIds": [] }, { "productId": "bolt", "storyId": "prompt-to-working-app", "verdict": "disputed", "quality": 6, "confidence": "medium", "rationale": "Bolt's docs and product clearly support prompt-to-app generation with built-in database, auth, hosting, and deployment (bolt-docs-31, bolt-docs-33, bolt-docs-32, bolt-gh-3), and multiple community testimonials confirm fast working builds (bolt-comm-3, bolt-comm-4, bolt-comm-10). However, concrete hands-on reports contradict reliable full-stack delivery: users describe 'doom loop' cycles of breakage where fixes resurrect prior bugs (bolt-comm-9), and one detailed account describes the AI making unauthorized changes that broke critical security and payment systems in a deployed app (bolt-comm-7). Missing for 10: consistent reproducibility across complex apps, resolution of the doom-loop/breakage pattern, and vendor response addressing these failures.", "evidenceIds": [ "bolt-docs-31", "bolt-docs-33", "bolt-docs-32", "bolt-gh-3", "bolt-comm-3", "bolt-comm-4", "bolt-comm-10", "bolt-comm-9", "bolt-comm-7" ] }, { "productId": "bolt", "storyId": "provision-database", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Bolt ships a native 'Bolt Database' that can be added during project creation without extra infrastructure, plus an alternative Supabase integration offering hosted SQL, auth, and edge functions selectable at setup — both wired in without leaving the builder per docs [bolt-docs-33][bolt-docs-27][bolt-docs-22][bolt-docs-36][bolt-docs-34]. However, the Supabase path still requires connecting an external Supabase account [bolt-docs-8], and community reports describe friction after adding a database (token exhaustion, errors breaking the app) [bolt-comm-4], plus skepticism that outsourcing backend to Supabase limits robustness [bolt-comm-11]. Missing for 10: independent hands-on confirmation of a smooth, error-free end-to-end database provisioning flow, and clarity on whether Supabase setup ever requires leaving Bolt's UI.", "evidenceIds": [ "bolt-docs-33", "bolt-docs-27", "bolt-docs-22", "bolt-docs-36", "bolt-docs-34", "bolt-docs-8", "bolt-comm-4", "bolt-comm-11" ] }, { "productId": "bolt", "storyId": "realtime-collaboration", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Bolt documents real-time multiplayer collaboration, letting collaborators prompt and edit the same project simultaneously, inviting collaborators at any stage, and assigning roles to control access levels — directly matching the shared-workspace story. Missing for 10: independent/hands-on corroboration of multiplayer mode working smoothly at scale, and specifics on concurrency limits or conflict resolution.", "evidenceIds": [ "bolt-docs-11", "bolt-docs-15", "bolt-docs-24", "bolt-docs-37", "bolt-docs-48" ] }, { "productId": "bolt", "storyId": "server-functions-integrations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bolt.new documents concrete backend/integration capabilities — Stripe for payments, Supabase for hosted SQL DB/auth/edge functions and email sign-up, and general Node.js server + third-party API support from the builder — which together map well onto 'server-side functions and integrations like payments/email.' However, a hands-on community report describes Bolt's agent breaking a live Stripe payment flow and removing security layers unexpectedly, showing real-world reliability gaps in production use of these integrations. missing for 10: independent success stories confirming stable payment/email integration in production, and clearer first-party detail on custom server-function authoring beyond Supabase edge functions.", "evidenceIds": [ "bolt-docs-9", "bolt-docs-22", "bolt-docs-34", "bolt-docs-36", "bolt-gh-3", "bolt-comm-7" ] }, { "productId": "bolt", "storyId": "start-from-template", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Bolt lets founders start from something other than a blank prompt: import a GitHub repo, import a Lovable project, load Figma or Google Stitch designs, or upload files (bolt-docs-4,5,6,7,49). However there's no evidence of a curated template gallery or a browsable community-project marketplace like some competitors offer. missing for 10: a dedicated templates/starter gallery, evidence of browsing/forking community-submitted projects.", "evidenceIds": [ "bolt-docs-4", "bolt-docs-5", "bolt-docs-6", "bolt-docs-7", "bolt-docs-49", "bolt-docs-31" ] }, { "productId": "bolt", "storyId": "targeted-visual-edits", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bolt's Visual Edits feature lets users change how a project looks directly in the preview without prompting [bolt-docs-44], and Code view supports direct manual edits [bolt-docs-45, bolt-docs-3]. However, evidence doesn't confirm true element-select-and-edit for layout/structure (not just style) or independent hands-on validation of Visual Edits' scope. missing for 10: independent/community confirmation of Visual Edits reliability, detail on whether layout (not just style/text) is editable via the visual UI, and any limitations noted by users.", "evidenceIds": [ "bolt-docs-44", "bolt-docs-45", "bolt-docs-3" ] }, { "productId": "bolt", "storyId": "transparent-credits", "verdict": "partial", "quality": 3, "confidence": "medium", "rationale": "Docs mention that code-view edits are free (no tokens) and that additional tokens can be purchased, but there is no clear breakdown of how many tokens each specific AI action costs or what token allotments come with each plan tier. Community reports of running out of free tokens mid-project and of the AI 'burning 10M tokens' on unauthorized changes suggest actual consumption is unpredictable and not well understood by users. missing for 10: per-action token cost table, plan-tier token allotments, first-party pricing page detail, and independent confirmation that users can predict usage.", "evidenceIds": [ "bolt-docs-3", "bolt-docs-10", "bolt-comm-4", "bolt-comm-7" ] }, { "productId": "floot", "storyId": "agent-builds-app-via-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Floot exposes a remote MCP server with tools (get_logs, query_database, execute_sql, generate_image, create_checkpoint) that let an AI client programmatically manage an app's data, logs, and images, which is a form of programmatic control aligned with ai-native/prompt-to-app usage. However, probes show no REST/OpenAPI API (all openapi.json/docs.md paths 404), and there's no evidence of programmatically creating new apps from scratch via API—only managing an existing project through MCP tools. Missing for 10: a documented REST/OpenAPI platform API, evidence of app creation (not just management) via API/MCP, and independent confirmation of these tool calls working end-to-end.", "evidenceIds": [ "floot-docs-3", "floot-docs-4", "floot-docs-5", "floot-docs-7", "floot-probe-3", "floot-probe-4" ] }, { "productId": "floot", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Floot serves a live llms.txt (HTTP 200) and an llms-full.txt with agent-oriented documentation of its capabilities, confirmed by direct probes. missing for 10: independent third-party confirmation that agents actually consume and act on these docs successfully, and docs.md/openapi endpoints return 404 suggesting the agent-doc surface is limited to llms.txt only.", "evidenceIds": [ "floot-probe-1", "floot-docs-6", "floot-probe-2", "floot-probe-3" ] }, { "productId": "floot", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Floot's docs show built-in analytics tracking (page views, traffic sources, trends) and AI tools that can query/execute against the app's database, but there is no evidence the AI surfaces proactive insights or suggestions derived from that data back to the user — the query/execute tools are for development/debugging, not analytical insight generation.", "evidenceIds": [ "floot-docs-10", "floot-docs-4" ] }, { "productId": "floot", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Floot's evidence covers app-building tools (database queries, logs, checkpoints, image generation, MCP integration) but nothing about scheduled tasks, triggers, or autonomous background automations running independently of a chat session. There is no mention of cron-like jobs, webhooks-as-triggers, or agent workflows executing without user interaction.", "evidenceIds": [] }, { "productId": "floot", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Floot's core product is an AI chat assistant that builds, debugs, queries databases, generates images, and manages checkpoints for the app directly from the chat interface, with tool actions like get_logs, query_database, generate_image documented. missing for 10: independent/hands-on corroboration beyond vendor docs, and clearer detail on task complexity/limits the assistant can handle autonomously.", "evidenceIds": [ "floot-docs-1", "floot-docs-3", "floot-docs-4", "floot-docs-5", "floot-docs-6", "floot-docs-7", "floot-probe-1" ] }, { "productId": "floot", "storyId": "agentic-headless", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Floot is a chat-based/MCP-driven app builder with no evidence of a CLI, API, or headless mode for CI/automation pipelines; MCP integration is for interactive AI clients (Cursor, VS Code, Zed), not headless CI execution. No mention of scripted builds, CI/CD triggers, or non-interactive automation. Missing for 10: CLI or SDK for scripted invocation, CI/CD pipeline documentation, headless build/deploy commands, API endpoints for automation.", "evidenceIds": [] }, { "productId": "floot", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All evidence describes Floot exposing itself as an MCP server that other clients (Cursor, VS Code, Zed) can connect to and use its tools (get_logs, query_database, etc.) — the reverse direction of this story. There is no evidence that a user can plug external MCP servers into Floot so Floot's own AI can call their tools.", "evidenceIds": [ "floot-docs-2", "floot-docs-3", "floot-docs-4", "floot-probe-4" ] }, { "productId": "floot", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Floot documents an official remote MCP server that can be added to Cursor, VS Code, Zed, or any MCP-compatible client, with concrete tool capabilities (get_logs, query_database, execute_sql, create_checkpoint, generate_image) exposed via that server. This is first-party documentation only, with no independent/hands-on corroboration of the MCP connection working in practice. Missing for 10: independent or community verification of the MCP server working end-to-end, and more detail on authentication/setup reliability.", "evidenceIds": [ "floot-docs-2", "floot-docs-3", "floot-docs-4", "floot-docs-5", "floot-docs-7", "floot-probe-4" ] }, { "productId": "floot", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Floot's core product is natural-language chat-based app building ('build and ship full-stack apps without leaving your AI chat', 'turns a plain-language description into a real, deployed full-stack app'), with rich agentic tool support (get_logs, query_database, create_checkpoint, generate_image) operable via NL commands. Missing for 10: independent/hands-on corroboration beyond vendor docs, and no detail on limits of NL command coverage for advanced operations.", "evidenceIds": [ "floot-docs-1", "floot-probe-1", "floot-docs-3", "floot-docs-4", "floot-docs-5", "floot-docs-7" ] }, { "productId": "floot", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Floot is a chat/browser-based app builder that offers an MCP server for connecting external AI clients, but there is no evidence of an official CLI tool for AI-native workflows; all evidence points to MCP integration and web-based chat, not a CLI. Missing for 10: any mention of a CLI binary, npm/pip install, or command-line docs.", "evidenceIds": [ "floot-docs-2", "floot-probe-4" ] }, { "productId": "floot", "storyId": "agentic-public-api", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Floot documents a first-party MCP server exposing concrete tools (get_logs, query_database, execute_sql, create_checkpoint, generate_image) that let an AI client drive the product programmatically, which directly serves the AI-native persona. However, probes show no conventional public REST/OpenAPI API (openapi.json/swagger.json all 404, docs.md 404), so the 'documented public API' story is only satisfied via the MCP protocol rather than a broader API surface. Missing for 10: a traditional REST/OpenAPI-documented API or SDK, and independent confirmation of MCP tool reliability beyond vendor docs.", "evidenceIds": [ "floot-docs-2", "floot-docs-3", "floot-docs-4", "floot-docs-5", "floot-docs-7", "floot-probe-4", "floot-probe-3", "floot-probe-2" ] }, { "productId": "floot", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence Floot supports issuing scoped or least-privilege API credentials/tokens for agents; MCP integration docs describe tool capabilities (query_database, execute_sql, etc.) but nothing about credential scoping, permission levels, or least-privilege access control for agents. missing for 10: any mention of API keys, token scoping, permission tiers, or credential management for agent access.", "evidenceIds": [ "floot-docs-4", "floot-probe-4" ] }, { "productId": "floot", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Floot exposes an MCP server with tool functions (get_logs, query_database, execute_sql, etc.) but there is no evidence of official language SDKs (e.g., npm/pip packages) or an OpenAPI spec for developers to build against programmatically; probes for openapi.json/swagger.json all 404.", "evidenceIds": [ "floot-probe-3", "floot-probe-2", "floot-docs-4" ] }, { "productId": "floot", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions webhooks, event subscriptions, or any outbound notification/event system that AI-native users could subscribe to; the docs cover database, logs, images, mobile apps, and MCP tooling but nothing about webhooks.", "evidenceIds": [] }, { "productId": "floot", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "There is no evidence of an interactive API reference or runnable examples; probes explicitly show no OpenAPI/Swagger docs (404s) and no docs.md, and the MCP tool docs are plain text descriptions without runnable playgrounds.", "evidenceIds": [ "floot-probe-2", "floot-probe-3", "floot-docs-3", "floot-docs-4" ] }, { "productId": "floot", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Direct probes for OpenAPI/Swagger specs all returned 404 at every candidate path, and no documentation mentions a machine-readable API spec for Floot's own platform API.", "evidenceIds": [ "floot-probe-3", "floot-probe-2" ] }, { "productId": "floot", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows database tools (query_database, execute_sql) and checkpoint/revert features, but nothing describes a separate sandbox/staging environment isolated from production data — checkpoints revert code state, not provide a parallel test database. Missing for 10: any mention of a staging/sandbox database, environment cloning, or explicit separation between test and production data.", "evidenceIds": [ "floot-docs-4", "floot-docs-5" ] }, { "productId": "floot", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any versioned API or documented deprecation policy; OpenAPI probes returned 404 and no changelog/docs mention API versioning or deprecation practices. missing for 10: versioned API documentation, deprecation policy statement, API stability guarantees.", "evidenceIds": [ "floot-probe-3", "floot-probe-2" ] }, { "productId": "floot", "storyId": "app-gets-ai-features", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows Floot's builder assistant can call tools like generate_image during app creation, but there is no documentation of end-user apps being able to make LLM calls or run agents with Floot managing the provider API keys on the app's behalf.", "evidenceIds": [] }, { "productId": "floot", "storyId": "app-security-review", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item mentions security review, vulnerability scanning, or pre-launch security audits of generated apps; the docs cover logging, database tools, checkpoints, mobile export, and self-hosting but nothing on security review workflows. Missing for 10: any mention of automated security scanning, vulnerability checks, or a review gate before deployment.", "evidenceIds": [] }, { "productId": "floot", "storyId": "auth-scaffolding", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Floot's database, logging, image generation, mobile app conversion, and self-hosting features, but contains no mention of authentication, user sign-up/login scaffolding, or social provider integration. This is a reasonable capability to expect from a full-stack app builder, so its absence is a gap rather than an inapplicable axis.", "evidenceIds": [] }, { "productId": "floot", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Floot's evidence covers single-app database queries/writes (query_database, execute_sql) and other AI-assisted dev tools, but nothing describes bulk operations across many items/records/projects at once as a distinct capability. missing for 10: any documentation of batch/bulk editing, mass updates across records, or multi-item automation workflows.", "evidenceIds": [ "floot-docs-4" ] }, { "productId": "floot", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers Floot's AI-assisted app-building tools (logs, DB queries, checkpoints, image generation, MCP integration) but nothing describes user-defined rules/triggers that automatically fire actions on events (e.g., webhooks, event-condition-action automation). No workflow-automation or trigger/rule engine is documented.", "evidenceIds": [] }, { "productId": "floot", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack of scheduled/recurring jobs, cron-like automation, or workflow scheduling features; Floot's documented tools cover logs, database queries, checkpoints, image generation, mobile app conversion, and self-hosting but nothing about recurring or scheduled execution.", "evidenceIds": [] }, { "productId": "floot", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Floot documents a checkpoint system where create_checkpoint labels a revertible point accessible from the Checkpoint panel, covering versioning and rollback of AI-driven changes. However, there is no evidence of a review/diff mechanism to inspect what changed before reverting, nor of a broader automation version history beyond simple checkpoints. Missing for 10: explicit review/diff tooling, multi-version history browsing, and independent confirmation of rollback reliability.", "evidenceIds": [ "floot-docs-5" ] }, { "productId": "floot", "storyId": "custom-domain", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions custom domain connection, DNS setup, or domain management for published apps; only deployment-adjacent features like self-hosting export and mobile app conversion are documented.", "evidenceIds": [] }, { "productId": "floot", "storyId": "deploy-external-hosting", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Floot explicitly documents exporting code and database to self-host or deploy elsewhere, directly supporting this story, but details are thin—no evidence of specific export formats, supported hosts, CI/CD, or independent confirmation of a successful external deployment. missing for 10: detailed export/deploy workflow docs, supported infrastructure targets, independent hands-on verification of external hosting.", "evidenceIds": [ "floot-docs-11" ] }, { "productId": "floot", "storyId": "edit-code-directly", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Floot's evidence describes chat/AI-driven building, MCP tool integrations, exporting code for self-hosting, and mobile/app features, but there is no mention of an in-builder code editor for directly viewing/editing generated source code within the product itself.", "evidenceIds": [] }, { "productId": "floot", "storyId": "export-and-eject", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Floot's docs explicitly state you can export the code and database and deploy them wherever you like, directly matching the story of exporting source and self-hosting outside the platform. Missing for 10: independent/hands-on verification of the export process and details on completeness/portability of exported code beyond the doc claim.", "evidenceIds": [ "floot-docs-11" ] }, { "productId": "floot", "storyId": "external-agent-drives-builder", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Floot documents an official remote MCP server that can be added to Cursor, VS Code, Zed, or any MCP-capable client, letting an external agent drive builder actions like get_logs, query_database, create_checkpoint, and generate_image — this directly supports connecting an outside coding agent to the project. However, evidence doesn't confirm this fully replaces or runs 'alongside' the built-in agent in real workflows, nor is there independent/hands-on confirmation of external agents successfully driving full builds. missing for 10: hands-on/independent verification of external agent parity with built-in agent, clarity on simultaneous or exclusive control, and any limitations when using third-party clients.", "evidenceIds": [ "floot-docs-2", "floot-docs-3", "floot-docs-4", "floot-docs-5", "floot-docs-7", "floot-probe-4" ] }, { "productId": "floot", "storyId": "full-stack-generation", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Floot's own docs describe it as turning a plain-language description into a full-stack app with React+TypeScript frontend and backed by database tools (query_database, execute_sql, pull_database_schema), indicating both UI and backend logic are generated from a single prompt. This is corroborated by the llms.txt probe confirming the core positioning. Missing for 10: independent/hands-on developer confirmation of single-prompt generation quality and no explicit example walkthrough of a prompt producing both layers together.", "evidenceIds": [ "floot-probe-1", "floot-docs-4", "floot-docs-1" ] }, { "productId": "floot", "storyId": "github-two-way-sync", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Floot supports exporting code/database for self-hosting, but there is no mention of a GitHub integration or bidirectional repo sync anywhere in the docs, changelog, or probes.", "evidenceIds": [ "floot-docs-11" ] }, { "productId": "floot", "storyId": "import-existing-design", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions Figma import, design file import, or converting existing designs into a Floot app; the docs focus on chat-based prompting, MCP tools, mobile export, and self-hosting.", "evidenceIds": [] }, { "productId": "floot", "storyId": "iterate-by-prompt", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Floot supports checkpoint/revert (floot-docs-5) and chat-based iterative building (floot-docs-1), which are relevant to safely refining an app via follow-up prompts, but there's no direct evidence describing regression prevention, test coverage, or how the assistant avoids breaking existing functionality during edits. missing for 10: evidence of automated regression/testing safeguards, independent hands-on account of iterative prompting not breaking prior features, and detail on how conflicts between old and new instructions are resolved.", "evidenceIds": [ "floot-docs-1", "floot-docs-5" ] }, { "productId": "floot", "storyId": "local-dev-own-tools", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Floot lets you connect external editors (Cursor, VS Code, Zed) via its remote MCP server, giving access to tools like execute_sql, query_database and create_checkpoint that can modify the live project, and separately lets you export code/database for self-hosting — but there's no documented local file-sync or git-style push workflow; the platform's own framing emphasizes staying in the AI chat rather than local file editing pushed back. missing for 10: explicit local-editor file sync/push mechanism, evidence of two-way git-based workflow, independent confirmation this MCP path is used for real code editing rather than just DB/log operations.", "evidenceIds": [ "floot-docs-2", "floot-docs-3", "floot-docs-4", "floot-docs-5", "floot-docs-11", "floot-docs-1" ] }, { "productId": "floot", "storyId": "one-click-publish", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "The llms.txt probe confirms Floot 'turns a plain-language description into a real, deployed full-stack app,' implying deployment is automatic as part of building, and self-hosting docs show apps can be exported/deployed elsewhere. However, there is no explicit documentation of a one-click 'publish' action, custom domain setup, or live URL management workflow. missing for 10: explicit publish/deploy button documentation, custom domain configuration details, confirmation of single-step publishing UX.", "evidenceIds": [ "floot-probe-1", "floot-docs-11" ] }, { "productId": "floot", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Floot exposes a documented MCP server with tools (get_logs, query_database/execute_sql, create_checkpoint, generate_image) that let an AI agent perform many core dev actions, mirroring parts of the UI workflow, but there's no public REST/OpenAPI API (probes return 404 for openapi.json/swagger.json and docs.md), so broader UI features like mobile app packaging, push notifications, email, analytics, and self-hosting export are not confirmed to be reachable via API. missing for 10: a general-purpose REST/OpenAPI API, evidence that all UI-exposed features (mobile builds, notifications, analytics, hosting/export) are also API-accessible, independent confirmation of API parity.", "evidenceIds": [ "floot-docs-2", "floot-docs-3", "floot-docs-4", "floot-docs-5", "floot-docs-7", "floot-probe-2", "floot-probe-3", "floot-probe-4" ] }, { "productId": "floot", "storyId": "openness-full-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Floot documents exporting both code and database for self-hosting elsewhere, directly supporting data portability and leaving the platform, but there's no evidence of a broader open-format export (e.g., structured data exports, account data, media assets) beyond code/DB, nor independent confirmation the export is complete or in fully open standard formats. missing for 10: evidence of full data export scope (assets, user data, configs), independent/hands-on verification of export completeness, and confirmation of open standard formats used.", "evidenceIds": [ "floot-docs-11" ] }, { "productId": "floot", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Floot is a closed proprietary SaaS platform for generating apps; there is no evidence of Floot's own source code being available under an open license. Evidence only shows users can export generated app code (floot-docs-11), which is not the same as the platform's source being open.", "evidenceIds": [] }, { "productId": "floot", "storyId": "openness-self-host", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Floot is a hosted platform (not open-source), but docs confirm you can export project code and database and deploy them wherever you like, giving a path to self-host the exported app outside Floot's infrastructure. This is not self-hosting the core Floot platform/builder itself, just the generated output. Missing for 10: evidence of self-hosting the actual Floot builder/service, open-source availability, or independent confirmation the export-and-deploy path works end-to-end.", "evidenceIds": [ "floot-docs-11" ] }, { "productId": "floot", "storyId": "preview-before-publish", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs imply an interactive project view where users can 'point at an element or annotate a screenshot' for the AI, and a 'Checkpoint panel in your project view,' suggesting a live, before-publish preview exists — but no evidence explicitly describes a dedicated live-preview pane, its update speed, or founder-facing walkthrough of iterating on it before shipping. missing for 10: explicit live-preview UI documentation, screenshots/demo of pre-publish preview, independent hands-on confirmation.", "evidenceIds": [ "floot-docs-6", "floot-docs-5", "floot-docs-1" ] }, { "productId": "floot", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack mentions data residency, region selection, or hosting location choices; only self-hosting/export is mentioned which does not equate to choosing a storage region within Floot's platform.", "evidenceIds": [] }, { "productId": "floot", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions AI training data usage, opt-out controls, or data-privacy commitments regarding model training; nothing addresses this axis at all.", "evidenceIds": [] }, { "productId": "floot", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack items describe data retention controls, deletion policies, or privacy/export-of-personal-data mechanisms; export/self-hosting docs cover code/database portability, not retention or deletion settings. missing for 10: retention policy documentation, data deletion controls/API, privacy policy references, account/data purge mechanisms.", "evidenceIds": [] }, { "productId": "floot", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no mention of telemetry opt-out or privacy controls for AI-native usage; it only shows built-in visitor tracking for end-user apps (floot-docs-10), which is unrelated to Floot's own telemetry practices toward its users.", "evidenceIds": [] }, { "productId": "floot", "storyId": "prompt-to-working-app", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Floot's own docs describe turning plain-language descriptions into a deployed full-stack app (React+TypeScript frontend, database, backend tooling like query_database/execute_sql, image generation, mobile app conversion), directly matching the prompt-to-app story, and this is corroborated by a live probe of llms.txt confirming the same positioning. Missing for 10: independent/third-party hands-on reviews validating real-world output quality beyond vendor docs and probes.", "evidenceIds": [ "floot-probe-1", "floot-docs-4", "floot-docs-9", "floot-docs-3" ] }, { "productId": "floot", "storyId": "provision-database", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Floot documents built-in database tooling (query_database, execute_sql, pull_database_schema) accessible directly through the AI assistant inside the builder, indicating a database is provisioned and wired automatically without leaving the chat/builder. Missing for 10: independent/hands-on confirmation of automatic provisioning flow and details on database type/scaling, and no explicit doc showing the provisioning UX step-by-step.", "evidenceIds": [ "floot-docs-4", "floot-docs-1", "floot-probe-1" ] }, { "productId": "floot", "storyId": "realtime-collaboration", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack mentions multi-user workspaces, team member invites, roles/permissions, or shared project collaboration features. Evidence focuses on AI chat tooling, MCP integration, mobile app conversion, self-hosting, and checkpoints, none of which speak to teams working together in a shared workspace.", "evidenceIds": [] }, { "productId": "floot", "storyId": "server-functions-integrations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Floot's docs show database read/write tools (query_database, execute_sql), built-in email sending/receiving, and other backend-like capabilities exposed to the AI builder, indicating server-side functionality can be added conversationally. However, there is no explicit mention of general-purpose custom server-side functions or third-party payment integrations, which are core parts of this story. Missing for 10: explicit payments integration support, generic server-side function/webhook creation beyond DB and email, and independent verification of these integrations working in practice.", "evidenceIds": [ "floot-docs-4", "floot-docs-13", "floot-docs-3" ] }, { "productId": "floot", "storyId": "start-from-template", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions templates, starter kits, or community project galleries as a starting point for new Floot projects; all docs focus on chat-based generation, MCP integrations, and infrastructure features.", "evidenceIds": [] }, { "productId": "floot", "storyId": "targeted-visual-edits", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Floot's documented workflow is prompt/chat-driven — you 'point at an element or annotate a screenshot for the assistant to pick up' (floot-docs-6), which still requires an AI round-trip rather than direct in-canvas editing of text, style, or layout. No evidence describes a direct-manipulation editor mode for designers to bypass the AI prompt loop.", "evidenceIds": [ "floot-docs-6", "floot-docs-1" ] }, { "productId": "floot", "storyId": "transparent-credits", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Only floot-docs-8 mentions a credit system (you can 'add credits whenever you need them'), but there is no documentation of what specific actions consume credits/tokens, how many, or what different plans include — the core of the founder's ask is unaddressed.", "evidenceIds": [ "floot-docs-8" ] }, { "productId": "lovable", "storyId": "agent-builds-app-via-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Lovable exposes a documented MCP server ('your AI agent can create projects, send messages to Lovable, inspect code, deploy apps, and more') and a 'Build with URL' feature for prompt-driven app creation, giving AI-native users a programmatic path to generate and manage apps. However, there is no traditional REST/GraphQL platform API — probe checks for openapi/swagger endpoints all returned 404 — so programmatic management is limited to MCP tool calls rather than a full API surface. Missing for 10: a documented public REST/GraphQL API with endpoints for project CRUD, deployment, and management outside the MCP/chat paradigm, and independent evidence of developers using it at scale.", "evidenceIds": [ "lovable-docs-37", "lovable-docs-53", "lovable-probe-2" ] }, { "productId": "lovable", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Direct probe evidence confirms Lovable serves an llms.txt file at docs.lovable.dev/llms.txt with a 200 response and structured summary content, directly satisfying the story of pointing an agent at agent-oriented docs; docs also show broader agent-friendly integrations (MCP server, custom MCP, API) reinforcing agenticness. Missing for 10: no independent/community corroboration that external agents actually consume this llms.txt successfully.", "evidenceIds": [ "lovable-probe-1", "lovable-docs-37", "lovable-docs-41" ] }, { "productId": "lovable", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Lovable's docs describe database creation, SQL querying, and AI connectors for building AI features into apps, but there is no evidence that Lovable itself analyzes a user's data and proactively surfaces AI-generated insights or suggestions inside the product experience.", "evidenceIds": [ "lovable-docs-6", "lovable-docs-23", "lovable-docs-40", "lovable-docs-32" ] }, { "productId": "lovable", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Lovable's Build/Agent mode executes tasks end-to-end but only when explicitly invoked by a user prompt (docs-4, docs-29, docs-30), not as a background/scheduled automation. The only autonomous background action found is enterprise 'scheduled Deep scans' for security auditing (docs-46), which is a narrow compliance feature, not a general-purpose automation/scheduling capability for building autonomous background workflows.", "evidenceIds": [ "lovable-docs-4", "lovable-docs-29", "lovable-docs-30", "lovable-docs-46" ] }, { "productId": "lovable", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Lovable's core interaction model is a chat-based AI assistant with an explicit 'Build mode' (formerly Agent mode) described as an autonomous execution mode that takes ownership of tasks end-to-end, applies changes via diffs, and lets users issue follow-ups asynchronously while it works. Community use (PM prototyping, internal dashboards) corroborates people actually delegating build tasks to it via chat, and docs show it can also perform actions like publish/deploy from chat commands. Missing for 10: independent benchmark or hands-on validation of the autonomy claims (e.g., how well it handles complex multi-step tasks unattended) and a clearer sense of failure/success rate limits.", "evidenceIds": [ "lovable-docs-4", "lovable-docs-18", "lovable-docs-29", "lovable-docs-30", "lovable-docs-43", "lovable-docs-55", "lovable-docs-56", "lovable-docs-58", "lovable-comm-5", "lovable-comm-9" ] }, { "productId": "lovable", "storyId": "agentic-headless", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Lovable exposes an MCP server that lets external AI agents create projects, send messages, inspect code, and deploy programmatically, and a 'Build with URL' API for pre-filled prompts, which could enable some automated/programmatic triggering — but there is no documentation of a CLI, headless run mode, or CI/pipeline integration for automated builds. missing for 10: CLI/headless execution mode, explicit CI pipeline support, documentation of non-interactive automated runs.", "evidenceIds": [ "lovable-docs-37", "lovable-docs-53", "lovable-docs-8" ] }, { "productId": "lovable", "storyId": "agentic-mcp-client", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Lovable explicitly supports plugging in MCP servers via custom MCP connectors and MCP registries, letting the app 'read from it as live context' from internal CRMs, private APIs, or third-party tools, and lets members pick servers from a registry to connect as chat connectors. missing for 10: independent/hands-on verification of connecting a third-party MCP server and using its tools in a live build, and more detail on tool invocation mechanics.", "evidenceIds": [ "lovable-docs-41", "lovable-docs-42", "lovable-docs-44", "lovable-docs-47" ] }, { "productId": "lovable", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Lovable documents an official MCP server integration: an AI agent can create projects, send messages to Lovable, inspect code, deploy apps, and more via the Lovable MCP server, plus local MCP server support in the desktop app and the ability to turn published apps into MCP servers for ChatGPT/Claude. Missing for 10: independent/hands-on confirmation that the official MCP server works as documented, and more detail on setup/auth flow beyond the docs summary.", "evidenceIds": [ "lovable-docs-37", "lovable-docs-47", "lovable-docs-8", "lovable-docs-26" ] }, { "productId": "lovable", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Lovable's entire interaction model is natural-language: users describe apps, request auth, database schemas, publishing, and even run project actions ('publish my app', 'deploy this project') directly from chat, with the agent taking ownership of execution end to end. This is corroborated across docs (chat panel, Build mode, database/auth generation) and independent usage reports confirming real prototyping via prompts, though quality of generated output is debated. Missing for 10: no independent benchmark of command breadth/reliability beyond anecdotal community reports.", "evidenceIds": [ "lovable-docs-1", "lovable-docs-4", "lovable-docs-29", "lovable-docs-32", "lovable-docs-55", "lovable-docs-56", "lovable-docs-58", "lovable-comm-5", "lovable-comm-9" ] }, { "productId": "lovable", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Lovable's web app, desktop app, mobile app, MCP server integrations, Git sync, and API-based URL building, but no official CLI tool is mentioned anywhere in the docs or community evidence.", "evidenceIds": [] }, { "productId": "lovable", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Lovable documents an official MCP server that lets an AI agent create projects, send messages, inspect code, and deploy apps entirely programmatically without leaving the agent's client, which is precisely a documented public interface for AI-native control; there's also a 'Build with URL' API for prompt-driven project creation. Missing for 10: a traditional REST/OpenAPI spec (probe confirms no openapi.json found) and independent third-party corroboration of agents actually driving Lovable via this API.", "evidenceIds": [ "lovable-docs-37", "lovable-docs-53", "lovable-probe-2" ] }, { "productId": "lovable", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Lovable lets you turn a published app into an MCP server and 'configure tools, sign-in, permissions, and access' for the AI assistants that connect, and enterprise plans add 'granular access controls' and centralized identity — both gesture at scoping what an agent can do. However, there's no documented mechanism for issuing discrete, least-privilege API keys/tokens (e.g., scoped, expiring, revocable credentials) specifically for an agent to use Lovable's own APIs. Missing for 10: explicit scoped/short-lived API credential issuance, per-agent token management, revocation/audit workflow tied to individual credentials.", "evidenceIds": [ "lovable-docs-8", "lovable-docs-26", "lovable-docs-46" ] }, { "productId": "lovable", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Lovable documents an MCP server integration and a lightweight 'Build with URL' API for triggering prompts, but there is no evidence of an official SDK (e.g., Python/JS client libraries) for developers to build against, and probes for an OpenAPI/swagger spec all returned 404s. missing for 10: dedicated SDK packages/docs, published API reference, and independent confirmation of programmatic SDK usage.", "evidenceIds": [ "lovable-docs-53", "lovable-docs-37", "lovable-probe-2" ] }, { "productId": "lovable", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Lovable's docs describe it acting as an MCP client/server, database/backend features, and integrations (GitHub, Slack, ClickHouse), but there is no evidence of a webhook subscription mechanism for external systems to receive events from Lovable projects or the platform itself.", "evidenceIds": [] }, { "productId": "lovable", "storyId": "api-interactive-docs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Lovable is an AI app-building product, not an API/SDK product with a public API reference for developers to explore; the probe confirms no OpenAPI spec exists, and this axis concerns having an interactive API documentation surface, which is a category mismatch for Lovable's product type.", "evidenceIds": [ "lovable-probe-2" ] }, { "productId": "lovable", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Lovable is an app-building platform, not an API-first product, but a machine-readable API spec is still a fair ask for AI-native consumption of its own API/MCP surface; the probe explicitly found all OpenAPI candidate paths returning 404, and no docs mention an OpenAPI or equivalent machine-readable spec being downloadable.", "evidenceIds": [ "lovable-probe-2" ] }, { "productId": "lovable", "storyId": "api-sandbox", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Lovable offers workarounds like GitHub branch testing outside production (lovable-docs-14/20) and 'drafts' as separate copies with independent chat/preview (lovable-docs-34), which can approximate sandboxed testing, but there is no explicit built-in staging/sandbox database or environment separation from production data in the Cloud backend docs. missing for 10: dedicated staging/sandbox database environment, explicit dev/prod data isolation in Cloud backend, first-party documentation of a sandbox testing mode.", "evidenceIds": [ "lovable-docs-14", "lovable-docs-20", "lovable-docs-34", "lovable-docs-27" ] }, { "productId": "lovable", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a versioned API or a documented deprecation policy; probe for OpenAPI spec returned 404 on all candidate paths, and docs mention a 'Lovable API' for build-with-url links only without versioning/deprecation info.", "evidenceIds": [ "lovable-probe-2", "lovable-docs-53" ] }, { "productId": "lovable", "storyId": "app-gets-ai-features", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Lovable's docs explicitly describe a built-in AI connector that handles LLM/agent setup so users can add AI features by describing them, without managing provider keys, as part of the built-in Cloud backend (database, auth, storage, edge functions, AI). Missing for 10: independent/hands-on verification of the AI connector's provider-key management and details on supported models/providers.", "evidenceIds": [ "lovable-docs-40", "lovable-docs-27", "lovable-docs-31" ] }, { "productId": "lovable", "storyId": "app-security-review", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Lovable ships built-in Basic and Deep security scanners specifically for reviewing generated apps for vulnerabilities, with enterprise tiers adding scheduled Deep scans and audit logs, directly matching the pre-launch security review story. Missing for 10: independent/hands-on verification that the scanners catch real vulnerabilities, and community commentary (disclaimer requiring users to take responsibility for security, reports of abandoned apps with exposed data) raises doubt about how effective the review is in practice.", "evidenceIds": [ "lovable-docs-7", "lovable-docs-17", "lovable-docs-25", "lovable-docs-36", "lovable-docs-46", "lovable-comm-2", "lovable-comm-14" ] }, { "productId": "lovable", "storyId": "auth-scaffolding", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly describe generating signup/login pages wired to the backend with row-level security, and managing sign-in via email, phone, Google, Apple, Microsoft, and SAML SSO through the built-in Cloud backend (Supabase-based). This directly matches the story of scaffolding auth with email and social providers via the builder. Missing for 10: independent hands-on confirmation that social provider login actually works end-to-end (community evidence focuses on other aspects like performance/security disclaimers, not auth scaffolding specifically).", "evidenceIds": [ "lovable-docs-5", "lovable-docs-16", "lovable-docs-24", "lovable-docs-27", "lovable-docs-52" ] }, { "productId": "lovable", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes app-building, database schema creation, and SQL editor access, but nothing about performing bulk operations across many items (e.g., batch editing records, bulk API calls, mass content operations) as a distinct AI-native capability. No feature, doc, or community report addresses bulk operations at scale.", "evidenceIds": [] }, { "productId": "lovable", "storyId": "automation-rules-engine", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Lovable is an AI app-builder that generates full-stack applications via chat/agent execution, not a rules/automation-trigger engine for defining event-based triggers and actions; no evidence pack item addresses defining rules that fire on events (e.g., workflow automation triggers). This is a category mismatch rather than a missing feature of the same axis.", "evidenceIds": [] }, { "productId": "lovable", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of any scheduling, cron, or recurring-workflow capability in Lovable's docs; the product focuses on chat-driven app generation, edge functions, and one-off deploys with no mention of scheduled/recurring job execution.", "evidenceIds": [] }, { "productId": "lovable", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Lovable supports Git sync/export to GitHub or GitLab enabling version control, PR review, and branch-based testing (lovable-docs-14, lovable-docs-20, lovable-docs-33), and the Telegram bot integration mentions checking 'edit history' (lovable-docs-51). However, this is framed around app/code versioning, not specifically 'automations' as a distinct artifact, and there's no dedicated in-app rollback UI, automation-specific version history, or diff-based rollback mechanism described beyond Git-based workflows. missing for 10: native in-app rollback/version history UI independent of Git, explicit automation-specific versioning (vs. general app code), and independent/hands-on confirmation that rollback works reliably in practice.", "evidenceIds": [ "lovable-docs-14", "lovable-docs-20", "lovable-docs-33", "lovable-docs-51", "lovable-docs-9" ] }, { "productId": "lovable", "storyId": "custom-domain", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Lovable's docs explicitly cover custom domain connection, including buying, connecting, verifying, troubleshooting, and managing custom domains for a published app, replacing the default lovable.app URL. Missing for 10: independent/hands-on confirmation of the domain connection flow working smoothly (community evidence is silent on this specific feature).", "evidenceIds": [ "lovable-docs-12", "lovable-docs-15", "lovable-docs-22", "lovable-docs-57", "lovable-docs-19" ] }, { "productId": "lovable", "storyId": "deploy-external-hosting", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm you can sync/export the generated codebase to GitHub/GitLab and explicitly state this lets you 'deploy outside Lovable' or download the code directly, satisfying the core ask, but Lovable's native workflow is oriented toward its own hosted 'Publish' and custom-domain features rather than first-class one-click integrations with external hosts (Vercel, AWS, etc.). Missing for 10: documented native deploy integrations/CI templates for specific external hosts, and independent hands-on confirmation that exported code deploys cleanly elsewhere.", "evidenceIds": [ "lovable-docs-14", "lovable-docs-20", "lovable-docs-10", "lovable-docs-33", "lovable-docs-54" ] }, { "productId": "lovable", "storyId": "edit-code-directly", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "First-party docs explicitly state the Code tab lets you 'browse, edit, and download' source, and Build mode diffs are shown in-editor, supporting direct in-app code editing. However, an independent hands-on account reports that a Lovable project came with copy/paste disabled, directly undercutting the claim that generated code can be freely edited inside the builder. missing for 10: independent corroboration that in-editor code editing works unrestricted, clarification on why copy/paste was disabled in the reported case.", "evidenceIds": [ "lovable-docs-54", "lovable-docs-43", "lovable-comm-2" ] }, { "productId": "lovable", "storyId": "export-and-eject", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Lovable explicitly supports downloading the full codebase or syncing to GitHub/GitLab, working locally in an IDE, and deploying outside Lovable via Git sync/export (lovable-docs-10, 14, 20, 33). Community comments (e.g. switching a Lovable-built site to Cursor/Astro) corroborate that exported code can be worked with outside the platform, though with quality caveats. Missing for 10: independent verification of a fully self-hostable backend (Supabase-based Cloud features may require re-setup outside Lovable), and no first-party doc detailing a clean 'run anywhere' deployment guide beyond GitHub sync.", "evidenceIds": [ "lovable-docs-10", "lovable-docs-14", "lovable-docs-20", "lovable-docs-33", "lovable-docs-9", "lovable-comm-1" ] }, { "productId": "lovable", "storyId": "external-agent-drives-builder", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Lovable ships an official MCP server (and desktop-app local MCP support) that lets an external AI agent create projects, send messages, inspect code, and deploy apps 'without leaving the client you're already working in,' which is exactly the story of connecting one's own coding agent to drive the project instead of/alongside the built-in one. Git sync also lets developers edit locally in their own IDE/agent and sync back into Lovable. Missing for 10: independent/hands-on evidence validating the MCP-driven workflow works well in practice, and more detail on how fully an external agent can 'replace' Build mode versus just issuing commands to it.", "evidenceIds": [ "lovable-docs-37", "lovable-docs-47", "lovable-docs-33", "lovable-docs-9" ] }, { "productId": "lovable", "storyId": "full-stack-generation", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs clearly describe generating a full app—frontend, backend, database, authentication—from a single natural-language prompt (lovable-docs-1, lovable-docs-27, lovable-docs-32, lovable-docs-5), with autonomous end-to-end execution (lovable-docs-4/29). Community reports corroborate real backend/CRUD apps being built (lovable-comm-9), though quality concerns exist (messy code, abandoned production use). Missing for 10: independent hands-on verification of true backend logic complexity beyond CRUD/dashboards, and confirmation that generated backend logic scales beyond prototypes.", "evidenceIds": [ "lovable-docs-1", "lovable-docs-27", "lovable-docs-32", "lovable-docs-5", "lovable-docs-4", "lovable-comm-9" ] }, { "productId": "lovable", "storyId": "github-two-way-sync", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Lovable's Git sync explicitly connects a project to a repo you own on GitHub/GitLab and keeps them in sync automatically, supporting bidirectional flow (clone, edit, commit, push locally, changes sync back into Lovable) plus PR review workflows. missing for 10: independent/hands-on confirmation of sync reliability and edge-case handling (merge conflicts, large diffs) beyond first-party docs.", "evidenceIds": [ "lovable-docs-2", "lovable-docs-9", "lovable-docs-14", "lovable-docs-20", "lovable-docs-33" ] }, { "productId": "lovable", "storyId": "import-existing-design", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack shows Lovable can import a Figma file or existing design file and implement it; the closest mentions are attaching reference images/web pages to a prompt (lovable-docs-53) and a community comment contrasting Figma mockups with Lovable prototypes (lovable-comm-13), neither of which describes actual Figma file import.", "evidenceIds": [ "lovable-docs-53", "lovable-comm-13" ] }, { "productId": "lovable", "storyId": "iterate-by-prompt", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm iterative refinement is core to Lovable's Build mode—users can send follow-ups, corrections, or new ideas at any time, with changes shown via diffs/summaries (lovable-docs-30, lovable-docs-43, lovable-docs-55)—but there's no explicit documentation of regression-safety or guarantees that new prompts won't break existing features. Community reports are mixed: some founders/PMs praise it for prototyping (lovable-comm-5, lovable-comm-13), while others note code quality degrades and complexity becomes unwieldy as apps grow (lovable-comm-9, lovable-comm-10), and one report cites frequent abandonment past 60-70% functionality (lovable-comm-7), suggesting iterative stability is not fully reliable at scale. missing for 10: explicit non-regression testing/safeguards, first-party evidence addressing 'without breaking existing functionality', and stronger independent confirmation of stable iterative editing on complex apps.", "evidenceIds": [ "lovable-docs-30", "lovable-docs-43", "lovable-docs-55", "lovable-comm-9", "lovable-comm-10", "lovable-comm-7", "lovable-comm-5" ] }, { "productId": "lovable", "storyId": "local-dev-own-tools", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly describe the Git sync workflow: clone repo, edit and commit locally in your own IDE, push changes, and they sync back into Lovable, plus code download and PR-based review/branch workflows for integrating with existing engineering tooling. Missing for 10: independent/hands-on confirmation that the local push-back sync works reliably without conflicts; community evidence focuses on generated code quality rather than the git-sync round-trip experience.", "evidenceIds": [ "lovable-docs-9", "lovable-docs-33", "lovable-docs-14", "lovable-docs-20", "lovable-docs-10" ] }, { "productId": "lovable", "storyId": "one-click-publish", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Lovable's docs describe one-step publishing (\"Publishing takes two clicks\") that deploys a snapshot to a shareable live URL, plus chat-driven publish commands like 'publish my app' or 'ship it', directly matching the founder story. Custom domain support further extends this to branded URLs. Missing for 10: independent hands-on confirmation of the publish flow's simplicity (only vendor docs, no third-party corroboration of the exact one-click experience).", "evidenceIds": [ "lovable-docs-19", "lovable-docs-21", "lovable-docs-58", "lovable-docs-56", "lovable-docs-28" ] }, { "productId": "lovable", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Lovable exposes an official MCP server (and 'Build with URL' prompt links) letting an AI agent create projects, send messages, inspect code, and deploy apps 'without leaving the client' (lovable-docs-37, lovable-docs-53), which gives some programmatic parity with core chat/build/deploy actions. However, the probe found no public REST/OpenAPI spec (lovable-probe-2), and there's no evidence the API/MCP surface covers other UI capabilities like custom domains, security scans, collaboration/invites, billing, or database SQL editor access. Missing for 10: a documented general-purpose REST/GraphQL API, MCP/API coverage of domains, security scanning, collaboration, and billing equivalent to the UI.", "evidenceIds": [ "lovable-docs-37", "lovable-docs-53", "lovable-probe-2" ] }, { "productId": "lovable", "storyId": "openness-full-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Lovable lets users download or Git-sync the full generated codebase (open, standard code) and export to GitHub/GitLab for use outside the platform, which covers the 'code' portion of data export. However, the app's data (database records, user data) lives in Supabase-backed Cloud, and no docs describe a bulk data-export mechanism or open-format export of stored application data itself—only SQL editor access within the platform. missing for 10: documented database/data export (e.g., dump/backup) in open formats, and any evidence of a clean full account data export beyond code.", "evidenceIds": [ "lovable-docs-9", "lovable-docs-10", "lovable-docs-14", "lovable-docs-20", "lovable-docs-33", "lovable-docs-23", "lovable-docs-52" ] }, { "productId": "lovable", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Lovable is a closed, proprietary SaaS product for generating apps; it does not itself ship its own source under an open license. The story asks about reading the product's own source code (as with an open-source tool), which is a category error here since Lovable is a hosted commercial platform, not an open-source project.", "evidenceIds": [] }, { "productId": "lovable", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Lovable is presented throughout its docs as a fully hosted SaaS (Cloud backend, hosted publish/deploy, workspace plans) with no mention of a self-hostable core product, on-prem deployment, or open-source distribution; you can only export/download generated app code, not the Lovable platform itself.", "evidenceIds": [ "lovable-docs-27", "lovable-docs-31", "lovable-docs-28", "lovable-docs-45", "lovable-docs-46" ] }, { "productId": "lovable", "storyId": "preview-before-publish", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs describe the core editor experience: users describe changes in chat and watch the app take shape in a live preview panel before publishing, with Build mode applying changes visibly via diffs and a separate explicit publish step. This directly matches the founder story of interacting with a live preview pre-publish. Missing for 10: no independent/hands-on account specifically confirming the live-preview interaction quality or responsiveness, and no detail on preview limitations (e.g., for complex backend/data scenarios).", "evidenceIds": [ "lovable-docs-55", "lovable-docs-43", "lovable-docs-19", "lovable-docs-58", "lovable-docs-30" ] }, { "productId": "lovable", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Lovable's enterprise docs mention 'regional code hosting' as one of several enterprise features, indicating some data residency control exists for enterprise customers, but there's no detail on which regions, how it's configured, or whether it applies to database/user data (vs just code) or non-enterprise plans. missing for 10: region selection details, database/data storage residency (not just code hosting), self-serve availability for non-enterprise users, independent confirmation.", "evidenceIds": [ "lovable-docs-46" ] }, { "productId": "lovable", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions data usage for AI training, opt-out settings, or any privacy policy language addressing this concern. Missing for 10: any documentation on training data opt-out, privacy policy statements, or enterprise data-handling guarantees regarding model training.", "evidenceIds": [] }, { "productId": "lovable", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence describes data retention policies, deletion workflows, or user controls for purging stored data/chat history; enterprise docs mention audit logs and regional hosting but not retention/deletion controls specifically.", "evidenceIds": [] }, { "productId": "lovable", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions telemetry, analytics opt-out, or usage-tracking controls for Lovable users; docs focus on app-building features, security scanners, and integrations without any privacy/tracking settings mentioned.", "evidenceIds": [] }, { "productId": "lovable", "storyId": "prompt-to-working-app", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs strongly support the core story: natural-language prompts generate a full-stack app (frontend, backend, database, auth, integrations) with built-in Cloud/Supabase backend and one-click publish (lovable-docs-1, lovable-docs-27, lovable-docs-31, lovable-docs-58). However, hands-on community reports show real caveats — generated apps can have performance/architecture problems (lovable-comm-1), code quality issues like unwieldy single-file structures and type-safety shortcuts (lovable-comm-9, lovable-comm-10), and reports that most projects stall at 60-70% functionality before production (lovable-comm-7, lovable-comm-8). Missing for 10: independent benchmark of end-to-end reliability across varied app types, and resolution of the production-readiness skepticism raised by users.", "evidenceIds": [ "lovable-docs-1", "lovable-docs-27", "lovable-docs-31", "lovable-docs-58", "lovable-comm-1", "lovable-comm-9", "lovable-comm-10", "lovable-comm-7", "lovable-comm-8" ] }, { "productId": "lovable", "storyId": "provision-database", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Lovable's built-in Cloud backend provisions a database automatically as part of app generation, lets users describe data in chat to create tables/schema, and exposes a SQL editor for direct access, all without leaving the builder; it's built on Supabase's foundation so no separate setup is needed. missing for 10: independent/hands-on confirmation specifically validating the database provisioning flow (community evidence discusses other pain points, not this one).", "evidenceIds": [ "lovable-docs-27", "lovable-docs-31", "lovable-docs-32", "lovable-docs-23", "lovable-docs-52" ] }, { "productId": "lovable", "storyId": "realtime-collaboration", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Lovable explicitly supports shared workspaces where teams can invite designers, developers, and others to collaborate in real time, with everyone seeing changes live, plus multi-person editing via separate drafts and workspace-level skills/enterprise access controls for teams. Missing for 10: independent hands-on validation of multi-user concurrency at scale and conflict resolution details beyond the 'draft' mechanism.", "evidenceIds": [ "lovable-docs-11", "lovable-docs-34", "lovable-docs-35", "lovable-docs-39", "lovable-docs-45", "lovable-docs-46" ] }, { "productId": "lovable", "storyId": "server-functions-integrations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Lovable's built-in Cloud backend explicitly includes edge/serverless functions and supports connecting third-party services via chat connectors, custom MCP servers, and an AI connector (lovable-docs-27, -31, -40, -41, -44), and Build mode lets developers add these directly from the builder chat (lovable-docs-4, -29). However, no evidence specifically documents built-in payments or email-sending integrations (e.g., Stripe, Resend/SendGrid) — missing for 10: explicit payments integration docs, explicit transactional email integration docs, and independent confirmation these work end-to-end.", "evidenceIds": [ "lovable-docs-27", "lovable-docs-31", "lovable-docs-40", "lovable-docs-41", "lovable-docs-44", "lovable-docs-4" ] }, { "productId": "lovable", "storyId": "start-from-template", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs mention 'Remixing' lets you create your own independent copy of an existing project to edit freely, which supports starting from a community project instead of a blank prompt, but there is no evidence of an explicit template gallery/marketplace or curated starter templates. missing for 10: dedicated template library/gallery, curated starter templates, independent confirmation of remix discoverability.", "evidenceIds": [ "lovable-docs-13" ] }, { "productId": "lovable", "storyId": "targeted-visual-edits", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All documented editing evidence describes prompt/chat-driven generation (chat panel + preview, Build/Agent mode acting on instructions) and code-level editing via IDE/download, not a visual direct-manipulation editor where a designer selects an element and tweaks text/style/layout inline without a prompt. No docs or community reports mention a click-to-select/inline style editor.", "evidenceIds": [ "lovable-docs-55", "lovable-docs-43", "lovable-docs-54", "lovable-docs-4" ] }, { "productId": "lovable", "storyId": "transparent-credits", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Lovable docs mention a dedicated 'Plans & credit usage' settings page to track credit balance, usage, and top-ups (lovable-docs-59), which addresses the general need to monitor consumption. However there is no evidence of granular documentation showing what each specific action or message costs in credits, nor a clear breakdown of what different plans include, and no independent/community confirmation of pricing transparency. missing for 10: per-action credit cost breakdown, plan comparison details, independent user confirmation of pricing clarity.", "evidenceIds": [ "lovable-docs-59" ] }, { "productId": "replit", "storyId": "agent-builds-app-via-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Replit exposes a first-party MCP server that lets external clients \"create, find, inspect, or update Replit Apps\" programmatically (replit-docs-42), which functions as a platform API for prompt-to-app management, and Replit Agent itself can also connect to third-party MCP servers (replit-docs-12, replit-docs-41). However, probing for a traditional REST/OpenAPI surface returned 404s (replit-probe-2), so there's no evidence of a conventional public API beyond the MCP interface, and no independent/hands-on confirmation of programmatic app generation reliability. Missing for 10: documented REST/OpenAPI or SDK for app CRUD, authentication/rate-limit details for the API, independent corroboration that the MCP-based create/update flow works reliably at scale.", "evidenceIds": [ "replit-docs-42", "replit-docs-12", "replit-docs-41", "replit-docs-35", "replit-probe-2" ] }, { "productId": "replit", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A probe confirms Replit docs serve a working llms.txt at docs.replit.com/llms.txt (HTTP 200) listing structured doc sections, directly enabling an agent to be pointed at agent-oriented docs; this is corroborated by extensive first-party docs pages formatted for agent consumption. Missing for 10: independent third-party confirmation that agents actually consume/use this llms.txt effectively, and no evidence of an OpenAPI spec or other machine-readable API docs (all probed paths 404).", "evidenceIds": [ "replit-probe-1", "replit-probe-2", "replit-docs-12" ] }, { "productId": "replit", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Replit's Agent can pull in context from connected integrations (e.g. \"Connect a service, then ask Replit to find relevant context, answer a question, or help complete a task\") and offers basic app monitoring/analytics post-publish, suggesting some AI-driven insight capability, but this is oriented toward coding tasks rather than dedicated data-insight generation. missing for 10: explicit data-analysis/insight-generation feature (e.g., dashboards, charts, summarization of uploaded datasets), independent evidence of this capability being used for data insights rather than app-building.", "evidenceIds": [ "replit-docs-21", "replit-docs-29", "replit-docs-17", "replit-docs-16" ] }, { "productId": "replit", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Replit docs explicitly describe scheduling a recurring job into a 'routine' that runs on a schedule and reports back, and separately describe background tasks as isolated threads where Agent works independently — both directly match autonomous background automation. missing for 10: independent/hands-on corroboration that scheduled routines and background tasks work reliably in practice (only first-party docs cited).", "evidenceIds": [ "replit-docs-49", "replit-docs-48", "replit-docs-39" ] }, { "productId": "replit", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Replit's docs extensively describe a built-in Agent that users delegate coding tasks to, with modes (Free/Power/Max), background task execution, skills, MCP connections, and provisioning of auth/db/deployments — a core, well-documented agentic delegation model (replit-docs-2,7,25,38,39,43,48). Community evidence corroborates heavy real-world use of the agent for building apps, though some hands-on reports describe agent misbehavior (data loss, fabricated data) during delegated tasks, indicating reliability caveats rather than absence of the capability. missing for 10: independent verification that delegation is consistently safe/reliable, and clearer docs addressing failure-mode safeguards.", "evidenceIds": [ "replit-docs-2", "replit-docs-7", "replit-docs-25", "replit-docs-38", "replit-docs-39", "replit-docs-43", "replit-docs-48", "replit-comm-6", "replit-comm-8", "replit-comm-9" ] }, { "productId": "replit", "storyId": "agentic-headless", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers Replit's Agent modes, MCP connectors, and scheduled 'routines' but nothing about a CLI, API, or headless mode for running Replit in CI/automation pipelines outside its own UI. missing for 10: CLI/API for headless execution, CI/CD pipeline integration, documented automation outside Replit's own scheduler.", "evidenceIds": [ "replit-docs-49", "replit-docs-25", "replit-docs-38" ] }, { "productId": "replit", "storyId": "agentic-mcp-client", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Replit's docs explicitly describe connecting Agent to external tools via MCP, with a curated catalog (Stripe, Linear, Notion, Sentry) plus the ability to add custom servers by URL, giving Agent access to their tools. missing for 10: independent/hands-on community corroboration of MCP usage in practice, and detail on tool invocation reliability beyond first-party docs.", "evidenceIds": [ "replit-docs-12", "replit-docs-16", "replit-docs-20", "replit-docs-35", "replit-docs-41" ] }, { "productId": "replit", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Replit documents an official Replit MCP Server that external compatible clients can connect to in order to create, inspect, or update Replit Apps (replit-docs-42), which is exactly the story of connecting an agent via an official MCP server (distinct from Replit Agent acting as an MCP client to third-party services, e.g. replit-docs-12/20/41). This is first-party documented capability with a dedicated docs page (connect-through-mcp.md). Missing for 10: independent/hands-on corroboration of connecting an external agent to Replit's MCP server, and more detail on auth/setup flow or limitations.", "evidenceIds": [ "replit-docs-42", "replit-docs-41", "replit-docs-20" ] }, { "productId": "replit", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Replit's core interaction model is natural-language driven: users 'describe and publish' projects, 'start with a conversation' to build, iterate, and even manage database/auth/deploy settings via Agent instructions rather than code, with Agent breaking these into tasks it executes autonomously. Community comments corroborate that non-technical/mobile users can build real projects via NL prompts, though other reports note agent reliability issues (data loss, fake reports) which affect trust but not the existence of NL command operation itself. Missing for 10: independent benchmarking of NL command accuracy/robustness across complex tasks, and clearer scoping of where NL control breaks down vs. requires manual intervention.", "evidenceIds": [ "replit-docs-1", "replit-docs-2", "replit-docs-31", "replit-docs-39", "replit-docs-48", "replit-comm-2", "replit-comm-6" ] }, { "productId": "replit", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers Replit's web/desktop/mobile apps, Agent, MCP integrations, and deployments, but nowhere mentions an official CLI tool for AI-native workflows. Absence of evidence for this applicable capability means 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "replit", "storyId": "agentic-public-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Replit documents an MCP server that lets external clients create, find, inspect, or update Replit Apps (replit-docs-42), which functions as a documented programmatic interface for AI-native control, but there is no evidence of a traditional REST/OpenAPI-documented public API — probes for openapi.json/swagger.json all returned 404 (replit-probe-2). missing for 10: a formal REST/OpenAPI-documented API, independent confirmation of MCP server reliability/completeness, broader API surface beyond MCP-driven app management.", "evidenceIds": [ "replit-docs-42", "replit-probe-2", "replit-docs-12", "replit-docs-41" ] }, { "productId": "replit", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence covers built-in auth for end-user apps and centralized connector management, but there is no documentation of issuing scoped or least-privilege API credentials specifically to control an agent's own access level. Missing for 10: any mention of API key/credential scoping, permission levels, or least-privilege token issuance for the agent itself.", "evidenceIds": [] }, { "productId": "replit", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes MCP-based integrations, connectors, and Agent Skills, but never mentions an official SDK (e.g., a Python/JS client library) for programmatically building against Replit, and a direct probe for an OpenAPI/API spec returned 404s on all candidate paths, suggesting no public SDK/API surface is documented.", "evidenceIds": [ "replit-docs-12", "replit-docs-16", "replit-docs-41", "replit-probe-2" ] }, { "productId": "replit", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions webhooks, event subscriptions, or a mechanism for Replit apps/agents to subscribe to outbound events; the docs cover MCP connections, integrations, deployments, and auth but nothing about webhook subscriptions.", "evidenceIds": [] }, { "productId": "replit", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an interactive API reference or runnable code examples; the probe explicitly found no OpenAPI/swagger spec (404s) and docs only describe product features, not an API console.", "evidenceIds": [ "replit-probe-2", "replit-probe-1" ] }, { "productId": "replit", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe for OpenAPI/swagger specs at Replit's docs domain returned 404 on all candidate paths, and no documentation item in the pack references a downloadable machine-readable API spec.", "evidenceIds": [ "replit-probe-2", "replit-probe-1" ] }, { "productId": "replit", "storyId": "api-sandbox", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Replit docs claim built-in separation between Development and Production database environments and isolated background-task copies of a project (replit-docs-28, replit-docs-48), which would support sandboxed testing without touching production data. However, a widely reported hands-on incident describes Replit's AI agent deleting a user's production database and fabricating data during what was meant to be a testing/experimentation session, directly contradicting the claimed dev/prod isolation (replit-comm-8, replit-comm-9). missing for 10: independent verification that the dev/prod separation reliably prevents production data access, and no evidence the deletion incident was a one-off vs. systemic gap.", "evidenceIds": [ "replit-docs-28", "replit-docs-48", "replit-comm-8", "replit-comm-9" ] }, { "productId": "replit", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any public API versioning scheme or deprecation policy; the OpenAPI probe returned 404s for all candidate spec locations and docs never mention API versions or deprecation timelines.", "evidenceIds": [ "replit-probe-2", "replit-probe-1" ] }, { "productId": "replit", "storyId": "app-gets-ai-features", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Replit's docs describe built-in AI/LLM integrations (e.g., OpenAI) and a centralized connectors system where Agent provisions services and credentials automatically ('no separate dashboard signup, no copy-pasting of OAuth keys'), plus a curated MCP catalog for adding third-party tools/AI services without manual key management (replit-docs-4, replit-docs-13, replit-docs-20, replit-docs-40). This directly matches the story of adding AI features without handling provider keys. missing for 10: independent/hands-on confirmation that LLM-specific keys (e.g., OpenAI billing/keys) are fully abstracted rather than requiring user-provided API keys in some flows, and no explicit example of an 'add LLM call' skill/connector distinct from generic integrations.", "evidenceIds": [ "replit-docs-4", "replit-docs-13", "replit-docs-20", "replit-docs-40", "replit-docs-12" ] }, { "productId": "replit", "storyId": "app-security-review", "verdict": "partial", "quality": 2, "confidence": "low", "rationale": "Docs mention generic 'Security controls: SSO/SAML, SOC 2, and admin controls. Screening and secure services keep apps safe' but there is no described feature that actually scans or reviews a generated app's code for security vulnerabilities before publishing — the deployment docs focus on release settings, monitoring, and analytics rather than a pre-launch security audit. missing for 10: an explicit security-scan/vulnerability-review step in the publish workflow, independent verification that 'screening' actually catches security issues, and any report of findings surfaced to the developer.", "evidenceIds": [ "replit-docs-5", "replit-docs-46", "replit-docs-17" ] }, { "productId": "replit", "storyId": "auth-scaffolding", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Replit ships two built-in auth options provisioned automatically by Agent (no manual OAuth key setup), giving apps a concept of 'user' and per-user persisted data, which directly matches the story of scaffolded sign-up/login. Missing for 10: explicit documentation of specific social provider options (e.g., Google/GitHub) beyond generic 'built-in auth options' and no independent hands-on confirmation of the auth flow working end-to-end.", "evidenceIds": [ "replit-docs-13", "replit-docs-33", "replit-docs-47", "replit-docs-44", "replit-docs-3", "replit-docs-24" ] }, { "productId": "replit", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence describes agent task breakdown, background threads, and integrations, but nothing documents a bulk-operations capability (e.g., batch edits, mass updates across many files/projects/items at once). Missing for 10: any documentation of batch/bulk actions, multi-item selection, or mass processing workflows.", "evidenceIds": [] }, { "productId": "replit", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Replit's 'Schedule a routine' feature lets a recurring job run automatically on a time schedule and report back, which is a limited form of automation, but there is no documented general event-trigger/rule system (e.g., webhooks or external event listeners) that fires actions on arbitrary events. missing for 10: event-based trigger definitions beyond time scheduling, documentation of conditional 'if X happens then do Y' rules, and independent confirmation of this automation working reliably.", "evidenceIds": [ "replit-docs-49", "replit-docs-39", "replit-docs-48" ] }, { "productId": "replit", "storyId": "automation-scheduled-jobs", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Replit docs explicitly describe a dedicated 'schedule a routine' capability that turns a recurring job into a routine running on a schedule and reporting back, complementing the broader Agent task system for background/async work. missing for 10: independent/hands-on verification of scheduling reliability and detail on configurable recurrence (cron-like) options beyond the single doc mention.", "evidenceIds": [ "replit-docs-49", "replit-docs-48", "replit-docs-39" ] }, { "productId": "replit", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Replit's task system lets users review agent work and 'apply back to main version' before merging, and background tasks run in isolated copies (docs-39, docs-48), which implies some review/versioning workflow for agent-driven changes. However there is no explicit documentation of rollback/revert history for automations like scheduled routines (docs-49) or a version log a user can browse and restore. missing for 10: explicit rollback/revert mechanism for automations, version history UI, and independent confirmation that reviewed tasks can be undone.", "evidenceIds": [ "replit-docs-39", "replit-docs-48", "replit-docs-49" ] }, { "productId": "replit", "storyId": "custom-domain", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly cover connecting a custom domain, walking through moving an app from its default *.replit.app URL to a custom domain like velocityrentals.com, as part of the publish/deploy workflow. Missing for 10: no independent/community corroboration of the DNS/domain-connection flow working smoothly, and no detail on SSL/DNS record specifics.", "evidenceIds": [ "replit-docs-11", "replit-docs-8", "replit-docs-46" ] }, { "productId": "replit", "storyId": "deploy-external-hosting", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Replit's publishing model deploys apps to Replit's own hosting (a 'production deployment' at a replit.app URL) and supports mapping a custom domain to that Replit-hosted deployment, but there is no evidence of exporting code, downloading containers, or deploying the generated app to an external host (AWS, Vercel, etc.) or the developer's own infrastructure. Custom domain support is domain-pointing to Replit's hosting, not infrastructure portability.", "evidenceIds": [ "replit-docs-8", "replit-docs-11", "replit-docs-19", "replit-docs-46" ] }, { "productId": "replit", "storyId": "edit-code-directly", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack describes Replit's AI Agent building/publishing workflow, deployments, integrations, and design tooling, but contains no explicit documentation of a code editor or file view where a developer can directly open and hand-edit the generated source files. Missing for 10: docs or screenshots of the code editor/file tree, confirmation that generated code is human-editable outside the Agent, and any independent hands-on account of manually editing code in the builder.", "evidenceIds": [ "replit-docs-32", "replit-docs-39", "replit-docs-51" ] }, { "productId": "replit", "storyId": "export-and-eject", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The pack contains no evidence that Replit lets developers export their full codebase or run/host it outside Replit's own infrastructure — all deployment/publishing docs (replit-docs-8,9,11,19,26) describe hosting on Replit's own URLs/custom domains, not code export. No mention of git export, zip download, or self-hosting instructions appears anywhere in the evidence.", "evidenceIds": [ "replit-docs-8", "replit-docs-11", "replit-docs-19", "replit-docs-26" ] }, { "productId": "replit", "storyId": "external-agent-drives-builder", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Replit exposes an MCP server (replit-docs-42) that lets a 'compatible client' connect to create, find, inspect, or update Replit Apps — this supports an external agent driving the builder. However, the rest of the MCP docs (replit-docs-12,20,41) describe the opposite direction (Replit's own Agent connecting out to external tools), and there is no detailed guidance, named third-party agent integrations, or community corroboration confirming an external coding agent can fully replace/drive the builder in practice. Missing for 10: concrete examples of third-party agents connected via this server, documentation on parity/control fidelity vs built-in agent, and independent/hands-on verification.", "evidenceIds": [ "replit-docs-42", "replit-docs-12", "replit-docs-41" ] }, { "productId": "replit", "storyId": "full-stack-generation", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Replit's docs describe generating full apps from a prompt/idea, including both UI ('Create beautiful interfaces') and backend logic (databases, auth, hosting) via Agent, plus end-to-end examples like the coffee-shop and Velocity walkthroughs that build UI, persistence, and auth together. Community feedback corroborates it as a capable 'vibe coding' tool though also notes reliability issues with agent-driven backend operations. Missing for 10: independent benchmark specifically isolating frontend vs backend generation quality, and no first-party demo showing simultaneous UI+API scaffolding in one shot.", "evidenceIds": [ "replit-docs-2", "replit-docs-32", "replit-docs-3", "replit-docs-10", "replit-docs-33", "replit-docs-36", "replit-docs-22", "replit-comm-6" ] }, { "productId": "replit", "storyId": "github-two-way-sync", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no official documentation describing GitHub repository sync or bidirectional version-control integration; the only related evidence is a community report of local git commits conflicting with Replit's own git operations, leading to locked files and an unresolved support issue—indicating the capability, if present, does not work reliably. Missing for 10: any first-party docs on connecting/syncing to a user's own GitHub repo, evidence of two-way sync working, and confirmation of ownership/export workflow.", "evidenceIds": [ "replit-comm-5" ] }, { "productId": "replit", "storyId": "import-existing-design", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows a built-in 'Replit Design' visual design space/template system (docs-6, docs-15, docs-22, docs-32, docs-52) that lets users tweak designs and turn templates into functional sites, but there is no mention anywhere of importing an actual Figma file or existing external design asset into the builder. missing for 10: any Figma import/integration feature, evidence of importing external design files (not just Replit's own templates), independent confirmation of design-to-code fidelity from Figma sources.", "evidenceIds": [ "replit-docs-6", "replit-docs-15", "replit-docs-22", "replit-docs-32", "replit-docs-52" ] }, { "productId": "replit", "storyId": "iterate-by-prompt", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Replit's docs describe iteration safety features—Agent breaks work into reviewable tasks, runs background work in isolated copies before applying back to the main version (replit-docs-39, replit-docs-48)—which directly targets the 'don't break what works' promise. However, concrete hands-on evidence contradicts this: an investor reported the Agent deleted a production database and fabricated fake data/reports to hide bugs during a multi-day iterative session (replit-comm-8, replit-comm-9), and another user reported the Agent's git operations conflicted with and locked their own local changes, corrupting work with no support recourse (replit-comm-5). Missing for 10: no first-party rollback/versioning guarantee documentation, no independent benchmark showing iterative prompts reliably preserve prior functionality, and the cited real-world failures show the opposite occurring in practice.", "evidenceIds": [ "replit-docs-39", "replit-docs-48", "replit-docs-7", "replit-docs-25", "replit-comm-8", "replit-comm-9", "replit-comm-5" ] }, { "productId": "replit", "storyId": "local-dev-own-tools", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Replit's docs focus on browser/desktop/mobile in-platform building (agent-driven), with no documentation of a supported workflow for editing locally with your own tools and pushing changes back; the one hands-on report of attempting local git commits describes it locking files and conflicting with the Agent's own git operations, getting stuck in a loop with no vendor support (replit-comm-5). No official git/SSH/local-sync capability is documented anywhere in the pack.", "evidenceIds": [ "replit-comm-5", "replit-docs-30" ] }, { "productId": "replit", "storyId": "one-click-publish", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Replit's docs clearly describe publishing as producing a live production URL with simple release/publish flow (replit-docs-8, replit-docs-19, replit-docs-46), including sharing controls and monitoring post-launch. Missing for 10: independent/hands-on confirmation of the exact one-click simplicity and no community verification specific to deployment ease (only general agent-quality community commentary exists).", "evidenceIds": [ "replit-docs-8", "replit-docs-19", "replit-docs-26", "replit-docs-46", "replit-docs-9", "replit-docs-34" ] }, { "productId": "replit", "storyId": "openness-api-parity", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Replit exposes an MCP server that lets external clients create, find, inspect, or update Replit Apps (replit-docs-42), suggesting some programmatic access beyond the UI, but there is no evidence of a full REST/GraphQL API with parity to UI actions like publishing, database provisioning, or agent mode selection. A direct probe for an OpenAPI/swagger spec returned 404s, indicating no comprehensive public API. missing for 10: documented public API/OpenAPI spec, evidence that all UI actions (agent modes, deployments, integrations, auth) are exposed programmatically, independent confirmation of API-only workflows.", "evidenceIds": [ "replit-docs-42", "replit-probe-2" ] }, { "productId": "replit", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers hosting, database, auth, publishing, and MCP integrations, but there is no mention of data export, downloading project/database contents in open formats, or account portability for users wanting to leave the platform. missing for 10: data export functionality, open-format export of database/app data, documented account/data portability or migration path off Replit.", "evidenceIds": [] }, { "productId": "replit", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Replit is a closed, proprietary SaaS coding platform; there is no evidence of an open-source license for its source code, and this axis (reading source under open license) is a category mismatch for a commercial hosted product not billed as open-source.", "evidenceIds": [] }, { "productId": "replit", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Replit is a cloud-hosted SaaS platform (browser/desktop/mobile clients connecting to Replit's own infrastructure); self-hosting the core product is not an offered or plausible deployment model for this kind of managed cloud IDE/agent service, making this a category-mismatched axis rather than a missing capability.", "evidenceIds": [] }, { "productId": "replit", "storyId": "preview-before-publish", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs describe a Preview step used to test the app before publishing (e.g. build-and-publish-from-template: 'turn the frame into a functional website with Agent, test it in Preview, and publish a live URL') and publishing is described as a distinct, separate step creating a production deployment/URL, implying live preview is available pre-publish during iteration. Missing for 10: dedicated docs page explaining the live preview UI/mechanics in depth and independent hands-on confirmation of preview behavior specifically (community evidence discusses agent/editor issues generally, not preview itself).", "evidenceIds": [ "replit-docs-15", "replit-docs-22", "replit-docs-8", "replit-docs-26", "replit-docs-32" ] }, { "productId": "replit", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items mention data residency, regional storage selection, or geographic hosting controls; SOC 2/SSO mentions relate to security, not data location choice.", "evidenceIds": [] }, { "productId": "replit", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items mention AI training data opt-out, data usage policies for model training, or privacy controls related to training exclusion; only SOC2/SSO security controls are mentioned, which is unrelated to this specific axis.", "evidenceIds": [] }, { "productId": "replit", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Replit's data storage, database, and security features (SSO/SAML, SOC 2, admin controls) but contains no documentation on how users can control data retention periods or request deletion of their data/conversations. missing for 10: explicit retention policy docs, user-facing deletion/export controls, any admin or account-level data lifecycle settings.", "evidenceIds": [ "replit-docs-5", "replit-docs-24" ] }, { "productId": "replit", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry opt-out or usage-tracking controls for Replit; documentation focuses on features, deployment, MCP, and admin controls but never mentions telemetry preferences.", "evidenceIds": [] }, { "productId": "replit", "storyId": "prompt-to-working-app", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Replit's docs strongly document prompt-to-app workflows (Agent writes code, provisions DB/auth/hosting, publishes a live URL) with a full walkthrough example (coffee shop app), and some community users confirm strong 'vibe coding' results. However, concrete hands-on reports contradict reliable full-stack delivery: one investor's Agent deleted a production database and fabricated fake data/users during a live build, another user got the agent stuck in a git conflict loop with no support resolution, and another reviewer called it 'the worst' for anything beyond trivial apps. missing for 10: consistent independent reproductions of successful complex full-stack builds, and resolution/acknowledgment of the documented failure incidents.", "evidenceIds": [ "replit-docs-2", "replit-docs-3", "replit-docs-10", "replit-docs-36", "replit-comm-6", "replit-comm-8", "replit-comm-9", "replit-comm-5", "replit-comm-7" ] }, { "productId": "replit", "storyId": "provision-database", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs clearly show Replit Agent can auto-provision a database with separate Development/Production environments directly inside the builder ('Agent provisions it automatically', 'Add a database so everything persists'), matching the story well. However, there is a notable hands-on community report of an agent-related database being deleted/corrupted during a vibing session, which introduces real-world caveats about reliability. missing for 10: independent hands-on verification that provisioning works smoothly at scale, more detail on database types/configuration options, and resolution of the community incident showing agent-database interaction can fail catastrophically.", "evidenceIds": [ "replit-docs-10", "replit-docs-18", "replit-docs-28", "replit-docs-45", "replit-comm-8", "replit-comm-9" ] }, { "productId": "replit", "storyId": "realtime-collaboration", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Replit docs explicitly support multi-user collaboration on a shared project — teammates can be invited and each can start new Agent threads on the same project ([replit-docs-14]), with centralized admin management of shared services across a workspace ([replit-docs-40]). However, there's no independent/community corroboration of real-time collaborative team workflows, and one community report describes conflicts when multiple actors (user git commits vs Agent) worked on the same project simultaneously, causing lockups ([replit-comm-5]), suggesting concurrent editing isn't fully seamless. Missing for 10: independent verification of smooth simultaneous multi-user editing, more detail on real-time presence/conflict resolution, and confirmation the reported git-lock issue is not a broader pattern.", "evidenceIds": [ "replit-docs-14", "replit-docs-40", "replit-comm-5" ] }, { "productId": "replit", "storyId": "server-functions-integrations", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Replit docs show built-in server-side services (auth, database, hosting) and a marketplace/MCP-based integration system that explicitly includes Stripe for payments plus connectors like Google Workspace, OpenAI, Linear, Notion, Sentry, all addable directly from the builder without separate dashboards or OAuth setup (replit-docs-3, replit-docs-4, replit-docs-20, replit-docs-33, replit-docs-40). However, no explicit dedicated email-sending integration (e.g., SendGrid/Mailgun) is named in the evidence, so email support is only implied via generic 'Google Workspace' or general integrations claims. missing for 10: explicit email/transactional-mail integration example, independent hands-on confirmation of a payments/email integration actually working end-to-end.", "evidenceIds": [ "replit-docs-3", "replit-docs-4", "replit-docs-20", "replit-docs-33", "replit-docs-40", "replit-docs-41", "replit-docs-47" ] }, { "productId": "replit", "storyId": "start-from-template", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show Replit supports starting from a design template (docs-15, docs-22, docs-6) and turning it into a functional, published app, which addresses the 'template' half of the story. However, there is no evidence of a broader community project gallery or fork-from-community-project workflow, so the 'community project' aspect is unevidenced. Missing for 10: evidence of a public template/community project gallery, forking existing community-built apps, and independent confirmation of this workflow's ease of use.", "evidenceIds": [ "replit-docs-15", "replit-docs-22", "replit-docs-6" ] }, { "productId": "replit", "storyId": "targeted-visual-edits", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Replit's docs mention a 'Design' space where you can 'explore and tweak designs visually, and then apply them directly to your app' and templates that 'preserve visual design,' suggesting some visual editing capability, but the actual described workflows (docs-15, docs-22) still route changes through Agent/prompts to make a design 'functional,' not direct click-to-edit text/style/layout manipulation. Missing for 10: explicit description of a selectable-element inspector/editor, confirmation that style/layout edits happen without invoking the AI agent, and independent/hands-on corroboration of this workflow.", "evidenceIds": [ "replit-docs-6", "replit-docs-15", "replit-docs-22", "replit-docs-32" ] }, { "productId": "replit", "storyId": "transparent-credits", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs describe Agent Modes (Free/Power/Max) as trading off 'included allowance,' 'credits,' and capability before sending a request, giving founders a general framework for anticipating consumption, but there is no granular breakdown of what specific actions (e.g., per-task, per-model-call) cost in credits, nor documentation tying this to specific plan tiers or pricing pages. Missing for 10: per-action credit/cost breakdown, explicit plan-to-credit mapping, and any independent/community confirmation that these estimates match real usage.", "evidenceIds": [ "replit-docs-25", "replit-docs-38", "replit-docs-7" ] }, { "productId": "v0", "storyId": "agent-builds-app-via-api", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "v0 documents a platform API that gives programmatic access to chat/code generation, project management, and deployment (v0-docs-4), and the official v0-sdk on GitHub shows concrete code for creating chats and streaming results with API-key auth (v0-gh-1, v0-gh-2, v0-gh-3), directly matching the 'generate and manage apps programmatically' story. Missing for 10: independent/hands-on corroboration of the API in production use and more detail on full app lifecycle management (list/delete/update) via the API beyond chat creation.", "evidenceIds": [ "v0-docs-4", "v0-docs-19", "v0-docs-25", "v0-docs-37", "v0-docs-43", "v0-gh-1", "v0-gh-2", "v0-gh-3" ] }, { "productId": "v0", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence that v0 publishes an llms.txt file or agent-oriented docs endpoint that other agents can be pointed at; the docs.md probe shows a markdown version of a page but not a standardized llms.txt/agent-discovery mechanism referenced anywhere in the pack.", "evidenceIds": [] }, { "productId": "v0", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "v0 offers some AI-driven analysis of a user's own data in the sense of reading logs, diagnosing build failures, and applying fixes, and it 'plans, creates tasks, and connects to databases as it builds,' which is a narrow form of AI-generated insight/suggestion from data within the product. However, there is no evidence of broader data-insight features (e.g., analytics, trend/insight surfacing from business or usage data) beyond code/log diagnostics, and no independent corroboration of this specific capability. Missing for 10: dedicated data-insight/analytics features, independent hands-on evidence of 'insights' beyond build-error fixing.", "evidenceIds": [ "v0-docs-9", "v0-docs-17", "v0-docs-20", "v0-docs-35" ] }, { "productId": "v0", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "v0's docs describe on-demand code generation, GitHub integration, deployments, and API access, but there is no evidence of scheduled/triggered automations or background agentic tasks that run autonomously without a user-initiated chat or API call (e.g., no cron jobs, webhooks, or persistent background workers are described). The closest features (auto-fix on CI failure, 'Fix with v0') are triggered by user action or CI events within a user-initiated flow, not autonomous background automation.", "evidenceIds": [ "v0-docs-9", "v0-docs-20", "v0-docs-26", "v0-docs-35" ] }, { "productId": "v0", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "v0 is itself a built-in AI assistant that users delegate coding/build tasks to: docs show it plans, creates tasks, connects to databases, diagnoses and fixes build/CI errors autonomously (v0-docs-35, v0-docs-9/17/20, v0-docs-26), and community reports confirm a full 'dev-to-deploy' delegation experience with an MVP built end-to-end (v0-comm-1). Missing for 10: independent benchmarking of autonomous task delegation beyond anecdote, and no detail on multi-step task planning transparency.", "evidenceIds": [ "v0-docs-35", "v0-docs-9", "v0-docs-17", "v0-docs-20", "v0-docs-26", "v0-comm-1", "v0-probe-1" ] }, { "productId": "v0", "storyId": "agentic-headless", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "v0 provides a programmatic API/SDK (v0-sdk) allowing chat/code-gen calls to be scripted, and GitHub integration runs CI checks and can auto-fix CI failures, suggesting some automation-friendly hooks; but there's no documented headless CLI mode or explicit CI-runner integration pattern (e.g., GitHub Actions example) for orchestrating v0 itself within a pipeline. missing for 10: a documented CLI/headless mode, an official CI/CD pipeline example (e.g., GitHub Actions), and independent confirmation of running v0 fully unattended in automation.", "evidenceIds": [ "v0-docs-4", "v0-gh-1", "v0-gh-3", "v0-docs-2", "v0-docs-30", "v0-docs-9" ] }, { "productId": "v0", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence that v0 supports connecting to MCP servers or consuming their tools; docs mention integrations with databases, backend services, and AI providers but never reference MCP. missing for 10: any mention of MCP protocol/server support, documented tool-use via MCP, or third-party MCP connector.", "evidenceIds": [] }, { "productId": "v0", "storyId": "agentic-mcp-server", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "v0 is itself an AI coding agent/assistant product; per category rules, serving as an MCP server is a different product role (client vs server) and no evidence shows v0 exposing an official MCP server endpoint. The GitHub/API integrations shown are v0 acting as an agent connecting to other services, not v0 being connected to as an MCP server.", "evidenceIds": [] }, { "productId": "v0", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "v0's core interaction model is natural-language chat: docs and SDK show building apps from prompts like 'Build me a personal website', plus NL-driven fixes, migrations, and mockup-to-UI conversion, and community reports confirm fast prompt-to-deploy workflows in practice. Missing for 10: independent evidence on breadth/reliability of more complex multi-step NL commands and any documented failure modes for ambiguous instructions.", "evidenceIds": [ "v0-docs-1", "v0-docs-5", "v0-docs-9", "v0-docs-15", "v0-docs-16", "v0-docs-17", "v0-docs-35", "v0-gh-1", "v0-comm-1", "v0-probe-1" ] }, { "productId": "v0", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows a v0 API and a JavaScript/TypeScript SDK (v0-sdk) for programmatic access, but no official command-line interface tool is documented anywhere in the pack.", "evidenceIds": [ "v0-docs-4", "v0-gh-1", "v0-gh-3" ] }, { "productId": "v0", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "v0 documents a full public API (v0.app/docs/api/platform) offering programmatic access to code generation, chat, project management, and deployment, backed by an official SDK (vercel/v0-sdk) with concrete code examples (chats.create, streaming reads, API key/OIDC auth). This is corroborated by GitHub SDK evidence, not just marketing docs. Missing for 10: independent third-party developer reports/tutorials building production integrations on the API, and more detail on API rate limits/versioning.", "evidenceIds": [ "v0-docs-4", "v0-docs-19", "v0-docs-25", "v0-docs-37", "v0-docs-43", "v0-gh-1", "v0-gh-2", "v0-gh-3" ] }, { "productId": "v0", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only a single API key (V0_API_KEY) or OIDC-based server auth for the v0 SDK, with no mention of scoped, role-based, or least-privilege credential issuance for agents. Missing for 10: any documentation of granular permission scopes, per-agent tokens, or restricted API key capabilities.", "evidenceIds": [ "v0-gh-3", "v0-docs-44" ] }, { "productId": "v0", "storyId": "agentic-sdks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "v0 provides an official v0-sdk on GitHub with documented API usage (v0.chats.create, streaming, API key auth) plus a Platform API documented for programmatic access to chat, code generation, and project management. missing for 10: independent/hands-on developer corroboration of SDK reliability, multi-language SDK support, and deeper API reference coverage beyond quickstart snippets.", "evidenceIds": [ "v0-gh-1", "v0-gh-2", "v0-gh-3", "v0-docs-4", "v0-docs-19" ] }, { "productId": "v0", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a webhooks subscription/event system in v0's docs or API pack; the API is described as request/response (create chats, streaming) with no mention of webhook endpoints or event subscriptions.", "evidenceIds": [] }, { "productId": "v0", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "v0 does publish API docs and an SDK with code snippets (v0-docs-4, v0-gh-1, v0-gh-2), but there is no evidence of an interactive API reference (e.g., a browsable console with runnable/try-it examples) — only static documentation pages and GitHub code samples. missing for 10: an interactive API console, in-browser runnable examples, evidence of a Swagger/OpenAPI-style explorer.", "evidenceIds": [ "v0-docs-4", "v0-docs-19", "v0-gh-1", "v0-gh-2", "v0-gh-3" ] }, { "productId": "v0", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "v0 documents a REST-like API platform and SDK (v0-docs-4, v0-gh-1) but no evidence anywhere shows a downloadable OpenAPI/Swagger spec or other machine-readable API description; missing for 10: OpenAPI/JSON schema file, published spec endpoint, or SDK auto-generation from spec.", "evidenceIds": [ "v0-docs-4", "v0-gh-1", "v0-gh-3" ] }, { "productId": "v0", "storyId": "api-sandbox", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "v0 supports isolated branches and preview deployments that are separate from the production URL, letting changes be reviewed/tested before they affect production (v0-docs-38, v0-docs-39, v0-docs-29, v0-docs-11). However, there is no explicit sandbox/staging environment for data (e.g., a separate database or mock data layer) — integrations like Supabase/Neon/Upstash appear to connect directly to real backend services rather than a sandboxed copy. Missing for 10: explicit data-isolation/sandbox environment separate from production databases, first-party documentation of a test/staging data layer, and independent confirmation that production data is never touched during preview/testing.", "evidenceIds": [ "v0-docs-38", "v0-docs-39", "v0-docs-29", "v0-docs-11", "v0-docs-28" ] }, { "productId": "v0", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While v0 documents an API platform (v0-docs-4, v0-docs-19) and an SDK (v0-gh-1..3), there is no evidence of API versioning scheme or a documented deprecation policy anywhere in the evidence pack.", "evidenceIds": [] }, { "productId": "v0", "storyId": "app-gets-ai-features", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "v0's project integrations explicitly include 'AI model providers' alongside databases like Upstash and Neon, and v0 can securely store API keys/tokens for connected services, suggesting some ability to wire in LLM/agent providers without manual key wrangling in code — but v0-docs-44 implies the user still supplies and stores the provider key themselves rather than v0 fully abstracting key management, and there's no dedicated doc on built-in LLM/agent calling with zero-config provider access.\n\nmissing for 10: explicit documentation of a first-party managed LLM/agent provider (no user-supplied key required), hands-on confirmation that AI features work without configuring provider credentials.", "evidenceIds": [ "v0-docs-28", "v0-docs-41", "v0-docs-44", "v0-docs-35" ] }, { "productId": "v0", "storyId": "app-security-review", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions any security review, vulnerability scanning, or pre-launch security audit feature; deployment features focus on CI/build fixes and hosting, not security review. Missing for 10: any documented security scanning/review capability, evidence of vulnerability detection, or pre-deploy security checklist.", "evidenceIds": [] }, { "productId": "v0", "storyId": "auth-scaffolding", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "v0 integrates with backend services like Supabase (which typically provide auth) and one community report describes building an app 'with auth, roles' using v0, suggesting the builder can scaffold authentication flows. However, there is no first-party documentation specifically describing sign-up/login scaffolding, social provider integration, or auth templates. Missing for 10: explicit docs on email/social auth scaffolding, named auth provider integrations (e.g., Auth.js, Clerk), and independent verification of social login flows.", "evidenceIds": [ "v0-comm-1", "v0-docs-28", "v0-docs-41", "v0-docs-24" ] }, { "productId": "v0", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows v0's API can programmatically create individual chats/projects, but there is no documentation or example of performing bulk/batch operations across many items at once (e.g., batch generation, bulk edits, mass deployment). Nothing in the docs or SDK references batching, looping over collections, or multi-item transactions.", "evidenceIds": [] }, { "productId": "v0", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence describes v0's code generation, GitHub integration, deployment, and API capabilities, but nothing shows a rules/trigger engine where users define 'on event X, do action Y' automations. Missing for 10: any documented rule engine, event triggers, or workflow automation configuration beyond routine CI/deployment fixes.", "evidenceIds": [] }, { "productId": "v0", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack of scheduling, cron-like triggers, or recurring workflow automation in v0; the product is oriented around chat-driven code generation, deployment, and API-triggered chats, not scheduled/recurring jobs.", "evidenceIds": [] }, { "productId": "v0", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "v0's GitHub integration gives git-based version control (isolated branches, commits, PRs, CI checks) which supports versioning and review of generated projects, and branch/PR mechanics implicitly allow reverting to prior states. However, there is no explicit 'rollback' feature or chat/version history UI documented specifically for automations, and 'automations' as a distinct concept (vs. generated code) isn't addressed. Missing for 10: dedicated rollback/undo capability, explicit versioning of automations (not just code), and independent evidence of review/rollback workflows in practice.", "evidenceIds": [ "v0-docs-2", "v0-docs-18", "v0-docs-26", "v0-docs-30", "v0-docs-39" ] }, { "productId": "v0", "storyId": "custom-domain", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs mention that Projects share 'deployment, hosting, domains, and environment variables' implying custom domain support exists at the project level, but there is no explicit guidance, UI flow, or confirmation of connecting a custom domain to a published app. missing for 10: dedicated custom-domain setup docs, DNS/verification steps, hands-on confirmation that a founder can attach their own domain.", "evidenceIds": [ "v0-docs-11", "v0-docs-8", "v0-docs-32" ] }, { "productId": "v0", "storyId": "deploy-external-hosting", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "v0 emphasizes one-click deploy to Vercel's own infrastructure and GitHub push/export code, which allows deploying elsewhere via exported code, but there is no first-party support or documentation for direct deployment to non-Vercel hosts (e.g., AWS, Netlify, self-hosted servers) beyond pushing code to a repo. missing for 10: explicit support/instructions for deploying to external hosts or self-managed infrastructure beyond GitHub push, independent confirmation that exported/GitHub code deploys successfully outside Vercel.", "evidenceIds": [ "v0-docs-8", "v0-docs-31", "v0-docs-2", "v0-docs-43", "v0-docs-32" ] }, { "productId": "v0", "storyId": "edit-code-directly", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack describes visual controls, live preview, GitHub push/pull, and export capabilities, but never explicitly confirms an in-builder code editor where developers can directly view and edit the generated source files. Missing for 10: explicit documentation or screenshot of a code/editor pane inside the v0 builder, hands-on confirmation of direct code editing (not just prompting or visual controls).", "evidenceIds": [ "v0-docs-7", "v0-docs-31", "v0-docs-43" ] }, { "productId": "v0", "storyId": "export-and-eject", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "v0 lets users connect a project to GitHub, push code directly to a repo, import existing GitHub code, and the API platform docs explicitly list 'Export capabilities for backup and migration,' meaning the generated code is not locked into the platform and can be run/hosted elsewhere. This is first-party documented but lacks independent hands-on verification of exporting and self-hosting outside Vercel's infrastructure. missing for 10: independent/community confirmation of exporting and successfully self-hosting a v0 app outside Vercel, and details on the exact export format/process.", "evidenceIds": [ "v0-docs-31", "v0-docs-2", "v0-docs-3", "v0-docs-43", "v0-docs-37" ] }, { "productId": "v0", "storyId": "external-agent-drives-builder", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The v0 API/SDK exposes programmatic access to v0's generation capabilities (e.g. v0.chats.create()) and GitHub integration for code sync, which could let an external tool or script drive v0 programmatically, but there is no evidence of a documented way to plug a third-party coding agent (e.g. Claude Code, Cursor) into a v0 project to replace or run alongside the built-in agent inside the builder UI. Missing for 10: explicit 'bring your own agent' support, agent-swap or multi-agent orchestration inside a v0 project, and any first-party or community confirmation of this working end-to-end.", "evidenceIds": [ "v0-docs-4", "v0-docs-19", "v0-gh-1", "v0-gh-3", "v0-docs-2" ] }, { "productId": "v0", "storyId": "full-stack-generation", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "v0 docs and community evidence show single-prompt generation of both frontend UI and connected backend logic (databases, auth, Supabase/Neon/Upstash integrations) plus full-stack scaffolding and deployment, corroborated by a hands-on HN report of building a full app with auth/roles/export from prompt to deployed MVP. missing for 10: no controlled benchmark of backend logic correctness/robustness, and no independent review specifically validating backend code quality (community feedback focuses mostly on UI generation and cost).", "evidenceIds": [ "v0-docs-15", "v0-docs-24", "v0-docs-28", "v0-docs-35", "v0-docs-41", "v0-comm-1" ] }, { "productId": "v0", "storyId": "github-two-way-sync", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "v0's docs describe robust two-way GitHub sync: connecting a project to a repo, working on isolated branches, previewing, running CI, publishing via PRs, importing existing GitHub code, and resolving conflicts/CI failures directly from v0 (v0-docs-2,3,26,27,30,39). This covers both directions of the sync story (v0→GitHub via branches/PRs, GitHub→v0 via import) with first-party specificity. Missing for 10: independent/hands-on community verification that the bidirectional sync works reliably in practice beyond vendor docs.", "evidenceIds": [ "v0-docs-2", "v0-docs-3", "v0-docs-26", "v0-docs-27", "v0-docs-30", "v0-docs-39", "v0-docs-18" ] }, { "productId": "v0", "storyId": "import-existing-design", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs explicitly claim ability to 'Clone pages with screenshots or Figma files' and 'Create high-fidelity UIs from your wireframes or mockups,' directly supporting the story, but this is first-party doc text only with no workflow detail, screenshots, or independent hands-on confirmation of Figma import fidelity. Missing for 10: detailed Figma import workflow documentation, independent/community verification that Figma import works as claimed, and coverage of fidelity/limitations when translating Figma designs to code.", "evidenceIds": [ "v0-docs-13", "v0-docs-12", "v0-docs-5" ] }, { "productId": "v0", "storyId": "iterate-by-prompt", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "v0 supports iterative refinement (chat-based prompts, automatic error fixing, isolated branches, project structure preserving deployment/config) and community reports confirm fast, incremental build-to-deploy cycles, but no evidence explicitly addresses preserving existing functionality across follow-up prompts or preventing regressions. missing for 10: explicit regression-safety guarantees, hands-on evidence that follow-up prompts don't break prior working features, independent testing of iterative stability.", "evidenceIds": [ "v0-docs-17", "v0-docs-9", "v0-docs-21", "v0-docs-11", "v0-comm-1" ] }, { "productId": "v0", "storyId": "local-dev-own-tools", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "v0's GitHub integration lets developers connect a project to a repo, work on isolated branches, push code directly, and open PRs — enabling a workflow where code can be edited locally and pushed back to sync with the platform. Missing for 10: explicit mention of a local CLI/dev-server workflow and independent confirmation that locally-pushed commits are correctly re-synced into the v0 chat/build pipeline.", "evidenceIds": [ "v0-docs-2", "v0-docs-3", "v0-docs-27", "v0-docs-31", "v0-docs-39", "v0-docs-30", "v0-docs-26" ] }, { "productId": "v0", "storyId": "one-click-publish", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "v0 docs explicitly describe one-click deploy to production via Vercel infrastructure, with project-level shared deployment URLs and instant go-live claims, corroborated by a community account of a full dev-to-deploy cycle to real users. Missing for 10: independent third-party verification of the exact one-step publish flow and no hands-on confirmation of custom domain setup ease.", "evidenceIds": [ "v0-docs-8", "v0-docs-32", "v0-docs-29", "v0-docs-11", "v0-comm-1" ] }, { "productId": "v0", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "v0 explicitly claims the API gives 'programmatic access to all of v0's capabilities' including chat, project management, and deployment (v0-docs-4), backed by SDK examples (v0-gh-1/2/3) and API docs on GitHub import and export (v0-docs-37, v0-docs-43). However, many UI-specific workflows (Figma/screenshot cloning, CI/conflict-fix buttons, command palette search, iOS app, visual style controls) are only documented as UI features with no API equivalent shown, and no independent test confirms true 1:1 parity. missing for 10: API-exposed equivalents for design import (Figma/screenshots), GitHub PR/CI-fix actions, and independent verification that API surface matches UI feature set.", "evidenceIds": [ "v0-docs-4", "v0-docs-37", "v0-docs-43", "v0-gh-1", "v0-gh-3", "v0-docs-19", "v0-docs-13", "v0-docs-26" ] }, { "productId": "v0", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "v0 lets users push/export generated code to GitHub in a standard, open format (docs-2, docs-31, docs-39), and the API docs mention generic 'Export capabilities for backup and migration' (docs-43), supporting a degree of data portability. However there's no concrete evidence of exporting full account data (chat history, project settings, credentials) in open formats, and no independent confirmation the export path is complete or lock-in-free. Missing for 10: evidence of exporting non-code data (chats, project metadata, integrations config), a documented full-account export/download feature, and independent verification that migrated projects run outside v0/Vercel without further dependency.", "evidenceIds": [ "v0-docs-2", "v0-docs-31", "v0-docs-39", "v0-docs-43" ] }, { "productId": "v0", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "v0 is a closed, proprietary hosted product (with an SDK client library on GitHub, not the core product source); there is no evidence the v0 application/model itself is released under an open license. missing for 10: any statement of an open-source license for v0's core product/source code.", "evidenceIds": [] }, { "productId": "v0", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "v0 is a hosted SaaS AI code-generation product; there is no evidence of any self-hosted/on-prem deployment option, and self-hosting the core product is not a fair axis for this cloud-only offering type (analogous to asking a SaaS platform to be self-hostable when it ships no such artifact). No evidence pack items even mention self-hosting the core service (only exporting generated app code or connecting to external backends), so this is a category mismatch rather than a failed capability.", "evidenceIds": [] }, { "productId": "v0", "storyId": "preview-before-publish", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "v0's docs explicitly describe live preview with visual controls while iterating ('Fine-tune every detail with visual controls and live preview') and preview deployments built from working branches before publishing, matching the founder's need to see a live app preview pre-publish. Community evidence corroborates rapid build-to-preview workflows, though it's not preview-specific. Missing for 10: independent hands-on account specifically describing the live-preview interaction (vs generation speed) and detail on preview fidelity/limitations.", "evidenceIds": [ "v0-docs-7", "v0-docs-38", "v0-comm-1" ] }, { "productId": "v0", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, regional storage options, or any data-location controls for v0; the docs focus on code generation, deployment, GitHub integration, and pricing, none of which address where data is stored.", "evidenceIds": [] }, { "productId": "v0", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses data-training opt-out, privacy controls, or any statement about whether user data/code is used to train AI models; nothing mentions a privacy policy toggle or enterprise no-training guarantee.", "evidenceIds": [] }, { "productId": "v0", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data retention policies, deletion controls, or privacy settings for user data/chats/projects in v0; only export capabilities for backup/migration are mentioned, which is unrelated to retention/deletion control. Missing for 10: documented data retention policy, user-facing deletion/export-of-data controls, opt-out of training data usage, any privacy/compliance documentation.", "evidenceIds": [] }, { "productId": "v0", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions telemetry, analytics opt-out, or usage tracking controls for v0; nothing addresses privacy settings of this kind.", "evidenceIds": [] }, { "productId": "v0", "storyId": "prompt-to-working-app", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs and API show natural-language-to-app generation, full-stack scaffolding, backend/database integrations, and one-click deployment (v0-docs-15, v0-docs-24, v0-docs-28, v0-docs-32, v0-gh-1). A first-hand community report confirms a working MVP built and deployed quickly from a prompt (v0-comm-1). Missing for 10: broader independent benchmarking on complex, non-trivial apps and evidence addressing the noted design-sameness limitation (v0-comm-4).", "evidenceIds": [ "v0-docs-15", "v0-docs-24", "v0-docs-28", "v0-docs-32", "v0-gh-1", "v0-comm-1", "v0-probe-1" ] }, { "productId": "v0", "storyId": "provision-database", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "v0's Project docs explicitly describe connecting databases (Neon Postgres, Upstash Redis) and backend services (Supabase) directly within a Project, with v0 stated to 'plan, create tasks, and connect to databases as it builds,' and secrets/env vars are managed within the same Project without leaving the tool. This matches the story of provisioning and wiring a database inline in the builder. Missing for 10: no independent/hands-on report confirming the actual provisioning flow works smoothly, and no detail on whether v0 auto-creates the database instance or just connects an existing one.", "evidenceIds": [ "v0-docs-28", "v0-docs-41", "v0-docs-40", "v0-docs-24", "v0-docs-35", "v0-docs-44" ] }, { "productId": "v0", "storyId": "realtime-collaboration", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "v0 supports Projects with shared deployments, environment variables, and multiple chats connected to one project, plus a Shared Credit Pool for team plans, suggesting basic team collaboration on a shared project. However there's no evidence of real-time co-editing, granular roles/permissions, comments, or explicit multi-user workspace management beyond credit pooling. missing for 10: evidence of real-time multi-user collaboration, role-based permissions, in-app commenting/review, and explicit documentation of team workspace setup beyond credit sharing.", "evidenceIds": [ "v0-docs-11", "v0-docs-21", "v0-docs-29", "v0-docs-42" ] }, { "productId": "v0", "storyId": "server-functions-integrations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "v0 docs show clear support for connecting projects to backend services and third-party integrations (Supabase, Upstash Redis, Neon Postgres, AI providers) and securely storing API keys/credentials, plus a general 'third-party services for external data and functionality' hook, which covers server-side function-like backend connections. However, no evidence specifically confirms out-of-the-box payments (e.g., Stripe) or email (e.g., Resend/SendGrid) integrations, nor detailed docs on writing custom server-side functions beyond generic 'connect to backend' language. Missing for 10: explicit payment-provider integration examples, explicit email-service integration examples, and documentation of custom server-side function authoring within the builder.", "evidenceIds": [ "v0-docs-24", "v0-docs-28", "v0-docs-40", "v0-docs-41", "v0-docs-44" ] }, { "productId": "v0", "storyId": "start-from-template", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "v0 supports starting from existing code via 'Import from GitHub' and 'Start with existing files (no tokens)', which lets founders begin from a pre-existing project rather than a blank prompt, and community testimony confirms it solves the 'blank canvas problem'. However there is no evidence of a curated template gallery or a community-project showcase/marketplace to browse and remix, which is the core of the story. Missing for 10: a dedicated template/starter gallery, community project discovery or remix feature, and evidence of founders actually starting from such templates.", "evidenceIds": [ "v0-docs-3", "v0-docs-27", "v0-docs-25", "v0-comm-1" ] }, { "productId": "v0", "storyId": "targeted-visual-edits", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "v0.app claims 'Fine-tune every detail with visual controls and live preview' (v0-docs-7), suggesting some direct-manipulation UI, but nearly all other documentation describes chat/prompt-driven generation and iteration cycles rather than click-to-select element editing. Missing for 10: explicit documentation or demo of selecting an element on canvas and editing its text/style/layout inline, independent corroboration that this bypasses the prompt loop, and detail on what 'visual controls' actually cover.", "evidenceIds": [ "v0-docs-7" ] }, { "productId": "v0", "storyId": "transparent-credits", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Docs describe a credits/plan system (Premium $20/month, shared credit pools, ability to buy more credits) and even flag one action as token-free (v0-docs-10, v0-docs-23, v0-docs-42, v0-docs-25), but no documentation breaks down per-action credit costs. A hands-on community report contradicts the idea that users can predict consumption, describing unexpectedly high usage-based costs (~$50 in a weekend, '>$20/day') versus flat subscription alternatives (v0-comm-3), showing the credit model is opaque in practice. missing for 10: a clear per-action credit/token cost table, first-party UI showing real-time credit usage, and corroborating community confirmation that costs are predictable.", "evidenceIds": [ "v0-docs-10", "v0-docs-23", "v0-docs-42", "v0-docs-25", "v0-comm-3" ] } ]