[ { "productId": "appwrite", "storyId": "agent-branch-migrations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of branch/preview environments for schema changes—Appwrite offers CLI, MCP server, and self-hosting, but nothing about isolated branches or preview environments for testing schema migrations before production. missing for 10: branching/preview environment feature, schema diff/staging workflow, agent-driven isolated testing capability.", "evidenceIds": [] }, { "productId": "appwrite", "storyId": "agent-provisions-backend", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Appwrite's CLI explicitly supports non-interactive/headless scripted usage (appwrite-docs-42), can create/manage projects, resources, users, functions, deployments (appwrite-docs-12/18/32), and databases/auth/policies are all provisionable via the CLI/API per the docs; this is exactly the kind of scriptable backend-standup an agent could drive headlessly. missing for 10: no end-to-end hands-on example/transcript of an agent actually scripting full schema+auth+policy setup in one script, and no independent/community verification of headless CLI workflows.", "evidenceIds": [ "appwrite-docs-42", "appwrite-docs-12", "appwrite-docs-18", "appwrite-docs-32", "appwrite-docs-6", "appwrite-docs-11", "appwrite-docs-38", "appwrite-probe-5" ] }, { "productId": "appwrite", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Appwrite explicitly hosts an llms.txt (confirmed via live probe returning HTTP 200) and docs.md, plus supports appending .md to any doc page for raw Markdown, and provides Skills files for agent context — directly matching the story. missing for 10: no independent/community corroboration of agents actually consuming these files.", "evidenceIds": [ "appwrite-probe-1", "appwrite-probe-2", "appwrite-docs-40", "appwrite-docs-39" ] }, { "productId": "appwrite", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Appwrite's AI-related docs describe building blocks for developers to add AI features to their own apps (embeddings storage, image labeling, functions) and tooling for external AI agents (MCP server, CLI, skills) — not Appwrite itself surfacing AI-generated insights or suggestions about the user's own data inside the product (e.g., an analytics/insights dashboard). No evidence of such a first-party insights feature was found.", "evidenceIds": [ "appwrite-docs-7", "appwrite-docs-16", "appwrite-docs-17", "appwrite-docs-37" ] }, { "productId": "appwrite", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Appwrite Functions can be triggered by scheduled executions, server events, and webhooks (appwrite-docs-9), which enables autonomous background automations, and the CLI supports non-interactive/headless automation for CI use (appwrite-docs-42). However, there's no dedicated 'agent automation' or workflow-orchestration framework documented specifically for AI-native autonomous agents beyond scheduled Functions — missing for 10: no evidence of long-running agent orchestration, multi-step autonomous agent workflows, or independent hands-on validation of scheduled/background automation reliability.", "evidenceIds": [ "appwrite-docs-9", "appwrite-docs-42", "appwrite-docs-11", "appwrite-docs-38" ] }, { "productId": "appwrite", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Appwrite's evidence shows it exposes AI-related APIs (image labeling, embeddings), an MCP server for external AI agents/tools to control Appwrite, and CLI/skills for agentic tooling — but nothing describes a built-in assistant inside the Appwrite product/console that a user can delegate tasks to. The MCP and CLI integrations are about external AI agents driving Appwrite, not Appwrite hosting its own assistant.", "evidenceIds": [ "appwrite-docs-11", "appwrite-docs-37", "appwrite-docs-16", "appwrite-docs-17" ] }, { "productId": "appwrite", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Appwrite explicitly documents a non-interactive/headless CLI mode designed for CI automation (appwrite-docs-42), backed by a full CLI for managing projects, resources, and Functions (appwrite-docs-12, appwrite-docs-18), self-hosting via Docker for full infra control (appwrite-docs-14/19/33), and API-driven triggers for functions via HTTP/SDK/webhooks/schedules (appwrite-docs-9). missing for 10: independent/hands-on CI pipeline examples or third-party confirmation of headless CLI use in production.", "evidenceIds": [ "appwrite-docs-42", "appwrite-docs-12", "appwrite-docs-14", "appwrite-docs-9", "appwrite-docs-22", "appwrite-probe-5" ] }, { "productId": "appwrite", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence only shows Appwrite exposing its own MCP server so external AI agents (Claude Code, Cursor, etc.) can call Appwrite's API — the opposite direction from the story, which asks whether Appwrite itself can plug in and consume other MCP servers' tools. No documentation or CLI feature describes Appwrite acting as an MCP client or importing external MCP tool servers.", "evidenceIds": [ "appwrite-docs-11", "appwrite-docs-38", "appwrite-probe-4" ] }, { "productId": "appwrite", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Appwrite documents an official MCP server that lets LLMs/agents (Claude Code, Cursor, Claude Desktop, etc.) directly interact with Appwrite's API and docs, covering project/resource management and function deployment. This is first-party documented and probe-confirmed, though there's no independent hands-on verification of the MCP server's reliability. Missing for 10: independent/community corroboration of the MCP server working in practice, and more detail on setup/auth flow.", "evidenceIds": [ "appwrite-docs-11", "appwrite-docs-38", "appwrite-probe-4" ] }, { "productId": "appwrite", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Appwrite ships an official MCP server plus open-source \"Skills\" markdown files that let AI agents/LLMs (via Claude Desktop, Cursor, Codex, etc.) directly operate on an Appwrite project—creating/managing projects, resources, and functions—using natural-language driven tool calls rather than manual API/CLI work, and docs are also served as raw markdown for agent consumption. Missing for 10: independent/hands-on user reports confirming the MCP server works reliably in practice, and no evidence of a broader natural-language chat/command interface beyond the MCP/agent-tooling layer.", "evidenceIds": [ "appwrite-docs-11", "appwrite-docs-38", "appwrite-docs-39", "appwrite-docs-31", "appwrite-docs-40", "appwrite-probe-4" ] }, { "productId": "appwrite", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Appwrite documents an official CLI covering project/resource management, function deployment, non-interactive/headless CI use, SDK generation, and self-hosted endpoint targeting, which is well suited to AI-native/agentic automation workflows. missing for 10: independent hands-on community corroboration of the CLI itself (evidence is entirely first-party docs) and no explicit example of an AI agent scripting the CLI end-to-end.", "evidenceIds": [ "appwrite-docs-12", "appwrite-docs-13", "appwrite-docs-22", "appwrite-docs-41", "appwrite-docs-42", "appwrite-probe-5" ] }, { "productId": "appwrite", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Appwrite exposes a full public API surface (auth, databases, storage, functions) with SDKs, a documented CLI for automation/CI, docs-as-markdown and llms.txt for machine consumption, and an official MCP server letting agents call the API accurately. This is strong first-party documentation of programmatic/agentic access, though an OpenAPI spec probe returned 404s. Missing for 10: a publicly discoverable OpenAPI/swagger spec and independent (non-vendor) hands-on confirmation of agentic API usage.", "evidenceIds": [ "appwrite-docs-11", "appwrite-docs-12", "appwrite-docs-38", "appwrite-docs-39", "appwrite-docs-40", "appwrite-docs-42", "appwrite-probe-1", "appwrite-probe-2", "appwrite-probe-3", "appwrite-probe-4", "appwrite-probe-5" ] }, { "productId": "appwrite", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Appwrite documents JWT-based delegated access for users (appwrite-docs-35) and a permissions-based database/resource model (appwrite-docs-6, appwrite-docs-23), which could underpin least-privilege credentialing, but there is no explicit documentation of issuing scoped API keys/tokens specifically for AI agents. missing for 10: explicit docs on creating agent-specific scoped API keys, examples of least-privilege key scoping for MCP/agent use, and independent confirmation this works in practice for agent credentials.", "evidenceIds": [ "appwrite-docs-35", "appwrite-docs-6", "appwrite-docs-23", "appwrite-docs-11" ] }, { "productId": "appwrite", "storyId": "agentic-sdks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Appwrite explicitly supports SDK-based development with 'SDK methods' triggering Functions, a CLI 'generate' command that creates type-safe SDKs from project schema, and dedicated AI-native tooling (MCP server for 'SDK-accurate calls', Skills markdown files, docs-as-markdown, llms.txt) that teaches agents to use its SDKs correctly. This shows first-party investment in making SDKs agent-buildable, though the pack lacks an explicit enumerated list of supported SDK languages or independent hands-on validation of SDK quality. Missing for 10: explicit list/count of official SDK languages, independent developer corroboration of SDK reliability.", "evidenceIds": [ "appwrite-docs-9", "appwrite-docs-41", "appwrite-docs-31", "appwrite-docs-38", "appwrite-docs-39", "appwrite-docs-40", "appwrite-probe-1", "appwrite-probe-2" ] }, { "productId": "appwrite", "storyId": "agentic-webhooks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Appwrite Functions can be triggered by webhooks, and Appwrite supports responding to server events in realtime, indicating webhook-based event subscription exists, but there's no dedicated documentation detailing webhook subscription setup, payload format, or management specifically for AI-native agent use. missing for 10: dedicated webhook subscription docs/API reference, examples of AI agents configuring webhook subscriptions, independent hands-on verification of webhook delivery.", "evidenceIds": [ "appwrite-docs-9", "appwrite-docs-28" ] }, { "productId": "appwrite", "storyId": "ai-dashboard-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Appwrite ships an MCP server and CLI/skills so external AI coding agents (Claude Code, Cursor, etc.) can operate on an Appwrite project, but there is no evidence of an in-dashboard/console AI chat assistant that users can converse with to write queries, debug errors, or get explanations directly inside the Appwrite UI.", "evidenceIds": [ "appwrite-docs-11", "appwrite-docs-38", "appwrite-docs-39", "appwrite-docs-37" ] }, { "productId": "appwrite", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Appwrite documentation, CLI, and MCP tooling for AI agents, but nothing describes an interactive API reference or console where users can run live code examples. A probe for OpenAPI/Swagger endpoints returned 404s, further suggesting no exposed interactive reference is confirmed.", "evidenceIds": [ "appwrite-probe-3" ] }, { "productId": "appwrite", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Appwrite is an API-driven backend platform, so publishing a machine-readable API spec (OpenAPI/Swagger) is a fair and applicable ask. A direct probe for standard OpenAPI/Swagger endpoints (openapi.json, swagger.json, etc.) returned 404 on all candidates, and no docs evidence points to a downloadable spec file, only CLI/MCP/markdown docs tooling.", "evidenceIds": [ "appwrite-probe-3", "appwrite-probe-1", "appwrite-probe-2" ] }, { "productId": "appwrite", "storyId": "api-sandbox", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Appwrite's self-hosting capability (docs-14/19/33/34) lets a user spin up an isolated instance separate from production, and CLI non-interactive mode (docs-42) supports automated CI testing, which together imply a way to test without touching production data. However, there is no explicit 'sandbox' or staging-environment feature, no seed/reset-data tooling, and no documentation calling out this exact workflow.\nmissing for 10: explicit sandbox/staging environment feature, data-seeding/reset tooling, first-party guidance on isolating test vs prod data.", "evidenceIds": [ "appwrite-docs-14", "appwrite-docs-19", "appwrite-docs-34", "appwrite-docs-42" ] }, { "productId": "appwrite", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item references API versioning scheme or a documented deprecation policy; the openapi.json probe even returned 404s, suggesting no discoverable versioned API spec. No mention of version numbers, deprecation timelines, or migration guides for breaking changes.", "evidenceIds": [ "appwrite-probe-3" ] }, { "productId": "appwrite", "storyId": "auth-email-social", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly cover email/password login with Argon2 hashing in a few lines of code, plus OAuth sign-in with GitHub, Google, Facebook and 30+ providers via SDK, matching the story closely. Missing for 10: independent/hands-on developer confirmation of the 'minutes' setup time, as evidence is entirely vendor docs.", "evidenceIds": [ "appwrite-docs-1", "appwrite-docs-26", "appwrite-docs-29" ] }, { "productId": "appwrite", "storyId": "auth-mfa-sso", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Appwrite's docs explicitly cover implementing MFA (appwrite-docs-4) and OAuth-based logins with 30+ providers (appwrite-docs-1, appwrite-docs-26), satisfying the MFA half of the story. However, there is no evidence of enterprise SSO protocols like SAML or OIDC being supported — only generic OAuth (Google, GitHub, Facebook, etc.) is documented, which is not the same as enterprise-grade SAML/OIDC federation. Missing for 10: explicit SAML/OIDC enterprise SSO support, admin-enforced MFA/SSO policy controls, and any enterprise identity-provider integration docs.", "evidenceIds": [ "appwrite-docs-4", "appwrite-docs-1", "appwrite-docs-26" ] }, { "productId": "appwrite", "storyId": "automated-backups-pitr", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions automated backups, snapshotting, or point-in-time recovery features for Appwrite databases; the closest related material covers self-hosting, migrations between platforms, and general database product features, none of which address backup/PITR. Missing for 10: any mention of backup scheduling, retention policies, restore/point-in-time recovery tooling, or disaster-recovery documentation.", "evidenceIds": [] }, { "productId": "appwrite", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Appwrite's CLI can push/pull configuration and manage resources, and its MCP server lets LLMs call SDK methods, but there is no documentation of a bulk/batch API (e.g., batch create, update, or delete across many rows/files/users) that would let an AI-native user perform true bulk operations. Missing for 10: explicit bulk create/update/delete endpoints or CLI commands operating on many items in one call, and any hands-on confirmation of such batch semantics.", "evidenceIds": [ "appwrite-docs-12", "appwrite-docs-13", "appwrite-docs-42", "appwrite-docs-11" ] }, { "productId": "appwrite", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Appwrite Functions can be triggered automatically by server events, webhooks, HTTP requests, SDK calls, and scheduled executions, and the docs mention responding to server events in realtime — this supports rule-like automation on events. However, there's no evidence of a dedicated declarative 'rules engine' (e.g., conditional trigger definitions, if-this-then-that style automation) beyond function triggers. Missing for 10: a first-class rules/automation builder distinct from writing custom Function code, and independent/hands-on validation of event-triggered automation reliability.", "evidenceIds": [ "appwrite-docs-9", "appwrite-docs-28" ] }, { "productId": "appwrite", "storyId": "automation-scheduled-jobs", "verdict": "full", "quality": 6, "confidence": "medium", "rationale": "Appwrite Functions explicitly support 'scheduled executions' as a trigger type alongside HTTP, SDK, and webhook triggers, enabling recurring job/workflow automation directly in the platform (appwrite-docs-9). This is corroborated by the general functions/automation docs and CLI's ability to deploy functions programmatically (appwrite-docs-12, appwrite-docs-42), fitting an AI-native automation workflow.\n\nmissing for 10: no docs detailing cron syntax or scheduling UI specifics, no mention of multi-step workflow orchestration beyond single-function triggers, and no independent/hands-on confirmation that scheduled executions work reliably in practice.", "evidenceIds": [ "appwrite-docs-9", "appwrite-docs-12", "appwrite-docs-42" ] }, { "productId": "appwrite", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of any version history, review workflow, or rollback mechanism for automations/functions in Appwrite. The CLI supports deploying/managing functions and configs, but there's no documented versioning, diffing, review process, or rollback capability for automations. missing for 10: version history for functions/automations, review/approval workflow, rollback/revert mechanism, audit trail of automation changes.", "evidenceIds": [] }, { "productId": "appwrite", "storyId": "db-event-triggers", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Appwrite's docs explicitly state that Functions can be triggered by server events (e.g., database and auth events) in addition to HTTP, SDK, webhooks, and scheduled triggers, directly matching the story of running functions automatically on database/auth events. Missing for 10: no explicit enumeration of specific database/auth event types (e.g., document.create, user.create) and no independent/hands-on confirmation of event-triggered execution.", "evidenceIds": [ "appwrite-docs-9", "appwrite-docs-28" ] }, { "productId": "appwrite", "storyId": "direct-db-connection", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Appwrite documents a 'dedicated, native PostgreSQL database you connect to directly' as an option distinct from its managed API-based database, implying some direct-connection capability, but there is no explicit documentation of a connection string, standard driver, or ORM (e.g., Prisma, TypeORM) compatibility — the primary interface described elsewhere is via SDKs/REST APIs. Missing for 10: explicit connection-string format, confirmation of standard Postgres client/ORM compatibility, and independent corroboration of this direct-connect feature.", "evidenceIds": [ "appwrite-docs-8", "appwrite-docs-23", "appwrite-docs-36" ] }, { "productId": "appwrite", "storyId": "file-storage-cdn", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs confirm core file storage APIs (upload, download, delete, list, plus unspecified utilities) but the evidence pack never explicitly documents per-file access/permission rules or CDN-backed delivery for storage. Missing for 10: explicit documentation of file-level permission/access rules, CDN or edge-caching delivery mechanism, and any hands-on/independent confirmation of these behaviors.", "evidenceIds": [ "appwrite-docs-10" ] }, { "productId": "appwrite", "storyId": "fulltext-search-builtin", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes Appwrite's database as supporting typed columns, relationships, and generic 'indexes' (appwrite-docs-6, appwrite-docs-23) and vector similarity search for embeddings (appwrite-docs-7), but nowhere documents a full-text search capability (e.g., fulltext index type or search query operators). Since no evidence confirms this specific capability, the axis applies but is unmet.", "evidenceIds": [ "appwrite-docs-6", "appwrite-docs-7", "appwrite-docs-23" ] }, { "productId": "appwrite", "storyId": "image-transformations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack only says Storage provides upload/download/delete/list APIs with vague 'added utilities' (appwrite-docs-10), but nowhere documents an image resize/transform/preview endpoint that would let a developer serve transformed images without a custom pipeline.", "evidenceIds": [ "appwrite-docs-10" ] }, { "productId": "appwrite", "storyId": "local-stack-emulator", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Appwrite is designed for self-hosting via Docker CLI and one-click marketplace installs, and the Appwrite CLI can manage projects, resources, and functions, including non-interactively — supporting local backend development before deploying to cloud. However, there is no explicit emulator/local-stack CLI command (like 'appwrite emulators start') mirroring the full backend for offline testing, and no direct evidence of a documented local dev workflow tying CLI + self-hosted Docker together as a single 'local emulator' story. missing for 10: dedicated local-emulator tooling or docs explicitly describing running the full stack locally via CLI for pre-deploy testing, independent hands-on confirmation of this workflow.", "evidenceIds": [ "appwrite-docs-14", "appwrite-docs-19", "appwrite-docs-25", "appwrite-docs-33", "appwrite-docs-12", "appwrite-docs-42", "appwrite-probe-5" ] }, { "productId": "appwrite", "storyId": "managed-database-queries", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Appwrite's Databases docs describe managed (shared-pool or dedicated) databases with permissions and realtime access (appwrite-docs-23, appwrite-docs-36), structured relational data with typed columns and indexes (appwrite-docs-6), and even a dedicated native Postgres option (appwrite-docs-8). The CLI can generate type-safe SDKs tied to your database schema for both client and server use (appwrite-docs-41), and Appwrite's core SDK model (client/server) is implied throughout its docs ecosystem. Missing for 10: an explicit code sample showing client vs server SDK read/write calls, and independent hands-on verification of database read/write reliability beyond vendor docs.", "evidenceIds": [ "appwrite-docs-6", "appwrite-docs-8", "appwrite-docs-23", "appwrite-docs-36", "appwrite-docs-41" ] }, { "productId": "appwrite", "storyId": "offline-first-sync", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items mention offline support, local write queuing, conflict resolution, or sync-on-reconnect capabilities; documentation only covers realtime server events broadcast, not client-side offline persistence. Appwrite's realtime features (appwrite-docs-5, appwrite-docs-28) address live sync while connected, but nothing indicates an offline-first SDK or local cache with automatic reconciliation.", "evidenceIds": [ "appwrite-docs-5", "appwrite-docs-28", "appwrite-docs-23" ] }, { "productId": "appwrite", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Appwrite's docs confirm the CLI/API can create and manage projects, resources (rows, files, users), and deploy Functions—core operations that mirror UI actions—and the platform is explicitly API/SDK-first with MCP and CLI tooling for automation (appwrite-docs-12, appwrite-docs-18, appwrite-docs-32, appwrite-probe-4, appwrite-probe-5). However, there's no explicit vendor statement or independent verification of full UI-API parity, and the OpenAPI spec probe returned 404s, leaving some doubt about complete coverage of every UI action. Missing for 10: an explicit API-completeness statement, a working public OpenAPI/spec reference, and independent confirmation that all UI-only settings (e.g., billing, org management) are also API-accessible.", "evidenceIds": [ "appwrite-docs-12", "appwrite-docs-18", "appwrite-docs-32", "appwrite-probe-4", "appwrite-probe-5", "appwrite-probe-3" ] }, { "productId": "appwrite", "storyId": "openness-full-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Appwrite documents self-hosting with 'full control over data location' and one-click migrations between Appwrite Cloud and self-hosted (and from Firebase/Supabase/NHost), plus a CLI that manages/exports resources (rows, files, users) into JSON files, all supporting data portability and open formats. However, there is no single documented 'export all my data' feature or explicit open-format export guarantee spanning all resource types. Missing for 10: an explicit full-data export/backup command or documented complete data-portability workflow, and independent verification that migrations/export are lossless and comprehensive.", "evidenceIds": [ "appwrite-docs-43", "appwrite-docs-14", "appwrite-docs-19", "appwrite-docs-24", "appwrite-docs-12", "appwrite-docs-13", "appwrite-docs-18" ] }, { "productId": "appwrite", "storyId": "openness-open-license", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Appwrite's own docs and llms.txt describe it as an 'open-source backend platform' and the project is hosted publicly on GitHub (appwrite-gh-1), with extensive self-hosting documentation confirming the source is available for anyone to run and inspect (appwrite-docs-14, appwrite-docs-19, appwrite-probe-1). This satisfies the core story that an AI-native user can read the source under an open license. Missing for 10: explicit citation of the specific license name (e.g. BSD-3-Clause) or a link to a LICENSE file, and independent confirmation of license terms beyond self-hosting claims.", "evidenceIds": [ "appwrite-gh-1", "appwrite-docs-14", "appwrite-docs-19", "appwrite-probe-1" ] }, { "productId": "appwrite", "storyId": "openness-self-host", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Appwrite documents itself as designed from the ground up for self-hosting via Docker, with one-click marketplace deployments, full control over data location, and explicit self-hosted CLI/endpoint support; community evidence corroborates a strong self-hosting experience. missing for 10: independent hands-on verification of feature parity between cloud and self-hosted beyond docs/community anecdotes.", "evidenceIds": [ "appwrite-docs-14", "appwrite-docs-15", "appwrite-docs-19", "appwrite-docs-24", "appwrite-docs-25", "appwrite-docs-34", "appwrite-docs-22", "appwrite-comm-1" ] }, { "productId": "appwrite", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Appwrite's self-hosting docs explicitly state 'full control over data location' and that it can be installed on any Docker-capable infrastructure, letting AI-native users choose where their data resides by hosting themselves (appwrite-docs-24, appwrite-docs-14/19/33). However, there is no evidence of a managed-cloud region-selection feature (e.g., picking an AWS/GCP region in Appwrite Cloud) for users who don't want to self-host. Missing for 10: cloud region/residency selector, explicit list of supported regions for hosted Appwrite, and independent confirmation of data-residency guarantees.", "evidenceIds": [ "appwrite-docs-24", "appwrite-docs-14", "appwrite-docs-19", "appwrite-docs-33", "appwrite-docs-34" ] }, { "productId": "appwrite", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses AI-training data usage policies, opt-out mechanisms, or any explicit privacy commitment regarding model training; self-hosting gives data control but that's not the same as an explicit no-AI-training policy. As a backend platform, this axis is plausible (buyers could ask whether their hosted data is used for AI training), so absence of evidence yields 'none'.", "evidenceIds": [] }, { "productId": "appwrite", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Appwrite exposes basic delete APIs for files (appwrite-docs-10) and CLI-driven resource management (users, rows, files) (appwrite-docs-12/18/32), and self-hosting gives 'full control over data location' (appwrite-docs-24), which together let an operator remove or relocate data. However there is no documented retention-policy tooling (TTL/expiry rules), no explicit GDPR-style 'delete all user data' workflow, and no data-export/erasure certification beyond generic SOC-2/HIPAA/BAA mentions (appwrite-docs-20). Missing for 10: explicit retention/expiry configuration, documented right-to-erasure workflow, and independent verification of deletion completeness.", "evidenceIds": [ "appwrite-docs-10", "appwrite-docs-12", "appwrite-docs-18", "appwrite-docs-24", "appwrite-docs-20" ] }, { "productId": "appwrite", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry opt-out or usage tracking controls for Appwrite; while self-hosting (full data control) is documented, there's no explicit statement about disabling telemetry/analytics collection.", "evidenceIds": [] }, { "productId": "appwrite", "storyId": "realtime-presence", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Appwrite's docs explicitly describe tracking online/typing/viewing presence and broadcasting realtime status between clients, backed by a general realtime event system for databases and server events. Missing for 10: independent hands-on verification of presence/typing broadcast specifically (only vendor docs cited), and no code example showing ephemeral state channels.", "evidenceIds": [ "appwrite-docs-5", "appwrite-docs-28", "appwrite-docs-23" ] }, { "productId": "appwrite", "storyId": "realtime-subscriptions", "verdict": "full", "quality": 6, "confidence": "medium", "rationale": "Appwrite's docs explicitly reference realtime capabilities for databases (broadcasting online/typing/viewing status, 'managed APIs with permissions and realtime', and 'respond to server events in realtime'), indicating a live-subscription mechanism for UI updates without polling. missing for 10: no first-party API/SDK code example for subscribing to database document changes, and no independent/hands-on verification of the realtime channel actually pushing updates.", "evidenceIds": [ "appwrite-docs-5", "appwrite-docs-23", "appwrite-docs-28" ] }, { "productId": "appwrite", "storyId": "relational-queries-joins", "verdict": "partial", "quality": 2, "confidence": "low", "rationale": "Appwrite's docs mention 'relationships' between typed columns/rows (appwrite-docs-6), implying some relational modeling, but there is no evidence of an actual join query syntax, constraint enforcement (e.g., foreign key/unique constraints), or multi-statement transactions. The dedicated PostgreSQL option (appwrite-docs-8) is a separate native database connection, not Appwrite's own query API, so it doesn't substantiate this capability either. missing for 10: explicit join query syntax/examples, constraint definitions (unique/foreign key), transactional guarantees or ACID transaction support, independent verification of any of these.", "evidenceIds": [ "appwrite-docs-6", "appwrite-docs-8", "appwrite-docs-23" ] }, { "productId": "appwrite", "storyId": "row-level-security", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Appwrite docs mention databases with 'permissions' as part of managed APIs, implying per-resource access control, but the evidence pack lacks detail on document/row-level permission rules, role-based read/write scoping, or policy syntax. Missing for 10: explicit documentation of row/document-level permission model, examples of user/team/role-based access rules, and any hands-on confirmation of enforcement.", "evidenceIds": [ "appwrite-docs-23" ] }, { "productId": "appwrite", "storyId": "serverless-functions", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Appwrite Functions are explicitly documented as serverless compute triggered by HTTP requests, SDK methods, server events, webhooks, and scheduled executions, and the CLI supports creating/deploying these functions, matching the story directly. Missing for 10: independent hands-on verification of function deployment/runtime behavior, and more detail on scaling/cold-start characteristics.", "evidenceIds": [ "appwrite-docs-9", "appwrite-docs-12", "appwrite-docs-32" ] }, { "productId": "appwrite", "storyId": "type-generation", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Appwrite's CLI has a documented `generate` command that detects the project's language and generates a type-safe SDK with typed helpers based on the database schema, directly matching the story of generating typed client code from schema. Missing for 10: no independent/hands-on corroboration of the generate command's output quality, and no detail on which languages/types are supported beyond TypeScript.", "evidenceIds": [ "appwrite-docs-41" ] }, { "productId": "appwrite", "storyId": "vector-similarity-search", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Appwrite's docs explicitly claim to \"Store embeddings and run similarity search to power AI features\" as part of its database product, but there is no detail on implementation (index types, distance metrics, API specifics) or any independent/hands-on corroboration that this works in practice. missing for 10: technical details on vector search API/config, independent or hands-on validation, performance/scale characteristics.", "evidenceIds": [ "appwrite-docs-7", "appwrite-docs-23", "appwrite-docs-37" ] }, { "productId": "appwrite", "storyId": "versioned-migrations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Appwrite's CLI can manage and push project resources (collections/attributes) defined in JSON config files, and supports a documented non-interactive/headless mode explicitly for CI automation, which covers schema-as-code applied via CLI/CI. However, there's no evidence of a dedicated migrations framework with versioned history, diffing, or rollback semantics — only generic 'push resources' and 'one-click migrate from other platforms' (data migration, not schema versioning). Missing for 10: explicit migration/version-history commands, rollback support, and independent confirmation of CI-based schema migration workflows in practice.", "evidenceIds": [ "appwrite-docs-12", "appwrite-docs-13", "appwrite-docs-18", "appwrite-docs-42", "appwrite-docs-43", "appwrite-docs-22" ] }, { "productId": "convex", "storyId": "agent-branch-migrations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Convex's CLI/dev deployment model (docs-6/17) automatically syncs schema changes to a separate dev deployment when developing, and the official MCP server explicitly blocks access to production deployments by default as a safety measure (docs-28), which together let an agent iterate on schema changes in an isolated, non-production environment. However, there is no explicit mention of a distinct 'branch' or 'preview environment' feature, nor documentation of testing/promotion workflow from that isolated environment into production. missing for 10: explicit preview/branch deployment feature, agent-driven promotion workflow from dev to production, independent corroboration of this workflow.", "evidenceIds": [ "convex-docs-6", "convex-docs-17", "convex-docs-28", "convex-docs-10" ] }, { "productId": "convex", "storyId": "agent-provisions-backend", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Convex's CLI (`npx convex dev/run/import/deploy`) and code-first schema/auth model mean an agent could script much of a backend setup, and the MCP server lets agents interact with a deployment programmatically. However, there is no documented example of a fully headless script that provisions auth (Clerk/WorkOS typically require external dashboard setup) and authorization policies end-to-end without manual/interactive steps. Missing for 10: a documented headless script or MCP flow that configures auth providers and authorization policies (not just schema/functions) without human interaction, and evidence this is done end-to-end in practice.", "evidenceIds": [ "convex-docs-4", "convex-docs-5", "convex-docs-15", "convex-docs-25", "convex-docs-32", "convex-docs-17", "convex-docs-18", "convex-docs-38", "convex-docs-10", "convex-probe-4" ] }, { "productId": "convex", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Convex confirmed hosts a live llms.txt (HTTP 200) plus a dedicated agent-setup.md file explicitly aimed at making 'your agent a Convex expert,' directly satisfying the story. missing for 10: no independent/third-party confirmation of agents actually consuming these files successfully, and per-page .md docs return 404 so coverage is limited to the root llms.txt/agent-setup files rather than full doc-tree markdown mirrors.", "evidenceIds": [ "convex-probe-1", "convex-docs-20", "convex-probe-2" ] }, { "productId": "convex", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Convex provides infrastructure (vector search, MCP server) that lets developers build AI features on top of their data, and a CLI 'insights' command that surfaces deployment health metrics — but there is no evidence Convex itself surfaces AI-generated insights or suggestions about the user's actual application data inside the product.", "evidenceIds": [ "convex-docs-27", "convex-docs-7", "convex-docs-10" ] }, { "productId": "convex", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack has no mention of Convex scheduled functions, cron jobs, or any mechanism for autonomous background automation; it only covers reactive queries, HTTP endpoints, and an MCP server for interactive agent access to a deployment. Missing for 10: any citation of cron/scheduled functions, background job triggers, or autonomous workflow execution.", "evidenceIds": [] }, { "productId": "convex", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Convex ships an MCP server and CLI that let external AI agents interact with a Convex deployment, and docs mention it 'works great with coding agents,' but this is agent-to-Convex tooling, not a built-in assistant embedded inside Convex itself that a user can delegate tasks to. No evidence of an in-product chat/assistant feature exists in the pack.", "evidenceIds": [ "convex-docs-10", "convex-docs-23", "convex-docs-28" ] }, { "productId": "convex", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Convex ships a documented CLI (`npx convex run`, `npx convex import`, `npx convex deploy` implied, `npx convex logs`) and HTTP API for calling functions, both of which are automatable/scriptable in CI, plus self-hosting options for full non-interactive control. However, there is no explicit documentation of a headless/non-interactive CI mode, no CI recipe or GitHub Actions example, and the dev CLI is described as a filesystem-watching interactive workflow rather than an explicit CI-first mode. missing for 10: explicit CI/automation docs or examples (e.g., GitHub Actions integration, non-interactive deploy flags), confirmation that `npx convex deploy` runs fully headlessly without auth prompts.", "evidenceIds": [ "convex-docs-18", "convex-docs-26", "convex-docs-34", "convex-docs-9", "convex-docs-31", "convex-docs-19", "convex-gh-1", "convex-probe-5" ] }, { "productId": "convex", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All MCP-related evidence describes Convex shipping its OWN MCP server so external AI agents can call Convex's tools (convex-docs-10, convex-probe-4, convex-docs-28) — this is the opposite direction from the story, which asks whether Convex itself can plug in and consume external MCP servers' tools. No evidence shows Convex acting as an MCP client or invoking third-party MCP tool servers from its backend/actions.", "evidenceIds": [ "convex-docs-10", "convex-probe-4", "convex-docs-28" ] }, { "productId": "convex", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Convex ships an official documented MCP server (docs.convex.dev/ai/convex-mcp-server) that exposes tools for AI agents to interact with a Convex deployment, with a safety note that it restricts production access by default, and this is corroborated by a direct probe confirming the doc page exists. Convex is a backend/database product (not itself an agent), so serving an MCP server is squarely within its category. Missing for 10: independent/hands-on community confirmation of the MCP server working in practice, and more detail on the specific tool set exposed.", "evidenceIds": [ "convex-docs-10", "convex-docs-28", "convex-probe-4" ] }, { "productId": "convex", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Convex offers an official MCP server so AI agents can inspect/query deployments in natural language, plus agent-setup guidance and messaging that it 'works great with coding agents' for scaffolding projects via natural-language requests. However, this relies entirely on external coding agents/MCP clients rather than a built-in NL interface, and production access is restricted by default. Missing for 10: a first-party conversational/NL interface within Convex itself, and independent hands-on evidence of natural-language operation succeeding end-to-end.", "evidenceIds": [ "convex-docs-10", "convex-docs-20", "convex-docs-23", "convex-docs-28", "convex-probe-4" ] }, { "productId": "convex", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Convex ships an official CLI (npx convex ...) with documented commands for dev workflow, running functions, importing data, logs, and insights, and it explicitly integrates with AI-native workflows via agent setup docs and MCP tooling. missing for 10: independent/hands-on third-party corroboration of the CLI's AI-native ergonomics beyond first-party docs, and no explicit versioning/stability notes for CLI commands.", "evidenceIds": [ "convex-docs-6", "convex-docs-17", "convex-docs-18", "convex-docs-26", "convex-docs-27", "convex-docs-34", "convex-docs-20", "convex-probe-5" ] }, { "productId": "convex", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Convex documents a public HTTP API that exposes deployment functions as callable endpoints, a full-featured CLI for scripting deployments, and an official MCP server specifically for AI agents to interact with the deployment — together giving an AI-native user multiple documented, programmatic ways to drive the product. Missing for 10: a formal OpenAPI/swagger spec (probes found only 404s) and independent hands-on corroboration of third-party API usage.", "evidenceIds": [ "convex-docs-9", "convex-docs-31", "convex-docs-18", "convex-docs-10", "convex-docs-28", "convex-probe-5", "convex-probe-4", "convex-probe-3" ] }, { "productId": "convex", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The only relevant evidence is that Convex's MCP server is restricted from production deployments by default as a safety measure, which is a coarse, binary dev/prod restriction rather than true scoped/least-privilege credential issuance for agents (e.g., granular API keys, role-based tokens, or fine-grained permission scopes). Missing for 10: documented ability to mint scoped API keys/tokens per agent, granular permission models, or per-function/table credential scoping.", "evidenceIds": [ "convex-docs-28", "convex-docs-10" ] }, { "productId": "convex", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Convex ships official SDKs/framework integrations for React, Next.js, Remix, TanStack Start, React Native, Vue, Nuxt, and Svelte, backed by typed client libraries, CLI tooling, and generated types, plus explicit AI-native tooling (MCP server, agent-setup guide) that lets AI-native developers build against these SDKs directly. Missing for 10: independent/hands-on third-party verification of SDK quality/completeness beyond vendor docs, and no OpenAPI/REST spec for non-JS ecosystems.", "evidenceIds": [ "convex-docs-39", "convex-docs-38", "convex-docs-17", "convex-docs-18", "convex-docs-10", "convex-docs-20", "convex-probe-4", "convex-probe-5" ] }, { "productId": "convex", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Convex exposes HTTP endpoints so external clients can call Convex functions (docs-9, docs-31), but there is no evidence of an outbound webhook mechanism where Convex notifies subscribers of database/event changes. This is a fair capability for a reactive backend to offer, but no such feature is documented.", "evidenceIds": [ "convex-docs-9", "convex-docs-31" ] }, { "productId": "convex", "storyId": "ai-dashboard-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows an official MCP server that lets external AI agents (e.g., coding assistants in an IDE) query/mutate/debug a Convex deployment, but nothing describes an AI assistant embedded within the Convex dashboard UI itself for writing queries, debugging errors, or explaining the backend. The story specifically asks about an in-dashboard assistant, which is not evidenced.", "evidenceIds": [ "convex-docs-10", "convex-docs-28", "convex-docs-20" ] }, { "productId": "convex", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of an interactive API reference or runnable-example playground; probes explicitly show no OpenAPI/Swagger spec (404s) and only static docs pages plus an llms.txt for AI consumption. Convex offers CLI, MCP server, and HTTP API docs, but nothing indicating a browsable, runnable API explorer.", "evidenceIds": [ "convex-probe-2", "convex-probe-3", "convex-docs-9", "convex-docs-31" ] }, { "productId": "convex", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe for OpenAPI/swagger spec files at all standard candidate paths returned 404, and no documentation mentions a downloadable API spec despite Convex exposing HTTP endpoints for functions.", "evidenceIds": [ "convex-probe-3", "convex-docs-9", "convex-docs-31" ] }, { "productId": "convex", "storyId": "api-sandbox", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Convex has separate dev/production deployments (npx convex dev pushes to dev deployment, not prod) and the official MCP server explicitly blocks access to production deployments by default as a safety measure to prevent accidental modifications to production data, directly enabling sandboxed testing for AI agents. missing for 10: independent/hands-on corroboration of this safety boundary actually holding in practice, and no explicit mention of a dedicated 'staging' environment distinct from dev/prod.", "evidenceIds": [ "convex-docs-28", "convex-docs-10", "convex-docs-6", "convex-docs-18" ] }, { "productId": "convex", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of API versioning scheme or a documented deprecation policy; OpenAPI spec probe returned 404s and no changelog/versioning docs are cited.", "evidenceIds": [ "convex-probe-3" ] }, { "productId": "convex", "storyId": "auth-email-social", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Convex docs confirm Convex Auth library supports password, social OAuth, OTP email/SMS sign-in, plus recommended third-party providers like Clerk/Auth0/WorkOS with SDK integration guidance for React/Next.js/React Native, but the evidence doesn't show a single unified 'minutes' quick-start flow or hands-on confirmation of setup speed. missing for 10: a concrete quick-start walkthrough/tutorial demonstrating minutes-level setup, independent developer testimony confirming ease/speed, and explicit code snippets for combined email/password + OAuth configuration.", "evidenceIds": [ "convex-docs-4", "convex-docs-5", "convex-docs-15", "convex-docs-16" ] }, { "productId": "convex", "storyId": "auth-mfa-sso", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Convex docs describe auth via third-party providers (Clerk, WorkOS AuthKit, Auth0) or its own Convex Auth library for social/email/password sign-in, but there is no evidence of built-in enterprise SSO (SAML/OIDC) or MFA enforcement controls for a platform engineer to configure directly; these would depend entirely on the chosen third-party provider's capabilities, which aren't documented here.", "evidenceIds": [ "convex-docs-4", "convex-docs-5", "convex-docs-15", "convex-docs-16" ] }, { "productId": "convex", "storyId": "automated-backups-pitr", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The pricing page lists 'Daily backups' as an included feature on paid plans, confirming some automated backup capability, but there is no mention of point-in-time recovery, backup retention windows, restore process, or how backups work for self-hosted deployments. Missing for 10: point-in-time recovery details, backup restore/testing docs, retention policy specifics, and independent verification of backup reliability.", "evidenceIds": [ "convex-docs-37" ] }, { "productId": "convex", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Convex supports bulk data import via `npx convex import` and lets mutations/queries run arbitrary JS over many documents, and the MCP server lets AI agents call these functions on a deployment. However, there's no documented explicit bulk-write/update/delete API, batch size limits, or a concrete example of an AI agent performing a bulk operation end-to-end. missing for 10: explicit bulk mutation/update/delete API docs, batch operation guarantees/limits, hands-on evidence of an AI agent executing bulk operations via MCP or CLI.", "evidenceIds": [ "convex-docs-26", "convex-docs-10", "convex-docs-18", "convex-docs-9" ] }, { "productId": "convex", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes Convex's reactive query model (queries automatically re-run when underlying data changes) and HTTP endpoints for calling functions, but there is no mention of user-definable rules, triggers, cron jobs, or scheduled/event-driven actions that a user could configure to fire automatically on arbitrary events. Missing for 10: any documentation of scheduled functions, cron/triggers, or a rule-definition mechanism for automated actions on events.", "evidenceIds": [ "convex-comm-2", "convex-comm-3", "convex-comm-4", "convex-docs-9", "convex-docs-31" ] }, { "productId": "convex", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Convex's reactive database, auth, CLI, MCP server, and self-hosting, but contains no mention of cron jobs, scheduled functions, or recurring workflow scheduling capabilities, which is a plausible and often-touted Convex feature but is absent from this evidence set.", "evidenceIds": [] }, { "productId": "convex", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Convex's docs describe deployments, CLI commands (deploy, logs, insights), and self-hosting, but there is no evidence of any built-in versioning, review, or rollback mechanism for functions/automations — deployments appear to be driven by git-based code pushes with no rollback tooling mentioned. missing for 10: deployment version history, rollback command, and any review/approval workflow for changes.", "evidenceIds": [ "convex-docs-6", "convex-docs-17", "convex-docs-18", "convex-docs-27", "convex-docs-34" ] }, { "productId": "convex", "storyId": "db-event-triggers", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence describes Convex's reactive query model (queries re-run when underlying data changes) and HTTP endpoints for calling functions, but there is no documentation of database/auth event triggers (e.g., 'on insert', 'on user created') that automatically invoke backend functions. Reactivity for client queries is a different mechanism than server-side function triggers implied by the story.", "evidenceIds": [] }, { "productId": "convex", "storyId": "direct-db-connection", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Convex is a TypeScript-native document database with its own query functions and explicitly states 'no need to write any SQL' (convex-docs-2), and self-hosting docs only mention using external stores like Postgres/Neon as backing infra, not exposing Convex data via standard connection strings or ORMs. No evidence of ODBC/JDBC/SQL client or ORM connection-string support is present anywhere in the pack.", "evidenceIds": [ "convex-docs-2", "convex-docs-8", "convex-gh-1" ] }, { "productId": "convex", "storyId": "file-storage-cdn", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of file upload, file storage APIs, access-controlled file serving, or CDN-backed delivery anywhere in the Convex docs, CLI, or community citations provided. While file storage is a plausible axis for a backend platform, nothing in this evidence pack substantiates it.", "evidenceIds": [] }, { "productId": "convex", "storyId": "fulltext-search-builtin", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack repeatedly documents Convex's vector search for embeddings-based similarity, but contains no mention of a full-text/keyword search index feature or API for text search without external tooling. Missing for 10: any docs reference to a full-text search index, search() query API, or tokenization/ranking behavior.", "evidenceIds": [ "convex-docs-7", "convex-docs-14", "convex-docs-29", "convex-docs-35" ] }, { "productId": "convex", "storyId": "image-transformations", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions Convex file storage supporting image resizing/transformation or any built-in image pipeline; storage-related evidence is entirely absent. This is a fair axis for a backend/storage platform, so lack of evidence yields 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "convex", "storyId": "local-stack-emulator", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Convex's standard CLI workflow (`npx convex dev`) watches your filesystem and pushes changes, but pushes to a cloud-hosted 'dev deployment' rather than a fully local emulator (convex-docs-6, convex-docs-17). Separately, Convex offers self-hosting where you 'run the Convex backend on your own servers' with 'the same fully up-to-date code the cloud service uses,' which could be used to run the whole stack locally before deploying to production (convex-docs-8, convex-docs-19, convex-gh-1), but the docs frame this as an alternative deployment target rather than an explicit local pre-deploy emulator/test loop. Missing for 10: explicit documentation of a local emulator mode built into the standard dev CLI workflow, and hands-on/community confirmation that self-hosted mode is commonly used as a local pre-deploy sandbox.", "evidenceIds": [ "convex-docs-6", "convex-docs-17", "convex-docs-8", "convex-docs-19", "convex-gh-1" ] }, { "productId": "convex", "storyId": "managed-database-queries", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Convex is a managed reactive database with no-setup provisioning (npm create convex@latest), automatic table creation, and client/server SDKs across React, Next.js, Remix, Vue, Svelte, React Native etc., plus HTTP endpoints for server-side access; ACID transactions and real-time sync are documented, and community/HN commentary corroborates the reactivity model internals. missing for 10: no independent hands-on benchmark of read/write latency or large-scale production usage report beyond docs and one HN thread.", "evidenceIds": [ "convex-docs-1", "convex-docs-2", "convex-docs-13", "convex-docs-21", "convex-docs-12", "convex-docs-31", "convex-docs-39", "convex-docs-38", "convex-comm-2", "convex-comm-3" ] }, { "productId": "convex", "storyId": "offline-first-sync", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Convex's evidence pack focuses on real-time reactive sync, ACID transactions, and cloud/self-hosted deployment, but nothing describes offline support, local persistence, or conflict-resolution/sync-on-reconnect for offline writes. Convex's model is explicitly server-driven reactivity (queries re-run on server-side data changes), which is a different capability than offline-first local writes.", "evidenceIds": [ "convex-comm-2", "convex-comm-3", "convex-docs-21", "convex-docs-22" ] }, { "productId": "convex", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Convex exposes deployment functions via HTTP API and offers a CLI (run, import, logs, insights) plus an MCP server for AI agents to interact with deployments, showing broad but not total API/CLI parity with the dashboard UI. There's no explicit documentation claiming full UI/API parity, and probes found no OpenAPI spec, suggesting some dashboard-only features (e.g., visual data browsing, deployment settings) may lack a documented API equivalent. missing for 10: explicit parity statement, full OpenAPI/reference coverage, evidence that all dashboard actions (schema editing, table browsing, settings) are scriptable via API/CLI.", "evidenceIds": [ "convex-docs-9", "convex-docs-31", "convex-docs-18", "convex-docs-26", "convex-docs-27", "convex-docs-34", "convex-docs-10", "convex-probe-3", "convex-probe-5" ] }, { "productId": "convex", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Convex documents open-source self-hosting (same backend as cloud) and an `npx convex import` CLI command, suggesting data portability and no vendor lock-in, but the evidence pack never documents an explicit data-export command or open export format (e.g., JSON/CSV dump) for leaving the platform. missing for 10: documented export command/tool, explicit open-format export (JSON/CSV), independent confirmation of a full data-export workflow.", "evidenceIds": [ "convex-docs-8", "convex-docs-19", "convex-docs-26", "convex-gh-1" ] }, { "productId": "convex", "storyId": "openness-open-license", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Convex explicitly documents itself as open source, with a public GitHub repo (get-convex/convex-backend) for the self-hosted backend containing 'the same fully up-to-date code the cloud service uses,' and Convex Components are described as open-source building blocks — giving AI-native users direct access to readable source under an open license. missing for 10: explicit statement of the specific open-source license terms and independent confirmation of license permissiveness.", "evidenceIds": [ "convex-docs-8", "convex-docs-19", "convex-docs-30", "convex-gh-1", "convex-docs-11" ] }, { "productId": "convex", "storyId": "openness-self-host", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Convex explicitly supports self-hosting the open-source backend on your own servers, with documentation confirming it runs the same code as the cloud service and integrates with common infra (Neon, Fly.io, Postgres, RDS, Sqlite, etc.). Missing for 10: independent hands-on reports of self-hosting at scale in production and more detail on feature parity/limitations versus the cloud offering.", "evidenceIds": [ "convex-docs-8", "convex-docs-19", "convex-docs-30", "convex-docs-36", "convex-gh-1" ] }, { "productId": "convex", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of region/data-residency selection controls in Convex's cloud offering; self-hosting is mentioned but that requires managing your own infrastructure rather than choosing a region within Convex's service, and no docs describe region pinning or residency options. missing for 10: any documentation of region/residency selection, data locality controls, or compliance-driven region choice in the managed cloud product.", "evidenceIds": [ "convex-docs-8", "convex-docs-19", "convex-docs-30", "convex-docs-36", "convex-gh-1", "convex-docs-37" ] }, { "productId": "convex", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Convex is a backend database/platform; it has no AI model training on user data to opt out of, and no evidence pack item addresses AI-training data usage or opt-out policy. This is a category error for a database product — the story would apply to AI model vendors, not to a data infrastructure tool.", "evidenceIds": [] }, { "productId": "convex", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Convex offers self-hosting (full control over your own data infrastructure) and lists HIPAA/SOC 2 reports plus daily backups as enterprise features, which implies some data governance controls, but there is no explicit documentation of user-facing data retention policies, deletion APIs, or right-to-erasure workflows. missing for 10: explicit data deletion/retention API or policy docs, GDPR/right-to-erasure support, retention configuration options.", "evidenceIds": [ "convex-docs-8", "convex-docs-19", "convex-docs-30", "convex-docs-37" ] }, { "productId": "convex", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item discusses telemetry, usage analytics, or an opt-out mechanism in Convex CLI/backend/docs; self-hosting is offered but that's a deployment choice, not a documented telemetry toggle.", "evidenceIds": [] }, { "productId": "convex", "storyId": "realtime-presence", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Convex's reactive query/mutation model and real-time sync primitives (automatic re-run of queries on data change, ACID mutations) provide the underlying building blocks needed to implement presence and ephemeral broadcast patterns, but the evidence pack contains no explicit presence feature, example, or component demonstrating this use case. Missing for 10: a documented presence/broadcast component or example, guidance on ephemeral (non-persisted) state handling, and TTL/expiry mechanisms for stale presence data.", "evidenceIds": [ "convex-docs-21", "convex-docs-22", "convex-comm-2", "convex-comm-3", "convex-comm-4", "convex-docs-11" ] }, { "productId": "convex", "storyId": "realtime-subscriptions", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Convex is built around reactive queries that automatically re-run and push updates to clients when underlying data changes, with first-party docs and detailed engineering explanations of the readset/writeset invalidation mechanism confirming no polling is used. Independent community corroboration from Convex engineers and outside commenters backs the reactivity model as real and functioning. Missing for 10: independent third-party benchmark or large-scale production case study explicitly measuring real-time update latency/reliability at scale.", "evidenceIds": [ "convex-docs-21", "convex-docs-22", "convex-comm-2", "convex-comm-3", "convex-comm-4", "convex-comm-5" ] }, { "productId": "convex", "storyId": "relational-queries-joins", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Convex explicitly forgoes SQL and relational joins ('no need to write any SQL'), instead letting developers express multi-read/compute logic in TypeScript query functions (convex-comm-2) and enforcing ACID transactions (convex-docs-12) and optional schema shape validation (convex-docs-3). There is no documented declarative join operator or relational constraint system (foreign keys, uniqueness enforcement) — joins/constraints must be hand-coded in application logic rather than expressed as first-class relational queries.\nmissing for 10: native join syntax/operator, declarative relational constraints (foreign keys/uniqueness), independent verification of complex multi-table query patterns.", "evidenceIds": [ "convex-docs-2", "convex-docs-3", "convex-docs-12", "convex-comm-2", "convex-comm-3" ] }, { "productId": "convex", "storyId": "row-level-security", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Convex explicitly rejects an RLS-style authorization framework, instead relying on developers to write access checks manually inside each query/mutation function ('Convex doesn't need an opinionated authorization framework like RLS... This flexibility lets you build and use an authorization framework for your needs'). This means per-user data restriction is possible but must be hand-coded per function rather than declared as reusable security rules/policies, unlike a dedicated RLS feature. missing for 10: no built-in declarative row-level security policy syntax, no documentation of a reusable rules layer, no independent/hands-on evidence of implementing or testing such access control.", "evidenceIds": [ "convex-docs-25", "convex-docs-32" ] }, { "productId": "convex", "storyId": "serverless-functions", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Convex functions (queries, mutations, actions) are deployed serverlessly and can be invoked via SDK calls or exposed as public HTTP endpoints, with no server management required — the platform handles deployment, scaling, and infrastructure. Docs confirm public HTTP endpoints for calling functions and SDK-based invocation is core to the product. missing for 10: independent hands-on benchmarking of production HTTP invocation latency/scaling behavior, and more detail on action-specific server-side execution guarantees beyond docs claims.", "evidenceIds": [ "convex-docs-9", "convex-docs-31", "convex-docs-18", "convex-docs-8", "convex-docs-30" ] }, { "productId": "convex", "storyId": "type-generation", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Convex's CLI dev process watches schema/function changes and automatically updates generated TypeScript types in convex/_generated, giving end-to-end type safety from schema to client code without manual codegen commands. This is well documented in official docs and reinforced by the 'end-to-end type safety' marketing claim. Missing for 10: independent/hands-on developer confirmation of the generated types workflow beyond first-party docs.", "evidenceIds": [ "convex-docs-17", "convex-docs-6", "convex-docs-33" ] }, { "productId": "convex", "storyId": "vector-similarity-search", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Convex documents native vector search over stored documents specifically framed for embeddings and LLM context/recommendations use cases (convex-docs-7, convex-docs-14, convex-docs-29, convex-docs-35), integrated directly into its database without extra infrastructure. Missing for 10: independent hands-on benchmarks or community validation of vector search at scale, and details on index configuration/limits.", "evidenceIds": [ "convex-docs-7", "convex-docs-14", "convex-docs-29", "convex-docs-35" ] }, { "productId": "convex", "storyId": "versioned-migrations", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Convex schemas are defined in code (schema.ts) and the CLI automatically pushes schema changes to a deployment when files change (convex-docs-6/17), which gives an implicit, version-controllable schema workflow, but the evidence never describes a dedicated migrations system (versioned migration files, up/down scripts) or explicit CI/CD deployment pipeline usage. missing for 10: explicit migration versioning/history tooling, documented CI integration for applying schema changes, rollback support.", "evidenceIds": [ "convex-docs-3", "convex-docs-6", "convex-docs-17", "convex-docs-18" ] }, { "productId": "firebase", "storyId": "agent-branch-migrations", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Firebase's Local Emulator Suite lets developers (or an agent driving the CLI) test Firestore/Security Rules changes locally without touching production data, and the MCP server exposes Firestore project/data management to AI agents — together giving a rough isolated-testing story. However, Firestore is schemaless (no formal schema-migration concept) and there is no documented branch/preview-environment feature (e.g., a project 'branch' with automatic promotion) analogous to database branching tools. Missing for 10: explicit schema/migration versioning, a true branch/preview-environment workflow, and evidence of an agent autonomously provisioning/testing such a branch before production.", "evidenceIds": [ "firebase-docs-9", "firebase-docs-24", "firebase-docs-20", "firebase-docs-34", "firebase-probe-3" ] }, { "productId": "firebase", "storyId": "agent-provisions-backend", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Firebase CLI (firebase-tools) supports scripted, headless deployment of Firestore/Realtime Database security rules, Cloud Functions, and Auth user import/export, and the MCP server also exposes project/auth/data management to agents, covering schema+auth+policies largely from the command line. However, there's no single documented 'init a full backend' one-shot command evidenced, and evidence doesn't show a non-interactive schema-creation flow (Firestore is schemaless, so 'schema' setup maps to rules/indexes only). missing for 10: a documented fully non-interactive scripted bootstrap command chaining project creation, schema/indexes, auth, and rules; independent hands-on confirmation of headless end-to-end setup.", "evidenceIds": [ "firebase-gh-1", "firebase-gh-3", "firebase-docs-10", "firebase-docs-18", "firebase-docs-20", "firebase-probe-4", "firebase-probe-3" ] }, { "productId": "firebase", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe shows firebase.google.com/llms.txt returns 404, and no evidence pack item shows any agent-oriented docs bundle or llms.txt equivalent; the MCP server (firebase-docs-12) is a separate tool-integration axis, not an agent-readable docs file.", "evidenceIds": [ "firebase-probe-1" ] }, { "productId": "firebase", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Firebase documents Gemini in Firebase providing AI-generated insights and suggestions from app data — e.g., actionable crash root-cause analysis and fix suggestions (Crashlytics) and summarization/recommendations for messaging campaign performance — directly matching the story's intent. However, this is scoped to specific data types (crash reports, messaging analytics) rather than general insights across all stored data (e.g., Firestore documents), and no independent/hands-on evidence corroborates the quality of these AI suggestions. Missing for 10: broader data-insight coverage (e.g., Firestore/database content analysis), independent user validation of insight quality.", "evidenceIds": [ "firebase-docs-14", "firebase-docs-15", "firebase-docs-25", "firebase-docs-13" ] }, { "productId": "firebase", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Firebase Cloud Functions explicitly support autonomous, event-driven background execution, including triggers from Firebase/Cloud events, HTTPS requests, and Cloud Scheduler jobs, letting users run scheduled or event-triggered automations without manual intervention. Docs also show one-command deployment and auto-scaling for these functions. Missing for 10: independent/hands-on corroboration of scheduler reliability and any dispute resolution for the older community complaint about lacking scheduled tasks (which predates Cloud Scheduler support).", "evidenceIds": [ "firebase-docs-7", "firebase-docs-8", "firebase-docs-29" ] }, { "productId": "firebase", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Firebase ships Gemini in Firebase, a built-in natural-language AI assistant embedded in the console that answers product questions, generates code, and provides crash/debugging insights and messaging analytics — directly fulfilling task delegation to a built-in AI assistant. missing for 10: independent/hands-on verification of Gemini in Firebase's task-execution quality beyond first-party docs, and clearer detail on the scope of tasks it can autonomously complete versus just advise on.", "evidenceIds": [ "firebase-docs-13", "firebase-docs-14", "firebase-docs-15", "firebase-docs-21", "firebase-docs-25" ] }, { "productId": "firebase", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Firebase ships a scriptable CLI (firebase-tools) that can deploy code, manage hosting, import/export auth data, and run a Local Emulator Suite for offline integration testing/QA and mocha-based unit tests — all of which are usable in a headless/CI context (firebase-docs-8, firebase-docs-9, firebase-docs-24, firebase-docs-31, firebase-gh-1/2/3). However, there is no explicit CI/CD pipeline documentation (e.g., GitHub Actions integration, service-account auth for non-interactive CLI use) or independent confirmation of CI usage. Missing for 10: explicit CI/CD integration docs (e.g., GitHub Actions workflow), non-interactive auth guidance for automated environments, and independent/hands-on evidence of running the emulator suite or CLI in a CI pipeline.", "evidenceIds": [ "firebase-docs-8", "firebase-docs-9", "firebase-docs-24", "firebase-docs-31", "firebase-gh-1", "firebase-gh-2", "firebase-gh-3", "firebase-probe-4" ] }, { "productId": "firebase", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows Firebase publishing its own MCP server for external AI tools (Claude, Cursor, VS Code Copilot, etc.) to consume — the opposite direction from this story, which asks whether Firebase itself (e.g., via Gemini in Firebase) can act as an MCP client and consume external MCP servers' tools. No evidence indicates Firebase's Gemini assistant or console supports plugging in external MCP servers. Missing for 10: any documentation of Firebase acting as an MCP client, any way to register external MCP servers within Firebase's own AI features.", "evidenceIds": [ "firebase-docs-12", "firebase-docs-20", "firebase-docs-34", "firebase-docs-13" ] }, { "productId": "firebase", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Firebase ships an official documented MCP server that works with any MCP client tool (Antigravity, Claude Code/Desktop, Cline, Cursor, VS Code Copilot, Windsurf, etc.), enabling agents to manage Firebase projects, Auth, Firestore, and more. This is corroborated by both docs and a dedicated probe confirming the official page. Missing for 10: independent hands-on community report validating the MCP server's real-world reliability.", "evidenceIds": [ "firebase-docs-12", "firebase-docs-20", "firebase-docs-34", "firebase-probe-3" ] }, { "productId": "firebase", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Firebase documents both a natural-language chat interface (Gemini in Firebase) for getting answers, generating code, and troubleshooting, and an official MCP server that lets AI editors/agents create and manage Firebase projects, Auth users, and Firestore data via natural-language commands routed through MCP clients like Claude Code, Cursor, VS Code Copilot, etc. Missing for 10: independent/hands-on validation that MCP-driven natural-language operations reliably perform complex actions, and confirmation Gemini in Firebase supports action-taking beyond Q&A/insights.", "evidenceIds": [ "firebase-docs-12", "firebase-docs-13", "firebase-docs-20", "firebase-docs-34", "firebase-probe-3" ] }, { "productId": "firebase", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Firebase ships an official CLI (firebase-tools) documented both in GitHub and Firebase docs, covering deploy, hosting, auth, and Firestore data management, and is corroborated by a dedicated CLI docs page (firebase-probe-4). This CLI is the standard tool for automating Firebase workflows, which AI-native/agentic users would drive programmatically or via the complementary MCP server for AI agents.\nmissing for 10: explicit documentation of AI-native features built into the CLI itself (e.g., natural-language command generation) beyond its use as an MCP-server backend.", "evidenceIds": [ "firebase-gh-1", "firebase-gh-2", "firebase-gh-3", "firebase-probe-4", "firebase-docs-12" ] }, { "productId": "firebase", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Firebase exposes a well-documented public surface for programmatic/agentic use: the official CLI (firebase-tools) for deploying, managing data, and auth; Admin/REST SDKs for Firestore, Auth, and Functions; and a dedicated first-party MCP server that lets AI agents (Claude, Cursor, VS Code Copilot, etc.) manage Firebase projects, auth, and Firestore data directly. missing for 10: a discoverable machine-readable spec (openapi.json/llms.txt both 404), and independent hands-on confirmation of the MCP server working reliably in production agent workflows", "evidenceIds": [ "firebase-gh-1", "firebase-docs-12", "firebase-docs-20", "firebase-probe-3", "firebase-probe-4", "firebase-docs-8" ] }, { "productId": "firebase", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Firebase has Security Rules and Identity Platform for general access control, but no evidence describes issuing scoped or least-privilege API credentials specifically for an AI agent (e.g., a restricted service-account/API-key flow tied to MCP or agent use). The MCP server docs only describe what an agent-connected editor can do, not how to scope its permissions. missing for 10: documentation of agent-specific scoped credentials/service accounts, least-privilege API key issuance workflow, any first-party or community confirmation of this capability.", "evidenceIds": [ "firebase-docs-10", "firebase-docs-18", "firebase-docs-20", "firebase-docs-34" ] }, { "productId": "firebase", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Firebase provides official SDKs across Firestore, Auth, Functions, Storage, and Rules, well documented with first-party docs and CLI (firebase-tools), plus emulator suite for local dev and testing. Community evidence corroborates real-world usage of these SDKs for building apps. missing for 10: independent hands-on validation specifically of SDK quality/versioning for AI-native workflows, and no explicit llms.txt/openapi spec found (probe results show 404s) which would strengthen AI-native discoverability.", "evidenceIds": [ "firebase-docs-7", "firebase-docs-8", "firebase-docs-5", "firebase-docs-6", "firebase-docs-10", "firebase-gh-1", "firebase-docs-24", "firebase-probe-4", "firebase-comm-8" ] }, { "productId": "firebase", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Firebase evidence shows event-driven Cloud Functions triggers and real-time listeners (Firestore/RTDB), but no documented webhook subscription mechanism for external endpoints to receive events; no first-party or community evidence of a webhooks feature.", "evidenceIds": [] }, { "productId": "firebase", "storyId": "ai-dashboard-assistant", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Firebase ships Gemini in Firebase, a natural-language chat interface in the dashboard that answers product questions, generates code, and provides AI-driven crash/debugging insights, directly matching the story of asking an AI assistant to write queries, debug, and explain the backend. missing for 10: independent hands-on user reports validating query-writing/debugging accuracy in Gemini in Firebase specifically (only vendor docs cited).", "evidenceIds": [ "firebase-docs-13", "firebase-docs-14", "firebase-docs-21", "firebase-docs-25" ] }, { "productId": "firebase", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of an interactive API reference with runnable examples; probes confirm llms.txt and openapi endpoints return 404, and docs pack contains no mention of a live/interactive reference or embedded runnable code samples.", "evidenceIds": [ "firebase-probe-1", "firebase-probe-2" ] }, { "productId": "firebase", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Probes for llms.txt and common OpenAPI/swagger spec paths all returned 404, and no documentation in the evidence pack references a downloadable OpenAPI/Swagger spec for Firebase APIs.", "evidenceIds": [ "firebase-probe-1", "firebase-probe-2" ] }, { "productId": "firebase", "storyId": "api-sandbox", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Firebase's Local Emulator Suite explicitly lets developers connect apps to emulators for integration testing or QA 'without touching production data,' covering Firestore, Auth, Functions, Storage, Hosting, and more, with unit test support via mocha. missing for 10: independent/hands-on corroboration of emulator fidelity and AI-native/agent-specific usage examples beyond generic dev testing.", "evidenceIds": [ "firebase-docs-9", "firebase-docs-24", "firebase-docs-31" ] }, { "productId": "firebase", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item documents API versioning schemes or a deprecation policy for Firebase's APIs/SDKs; probes for openapi.json/llms.txt both 404, and no docs reference version support windows or deprecation timelines.", "evidenceIds": [ "firebase-probe-1", "firebase-probe-2" ] }, { "productId": "firebase", "storyId": "auth-email-social", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Firebase Authentication SDK explicitly supports email/password plus Google, Facebook, Twitter, and GitHub OAuth sign-in, and FirebaseUI offers a drop-in solution handling these sign-in flows out of the box, matching the 'minutes' setup claim; community feedback also corroborates Firebase's ease of bootstrapping auth alongside other services. missing for 10: independent hands-on account of specifically timing/adding OAuth sign-in setup, and no mention of other providers like Apple/Microsoft.", "evidenceIds": [ "firebase-docs-5", "firebase-docs-6", "firebase-docs-23", "firebase-comm-8", "firebase-comm-2" ] }, { "productId": "firebase", "storyId": "auth-mfa-sso", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Firebase's own docs state that native Firebase Authentication does not include MFA or enterprise SSO by default — these require upgrading to 'Firebase Authentication with Identity Platform,' which unlocks multi-factor authentication, SAML and generic OpenID Connect support, and enterprise-level support (firebase-docs-28). This confirms the capability exists but only via a paid upgrade path, not as a standard included feature — a significant caveat for a platform engineer evaluating baseline auth. Missing for 10: first-party documentation detailing MFA/SSO configuration steps, enforcement policies, and independent/hands-on validation of enterprise SSO setup.", "evidenceIds": [ "firebase-docs-28" ] }, { "productId": "firebase", "storyId": "automated-backups-pitr", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions automated backups or point-in-time recovery for Firestore/Realtime Database; only security rules, querying, offline caching, and emulator suite are documented. This is a plausible axis for a database/backend platform, so absence of evidence yields 'none'.", "evidenceIds": [] }, { "productId": "firebase", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Firebase provides some bulk-oriented capabilities—bulk import/export of Auth users via the CLI (firebase-gh-1, firebase-gh-3), multi-filter/sort queries in Firestore (firebase-docs-2), and MongoDB-compatible API access that could support bulk driver operations (firebase-docs-4/16/27)—but there's no direct documentation of a native batch-write/bulk-update API for arbitrary large-scale item operations, and community feedback notes limitations like lacking count queries without full downloads (firebase-comm-16). missing for 10: explicit batch-write/bulk-update API documentation, evidence of scale/performance for bulk operations, and independent hands-on confirmation of bulk data manipulation beyond auth import/export.", "evidenceIds": [ "firebase-docs-2", "firebase-gh-1", "firebase-gh-3", "firebase-docs-4", "firebase-comm-16" ] }, { "productId": "firebase", "storyId": "automation-rules-engine", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Firebase's Cloud Functions is explicitly a serverless event-trigger system: it runs backend code automatically in response to background events, HTTPS requests, Admin SDK, Auth triggers, Storage triggers, or Cloud Scheduler jobs, directly matching the 'rules that trigger actions on events' story. This is well documented across multiple first-party pages with concrete trigger types listed. Missing for 10: independent/hands-on corroboration of trigger reliability at scale, and clarification of any AI-native-specific tooling for defining these rules beyond standard docs.", "evidenceIds": [ "firebase-docs-7", "firebase-docs-8", "firebase-docs-29", "firebase-docs-17", "firebase-docs-30" ] }, { "productId": "firebase", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Firebase docs confirm Cloud Functions can be triggered by Cloud Scheduler jobs, enabling recurring/scheduled workflows (firebase-docs-7, firebase-docs-29), but this relies on the external Cloud Scheduler service rather than a native, first-class 'scheduled job' primitive within Firebase itself, and there is only one high-level doc mention with no setup detail or independent validation of current experience. An older community comment even flagged the historical absence of scheduled-task support as a pain point, suggesting this capability was a later addition rather than a mature core feature.\nmissing for 10: dedicated documentation/tutorial on defining a scheduled function end-to-end, independent/hands-on confirmation that scheduling works reliably today, and any AI-native/agent-specific workflow scheduling support.", "evidenceIds": [ "firebase-docs-7", "firebase-docs-29", "firebase-comm-19" ] }, { "productId": "firebase", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Firebase's evidence describes Cloud Functions, Security Rules, and CLI deployment, but nothing documents version history, review workflows, or rollback specifically for these 'automations' (functions/rules/triggers). Missing for 10: documented versioning/rollback of Cloud Functions or Security Rules, review/approval workflow, and any audit trail for automation changes.", "evidenceIds": [ "firebase-docs-7", "firebase-docs-8", "firebase-docs-10", "firebase-gh-1" ] }, { "productId": "firebase", "storyId": "db-event-triggers", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Cloud Functions for Firebase is explicitly documented as a serverless framework that runs backend code automatically in response to background events, HTTPS requests, Admin SDK actions, or scheduler jobs, including Firebase Authentication triggers and Cloud Storage/Firestore triggers. This directly matches the story of running functions in response to database/auth events. missing for 10: independent hands-on report specifically confirming database/auth trigger reliability (community evidence focuses on other pain points, not this specific capability).", "evidenceIds": [ "firebase-docs-7", "firebase-docs-8", "firebase-docs-29" ] }, { "productId": "firebase", "storyId": "direct-db-connection", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Firebase's Firestore Enterprise edition offers a MongoDB-compatible API, letting developers reuse existing MongoDB drivers/tools/ORM ecosystem, which partially satisfies 'standard clients and ORMs' — but this applies only to the Enterprise edition, not the default Firestore or Realtime Database, and no connection-string mechanism is documented anywhere in the evidence pack. Missing for 10: explicit connection-string documentation, ORM support for the standard (non-Enterprise) Firestore/Realtime DB products, and independent/hands-on confirmation that MongoDB drivers actually connect successfully.", "evidenceIds": [ "firebase-docs-4", "firebase-docs-16", "firebase-docs-27" ] }, { "productId": "firebase", "storyId": "file-storage-cdn", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Evidence only mentions 'Cloud Storage' in passing as one of the products covered by Firebase Security Rules and the Emulator Suite (firebase-docs-10, firebase-docs-18, firebase-docs-24), with community mentions of 'storage rules' as a pain point (firebase-comm-5, firebase-comm-8). There is no documentation in the pack describing file upload/download APIs, access-rule granularity for files, or CDN-backed delivery specifics. missing for 10: docs on Cloud Storage upload/download SDK, CDN delivery details, access-rule examples for file objects, independent corroboration of CDN performance.", "evidenceIds": [ "firebase-docs-10", "firebase-docs-18", "firebase-docs-24", "firebase-comm-5", "firebase-comm-8" ] }, { "productId": "firebase", "storyId": "fulltext-search-builtin", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows Cloud Firestore supports filtering/sorting queries and vector (KNN) similarity search for embeddings, but nothing indicates native full-text search over string fields; Firestore is well known for requiring an external service (e.g., Algolia/Elasticsearch) for text search, and community comments even complain about limited query capabilities. No documentation or hands-on evidence shows a built-in full-text search feature.", "evidenceIds": [ "firebase-docs-2", "firebase-docs-11", "firebase-docs-19", "firebase-docs-33", "firebase-comm-4" ] }, { "productId": "firebase", "storyId": "image-transformations", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items mention Firebase Extensions for image resizing, Cloud Storage image transformation, or any built-in image pipeline capability; the pack covers Firestore, Auth, Functions, Rules, and MCP but nothing about serving resized/transformed images.", "evidenceIds": [] }, { "productId": "firebase", "storyId": "local-stack-emulator", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Firebase's Local Emulator Suite explicitly supports running Firestore, Realtime Database, Storage, Auth, Hosting, Functions, Pub/Sub, and Extensions locally for integration testing without touching production, and the firebase-tools CLI supports running a local hosting server and interacting with data/deploying. This directly matches the story of running the whole backend stack locally before deploying, with unit-test support via mocha also documented. Missing for 10: independent/hands-on verification of full-suite parity beyond docs, and no community corroboration specifically praising the emulator suite's completeness.", "evidenceIds": [ "firebase-docs-24", "firebase-docs-9", "firebase-docs-31", "firebase-gh-1", "firebase-gh-2", "firebase-probe-4" ] }, { "productId": "firebase", "storyId": "managed-database-queries", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Firebase offers managed NoSQL databases (Cloud Firestore, Realtime Database) with official client SDKs for web/mobile and Admin SDK for server-side access, backed by docs on data sync, offline caching, and security rules, plus CLI/GitHub tooling for data interaction; community hands-on reports confirm fast, real-time read/write from clients and servers. Missing for 10: independent benchmark on server SDK write/read latency, and community notes cite real limitations (query features, count queries, historical offline-persistence gaps) that keep it from a perfect score.", "evidenceIds": [ "firebase-docs-1", "firebase-docs-2", "firebase-docs-3", "firebase-docs-10", "firebase-docs-18", "firebase-gh-1", "firebase-comm-14", "firebase-comm-18", "firebase-comm-20", "firebase-comm-4", "firebase-comm-16", "firebase-comm-17" ] }, { "productId": "firebase", "storyId": "offline-first-sync", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Firestore docs explicitly describe local caching and offline write/read/sync capability that reconciles when connectivity returns, directly matching the story (firebase-docs-3). Confidence is tempered by an older community report claiming lack of JS offline persistence and by the absence of independent, recent hands-on corroboration of sync behavior. missing for 10: independent/hands-on confirmation of reconnect-sync behavior, and clarification on which SDKs currently support offline persistence.", "evidenceIds": [ "firebase-docs-3", "firebase-comm-17" ] }, { "productId": "firebase", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Firebase ships extensive first-party APIs/SDKs (Firestore, Auth, Functions, Rules) plus a CLI (firebase-tools) that can deploy, manage hosting, interact with database data, and import/export Auth users, and an official MCP server that lets AI tools create/manage projects, manage Auth users, and work with Firestore data — strong evidence of broad API/CLI parity with console actions. However, there is no explicit documentation or independent confirmation that *every* console capability (e.g., analytics/Gemini insights, billing, project-level admin settings) is exposed via API/CLI, and probes show no OpenAPI spec or llms.txt is published, weakening claims of full programmatic parity. Missing for 10: explicit full-parity documentation, independent hands-on confirmation that all UI actions map to API/CLI, and a published OpenAPI/machine-readable spec.", "evidenceIds": [ "firebase-gh-1", "firebase-gh-2", "firebase-gh-3", "firebase-docs-20", "firebase-docs-34", "firebase-docs-8", "firebase-probe-3", "firebase-probe-4", "firebase-probe-1", "firebase-probe-2" ] }, { "productId": "firebase", "storyId": "openness-full-export", "verdict": "disputed", "quality": 3, "confidence": "medium", "rationale": "Firebase CLI documents import/export of Auth users (open JSON) and one user notes deliberately keeping Firestore as a plain JSON store for portability, but there is no documented open-format export path for Firestore, Storage, or Security Rules, and multiple community accounts explicitly describe painful lock-in ('keenly aware of the cost of lock-in', 'need to reimplement storage rules — an absolute pain', 'the lock-in effect is super bad') when trying to migrate away. Missing for 10: comprehensive documented export of Firestore/Storage/Rules in open formats, first-party migration tooling, and independent confirmation that leaving is frictionless.", "evidenceIds": [ "firebase-gh-1", "firebase-gh-3", "firebase-comm-13", "firebase-comm-3", "firebase-comm-5", "firebase-comm-6" ] }, { "productId": "firebase", "storyId": "openness-open-license", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Evidence shows firebase-tools (the CLI) is hosted on a public GitHub repo, implying it is open source, but the core Firebase backend/service (Firestore, Auth, Functions runtime) is proprietary and no license or full source is documented for the platform itself. missing for 10: explicit open-source license statement, source availability for the core backend/service, independent confirmation of license terms for firebase-tools.", "evidenceIds": [ "firebase-gh-1", "firebase-gh-2", "firebase-gh-3" ] }, { "productId": "firebase", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Firebase is a fully-managed, closed Google Cloud service with no self-hosted/on-premise deployment option documented anywhere in the evidence; the emulator suite is explicitly for local testing/QA, not production self-hosting, and community comments highlight lock-in concerns rather than any self-hosting path.", "evidenceIds": [ "firebase-docs-9", "firebase-docs-24", "firebase-comm-3", "firebase-comm-6" ] }, { "productId": "firebase", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions data residency, region selection, or location settings for Firebase services (Firestore, Storage, Functions, etc.), even though Firebase/GCP products do typically allow region selection in reality. Absence of evidence for this applicable capability yields 'none'.", "evidenceIds": [] }, { "productId": "firebase", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item addresses AI-training data usage or opt-out policies for Firebase data; the docs cover storage, auth, functions, and AI-assistance features, but nothing about preventing data from being used to train AI models.", "evidenceIds": [] }, { "productId": "firebase", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Firestore querying, security rules, auth, functions, and emulators, but contains no documentation of data retention policies, TTL/expiration settings, or deletion/export APIs for privacy compliance (e.g., GDPR-style user data deletion). No citations address controlling how long data is kept or how it can be purged.", "evidenceIds": [] }, { "productId": "firebase", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items mention telemetry, usage tracking, or an opt-out mechanism for Firebase's own tooling/services; only unrelated docs on Firestore, Auth, Functions, and pricing are present.", "evidenceIds": [] }, { "productId": "firebase", "storyId": "realtime-presence", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Firebase's Realtime Database is built for syncing live data across connected clients, and community evidence describes real-time multi-client state sync (whiteboarding with synced positions, real-time notifications), which supports ephemeral broadcast use cases. However, the evidence pack never explicitly documents Firebase's presence-detection features (onDisconnect, .info/connected) which are the canonical mechanism for this story. Missing for 10: explicit docs on presence detection/onDisconnect API, independent verification of presence tracking specifically (not just generic realtime sync).", "evidenceIds": [ "firebase-docs-1", "firebase-comm-1", "firebase-comm-14" ] }, { "productId": "firebase", "storyId": "realtime-subscriptions", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Firebase's core value proposition includes real-time data sync — docs confirm apps can 'write, read, listen to, and query data' even offline, and multiple independent developers describe building live-updating apps (real-time whiteboard, real-time notifications, live sync across clients) confirming push-based updates rather than polling. missing for 10: explicit documentation of the onSnapshot/listener API surface and any first-party performance benchmarks for real-time delivery.", "evidenceIds": [ "firebase-docs-3", "firebase-comm-1", "firebase-comm-14", "firebase-comm-18" ] }, { "productId": "firebase", "storyId": "relational-queries-joins", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Firebase's Firestore/Realtime Database are NoSQL document stores; evidence shows only chained filter/sort queries (firebase-docs-2) and a MongoDB-compatible API option (firebase-docs-4), with no mention of relational joins or multi-document transactions, and community evidence explicitly calls out the lack of server-side query capability (firebase-comm-4) and missing features like count queries (firebase-comm-16). No documentation of join support or transaction semantics appears in the pack.", "evidenceIds": [ "firebase-docs-2", "firebase-docs-4", "firebase-comm-4", "firebase-comm-16" ] }, { "productId": "firebase", "storyId": "row-level-security", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Firebase Security Rules are extensively documented as the mechanism to restrict data access per user across Firestore, Realtime Database, and Cloud Storage, with granular, expression-based rule languages, and community evidence corroborates real-world use (e.g., failed attempts blocked by rules, separation of auth logic from data access). missing for 10: no independent hands-on audit of rule robustness/edge cases, and community notes rules can be painful to maintain/reimplement, slightly denting polish.", "evidenceIds": [ "firebase-docs-10", "firebase-docs-18", "firebase-docs-32", "firebase-comm-2", "firebase-comm-12", "firebase-comm-5" ] }, { "productId": "firebase", "storyId": "serverless-functions", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Cloud Functions for Firebase is explicitly documented as a serverless framework to run backend code triggered by HTTPS requests, Admin SDK/client SDK calls, or background events, with one-command CLI deployment and automatic scaling. Missing for 10: independent hands-on corroboration of production-scale reliability of Functions specifically (community evidence mostly discusses Firestore/Auth/pricing rather than Functions usage).", "evidenceIds": [ "firebase-docs-7", "firebase-docs-8", "firebase-docs-29", "firebase-gh-1" ] }, { "productId": "firebase", "storyId": "type-generation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of Firebase generating typed client code/TypeScript types from a schema; docs cover Firestore, Auth, Functions, emulators, security rules, and MCP/Gemini AI features but nothing about automatic type generation from schema.", "evidenceIds": [] }, { "productId": "firebase", "storyId": "vector-similarity-search", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Firebase documents dedicated Cloud Firestore vector search functionality with KNN vector search, including storing vector values and creating/managing KNN vector indexes, directly matching the story. Missing for 10: independent/hands-on developer corroboration of vector search in production and details on scale/performance limits.", "evidenceIds": [ "firebase-docs-11", "firebase-docs-19", "firebase-docs-33" ] }, { "productId": "firebase", "storyId": "versioned-migrations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Firebase's CLI can deploy static config like Firestore security rules and indexes, but there is no evidence of a schema-migration framework (versioned, incremental migrations with up/down scripts) applied via CLI/CI; in fact a community report explicitly notes the lack of a business-logic layer makes schema changes 'extremely challenging' for deployed clients, underscoring the absence of this capability.", "evidenceIds": [ "firebase-docs-10", "firebase-gh-1", "firebase-comm-15" ] }, { "productId": "supabase", "storyId": "agent-branch-migrations", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Supabase documents a first-class branching feature explicitly for testing schema changes in isolation before production (supabase-docs-10/17/24), and separately provides an MCP server plus CLI that let AI agents interact with and provision projects programmatically (supabase-docs-1/27, supabase-probe-4/6/7/8). However, there is no direct evidence tying agent/MCP workflows to the branching feature itself, and community feedback flags migration/local-dev tooling as a weak point ('local development & database migration story is Supabase's biggest weakness' — supabase-comm-7) and notes AI agents often run with elevated service_role access that bypasses RLS (supabase-comm-15), raising safety caveats for autonomous schema testing. Missing for 10: explicit documentation/example of an agent using MCP+CLI to create a branch, apply/test schema changes, and promote to production; independent hands-on validation of that full agent-branch loop.", "evidenceIds": [ "supabase-docs-10", "supabase-docs-17", "supabase-docs-24", "supabase-docs-1", "supabase-docs-27", "supabase-probe-4", "supabase-probe-6", "supabase-probe-7", "supabase-probe-8", "supabase-comm-7", "supabase-comm-15" ] }, { "productId": "supabase", "storyId": "agent-provisions-backend", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "The CLI supports fully headless project scaffolding (`supabase init`/`start`) with no login or key required, confirmed by hands-on probes, and Postgres RLS policies (schema/auth authorization) are defined declaratively in SQL that can be run via the CLI/migrations, matching the 'headless from a script' requirement. missing for 10: explicit hands-on evidence of scripting auth-provider configuration and RLS policy deployment together via `supabase db push`/migrations in one automated flow, and independent (non-vendor) confirmation of a full agent-driven backend bootstrap.", "evidenceIds": [ "supabase-docs-2", "supabase-docs-14", "supabase-docs-28", "supabase-probe-6", "supabase-probe-8", "supabase-docs-15", "supabase-docs-4", "supabase-docs-6", "supabase-docs-21" ] }, { "productId": "supabase", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A hands-on probe confirms Supabase serves a valid llms.txt at the root domain (HTTP 200) pointing to full documentation, and Supabase's own docs also document agent-oriented integration paths like MCP and CLI that an agent could follow. Missing for 10: independent/community corroboration of agents actually consuming llms.txt in practice.", "evidenceIds": [ "supabase-probe-1", "supabase-docs-19", "supabase-probe-4" ] }, { "productId": "supabase", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Supabase enables AI assistants (via MCP) to query and interact with a user's Supabase data, and provides a pgvector-based AI toolkit for building AI features into apps, but there is no evidence of a first-party in-dashboard 'insights and suggestions' feature that surfaces AI-generated analysis directly inside Supabase Studio. The MCP-based approach requires connecting an external AI assistant rather than the product itself generating insights natively. missing for 10: evidence of a built-in Studio AI insights/suggestions feature, evidence of automatic proactive suggestions (vs. user-initiated querying via external assistant), independent hands-on validation of insight quality.", "evidenceIds": [ "supabase-docs-1", "supabase-docs-9", "supabase-probe-4", "supabase-probe-7" ] }, { "productId": "supabase", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Edge Functions, Realtime, RLS, and MCP/CLI tooling, but contains no mention of scheduled jobs, cron triggers, database webhooks, or any mechanism for autonomous background automations that an AI-native user could set up and let run unattended. As a backend platform, this axis is plausible for Supabase, but no supporting evidence exists in the pack.", "evidenceIds": [] }, { "productId": "supabase", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All evidence describes Supabase's MCP server, which lets *external* AI assistants (Cursor, Claude, etc.) connect to and query a Supabase project — this is the client-connectivity story, not a built-in assistant living inside Supabase's own product/dashboard. No evidence pack item describes a first-party in-product AI assistant that users delegate tasks to within Supabase itself.", "evidenceIds": [] }, { "productId": "supabase", "storyId": "agentic-headless", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Supabase's official CLI explicitly documents running the entire stack locally or in CI (supabase-docs-21), and hands-on probes confirm the CLI installs and runs headlessly via npx and can scaffold a project with no login/keys (supabase-probe-6, supabase-probe-8), which is exactly the CI/automation use case. Missing for 10: no explicit first-party CI pipeline example (e.g., GitHub Actions workflow) or independent third-party CI report beyond the probe evidence.", "evidenceIds": [ "supabase-docs-21", "supabase-probe-6", "supabase-probe-8", "supabase-docs-2" ] }, { "productId": "supabase", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Supabase is a backend/database platform, not an agentic assistant that itself consumes external tools via MCP; the evidence shows the reverse relationship (Supabase exposes its own MCP server so other AI assistants can plug into Supabase, not that Supabase plugs into other MCP servers to use their tools). This story's axis—being an MCP client that ingests third-party MCP servers—doesn't fit Supabase's product category.", "evidenceIds": [] }, { "productId": "supabase", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Supabase documents and ships an official MCP server (mcp.supabase.com) that lets AI assistants query/interact with Supabase projects, and a live hands-on probe confirms the remote MCP endpoint is reachable and auth-gated as expected. Community discussion also confirms real-world usage (e.g., Cursor assistant connecting via MCP) alongside security mitigations for prompt injection risks. Missing for 10: independent end-to-end hands-on demonstration of a successful authenticated agent session beyond the 401 handshake check.", "evidenceIds": [ "supabase-docs-1", "supabase-probe-4", "supabase-probe-7", "supabase-comm-13", "supabase-comm-15" ] }, { "productId": "supabase", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Supabase ships an official, documented MCP server enabling AI assistants to query and operate projects via natural language, corroborated by a live hands-on handshake against the production MCP endpoint and documented prompt-injection mitigations from a Supabase engineer. Missing for 10: independent quality benchmarking of the NL interface itself and full resolution of security concerns (e.g., service_role bypassing RLS in some client configs).", "evidenceIds": [ "supabase-docs-1", "supabase-docs-27", "supabase-probe-4", "supabase-probe-7", "supabase-comm-13", "supabase-comm-15" ] }, { "productId": "supabase", "storyId": "agentic-official-cli", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Supabase ships an official CLI with clear docs and hands-on verification: `npx supabase --version` runs headlessly and `supabase init` scaffolds a project without login or keys, making it directly usable by AI agents/scripts. Missing for 10: no explicit agent-oriented CLI subcommands or independent third-party report of an AI agent driving the CLI end-to-end.", "evidenceIds": [ "supabase-docs-21", "supabase-probe-5", "supabase-probe-6", "supabase-probe-8", "supabase-docs-14" ] }, { "productId": "supabase", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Supabase auto-generates REST and GraphQL APIs from the database schema and a live probe confirms a real spec-compliant OpenAPI document at supabase.com/openapi.json, giving AI agents a documented machine-readable API surface. This is reinforced by CLI and MCP tooling that lets an agent programmatically provision and query projects headlessly. Missing for 10: independent third-party validation of API completeness/stability, and explicit versioning/rate-limit docs for programmatic API consumers.", "evidenceIds": [ "supabase-docs-13", "supabase-gh-2", "supabase-probe-3", "supabase-docs-18", "supabase-probe-6", "supabase-probe-8", "supabase-docs-27" ] }, { "productId": "supabase", "storyId": "agentic-scoped-keys", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Supabase offers RLS-based fine-grained authorization and says it encourages read-only access by default for its MCP integration (mitigating agent access scope), but a concrete hands-on community report shows the Cursor/Supabase MCP integration actually running with elevated service_role access that bypasses RLS entirely — the opposite of least-privilege scoping. This is a direct, concrete contradiction rather than general skepticism. Missing for 10: documented ability to mint scoped/least-privilege API keys or tokens specifically for agent use, and confirmation that MCP connections default to restricted roles rather than service_role.", "evidenceIds": [ "supabase-docs-4", "supabase-docs-15", "supabase-comm-13", "supabase-comm-15", "supabase-probe-7" ] }, { "productId": "supabase", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Supabase provides auto-generated REST/GraphQL APIs, TypeScript type generation, and quickstart docs for connecting projects, which support building against official client libraries, and its CLI/MCP tooling make it usable by AI agents. However, the evidence pack never explicitly documents a suite of official SDKs (e.g., JS, Python, Flutter) or their API surfaces beyond generated types. Missing for 10: explicit official SDK/client-library documentation across languages, code samples showing SDK usage, and independent confirmation that AI agents build reliably against these SDKs.", "evidenceIds": [ "supabase-docs-11", "supabase-docs-18", "supabase-gh-2", "supabase-docs-19", "supabase-docs-13", "supabase-probe-5", "supabase-probe-6" ] }, { "productId": "supabase", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes Supabase Realtime (Broadcast, Presence, Postgres Changes via websockets) but never mentions a webhooks feature for subscribing to events over HTTP callbacks, which is what the story asks for; missing for 10: any documentation of Database Webhooks or HTTP callback subscription mechanism.", "evidenceIds": [] }, { "productId": "supabase", "storyId": "ai-dashboard-assistant", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Evidence shows Supabase's MCP integration lets external AI assistants (e.g., Cursor) query and operate on Supabase projects, and community reports confirm real-world use of an AI assistant against the database (with RLS-bypass caveats). However, none of the evidence specifically documents a native AI assistant embedded inside the Supabase dashboard UI itself, nor its ability to explicitly debug errors or explain the backend within that context — the evidence centers on MCP-based external tool connections, not an in-dashboard chat assistant. Missing for 10: dedicated dashboard-embedded assistant UI documentation, explicit debug/explain feature descriptions, and independent hands-on confirmation of dashboard-based query writing.", "evidenceIds": [ "supabase-docs-1", "supabase-docs-27", "supabase-probe-4", "supabase-probe-7", "supabase-comm-15", "supabase-comm-13" ] }, { "productId": "supabase", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence shows Supabase auto-generates REST/GraphQL APIs and exposes a public OpenAPI spec (openapi.json), implying some form of API reference exists, but there is no direct evidence of an interactive reference page with runnable/executable code examples for AI-native exploration. Missing for 10: explicit documentation or screenshot of an interactive API docs page, evidence of runnable/testable examples in the browser, and any AI-specific tooling around that reference beyond the generic OpenAPI file.", "evidenceIds": [ "supabase-probe-3", "supabase-docs-13", "supabase-docs-25", "supabase-gh-2" ] }, { "productId": "supabase", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Probe evidence confirms a live, machine-readable OpenAPI spec at supabase.com/openapi.json returning a valid 'openapi' key, plus a documented REST API auto-generated from the database and generated TypeScript types. Missing for 10: no independent third-party confirmation of spec completeness or versioning stability.", "evidenceIds": [ "supabase-probe-3", "supabase-docs-13", "supabase-docs-18", "supabase-gh-2" ] }, { "productId": "supabase", "storyId": "api-sandbox", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Supabase offers both local development (via CLI 'supabase init/start', hands-on verified) and branching environments explicitly designed to test schema/config/feature changes without affecting production, directly matching the sandbox-without-touching-production story. Community evidence corroborates real usage of local dev/migrations, though some report local dev as a weak point (comm-7, comm-8). Missing for 10: independent hands-on confirmation of branching workflow specifically, and no direct evidence addressing data-seeding/isolation guarantees in the sandbox beyond docs claims.", "evidenceIds": [ "supabase-docs-10", "supabase-docs-17", "supabase-docs-24", "supabase-docs-21", "supabase-probe-6", "supabase-probe-8", "supabase-comm-7" ] }, { "productId": "supabase", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack has no mention of API versioning schemes or a documented deprecation policy for Supabase's REST/GraphQL/Auth/Storage APIs; there's an OpenAPI spec probe but nothing about version lifecycle or deprecation commitments. This is a fair axis for a backend platform with public APIs, so absence of evidence yields 'none'.", "evidenceIds": [] }, { "productId": "supabase", "storyId": "auth-email-social", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Supabase Auth docs explicitly list password, magic link, OTP, social login, and SSO as supported methods, and community evidence confirms real developers implementing RLS + OAuth providers in production apps, indicating SDK-based setup works in practice. Missing for 10: no direct evidence of the specific SDK code snippets/quickstart timing claim ('minutes') or independent hands-on verification of OAuth setup speed.", "evidenceIds": [ "supabase-docs-6", "supabase-comm-17", "supabase-docs-4" ] }, { "productId": "supabase", "storyId": "auth-mfa-sso", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs mention SSO as one of many supported auth methods, implying enterprise SSO (SAML/OIDC) support, but there is no evidence of MFA enforcement mechanisms or platform-level policy to require MFA/SSO for application users. missing for 10: explicit MFA requirement/enforcement docs, OIDC/SAML configuration details, admin controls to mandate SSO for all users, independent corroboration.", "evidenceIds": [ "supabase-docs-6" ] }, { "productId": "supabase", "storyId": "automated-backups-pitr", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no vendor-tier documentation describing automated backups or point-in-time recovery (PITR) features; the only related evidence is a community report (supabase-comm-9) describing nightly pg_dump backups causing write-contention problems and the team merely 'investigating' moving customers to PITR. Since a disputed verdict requires a vendor claim tier plus a contradicting tier, and no first-party docs claim exists here, there is insufficient evidence the capability is delivered at all.", "evidenceIds": [ "supabase-comm-9" ] }, { "productId": "supabase", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack documents Supabase's REST API generation, RLS, realtime, storage, and CLI/MCP tooling, but nothing explicitly addresses bulk/batch operations (e.g., batch insert/update/delete across many rows or items) via the API, CLI, or MCP server. Postgres and PostgREST can technically support bulk SQL operations, but this capability is never documented or demonstrated in the evidence provided.", "evidenceIds": [] }, { "productId": "supabase", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Supabase's Realtime feature lets clients listen to Postgres changes and Edge Functions can act as serverless handlers, which together could be combined to react to database events, but the evidence pack contains no explicit rule-engine, database triggers/webhooks, or automation configuration UI for defining 'if event X then action Y' rules as an AI-native user would expect. missing for 10: explicit database triggers/webhooks documentation, a rules/automation configuration interface, and evidence of AI-native automatic action execution tied to events.", "evidenceIds": [ "supabase-docs-5", "supabase-docs-8", "supabase-docs-22", "supabase-docs-23" ] }, { "productId": "supabase", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers Supabase's database, auth, storage, realtime, edge functions, CLI, and MCP integration, but contains no mention of a scheduling feature (e.g., pg_cron, scheduled Edge Functions, or workflow orchestration) for recurring jobs. No documentation, community report, or probe references any cron-like or recurring-job capability.", "evidenceIds": [] }, { "productId": "supabase", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Supabase's branching feature lets users create separate environments to test schema/config/feature changes without affecting production, which is a git-like versioning mechanism, but this applies to database schema/config rather than 'automations' specifically, and there is no explicit rollback or review (PR-style) workflow documented for Edge Functions, triggers, or other automation-like constructs. Missing for 10: explicit automation objects (workflows/triggers/functions) with versioning, diff/review, and one-click rollback; independent/hands-on confirmation of rollback working.", "evidenceIds": [ "supabase-docs-10", "supabase-docs-17", "supabase-docs-24" ] }, { "productId": "supabase", "storyId": "db-event-triggers", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Supabase does offer Edge Functions (supabase-docs-8) and real-time Postgres Changes streaming (supabase-docs-5/22/23), which are the building blocks for event-driven execution, but the evidence pack never documents the actual Database Webhooks / Auth Hooks mechanism that automatically invokes a function on insert/update/delete or auth events. Missing for 10: explicit docs or hands-on proof of Database Webhooks/Auth Hooks wiring DB/auth events directly to function invocation, and any developer confirmation that this works reliably in practice.", "evidenceIds": [ "supabase-docs-8", "supabase-docs-5", "supabase-docs-22", "supabase-docs-23" ] }, { "productId": "supabase", "storyId": "direct-db-connection", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Supabase is built on Postgres and its docs confirm full self-hosting/local Postgres access and RLS-based auth (implying a standard Postgres connection string is available), but no evidence pack item explicitly documents the connection string format, ORM compatibility (e.g., Prisma, Drizzle), or connection pooling (pgbouncer/Supavisor) setup that developers typically need for direct client/ORM access. missing for 10: explicit docs on connection string/pooling, ORM-specific integration guides (Prisma/Drizzle/TypeORM), and independent hands-on confirmation of connecting via standard Postgres clients.", "evidenceIds": [ "supabase-docs-3", "supabase-docs-29", "supabase-docs-4", "supabase-docs-21", "supabase-comm-7", "supabase-comm-10" ] }, { "productId": "supabase", "storyId": "file-storage-cdn", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Supabase Storage docs explicitly describe fine-grained access controls via RLS policies and 'optimized delivery' for files of any size, directly matching the story's core ask (upload, access rules, CDN-like delivery). Community feedback confirms real-world usage of RLS with storage but notes storage still lacks direct metadata support, a secondary limitation rather than a failure of access control or delivery. Missing for 10: explicit mention of CDN infrastructure/edge caching details and independent hands-on confirmation of CDN-backed delivery performance.", "evidenceIds": [ "supabase-docs-7", "supabase-docs-30", "supabase-docs-4", "supabase-comm-5" ] }, { "productId": "supabase", "storyId": "fulltext-search-builtin", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Supabase's database, realtime, auth, storage, vector/AI, and CLI/MCP capabilities extensively, but nowhere mentions Postgres full-text search (tsvector, GIN indexes, or a dedicated search feature). Full-text search is a fair axis for a database-as-a-service product, so absence of evidence yields 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "supabase", "storyId": "image-transformations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence only describes Supabase Storage generally as offering 'fine-grained access controls and optimized delivery' (supabase-docs-7, supabase-docs-30) but never mentions image resizing, transformation parameters, or a CDN-based image pipeline feature. Missing for 10: any documentation of an image transformation/resizing API or CDN, hands-on proof of resized image delivery.", "evidenceIds": [ "supabase-docs-7", "supabase-docs-30" ] }, { "productId": "supabase", "storyId": "local-stack-emulator", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Docs and a keyless hands-on probe confirm the Supabase CLI (`supabase init`/`supabase start`) can spin up the entire stack (DB, Auth, Storage, Realtime, Functions) locally before deploying, which is exactly what the story asks for. However, independent hands-on community reports directly contradict smooth delivery — one calls local dev/migration 'Supabase's biggest weakness' and another describes it as 'a massive pain with random bugs' making the whole platform feel 'like a hack'. Missing for 10: consistent independent corroboration that local stack runs reliably without the bugs/friction reported by hands-on users.", "evidenceIds": [ "supabase-docs-21", "supabase-docs-2", "supabase-probe-6", "supabase-probe-8", "supabase-comm-7", "supabase-comm-8" ] }, { "productId": "supabase", "storyId": "managed-database-queries", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Supabase provisions a managed Postgres database with instant REST/GraphQL APIs, auto-generated type-safe client SDKs, and server-side access via CLI/self-hosting, backed by docs, GitHub feature list, and hands-on probes confirming CLI/local provisioning works. Community evidence corroborates real-world usage building backends with RLS-secured reads/writes from clients (SvelteKit) and servers. Missing for 10: independent benchmark of production-scale read/write reliability and a fully hands-on client-SDK read/write test beyond CLI provisioning.", "evidenceIds": [ "supabase-docs-13", "supabase-docs-18", "supabase-docs-25", "supabase-gh-2", "supabase-docs-19", "supabase-probe-5", "supabase-probe-6", "supabase-probe-8", "supabase-comm-17", "supabase-comm-4" ] }, { "productId": "supabase", "storyId": "offline-first-sync", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Supabase's Realtime (Broadcast, Presence, Postgres Changes), Auth, Storage, and local dev CLI, but nothing addresses offline-first client SDKs, local write queuing, conflict resolution, or automatic sync-on-reconnect. Community threads even flag local development and migrations as weak points, with no mention of offline capability.", "evidenceIds": [ "supabase-docs-5", "supabase-docs-22", "supabase-docs-23", "supabase-comm-7", "supabase-comm-8" ] }, { "productId": "supabase", "storyId": "openness-api-parity", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Supabase auto-generates REST/GraphQL APIs directly from the Postgres schema, exposes an openapi.json spec (confirmed live via probe), and ships a CLI/MCP server that lets an AI agent provision, configure, and query projects headlessly (supabase-docs-13, supabase-gh-2, supabase-probe-3, supabase-probe-6, supabase-probe-8). This gives strong data/API-layer parity with the dashboard for CRUD, auth, storage, and realtime features. However, the evidence pack never confirms parity for dashboard-only administrative actions (billing, org/project creation, scaling settings) via a documented Management API, and community notes point to real friction (PostgREST/RLS limitations, CLI instability) that complicate full UI-equivalent control (supabase-comm-10, supabase-comm-12). Missing for 10: explicit Management API docs covering billing/org/project-lifecycle actions, and independent verification that all dashboard settings are scriptable.", "evidenceIds": [ "supabase-docs-13", "supabase-gh-2", "supabase-probe-3", "supabase-probe-5", "supabase-probe-6", "supabase-probe-8", "supabase-comm-10", "supabase-comm-12" ] }, { "productId": "supabase", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Supabase is built on Postgres and offers self-hosting for 'full control over your data,' implying standard SQL/pg_dump portability and no lock-in, but the evidence pack contains no explicit documentation of a data-export feature, backup/download tooling, or migration-out guide. Missing for 10: explicit export/backup documentation (e.g., pg_dump instructions or a dashboard 'export project' feature), independent confirmation that self-hosted migration actually works cleanly, and any community report of successfully exporting/leaving.", "evidenceIds": [ "supabase-docs-3", "supabase-docs-29", "supabase-gh-1", "supabase-comm-6" ] }, { "productId": "supabase", "storyId": "openness-open-license", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "The GitHub repo (supabase/supabase) is explicitly described as built with 'enterprise-grade open source tools' and aims to give a 'Firebase-like developer experience using open source tools,' and the docs describe self-hosting for full control over data, confirming the source is publicly available and open. Missing for 10: explicit citation of the license file/name (e.g., Apache 2.0) and independent confirmation of license terms beyond marketing language.", "evidenceIds": [ "supabase-gh-1", "supabase-gh-3", "supabase-docs-3", "supabase-docs-29" ] }, { "productId": "supabase", "storyId": "openness-self-host", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Supabase explicitly documents self-hosting as a supported deployment model for full data control/compliance, with docs and CLI tooling (supabase init/start) to run the entire stack locally or in CI, corroborated by a hands-on probe confirming headless local provisioning. missing for 10: independent hands-on verification of a full production self-hosted deployment (beyond local dev) and community reports raise friction/rough edges in self-hosting experience.", "evidenceIds": [ "supabase-docs-3", "supabase-docs-29", "supabase-docs-21", "supabase-probe-8", "supabase-comm-6" ] }, { "productId": "supabase", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Supabase docs claim self-hosting as an option for users needing 'full control over your data' or compliance requirements, which implies data residency control by running your own infrastructure (supabase-docs-3/29). However, the evidence pack contains no explicit documentation of region selection for hosted/managed Supabase projects (e.g., a region picker at project creation), which is the more common AI-native use case. Missing for 10: explicit hosted-plan region selection docs, confirmation of available regions, and independent verification of residency guarantees.", "evidenceIds": [ "supabase-docs-3", "supabase-docs-29" ] }, { "productId": "supabase", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Supabase is a backend/database platform, not an AI model vendor or AI product with training-data policies; there is no evidence of any AI-training opt-out control, and this is a category mismatch for a BaaS provider rather than an applicable-but-unmet privacy axis.", "evidenceIds": [] }, { "productId": "supabase", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Supabase's self-hosting option ('full control over your data') and RLS/storage access controls, but nothing documents explicit data-retention policies, project/data deletion tooling, backup expiry controls, or user-data erasure workflows that would let an AI-native user manage retention/deletion. Missing for 10: documented deletion APIs (e.g., project/user data deletion), retention policy settings, backup lifecycle controls, and GDPR-style data erasure guarantees.", "evidenceIds": [] }, { "productId": "supabase", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item discusses telemetry, usage tracking, opt-out settings, or privacy controls for CLI/dashboard analytics; the pack covers unrelated product features (auth, storage, realtime, MCP, self-hosting). Missing for 10: any docs or settings referencing telemetry collection, an opt-out flag/env var, or privacy policy statement on usage data.", "evidenceIds": [] }, { "productId": "supabase", "storyId": "realtime-presence", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Supabase Realtime explicitly documents Presence for tracking/synchronizing user state ('who's online') and Broadcast for sending low-latency ephemeral messages between clients, and a launch community post confirms real-world usage at scale (~2000 concurrent users). Missing for 10: independent hands-on verification of presence/broadcast APIs beyond docs and one launch anecdote.", "evidenceIds": [ "supabase-docs-5", "supabase-docs-22", "supabase-docs-23", "supabase-docs-16", "supabase-comm-16" ] }, { "productId": "supabase", "storyId": "realtime-subscriptions", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Supabase Realtime explicitly supports Postgres Changes (listen to DB changes in real-time), plus Broadcast and Presence, enabling UI updates without polling, and community evidence confirms real-world usage (Realtime Multiplayer launch with concurrent users, developers building chat/typing apps on it). Missing for 10: independent hands-on benchmark of Postgres Changes latency/reliability at scale beyond the CEO's launch post.", "evidenceIds": [ "supabase-docs-5", "supabase-docs-22", "supabase-docs-23", "supabase-docs-16", "supabase-gh-2", "supabase-comm-16", "supabase-comm-17" ] }, { "productId": "supabase", "storyId": "relational-queries-joins", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Supabase is built on Postgres, so full SQL relational capabilities (joins, constraints, transactions) are inherently available via direct Postgres connections, and evidence shows RLS policies and schema/migration support. However, the evidence pack focuses mainly on REST/PostgREST APIs, RLS, and CLI/branching rather than explicitly documenting joins, transactions, or foreign-key constraints, and community reports note PostgREST limitations pushing developers back to custom API servers. Missing for 10: explicit docs on PostgREST/join query syntax across relations, transaction support via API/client libraries, constraint definition examples, and independent confirmation these work smoothly at scale.", "evidenceIds": [ "supabase-docs-4", "supabase-docs-15", "supabase-docs-18", "supabase-comm-10", "supabase-docs-13" ] }, { "productId": "supabase", "storyId": "row-level-security", "verdict": "partial", "quality": 7, "confidence": "high", "rationale": "Supabase's docs clearly document RLS policy syntax (e.g. per-user select policies using auth.uid()) and extend fine-grained access control to Storage, directly matching the story. However, community evidence shows real-world friction: RLS limitations forcing workarounds (multiple user tables), an MCP integration that bypasses RLS via service_role, and reports of RLS being tricky to get right initially. missing for 10: independent hands-on verification that policies enforce correctly across all Supabase products, clearer documentation on avoiding service_role bypass pitfalls, and resolution of cited RLS/auth issues left open for over a year.", "evidenceIds": [ "supabase-docs-4", "supabase-docs-15", "supabase-docs-30", "supabase-comm-10", "supabase-comm-11", "supabase-comm-15", "supabase-comm-17" ] }, { "productId": "supabase", "storyId": "serverless-functions", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Supabase Edge Functions are explicitly documented as server-side TypeScript functions distributed globally, invocable via HTTP or client SDKs, without server management — matching the story directly. Community evidence corroborates broader platform reliability and real usage, though nothing specifically stress-tests Edge Functions at scale. missing for 10: independent hands-on validation of Edge Function invocation/scaling, and details on cold-start/latency limits.", "evidenceIds": [ "supabase-docs-8", "supabase-docs-20", "supabase-docs-31", "supabase-gh-2" ] }, { "productId": "supabase", "storyId": "type-generation", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Supabase documents generating type-safe TypeScript definitions (database.types.ts) directly from database introspection via CLI/API tooling, a core first-party feature. Missing for 10: independent/hands-on confirmation of the generated types working in a real project and coverage of edge cases (e.g., custom types, enums) beyond docs.", "evidenceIds": [ "supabase-docs-11", "supabase-docs-18" ] }, { "productId": "supabase", "storyId": "vector-similarity-search", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Supabase explicitly ships pgvector-based AI toolkit and lists 'Vector embeddings' as a core product feature alongside its Postgres stack, directly enabling embedding storage and similarity search. missing for 10: no hands-on probe demonstrating actual vector search query execution, and no independent/community corroboration of vector search performance or usage.", "evidenceIds": [ "supabase-docs-9", "supabase-docs-20", "supabase-docs-31" ] }, { "productId": "supabase", "storyId": "versioned-migrations", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Docs confirm the Supabase CLI can run the full stack locally and in CI environments (supabase-docs-21) and branching lets you test schema changes outside production (supabase-docs-10/17), implying a migrations-in-CI workflow, but the evidence pack never shows an actual migration/diff/push command being used. A hands-on community report explicitly states 'The local development & database migration story is Supabase's biggest weakness... I hate having to do migrations live in prod' (supabase-comm-7), directly contradicting the implied smooth CLI/CI migration workflow. missing for 10: explicit evidence of `supabase migration new`/`db push`/`db diff` commands, a documented CI pipeline example, and independent corroboration that migrations work reliably in production.", "evidenceIds": [ "supabase-docs-21", "supabase-docs-10", "supabase-docs-17", "supabase-comm-7" ] } ]