[ { "productId": "cloudflare", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A direct probe confirms Cloudflare serves a live llms.txt at developers.cloudflare.com/llms.txt returning HTTP 200 with structured agent-oriented documentation content, which is exactly the capability the story asks for. Missing for 10: no independent third-party confirmation of agents actually consuming/using this file successfully in the wild.", "evidenceIds": [ "cloudflare-probe-1" ] }, { "productId": "cloudflare", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Cloudflare Workers offers Workers AI for building AI agents/inference into apps (cloudflare-docs-3) and built-in observability (cloudflare-docs-22), but there is no evidence of the platform itself surfacing AI-generated insights or suggestions from a user's own data within a Cloudflare product interface (e.g., analytics dashboard insights).", "evidenceIds": [ "cloudflare-docs-3", "cloudflare-docs-22" ] }, { "productId": "cloudflare", "storyId": "agentic-autonomous-automation", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Cloudflare Workers docs advertise cron triggers, durable Workflows, and Queues for scheduling background automation (cloudflare-docs-9), which could support autonomous agent-like tasks. However, community evidence specifically counters this for long-running autonomous agent workloads: the 30s CPU limit on free tier and 15min cap even on paid plans 'don't work for long-running agent tasks,' pushing people to self-host instead (cloudflare-comm-9). missing for 10: evidence that Workflows/Durable Objects can sustain truly long-running autonomous agent loops without hitting CPU/time caps, and independent confirmation of successful production AI-agent automations on Workers.", "evidenceIds": [ "cloudflare-docs-9", "cloudflare-comm-9" ] }, { "productId": "cloudflare", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Cloudflare Workers documentation describes building AI inference/agents as capabilities you can deploy (cloudflare-docs-3, cloudflare-docs-12), but there is no evidence of a built-in AI assistant embedded in the Workers product/dashboard itself that a user can delegate tasks to.", "evidenceIds": [] }, { "productId": "cloudflare", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Wrangler is a first-party CLI (cloudflare-gh-2, cloudflare-docs-11/20, cloudflare-probe-3) explicitly designed for scripted/headless deploy, dev, and management commands, and Cloudflare explicitly advertises fitting into existing CI workflows like Git and GitHub Actions (cloudflare-docs-17) with no proprietary tooling lock-in. Missing for 10: a dedicated GitHub Actions/CI recipe or worked example, and independent hands-on confirmation of CI automation specifically (community evidence is about DX/runtime limits, not CI usage).", "evidenceIds": [ "cloudflare-gh-2", "cloudflare-docs-11", "cloudflare-docs-20", "cloudflare-docs-17", "cloudflare-probe-3" ] }, { "productId": "cloudflare", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "medium", "rationale": "Cloudflare Workers is a serverless compute/deployment platform, not an AI agent or assistant that itself consumes external tools; the evidence only shows it lets developers build and deploy MCP *servers* (cloudflare-docs-12), which is the opposite role (server, not client). There's no product-level 'agent' inside Workers that plugs in MCP servers to use their tools, so this client-consumption story is a category mismatch for this kind of product.", "evidenceIds": [ "cloudflare-docs-12" ] }, { "productId": "cloudflare", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Cloudflare's official docs explicitly state you can build and deploy MCP servers on Cloudflare Workers, directly supporting agent connectivity via an official MCP server. Missing for 10: independent/hands-on corroboration of MCP server deployment, and more detail on the connection/auth workflow for agents.", "evidenceIds": [ "cloudflare-docs-12" ] }, { "productId": "cloudflare", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Wrangler CLI and dashboard-based operation (docs-11, docs-20, probe-3) plus AI-inference/MCP-building capabilities for apps built on Workers (docs-3, docs-12), but nothing indicates Cloudflare Workers itself can be operated via natural-language commands (e.g., a chat-based deploy/manage interface). Missing for 10: any documented natural-language control plane, chatbot/agent for managing deployments, or NL-to-CLI translation feature.", "evidenceIds": [ "cloudflare-docs-11", "cloudflare-docs-20", "cloudflare-probe-3", "cloudflare-docs-3", "cloudflare-docs-12" ] }, { "productId": "cloudflare", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Wrangler is Cloudflare's official, well-documented CLI for creating, developing, deploying, and rolling back Workers, with dedicated docs, command references, and a GitHub repo, making it fully usable by AI-native/agentic tooling that shells out to CLIs. Missing for 10: explicit evidence of AI-agent-specific CLI ergonomics (e.g., machine-readable output modes, agent SDK wrappers) or independent hands-on confirmation of AI-native usage of Wrangler specifically.", "evidenceIds": [ "cloudflare-docs-11", "cloudflare-docs-20", "cloudflare-docs-23", "cloudflare-gh-2", "cloudflare-probe-3" ] }, { "productId": "cloudflare", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Cloudflare publishes a documented, OpenAPI-spec'd public API (confirmed live at openapi.json), token-scoped API auth docs, and Wrangler CLI/SDK tooling that wraps the API for programmatic management of Workers — all fair for AI-native/agentic driving of the platform. Missing for 10: independent hands-on verification of API completeness/reliability from third parties beyond the probe confirmation.", "evidenceIds": [ "cloudflare-probe-2", "cloudflare-docs-14", "cloudflare-docs-21", "cloudflare-docs-11", "cloudflare-probe-3", "cloudflare-gh-2" ] }, { "productId": "cloudflare", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Cloudflare's API token system lets users create scoped, least-privilege credentials (selecting specific resources/zones and permissions like 'Zone DNS Read' for a single zone), which an AI agent could use. However, this is a general platform credential feature not specifically tailored or documented for AI-agent use cases, and there's no evidence of agent-specific scoping guidance, short-lived/ephemeral agent tokens, or dedicated tooling for provisioning credentials to autonomous agents. Missing for 10: agent-specific credential issuance workflows, dynamic/ephemeral token minting for agents, and independent corroboration of this being used for AI-native agent scenarios.", "evidenceIds": [ "cloudflare-docs-14", "cloudflare-docs-21" ] }, { "productId": "cloudflare", "storyId": "agentic-sdks", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Cloudflare ships official SDK/tooling (workers-sdk GitHub repo, Wrangler CLI) plus AI-specific building blocks (Workers AI, agents, MCP servers) and machine-readable docs (llms.txt, openapi.json) that support AI-native development. However, community feedback flags gaps in language coverage (no full Python/Node runtime) and vendor lock-in concerns around bindings, tempering full delivery. Missing for 10: mature first-class SDKs in multiple languages (e.g. Python), independent validation that AI-agent workloads run smoothly without CPU/time limits, and confirmation the MCP/agent tooling is broadly adopted rather than experimental.", "evidenceIds": [ "cloudflare-docs-3", "cloudflare-docs-12", "cloudflare-gh-1", "cloudflare-gh-2", "cloudflare-probe-1", "cloudflare-probe-2", "cloudflare-probe-3", "cloudflare-comm-9", "cloudflare-comm-13", "cloudflare-comm-14" ] }, { "productId": "cloudflare", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While Cloudflare Workers is a general HTTP-handling serverless platform (and could technically host webhook receivers), the evidence pack contains no documentation, examples, or community reports about subscribing to or receiving webhooks, or any built-in event-subscription mechanism for AI agents.", "evidenceIds": [] }, { "productId": "cloudflare", "storyId": "ai-global-inference", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Cloudflare's docs explicitly state Workers AI lets you 'run AI inference globally with one API call... no GPUs to manage, no capacity planning' (cloudflare-docs-3), directly matching the story, and the broader Workers platform (global edge network, no server/cold-start management) supports this claim (cloudflare-docs-1, cloudflare-docs-19). However, evidence is entirely first-party marketing/docs with no independent hands-on benchmarks or user reports specifically validating Workers AI inference performance or global reach, and general community skepticism exists about doc accuracy on newer Cloudflare AI products (cloudflare-comm-6). Missing for 10: independent/hands-on validation of Workers AI inference specifically, details on model catalog/latency benchmarks, and confirmation from third-party users running production AI inference on Workers.", "evidenceIds": [ "cloudflare-docs-3", "cloudflare-docs-1", "cloudflare-docs-19", "cloudflare-comm-6" ] }, { "productId": "cloudflare", "storyId": "ai-model-gateway", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Cloudflare docs mention running AI inference 'globally with one API call' (Workers AI), but there is no evidence naming an AI Gateway product, a unified endpoint spec, or support for 'hundreds' of models from multiple providers. Missing for 10: explicit AI Gateway/unified endpoint documentation, list of supported model providers/count, and independent confirmation of breadth of model access.", "evidenceIds": [ "cloudflare-docs-3" ] }, { "productId": "cloudflare", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "There is a machine-readable OpenAPI spec (openapi.json) confirming a formal API reference exists, and an llms.txt endpoint signaling AI-friendly doc structuring, but no evidence pack item describes an interactive explorer or runnable code examples within the docs. missing for 10: explicit interactive API reference UI, runnable/executable code samples in docs, hands-on confirmation of 'try it' functionality.", "evidenceIds": [ "cloudflare-probe-2", "cloudflare-probe-1" ] }, { "productId": "cloudflare", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A probe confirms Cloudflare publishes a machine-readable OpenAPI spec at developers.cloudflare.com/openapi.json (HTTP 200, contains \"openapi\" key), plus an llms.txt for AI-native discovery. Missing for 10: independent third-party corroboration of spec completeness/usage beyond the probe.", "evidenceIds": [ "cloudflare-probe-2", "cloudflare-probe-1" ] }, { "productId": "cloudflare", "storyId": "api-sandbox", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Cloudflare provides Miniflare/workerd-based local simulator and `wrangler dev` for developing and testing Workers locally without touching production, as documented in the workers-sdk repo descriptions and Wrangler CLI docs. This directly supports testing in an isolated sandbox before deployment. Missing for 10: explicit first-party documentation framing this as a 'staging/sandbox environment' separate from prod data, and independent hands-on confirmation that local simulation fully mirrors production behavior (e.g., bindings to real data stores).", "evidenceIds": [ "cloudflare-gh-1", "cloudflare-gh-2", "cloudflare-gh-3", "cloudflare-docs-11", "cloudflare-docs-20", "cloudflare-probe-3" ] }, { "productId": "cloudflare", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence shows Wrangler/API tooling, an OpenAPI spec, and Worker version rollback, but nothing documents a formal API versioning scheme or deprecation policy for Cloudflare's APIs. Missing for 10: explicit versioned API endpoints, a documented deprecation/sunset policy, and any changelog or migration guidance referenced in the evidence pack.", "evidenceIds": [ "cloudflare-probe-2", "cloudflare-docs-23", "cloudflare-docs-11" ] }, { "productId": "cloudflare", "storyId": "automatic-tls-certs", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Cloudflare's docs confirm free, auto-issued and auto-renewed publicly trusted SSL certificates for all domains added to Cloudflare (Universal SSL), which would cover custom domains used with Workers. However, the evidence never explicitly confirms wildcard certificate coverage or independent/hands-on verification of renewal behavior. missing for 10: explicit wildcard-domain certificate confirmation, independent/hands-on corroboration of auto-renewal in practice.", "evidenceIds": [ "cloudflare-docs-13" ] }, { "productId": "cloudflare", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Workers documentation mentions Queues, cron jobs, and durable Workflows (cloudflare-docs-9), which are building blocks that could be used to orchestrate bulk/batch processing across many items, but the evidence pack contains no explicit documentation of a bulk-operations API, batch endpoint, or example of processing many items in a single call. Missing for 10: dedicated bulk API/CLI feature (e.g., bulk KV writes, batch job examples), first-party guide on batch processing patterns, and independent confirmation of bulk workflows at scale.", "evidenceIds": [ "cloudflare-docs-9" ] }, { "productId": "cloudflare", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Workers documentation confirms event-driven automation primitives — cron triggers, durable Workflows, and Queues integration — which let developers define actions that fire automatically on schedule or on message/event arrival, and Wrangler/dashboard tooling supports managing these. However there's no deeper documentation of a full conditional 'rules engine' (e.g., branching logic across multiple event types) and no independent/hands-on corroboration specifically validating this automation depth. missing for 10: detailed docs on complex rule/condition chaining across event types, independent hands-on validation of automated trigger reliability, and examples beyond cron/queues/workflows.", "evidenceIds": [ "cloudflare-docs-9", "cloudflare-docs-10", "cloudflare-docs-11", "cloudflare-docs-20" ] }, { "productId": "cloudflare", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Cloudflare Workers explicitly supports cron triggers, durable Workflows, and Queues for scheduling recurring jobs (cloudflare-docs-9), which directly satisfies the automation-depth story. However, community feedback notes CPU-time limits (30s free / 15min paid) make it unsuitable for long-running AI agent workloads, a significant caveat for AI-native recurring automation (cloudflare-comm-9). Missing for 10: independent hands-on validation of cron/Workflows specifically for AI agent scheduling, and no evidence addressing the long-running-task limitation being resolved.", "evidenceIds": [ "cloudflare-docs-9", "cloudflare-comm-9" ] }, { "productId": "cloudflare", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Cloudflare docs explicitly document version tracking and rollback of Worker deployments via Wrangler or dashboard, and note Git/GitHub Actions integration for standard workflows, which covers versioning and rollback for automations built on Workers. Missing for 10: explicit review/approval workflow tooling (e.g., staged review before promoting a version) and independent hands-on confirmation of rollback reliability.", "evidenceIds": [ "cloudflare-docs-23", "cloudflare-docs-17", "cloudflare-docs-11" ] }, { "productId": "cloudflare", "storyId": "cli-deploy", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Wrangler CLI is documented with commands for creating, developing, and deploying Workers directly from the command line, including rollback support, and community comments corroborate CLI-based workflows (e.g., 'API running in production' via CLI-based workflow). Missing for 10: no direct hands-on transcript showing a full `wrangler deploy` session output.", "evidenceIds": [ "cloudflare-docs-11", "cloudflare-docs-20", "cloudflare-docs-23", "cloudflare-probe-3", "cloudflare-gh-2" ] }, { "productId": "cloudflare", "storyId": "cli-scaffold-project", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Cloudflare documents `npm create cloudflare@latest my-app` for scaffolding and Wrangler CLI for deployment, with dedicated commands docs and rollback support; community feedback corroborates fast, easy setup and deployment (e.g. API running in production within an hour). Missing for 10: independent hands-on verification of the exact one-command scaffold-to-deploy flow and any noted friction points beyond generic praise.", "evidenceIds": [ "cloudflare-docs-2", "cloudflare-docs-11", "cloudflare-docs-20", "cloudflare-probe-3", "cloudflare-comm-3" ] }, { "productId": "cloudflare", "storyId": "condition-based-alerts", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The pack shows Cloudflare Workers has 'built-in observability' (cloudflare-docs-22) but no evidence of configurable alerting rules that notify via chat/email when specific conditions are met. Missing for 10: alert rule configuration, notification channel integrations (Slack/email/PagerDuty), and any docs or community confirmation that such alerting exists for Workers.", "evidenceIds": [ "cloudflare-docs-22" ] }, { "productId": "cloudflare", "storyId": "consistent-api-at-scale", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Cloudflare docs promote a unified model—Workers plus bindings to KV/D1/Queues/Workflows—that in principle lets code scale from prototype to production without changing runtime primitives, and per-request (not wall-time) billing plus Smart Placement are pitched as scale-friendly (cloudflare-docs-9, cloudflare-docs-10, cloudflare-docs-15, cloudflare-docs-16). However, hands-on community reports concretely contradict frictionless scaling: CPU-time caps (30s free/15min paid) are reported as unworkable for long-running agent workloads, forcing self-hosting (cloudflare-comm-9), and developers report needing to rearchitect away from Workers entirely due to lack of full Node runtime and platform lock-in around KV/D1 (cloudflare-comm-13, cloudflare-comm-10). missing for 10: evidence that identical code/bindings scale to heavy long-running or compute-intensive workloads without hitting execution limits, and independent case studies confirming zero-rearchitecture scale-up.", "evidenceIds": [ "cloudflare-docs-9", "cloudflare-docs-10", "cloudflare-docs-15", "cloudflare-docs-16", "cloudflare-comm-9", "cloudflare-comm-13", "cloudflare-comm-10" ] }, { "productId": "cloudflare", "storyId": "custom-domains", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack lacks any explicit documentation or community mention of attaching custom domains specifically to deployed Workers services (e.g., Workers Custom Domains or Routes feature); the closest evidence (cloudflare-docs-13) only describes SSL certificate issuance for domains added to Cloudflare generally, not domain-to-Worker binding. missing for 10: explicit docs on Workers Custom Domains/Routes setup, CLI/dashboard steps to bind a domain to a Worker, and any independent confirmation of this workflow.", "evidenceIds": [ "cloudflare-docs-13" ] }, { "productId": "cloudflare", "storyId": "custom-metrics-dashboards", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Cloudflare only vaguely claims 'built-in observability' for Workers, but there is no evidence of a dashboarding capability that lets a devops-lead combine metrics from arbitrary/external services into custom views. Missing for 10: documented Analytics/GraphQL API for pulling metrics into external dashboards, named integrations with Grafana/Datadog/other observability tools, and any UI for building custom multi-service dashboards.", "evidenceIds": [ "cloudflare-docs-22" ] }, { "productId": "cloudflare", "storyId": "default-ddos-waf", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Cloudflare docs explicitly state that DDoS protection, WAF, bot management, and Zero Trust controls are built into the network and run on every request from day one, directly matching the story for Workers deployed on that network. Missing for 10: independent/hands-on verification or granular configuration details showing default-on behavior in practice (e.g., WAF rule specifics for Workers routes).", "evidenceIds": [ "cloudflare-docs-18" ] }, { "productId": "cloudflare", "storyId": "docker-deploy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Cloudflare Workers is a V8-isolate serverless runtime; the evidence pack never mentions Docker or container-based deployment, and community feedback (comm-13) notes the lack of a full Node runtime, undermining any container-workflow support. This is a fair question for a deploy-workflow platform, but no evidence shows Workers supports deploying Docker-packaged apps.", "evidenceIds": [ "cloudflare-comm-13" ] }, { "productId": "cloudflare", "storyId": "documented-rate-limits", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Only indirect evidence exists: a community comment (comm-9) mentions concrete CPU-time limits (30s free tier, 15min paid) that could inform scaling decisions, but this is not from first-party docs and doesn't cover rate limits or concurrency caps generally. Missing for 10: official documentation pages on request-per-second rate limits, concurrent execution caps, subrequest limits, or a dedicated 'Limits' reference page.", "evidenceIds": [ "cloudflare-comm-9" ] }, { "productId": "cloudflare", "storyId": "durable-workflows", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Cloudflare explicitly advertises 'durable Workflows' as a built-in Workers feature integrated with Queues and cron scheduling, and its pricing model is designed to support long-running workflows without extra wall-time cost, which matches the story's intent. However, the evidence pack gives only a single sentence describing this feature with no detail on step/retry semantics, and a community comment raises concerns that Workers' CPU-time limits are unsuited to long-running agent-style workloads, adding a real caveat. missing for 10: detailed documentation of Workflows API (steps, automatic retries, state persistence), independent hands-on validation of durability guarantees, and resolution of the CPU-time-limit concern for long-running tasks.", "evidenceIds": [ "cloudflare-docs-9", "cloudflare-docs-15", "cloudflare-comm-9" ] }, { "productId": "cloudflare", "storyId": "edge-websocket-support", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Cloudflare docs reference billing for 'hibernating WebSockets' during long-running workflows, implying built-in support for persistent WebSocket connections (likely via Durable Objects), but the evidence pack lacks dedicated documentation of the WebSocket API, hibernation API, or Durable Objects mechanics. An older community comment notes WebSockets support was still 'in beta' at one point, adding some uncertainty about maturity, though this isn't a concrete current-state failure. missing for 10: explicit docs on WebSocket/Durable Objects APIs, hands-on developer confirmation of stable persistent connections, clarity on beta status resolution.", "evidenceIds": [ "cloudflare-docs-15", "cloudflare-comm-4" ] }, { "productId": "cloudflare", "storyId": "execution-time-limits", "verdict": "disputed", "quality": 6, "confidence": "medium", "rationale": "Community evidence gives concrete numbers (30s CPU on free tier, 15min on paid plans) and explicitly states these limits 'don't work for long-running agent tasks' (cloudflare-comm-9), directly contradicting Cloudflare's own marketing that Workers can 'build agents' and run 'long agent workflows' (cloudflare-docs-3, cloudflare-docs-15). The evidence pack itself never surfaces official documented CPU/wall-clock limit tables, so a devops-lead must rely on secondhand community reports rather than clear first-party limit docs. missing for 10: first-party docs page enumerating exact CPU-time/wall-clock limits per plan, and independent benchmarking confirming or refuting the 30s/15min figures for real agentic workloads.", "evidenceIds": [ "cloudflare-comm-9", "cloudflare-docs-15", "cloudflare-docs-3" ] }, { "productId": "cloudflare", "storyId": "external-service-bindings", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Cloudflare's own docs explicitly describe Bindings as the mechanism to 'connect to external services like databases, APIs, and storage... with just a few lines of code,' directly matching the story, and this is reinforced by docs on D1/KV requiring 'no connection management.' Community reports (HN) corroborate ease of connecting to KV/D1 storage from Workers in production with minimal setup. Missing for 10: deeper technical documentation of the full range of binding types (R2, Queues, Durable Objects, Vectorize) and more independent hands-on reviews specifically praising the boilerplate reduction versus manual SDK wiring.", "evidenceIds": [ "cloudflare-docs-10", "cloudflare-docs-4", "cloudflare-docs-8", "cloudflare-comm-3", "cloudflare-comm-11" ] }, { "productId": "cloudflare", "storyId": "fast-boot-scaling", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Cloudflare explicitly markets Workers as having 'no cold starts' running on isolates across its global network, and community reports corroborate very fast startup and easy scaling for production APIs (docs-1, comm-2, comm-3, comm-11). However, the pack lacks independent load-test data proving 'tens of thousands' concurrent scale, and some users note CPU-time limits complicating long-running workloads. missing for 10: independent benchmark/load-test evidence of concurrency at scale, third-party boot-time measurements beyond vendor claims.", "evidenceIds": [ "cloudflare-docs-1", "cloudflare-docs-19", "cloudflare-comm-2", "cloudflare-comm-3", "cloudflare-comm-11", "cloudflare-comm-9" ] }, { "productId": "cloudflare", "storyId": "framework-tool-continuity", "verdict": "disputed", "quality": 6, "confidence": "medium", "rationale": "Cloudflare explicitly claims Workers 'fits into your existing workflows: Git, GitHub Actions, VS Code, and any framework... No proprietary tools or vendor lock-in' and backs this with framework/language choice, Wrangler CLI, and a local dev simulator (workerd) that integrates with standard editor/CI tooling. However, hands-on community reports directly contradict the 'no lock-in' claim: developers note KV/D1 storage APIs and lack of a full Node runtime make it hard to port apps to/from Vercel/Netlify or self-hosted setups without rewrites, undermining the 'use your existing workflow freely' promise. missing for 10: independent verification that CI/editor integration is friction-free in practice, and resolution of the documented runtime/storage lock-in concerns.", "evidenceIds": [ "cloudflare-docs-17", "cloudflare-docs-6", "cloudflare-docs-7", "cloudflare-docs-11", "cloudflare-gh-1", "cloudflare-comm-10", "cloudflare-comm-13" ] }, { "productId": "cloudflare", "storyId": "full-nodejs-runtime-support", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Workers support JS/TS/Rust/Python via a V8-isolate runtime, but nothing claims unmodified Node.js/npm compatibility, and community feedback explicitly states the opposite ('Can we just get full node runtime?... without a full node runtime... can't switch from Vercel/Netlify').", "evidenceIds": [ "cloudflare-docs-7", "cloudflare-comm-13" ] }, { "productId": "cloudflare", "storyId": "full-stack-preview-envs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes Workers' general deployment, versioning, and rollback capabilities, but nowhere mentions automatic ephemeral preview environments tied to pull requests (a feature associated with Cloudflare Pages, not documented here for Workers). Missing for 10: any documentation of per-PR preview URLs, ephemeral environment provisioning, or GitHub PR integration producing full-stack previews.", "evidenceIds": [ "cloudflare-docs-17", "cloudflare-docs-23", "cloudflare-gh-1" ] }, { "productId": "cloudflare", "storyId": "git-push-deploy", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm Workers fits into Git/GitHub Actions workflows and supports CLI-based deploys plus rollbacks, implying CI/CD integration for git-push deploys, but there's no explicit first-party doc or hands-on account describing an automatic 'deploy on every git push' pipeline (e.g., Workers Builds) in this pack. Missing for 10: dedicated documentation/example of a git-integrated CI pipeline auto-triggering deploys, and independent user confirmation of this exact workflow.", "evidenceIds": [ "cloudflare-docs-17", "cloudflare-docs-23", "cloudflare-docs-11" ] }, { "productId": "cloudflare", "storyId": "global-edge-network", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Cloudflare Workers is explicitly documented and independently corroborated as a global edge compute platform, with docs on deployment to Cloudflare's global network, Smart Placement for latency optimization, and multiple hands-on community reports confirming fast, easy deployment across regions. Missing for 10: no independent latency benchmarks quantifying edge proximity to end users.", "evidenceIds": [ "cloudflare-docs-1", "cloudflare-docs-16", "cloudflare-docs-8", "cloudflare-comm-2", "cloudflare-comm-3", "cloudflare-comm-11" ] }, { "productId": "cloudflare", "storyId": "global-static-cdn", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence confirms Workers run on Cloudflare's global edge network and one community comment notes using 'CF Pages for static sites with great results,' implying edge-hosted static content, but the pack has no first-party documentation of Workers' static-assets feature, cache API, or edge-caching configuration specifics for static files. Missing for 10: explicit docs on Workers Static Assets/Cache API, cache-control/edge-caching configuration details, and performance benchmarks for static asset delivery via Workers.", "evidenceIds": [ "cloudflare-docs-1", "cloudflare-docs-19", "cloudflare-comm-14" ] }, { "productId": "cloudflare", "storyId": "hard-spending-limits", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item describes hard spending limits or budget caps for Workers usage; the only pricing-related note explains billing model (compute-only charges) but not spend-limiting controls.", "evidenceIds": [] }, { "productId": "cloudflare", "storyId": "idp-secured-access", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The evidence only vaguely mentions that 'Zero Trust controls are built into the network' (docs-18), with no detail on integrating with an external IdP (SAML/OIDC), configuring SSO for internal apps, or securing deployed agents specifically. Missing for 10: explicit IdP/SSO integration docs, Access policy examples tied to identity providers, and evidence of securing agent/deployment pipelines with org identity.", "evidenceIds": [ "cloudflare-docs-18" ] }, { "productId": "cloudflare", "storyId": "in-browser-dev-toolbar", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "An in-browser toolbar for feedback, feature flags, and live performance inspection is a specific product feature (like Vercel Toolbar) unrelated to Cloudflare Workers' role as a serverless compute platform; this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "cloudflare", "storyId": "incremental-rollout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Workers supports versioning and rollback to a previous deployment (cloudflare-docs-23) but contains no mention of percentage-based, canary, or gradual rollout mechanisms that would let a devops-lead incrementally shift traffic to a new deployment to limit blast radius.", "evidenceIds": [ "cloudflare-docs-23" ] }, { "productId": "cloudflare", "storyId": "infra-as-code-blueprint", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Wrangler's config (wrangler.toml/jsonc) plus CLI covers defining bindings/resources (docs-10), deploying (docs-11,20), and rolling back to prior versions (docs-23), which together approximate 'define, deploy, version' via one workflow. However, the evidence never explicitly shows the wrangler.toml file itself, multi-environment/staging config, or how complex multi-service architectures (Queues, Durable Objects, D1, R2) are all declared in a single file rather than via dashboard clicks or multiple bindings setup steps. Missing for 10: explicit wrangler.toml example, multi-environment config docs, and independent confirmation that full architecture (not just a single Worker) can be captured in one file.", "evidenceIds": [ "cloudflare-docs-10", "cloudflare-docs-11", "cloudflare-docs-20", "cloudflare-docs-23", "cloudflare-docs-9" ] }, { "productId": "cloudflare", "storyId": "instant-folder-deploy", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence describes CLI-based deployment via Wrangler (npm create cloudflare, wrangler commands, rollbacks) but nothing about a no-Git folder-upload flow with a private, unpublished draft state before publishing.", "evidenceIds": [] }, { "productId": "cloudflare", "storyId": "invisible-bot-protection", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Cloudflare's platform docs state that bot management and WAF are 'built into the network and run on every request from day one,' implying Workers apps inherit these protections, but there is no detail on invisible challenge mechanics (e.g., Turnstile) or Workers-specific configuration, and no independent corroboration in the evidence pack. missing for 10: specifics on invisible/challenge-based bot protection mechanics, Workers-specific setup docs, and hands-on/independent verification.", "evidenceIds": [ "cloudflare-docs-18" ] }, { "productId": "cloudflare", "storyId": "isolated-sandbox-execution", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes Workers as a general serverless/edge compute platform (deployment, bindings, frameworks, Wrangler CLI) but never mentions the underlying isolate/sandbox security model, hardware isolation guarantees, or a use case of running untrusted third-party code (e.g. Workers for Platforms, V8 isolate security boundaries). No citation addresses sandboxing untrusted code specifically. Missing for 10: any mention of V8 isolate/hardware isolation architecture, security boundary guarantees for untrusted code, or a multi-tenant 'Workers for Platforms' style product, and any independent security audit/hands-on evidence of sandbox isolation.", "evidenceIds": [] }, { "productId": "cloudflare", "storyId": "known-overage-pricing", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item documents overage fee schedules for exceeding bandwidth, invocations, or build-minute limits; only general pricing philosophy ('charges only for compute') is mentioned without any fee schedule detail.", "evidenceIds": [] }, { "productId": "cloudflare", "storyId": "load-based-autoscaling", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Workers' serverless, global-network architecture ('no servers to manage', 'no cold starts or region complexity') implies automatic scaling to traffic without ops intervention, and community posts corroborate quick, hands-off scaling in production use. However, there is no explicit documentation of autoscaling mechanics, burst-handling guarantees, or SLAs, and community evidence flags CPU-time limits (30s free/15min paid) as a real constraint for sustained high-load workloads. Missing for 10: explicit autoscaling/burst-capacity documentation, quantified traffic-spike case studies, and confirmation that limits don't throttle under load.", "evidenceIds": [ "cloudflare-docs-1", "cloudflare-docs-19", "cloudflare-comm-3", "cloudflare-comm-9" ] }, { "productId": "cloudflare", "storyId": "local-dev-simulator", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Cloudflare Workers ships Wrangler with a bundled local simulator (Miniflare) explicitly described as 'powered by workerd' — the same runtime that runs Workers in production — enabling local dev/test that mirrors production behavior, backed by dedicated Wrangler CLI docs and commands. Missing for 10: independent/hands-on confirmation that local and production behavior stay perfectly in sync, and documentation of known simulator-vs-production edge-case gaps.", "evidenceIds": [ "cloudflare-gh-1", "cloudflare-gh-3", "cloudflare-docs-11", "cloudflare-docs-20", "cloudflare-probe-3" ] }, { "productId": "cloudflare", "storyId": "managed-postgres", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Cloudflare's data offerings are serverless SQL (D1, SQLite-based), KV, and Hyperdrive for accelerating connections to external databases, not a fully-managed Postgres service with point-in-time recovery, replicas, and HA. No citation mentions Postgres provisioning, PITR, replication, or HA guarantees. missing for 10: any mention of a managed Postgres offering, PITR support, read replicas, or HA configuration.", "evidenceIds": [ "cloudflare-docs-4" ] }, { "productId": "cloudflare", "storyId": "migration-credits", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack mentions migration credits, competitive-switch incentives, or financial credits for moving workloads to Cloudflare Workers; pricing/lock-in discussion is limited to community complaints about vendor lock-in and CPU-time limits.", "evidenceIds": [] }, { "productId": "cloudflare", "storyId": "multi-protocol-native-support", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Evidence confirms Workers serve HTTP (implicit in all docs) and WebSockets (docs-15 mentions hibernating WebSockets; comm-4 references WebSocket support), but there is no evidence of TCP or gRPC serving capability, nor any claim that all four protocols run from a single deployed service without separate configuration. missing for 10: TCP support, gRPC support, unified same-service multi-protocol configuration documentation.", "evidenceIds": [ "cloudflare-docs-15", "cloudflare-comm-4" ] }, { "productId": "cloudflare", "storyId": "object-storage-uploads", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "The pack shows Cloudflare Images as a first-party service for storing/delivering media (cloudflare-docs-5) and generic 'storage' bindings (cloudflare-docs-10), suggesting Workers can persist uploads via a built-in storage binding, but there is no explicit mention of a general object-storage product (e.g. R2), upload APIs, size limits, or hands-on developer confirmation of using it for arbitrary user uploads. Missing for 10: explicit object-storage product docs (R2 or equivalent), upload flow/API examples, and independent/community confirmation of real-world use for user-generated uploads.", "evidenceIds": [ "cloudflare-docs-5", "cloudflare-docs-10" ] }, { "productId": "cloudflare", "storyId": "on-demand-image-optimization", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Cloudflare Images is documented as a product for storing, optimizing, and delivering images, which could be integrated with Workers, but the evidence pack gives only a one-line description with no detail on on-demand resize/transform APIs, integration specifics with Workers, or hands-on confirmation. Missing for 10: detailed docs on image transformation/resize parameters, explicit Workers integration examples, and independent/community corroboration of the image CDN working as claimed.", "evidenceIds": [ "cloudflare-docs-5" ] }, { "productId": "cloudflare", "storyId": "one-click-rollback", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Cloudflare docs explicitly confirm rollback to any previously deployed version of a Worker via Wrangler CLI or the dashboard, directly matching the story. Missing for 10: no independent/hands-on confirmation of the one-click UX or edge cases (e.g. rollback with bindings/config changes).", "evidenceIds": [ "cloudflare-docs-23" ] }, { "productId": "cloudflare", "storyId": "one-click-templates", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Cloudflare's `npm create cloudflare@latest` CLI (cloudflare-docs-2) and framework support for React, Vue, Svelte, Next, Astro etc. (cloudflare-docs-6) indicate scaffolding from starter templates, and Wrangler docs (cloudflare-docs-11, -20) cover creating/deploying projects. However, the pack lacks explicit evidence of a curated one-click template gallery or 'deploy to Cloudflare' button flow comparable to competitors, and one community note (cloudflare-gh) even flags a showcase project (Wildebeest) being abandoned shortly after launch. Missing for 10: an explicit template/starter gallery page, one-click deploy button UX, and independent hands-on confirmation of ease of using such templates.", "evidenceIds": [ "cloudflare-docs-2", "cloudflare-docs-6", "cloudflare-docs-11", "cloudflare-docs-20", "cloudflare-comm-7" ] }, { "productId": "cloudflare", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Cloudflare exposes a full REST API (confirmed via live openapi.json spec) and a CLI (Wrangler) that mirrors dashboard actions like deploying, rolling back, and managing Workers, plus fine-grained API token scoping — all pointing to strong API/UI parity. However, no evidence explicitly confirms 100% feature parity between the dashboard and API/CLI, and some community comments note friction with certain platform features (e.g., runtime limits) that could imply UI-only nuances. Missing for 10: explicit parity documentation or changelog confirming every dashboard feature has an API equivalent, and independent verification of parity gaps.", "evidenceIds": [ "cloudflare-probe-2", "cloudflare-docs-11", "cloudflare-docs-20", "cloudflare-docs-23", "cloudflare-docs-14", "cloudflare-docs-21", "cloudflare-gh-2" ] }, { "productId": "cloudflare", "storyId": "openness-full-export", "verdict": "disputed", "quality": 3, "confidence": "medium", "rationale": "Cloudflare claims workflows are open and free of vendor lock-in ('No proprietary tools or vendor lock-in' - cloudflare-docs-17), but hands-on community feedback specifically contradicts this for data services, calling out that KV, D1, etc. are 'locked-in things' that make it hard to switch or run the same code elsewhere (cloudflare-comm-10). There is no documented data-export tool, open-format export command, or migration guide for leaving the platform with your data. Missing for 10: any first-party data-export/migration documentation for D1/KV/R2, evidence of open-format export capability, and resolution of the lock-in complaint.", "evidenceIds": [ "cloudflare-docs-17", "cloudflare-comm-10" ] }, { "productId": "cloudflare", "storyId": "openness-open-license", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Cloudflare open-sources parts of its developer tooling (workers-sdk CLI, and references to the workerd runtime) on GitHub, but the evidence never states a license or confirms the entire Workers platform/runtime is open-source — the core edge service remains closed. Missing for 10: explicit license text, confirmation that the full production runtime (not just CLI/simulator) is open, and independent confirmation of open-source status beyond repo existence.", "evidenceIds": [ "cloudflare-gh-1", "cloudflare-gh-2", "cloudflare-gh-3" ] }, { "productId": "cloudflare", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Cloudflare Workers is presented throughout the evidence as a proprietary hosted platform running on Cloudflare's global network (docs-1, docs-19); there is no evidence of an official self-hosted distribution of the full Workers platform. Community comments explicitly note lock-in and desire to 'run the same code on your own stuff or a small VPS' as an unmet want (cloudflare-comm-10, cloudflare-comm-9), confirming self-hosting the core product is not currently offered.", "evidenceIds": [ "cloudflare-docs-1", "cloudflare-docs-19", "cloudflare-comm-9", "cloudflare-comm-10" ] }, { "productId": "cloudflare", "storyId": "pay-for-active-compute", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Cloudflare explicitly documents billing only for active CPU/compute time rather than wall-clock time, even for long-running agent workflows or hibernating WebSockets, aligning directly with the devops-lead pricing story. missing for 10: independent third-party billing analysis or hands-on cost comparison corroborating the claim beyond vendor docs.", "evidenceIds": [ "cloudflare-docs-15" ] }, { "productId": "cloudflare", "storyId": "pay-per-byte-storage", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Workers is a serverless compute platform with no attached-disk concept; storage needs are met via separate bound services (R2, D1, KV) rather than a growing disk volume, so this pricing axis doesn't map onto Workers itself.", "evidenceIds": [] }, { "productId": "cloudflare", "storyId": "persistent-agent-runtime", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Cloudflare docs reference 'durable Workflows' and agent-oriented billing that doesn't charge for idle/hibernating time (cloudflare-docs-9, cloudflare-docs-15), suggesting some support for persistent, checkpointed agent runtimes, but there's no explicit documentation of Durable Objects or automatic state checkpointing for coding agents. Community evidence directly contradicts the practical viability for long-running agent tasks, noting CPU time limits (30s free / 15min paid) make Workers unsuitable for long-running agent workloads, pushing developers to self-host instead (cloudflare-comm-9). missing for 10: explicit Durable Objects/state-checkpointing documentation for agents, confirmation that long-running agent tasks work without hitting CPU/time limits, first-party agent-specific persistence guide.", "evidenceIds": [ "cloudflare-docs-9", "cloudflare-docs-15", "cloudflare-comm-9" ] }, { "productId": "cloudflare", "storyId": "polyglot-edge-runtimes", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Cloudflare's own docs explicitly claim Workers support 'JavaScript, TypeScript, Python, Rust, and more' (cloudflare-docs-7), and Wrangler/workers-sdk tooling supports multi-language builds. However, a hands-on community report explicitly cites 'lack of Python support' as a blocker to building more on Cloudflare (cloudflare-comm-14), and no evidence in the pack shows working Go support or first-class (non-WASM) language support beyond JS/TS. missing for 10: independent confirmation of Python/Rust/Go working smoothly in production, clarification of Python's beta/Pyodide limitations, and any mention of Go support at all.", "evidenceIds": [ "cloudflare-docs-7", "cloudflare-comm-14" ] }, { "productId": "cloudflare", "storyId": "pr-preview-urls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions Cloudflare Workers automatically generating preview URLs for pull requests (e.g., via Workers Builds or Pages-style git integration); only generic deployment, Wrangler CLI, and rollback docs are cited. This is a fair and applicable axis for a serverless deploy platform, but the pack lacks any confirmation of automatic PR preview URLs.", "evidenceIds": [] }, { "productId": "cloudflare", "storyId": "preview-collaboration-comments", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Cloudflare Workers is a serverless compute/edge platform; collaborative commenting on preview/production deployments is a category error for this kind of infrastructure product, not a capability buyers evaluating Workers would expect.", "evidenceIds": [] }, { "productId": "cloudflare", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Workers' global deployment, latency optimization (Smart Placement), and general platform features, but contains no mention of data residency, regional storage selection, or jurisdictional controls for where data is stored. Missing for 10: any documentation of region-pinning, data localization suite, or jurisdiction restrictions for storage products (D1, KV, R2).", "evidenceIds": [ "cloudflare-docs-1", "cloudflare-docs-16" ] }, { "productId": "cloudflare", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "low", "rationale": "Cloudflare Workers is a compute/serverless platform, not an AI model provider or data-training controller; there is no evidence about training-data opt-out policies since this is not the relevant axis for a serverless execution platform.", "evidenceIds": [] }, { "productId": "cloudflare", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data retention policies, deletion controls, or privacy/data lifecycle management for data stored via Workers, KV, D1, R2, or related services; evidence covers deployment, tooling, pricing, and general dev experience but nothing about retention/deletion controls.", "evidenceIds": [] }, { "productId": "cloudflare", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry opt-out or usage-tracking controls for Cloudflare Workers or Wrangler CLI; this is a plausible axis for a developer platform/CLI but no documentation or community report confirms such a setting exists.", "evidenceIds": [] }, { "productId": "cloudflare", "storyId": "private-service-networking", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers Bindings for connecting to external databases/APIs (docs-10) and generic Zero Trust/WAF network security (docs-18), but nothing documents a specific mechanism (e.g., private Worker-to-Worker service bindings or a documented private-network route) for keeping internal service-to-service calls off the public internet without VPC setup. No first-party or community evidence directly addresses this specific networking pattern.", "evidenceIds": [] }, { "productId": "cloudflare", "storyId": "prod-nonprod-isolation", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence covers general Workers features (deployment, bindings, Wrangler, API token scoping to specific zones) but nothing addresses environment-level isolation (e.g., separate staging vs production networks, service bindings restricted by environment, or network policies preventing non-prod from reaching prod services). No documentation or community evidence describes this capability.", "evidenceIds": [] }, { "productId": "cloudflare", "storyId": "programmatic-multitenant-domains", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows general Workers platform capabilities (deployment, bindings, API tokens, SSL certs) but contains no mention of Cloudflare for SaaS, custom hostnames API, or Workers for Platforms — the specific mechanisms for programmatically provisioning and routing custom domains on behalf of multi-tenant SaaS customers.", "evidenceIds": [] }, { "productId": "cloudflare", "storyId": "public-incident-history", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack references a public status page, uptime history, or incident postmortems for Cloudflare Workers; the pack only covers product features, docs, and community opinions.", "evidenceIds": [] }, { "productId": "cloudflare", "storyId": "published-uptime-sla", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item references an SLA document, uptime guarantee, or service credit terms for Workers; documentation covers features, tooling, and pricing model but not a formal SLA.", "evidenceIds": [] }, { "productId": "cloudflare", "storyId": "redis-compatible-cache", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence confirms Cloudflare offers a 'globally distributed key-value' store (cloudflare-docs-4) and a separate Queues service (cloudflare-docs-9), and community mentions 'KV storage' (cloudflare-comm-3), but nothing indicates Workers KV is Redis-protocol-compatible or usable as a Redis substitute for caching/job-queue patterns. Missing for 10: any claim or documentation of Redis API/protocol compatibility, benchmarks showing low-latency Redis-like semantics, or job-queue patterns built directly on the KV store.", "evidenceIds": [ "cloudflare-docs-4", "cloudflare-docs-9", "cloudflare-comm-3" ] }, { "productId": "cloudflare", "storyId": "regional-deployment", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Workers deploy automatically across Cloudflare's global edge network, meaning code runs close to users worldwide by default (cloudflare-docs-1, cloudflare-docs-19), and Smart Placement further optimizes Worker placement relative to backend data for latency (cloudflare-docs-16, cloudflare-docs-8). However, this is an automatic proximity model rather than explicit region selection/pinning that a devops-lead could target — there's no evidence of manual region choice, data-residency controls, or region-restricted deployment commands. Missing for 10: explicit region-selection/pinning API or dashboard control, data residency guarantees, and independent verification that global auto-placement matches manual region targeting needs.", "evidenceIds": [ "cloudflare-docs-1", "cloudflare-docs-19", "cloudflare-docs-16", "cloudflare-docs-8" ] }, { "productId": "cloudflare", "storyId": "seat-based-pricing-minimums", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Cloudflare Workers pricing is usage/compute-based ('Cloudflare charges only for compute, not wall time') rather than per-seat, and no documentation or community evidence mentions seat-based pricing tiers or per-seat minimums that scale with team size. Missing for 10: any pricing page, seat-count minimums, or per-user cost documentation.", "evidenceIds": [ "cloudflare-docs-15" ] }, { "productId": "cloudflare", "storyId": "serverless-functions", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Cloudflare Workers is extensively documented as a no-server-management serverless platform supporting full-stack frameworks, multiple languages, bindings to data stores, CLI (Wrangler) deployment, and rollback, corroborated by independent developer testimonials describing production use with no server management. Missing for 10: fully independent (non-vendor) benchmark of 'no cold starts' claims and some community friction around limits for long-running tasks (community notes on CPU time limits and lock-in), which slightly tempers a perfect score.", "evidenceIds": [ "cloudflare-docs-1", "cloudflare-docs-6", "cloudflare-docs-7", "cloudflare-docs-10", "cloudflare-docs-11", "cloudflare-docs-23", "cloudflare-comm-2", "cloudflare-comm-3", "cloudflare-comm-11", "cloudflare-probe-3" ] }, { "productId": "cloudflare", "storyId": "serverless-sql-query", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Cloudflare docs explicitly state D1 (serverless SQL) can be queried directly from Workers with no connection management, backed by bindings that eliminate manual pooling/connection setup, and community evidence corroborates developers using D1 in production Workers apps. missing for 10: independent hands-on benchmarking of connection behavior at scale and more detail on D1 driver/ORM specifics beyond marketing docs.", "evidenceIds": [ "cloudflare-docs-4", "cloudflare-docs-10", "cloudflare-comm-11" ] }, { "productId": "cloudflare", "storyId": "site-password-protection", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Cloudflare's docs mention that 'Zero Trust controls are built into the network and run on every request from day one' (docs-18), which hints at Access-based authentication gating that could be used for password protection, but there's no direct documentation, tutorial, or example showing a simple password-gate on a Worker/site without building custom auth. missing for 10: explicit Cloudflare Access/password-protection walkthrough, developer confirmation it works as a lightweight auth layer, and any hands-on corroboration of ease-of-use for this specific use case.", "evidenceIds": [ "cloudflare-docs-18" ] }, { "productId": "cloudflare", "storyId": "smart-placement", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Cloudflare documents Smart Placement, a feature explicitly designed to automatically move Worker execution closer to backend data sources to optimize end-to-end latency, directly matching the story. This is first-party documented and reinforced across multiple docs pages, though there is no independent/hands-on benchmark confirming real-world latency gains. Missing for 10: independent performance validation or case study demonstrating measured latency improvement from Smart Placement.", "evidenceIds": [ "cloudflare-docs-8", "cloudflare-docs-16" ] }, { "productId": "cloudflare", "storyId": "standards-based-runtime-portability", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Cloudflare markets Workers as fitting 'existing workflows... no proprietary tools or vendor lock-in' (cloudflare-docs-17) and Workers do use some standard Web APIs (fetch/Request/Response), but hands-on community reports directly contradict the portability claim for storage/runtime: developers cite KV, D1, and other bindings as proprietary lock-in that 'always made it hard to switch,' explicitly wanting to 'run the same code on your own stuff or a small VPS without changing the storage' (cloudflare-comm-10), and others note the lack of a full Node runtime prevents switching between Vercel/Netlify/Cloudflare (cloudflare-comm-13), and Python support gaps limit portability further (cloudflare-comm-14). missing for 10: evidence of actual standards-based (e.g. WinterCG-compliant) runtime API parity independent of proprietary bindings, and confirmation that storage APIs are swappable/standard rather than Cloudflare-specific.", "evidenceIds": [ "cloudflare-docs-17", "cloudflare-comm-10", "cloudflare-comm-13", "cloudflare-comm-14", "cloudflare-docs-10" ] }, { "productId": "cloudflare", "storyId": "support-response-sla", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions SLA response times, support tiers, or escalation paths for Cloudflare Workers; the docs focus on product features, deployment, and dev workflows rather than support commitments.", "evidenceIds": [] }, { "productId": "cloudflare", "storyId": "team-secrets-env-vars", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Evidence only shows Cloudflare API tokens can be scoped to specific resources (e.g., Zone DNS read access), which supports granular access control, but there is no documentation of team-level role management, centralized secrets storage, or environment-variable management for a devops-lead. missing for 10: team/organization RBAC docs, secrets management (e.g., Secrets Store) evidence, environment variable management across environments, and any hands-on confirmation of these features.", "evidenceIds": [ "cloudflare-docs-14", "cloudflare-docs-21" ] }, { "productId": "cloudflare", "storyId": "unified-log-view", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs mention built-in observability (\"Gain deep visibility and insight with built-in observability\") and rollback tracking of deployments, implying some runtime and deploy visibility, but there is no explicit evidence of a unified dashboard combining build logs, deploy logs, and runtime logs in one place. Missing for 10: explicit documentation of a consolidated logs view spanning build, deploy, and runtime stages, and independent confirmation of this unified experience.", "evidenceIds": [ "cloudflare-docs-22", "cloudflare-docs-23" ] }, { "productId": "cloudflare", "storyId": "unlimited-team-environments", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Workers' architecture (isolate-based execution, Miniflare local simulator, quick project scaffolding via `npm create cloudflare`, and Wrangler CLI for deploy/rollback) supports fast, isolated dev setups, and Workers fit into existing Git/CI workflows implying branch-based deploys. However, there is no explicit documentation of per-branch/preview environments, environment-scoping features, or team-scale 'unlimited environments' guarantees in the evidence pack. Missing for 10: explicit preview/staging environment docs, team-scale environment provisioning evidence, and independent confirmation of simultaneous multi-developer isolated environments.", "evidenceIds": [ "cloudflare-gh-1", "cloudflare-gh-2", "cloudflare-docs-2", "cloudflare-docs-11", "cloudflare-docs-17", "cloudflare-docs-23", "cloudflare-comm-3" ] }, { "productId": "cloudflare", "storyId": "visitor-performance-analytics", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs mention Cloudflare Workers ships with 'built-in observability' for deep visibility and insight, which touches the observability theme, but there is no evidence of visitor-behavior tracking, traffic/analytics dashboards, or performance metrics presented to a devops-lead. Missing for 10: dedicated analytics/dashboard evidence, visitor behavior metrics (page views, referrers, geo), and independent confirmation of the observability feature's depth.", "evidenceIds": [ "cloudflare-docs-22" ] }, { "productId": "cloudflare", "storyId": "visual-infra-canvas", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Cloudflare Workers is a code-first serverless platform (Wrangler CLI, JS/TS/Python/Rust runtimes) with no visual canvas or no-code infrastructure editor evidenced anywhere in the pack; this axis is a category mismatch for this product type.", "evidenceIds": [] }, { "productId": "cloudflare", "storyId": "zero-config-repo-deploy", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs mention that Workers 'fits into your existing workflows: Git, GitHub Actions' implying repo-based CI/CD, and support for many frameworks out of the box, but the evidence pack lacks explicit description of a zero-config 'connect repo and deploy' flow (e.g., automatic framework/build detection like Pages does) or hands-on confirmation that no manual build setup is needed. missing for 10: explicit docs on automatic build configuration detection, dashboard 'connect GitHub repo' walkthrough, and independent confirmation of a truly no-config deploy experience.", "evidenceIds": [ "cloudflare-docs-17", "cloudflare-docs-6", "cloudflare-docs-2" ] }, { "productId": "fly-io", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A direct probe confirms Fly.io serves a working llms.txt file at https://fly.io/llms.txt (HTTP 200) with structured documentation content, and Fly.io's docs also explicitly cater to agent-oriented workflows (e.g. Sprites, MCP server hosting). Missing for 10: no evidence of docs.md fallback or broader agent-oriented docs discoverability beyond the single llms.txt file, and no independent confirmation of agents actually consuming it successfully.", "evidenceIds": [ "fly-io-probe-1", "fly-io-docs-4", "fly-io-docs-6" ] }, { "productId": "fly-io", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Fly.io is an infrastructure/compute platform (VMs, deployment, scaling) rather than a data product with an interface for surfacing AI-generated insights from a user's own data; this axis targets analytics/SaaS-type products, not a cloud hosting platform.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Fly.io markets 'Sprites' that sleep until needed, checkpoint themselves, and wake up remembering state, plus Machine restart policies that keep processes running autonomously — a reasonable base for background agent automation (fly-io-docs-4, fly-io-docs-5, fly-io-docs-26). However, community feedback explicitly flags that Fly lacks a built-in scheduler: 'the missing bit in Fly is scheduled tasks... basic cron-like scheduling should be part of the platform' (fly-io-comm-20), meaning real automation triggers require external glue rather than a native mechanism. Missing for 10: native cron/scheduled-trigger support, and independent hands-on evidence that long-running agent automations are reliable in production (reliability complaints exist elsewhere in the evidence pack, though not specific to automation).", "evidenceIds": [ "fly-io-docs-4", "fly-io-docs-5", "fly-io-docs-26", "fly-io-comm-20" ] }, { "productId": "fly-io", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Fly.io is an infrastructure/PaaS platform for hosting apps and agent sandboxes (Sprites, Machines); it has no built-in AI assistant that end-users delegate tasks to inside the product itself. This axis is a category error for a cloud infra provider, not a gap in an AI-product feature set.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Fly.io ships a scriptable CLI (flyctl) with single-line commands like `fly launch`, `fly machine run`, and `fly scale`, plus a full REST API for Machines that can provision and manage apps programmatically — both of which are usable headlessly in automation/CI pipelines. However, the evidence pack never explicitly documents CI integration (e.g., GitHub Actions, deploy tokens for automated pipelines) or shows a CI-specific workflow example. Missing for 10: explicit CI/CD integration docs (GitHub Actions, deploy tokens), independent hands-on confirmation of running flyctl in CI, and any automation-specific guide beyond general CLI/API docs.", "evidenceIds": [ "fly-io-docs-1", "fly-io-docs-16", "fly-io-docs-27", "fly-io-docs-31", "fly-io-docs-18", "fly-io-probe-4" ] }, { "productId": "fly-io", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "medium", "rationale": "Fly.io is a cloud infrastructure/hosting platform, not an AI agent or assistant that itself consumes tools; the evidence shows Fly.io as infrastructure for *running* MCP servers (docs-6, docs-17) or *exposing* an MCP server via flyctl (docs-29), not as a client that plugs in MCP servers to gain tool-use capabilities. This story's axis (product acting as an MCP client) doesn't apply to a compute/hosting platform.", "evidenceIds": [ "fly-io-docs-6", "fly-io-docs-17", "fly-io-docs-29" ] }, { "productId": "fly-io", "storyId": "agentic-mcp-server", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Fly.io ships an official, first-party MCP server via `flyctl mcp` that lets agents connect to Fly's platform, but the docs explicitly flag it as experimental with no independent corroboration of stability or broader agent-integration support. Missing for 10: evidence of production-readiness, hands-on validation from third parties, and details on capabilities/tools exposed beyond the experimental flag.", "evidenceIds": [ "fly-io-docs-29", "fly-io-docs-17" ] }, { "productId": "fly-io", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Fly.io exposes an experimental flyctl MCP server (docs-29) that lets AI agents/assistants issue natural-language-driven commands against the platform, and the broader Sprites/agent-hosting narrative (docs-4–13, docs-28) shows agent-first design. However, the primary interface remains the traditional flyctl CLI and fly.toml config (docs-1, docs-16, docs-27), with no dedicated natural-language command layer for end users themselves. Missing for 10: a first-party natural-language chat/command interface for operating Fly.io (not just an experimental MCP passthrough), and independent corroboration that the MCP server reliably interprets NL commands.", "evidenceIds": [ "fly-io-docs-29", "fly-io-docs-28", "fly-io-docs-4", "fly-io-docs-1" ] }, { "productId": "fly-io", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Fly.io ships flyctl, a fully documented official CLI (fly launch, fly machine run, fly scale, fly doctor, etc.) that is AI-native aware — it even has an experimental built-in MCP server mode (flyctl mcp) and the whole platform markets itself around agent workflows (Sprites for Claude/Codex harnesses). missing for 10: no independent/hands-on corroboration of the CLI's AI-native workflows specifically, and the MCP server mode is explicitly labeled experimental rather than GA.", "evidenceIds": [ "fly-io-docs-1", "fly-io-docs-16", "fly-io-docs-22", "fly-io-docs-27", "fly-io-docs-29", "fly-io-probe-4", "fly-io-docs-4" ] }, { "productId": "fly-io", "storyId": "agentic-public-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Fly.io documents a public REST API (\"Fly Machines REST API provides resources to provision and manage Fly Apps, Fly Machines, and Fly Volumes\" and \"fast-launching VMs with a simple REST API\"), giving AI-native users a genuine programmatic interface beyond the CLI. However, probes show no discoverable OpenAPI/swagger spec at standard locations, and the primary documented interface for automation is flyctl (CLI) with only an experimental MCP wrapper, not a fully machine-consumable API spec. missing for 10: a published OpenAPI/swagger schema, independent developer corroboration of API robustness, and clarity on API versioning/stability guarantees.", "evidenceIds": [ "fly-io-docs-31", "fly-io-docs-18", "fly-io-docs-17", "fly-io-probe-3", "fly-io-docs-29" ] }, { "productId": "fly-io", "storyId": "agentic-scoped-keys", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Fly.io docs describe least-privilege credential scoping for agent sandboxes ('Sprites'): grant access by Sprite name, label, or single endpoint so an agent 'can post messages and never touch admin,' plus per-Sprite rotation/revocation without affecting others, and secrets encrypted in a vault. This directly matches issuing scoped, revocable credentials to an agent. Missing for 10: independent/hands-on corroboration of this scoping in practice, and no detail on underlying token/macaroon API mechanics.", "evidenceIds": [ "fly-io-docs-11", "fly-io-docs-12", "fly-io-docs-13", "fly-io-docs-23" ] }, { "productId": "fly-io", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Fly.io provides a CLI (flyctl), a REST API (Fly Machines API), and an experimental MCP server, but no official language SDKs (e.g., Python/JS/Go client libraries) are mentioned anywhere in the evidence pack; probes for openapi/SDK-related pages also failed (404s).", "evidenceIds": [ "fly-io-docs-31", "fly-io-probe-3", "fly-io-probe-4" ] }, { "productId": "fly-io", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence Fly.io offers a webhook subscription/event system for its own platform events (deploys, machine state changes, etc.); the only webhook-related item describes Fly.io as a place to receive/ingest incoming webhooks, not a source of subscribable events.", "evidenceIds": [ "fly-io-docs-32" ] }, { "productId": "fly-io", "storyId": "ai-global-inference", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents Fly Machines/Sprites as general-purpose global compute VMs, agent sandboxes, and Postgres, but contains no mention of a managed AI inference/model-serving API, GPU provisioning, or 'single call' inference capability abstracting away GPU/capacity management. Absence of evidence for this applicable capability yields none.", "evidenceIds": [ "fly-io-docs-14", "fly-io-docs-18", "fly-io-docs-20" ] }, { "productId": "fly-io", "storyId": "ai-model-gateway", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Fly.io is an edge-compute/VM hosting platform, not an AI model gateway/router; nothing in the evidence pack indicates a unified model-access endpoint for hundreds of AI models. This capability is a category error for this product type.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an interactive API reference or runnable examples; the openapi probe returned 404s across all candidate paths and docs only describe REST API resources in prose form.", "evidenceIds": [ "fly-io-docs-31", "fly-io-probe-3" ] }, { "productId": "fly-io", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Fly.io documents a REST API for Machines (fly-io-docs-31) but explicit probes for OpenAPI/swagger spec files at standard locations all return 404 (fly-io-probe-3), and no downloadable machine-readable spec is referenced anywhere in the docs.", "evidenceIds": [ "fly-io-probe-3", "fly-io-docs-31" ] }, { "productId": "fly-io", "storyId": "api-sandbox", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Fly.io explicitly offers hardware-isolated, disposable VMs/Sprites with 'a clean baseline per request, so no two users ever share state,' plus checkpoint/rollback features that support spinning up isolated test environments without affecting production (fly-io-docs-7, fly-io-docs-9, fly-io-docs-4/5). However, this is all first-party marketing language with no independent or hands-on evidence confirming a documented staging/production separation workflow, and community threads focus on unrelated reliability complaints rather than sandbox isolation. Missing for 10: independent corroboration of sandbox isolation in practice, explicit staging-vs-production workflow documentation.", "evidenceIds": [ "fly-io-docs-7", "fly-io-docs-9", "fly-io-docs-4", "fly-io-docs-5" ] }, { "productId": "fly-io", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of API versioning scheme or a documented deprecation policy; OpenAPI spec probes returned 404s and no docs mention versioning/deprecation commitments.", "evidenceIds": [ "fly-io-probe-3", "fly-io-probe-2" ] }, { "productId": "fly-io", "storyId": "automatic-tls-certs", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions TLS certificate provisioning, automatic renewal, or wildcard domain support; the docs excerpts focus on Machines, Sprites, MCP servers, and pricing, not certificate management.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Fly.io exposes a REST API for provisioning/managing many Fly Apps, Machines, and Volumes (fly-io-docs-31) and CLI primitives like `fly machine run`/`fly scale` (fly-io-docs-27, fly-io-docs-24), which could be scripted for bulk actions, but there is no documented bulk/batch operation feature (e.g., a single command or endpoint acting on many resources at once) demonstrated in the evidence. missing for 10: explicit bulk/batch API endpoints or CLI flags for multi-resource operations, examples of scripted bulk workflows, independent confirmation of bulk automation use.", "evidenceIds": [ "fly-io-docs-31", "fly-io-docs-27", "fly-io-docs-24" ] }, { "productId": "fly-io", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Fly.io's evidence only shows infrastructure-level triggers like Machine restart policies (restart on process exit) — not a general-purpose rules/automation engine for reacting to arbitrary events. Community feedback explicitly notes the platform lacks built-in scheduled/cron-like automation, requiring users to build their own scheduler apps, indicating no native event-rule automation capability exists.", "evidenceIds": [ "fly-io-docs-26", "fly-io-comm-20" ] }, { "productId": "fly-io", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No documentation evidence shows native recurring job/cron scheduling on Fly.io; a community comment explicitly states scheduled tasks are missing from the platform and must be built manually.", "evidenceIds": [ "fly-io-comm-20" ] }, { "productId": "fly-io", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Fly.io documents checkpoint-based rollback for its Sprite/agent runtime ('Roll back to any checkpoint in seconds', 'A Sprite checkpoints itself while the agent works'), which covers rollback of agent state, but there is no evidence of version history, diffing, or a review workflow for automations themselves. missing for 10: version history/diff view, review workflow before rollback, independent hands-on confirmation of checkpoint rollback working reliably", "evidenceIds": [ "fly-io-docs-9", "fly-io-docs-4", "fly-io-docs-5" ] }, { "productId": "fly-io", "storyId": "cli-deploy", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Fly.io's core deploy workflow is CLI-first via flyctl: `fly launch` creates, configures, and deploys apps directly from the project directory, with additional CLI commands (`fly machine run`, `fly scale`, `fly doctor`) covering the full deploy lifecycle without needing a dashboard. Missing for 10: independent hands-on confirmation that dashboard is never required for any step (community evidence focuses on reliability, not on CLI-vs-dashboard workflow).", "evidenceIds": [ "fly-io-docs-1", "fly-io-docs-16", "fly-io-docs-27", "fly-io-docs-24", "fly-io-probe-4" ] }, { "productId": "fly-io", "storyId": "cli-scaffold-project", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "First-party docs explicitly describe `fly launch` as a single command run from the project directory that creates, configures, and deploys a new app, matching the story precisely (fly-io-docs-16, fly-io-docs-1). However, there is no independent/hands-on confirmation of this specific scaffold-and-deploy flow, and several community reports describe general deploy flakiness on the platform (fly-io-comm-5, fly-io-comm-19) that add some uncertainty to the smoothness of the workflow. missing for 10: independent hands-on verification of `fly launch` succeeding cleanly, and resolution of community reports of deploy failures.", "evidenceIds": [ "fly-io-docs-16", "fly-io-docs-1", "fly-io-comm-5", "fly-io-comm-19" ] }, { "productId": "fly-io", "storyId": "condition-based-alerts", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack of Fly.io offering alerting/notification features (chat or email) tied to conditions or metrics; documentation covers deployment, compute, and infrastructure but not alerting/observability notification setup.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "consistent-api-at-scale", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Fly.io explicitly claims consistent primitives across scale (\"Keep the same primitives at every scale... The API you start with is the API you keep\") and backs this with Fly Machines' fast-boot VM API, `fly scale`, and a unified REST API for Apps/Machines/Volumes usable from prototype to production. However, this is a first-party marketing claim with no independent case study or benchmark showing a real app scaling from prototype to 'massive scale' on the same primitives without changes. Missing for 10: independent verification of large-scale deployments retaining the original API/architecture, and case studies documenting zero-rearchitecture growth.", "evidenceIds": [ "fly-io-docs-15", "fly-io-docs-14", "fly-io-docs-24", "fly-io-docs-31", "fly-io-docs-18" ] }, { "productId": "fly-io", "storyId": "custom-domains", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of custom domain attachment, certificate management, or DNS configuration (e.g., no reference to `fly certs add` or similar). While Fly.io is a PaaS where this axis clearly applies, no documentation or community evidence in the pack demonstrates this capability.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "custom-metrics-dashboards", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a custom dashboarding capability or metrics/observability aggregation feature across services; the evidence pack covers deployment, compute, VMs, and agent 'Sprites' but nothing about building dashboards from metrics.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "default-ddos-waf", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions DDoS protection or a web application firewall being applied by default (or at all); docs cover secrets, scaling, and Machine isolation but never security perimeter features like WAF/DDoS mitigation.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "docker-deploy", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Fly.io's core docs clearly describe Docker-based deployment (fly launch/fly deploy converting Docker images into VMs, fly machine run, fly doctor checking Docker) [fly-io-docs-1, fly-io-docs-16, fly-io-docs-27, fly-io-docs-32, fly-io-docs-22], but hands-on community reports describe container deploys failing at high rates with unclear errors and stuck/unreachable instances, directly contradicting a smooth deploy workflow [fly-io-comm-5, fly-io-comm-19]. missing for 10: independent verification that Docker deploys work reliably at scale, and resolution of documented deploy failure reports.", "evidenceIds": [ "fly-io-docs-1", "fly-io-docs-16", "fly-io-docs-27", "fly-io-docs-32", "fly-io-docs-22", "fly-io-comm-5", "fly-io-comm-19" ] }, { "productId": "fly-io", "storyId": "documented-rate-limits", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack items document specific rate limits, concurrency caps, or throughput quotas for Fly.io's APIs, Machines, or MCP/Sprite offerings; docs focus on scaling primitives and pricing model, not documented numeric limits/caps for planning capacity.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "durable-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Fly.io evidence covers VM provisioning, restart policies, and infrastructure primitives (Machines, checkpoints, Postgres) but nothing about a durable workflow-as-code abstraction, built-in queueing, or automatic retry orchestration for long-running business workflows. The restart policy docs (fly-io-docs-26) only cover process-level VM restarts, not workflow state/retry semantics.", "evidenceIds": [ "fly-io-docs-26", "fly-io-docs-14", "fly-io-comm-20" ] }, { "productId": "fly-io", "storyId": "edge-websocket-support", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Fly Machines are persistent, hardware-isolated VMs (not typical ephemeral serverless functions) that boot fast and run close to users globally, which architecturally supports long-lived connections like WebSockets, but no evidence explicitly documents WebSocket handling or real-time connection APIs. missing for 10: explicit WebSocket/real-time API documentation, hands-on example of persistent socket handling in a Fly app, confirmation of behavior under Fly's edge/global routing for stateful connections.", "evidenceIds": [ "fly-io-docs-14", "fly-io-docs-18", "fly-io-docs-20" ] }, { "productId": "fly-io", "storyId": "execution-time-limits", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Fly.io's evidence pack markets Machines/Sprites explicitly for long-running and agentic workloads (e.g., fly-io-docs-4, fly-io-docs-5, fly-io-docs-28) but never states any CPU or wall-clock execution time limit — Fly Machines are persistent VMs, not time-boxed functions, and no documentation here quantifies compute/runtime ceilings a devops-lead could compare against. Missing for 10: any documented CPU quota, timeout, or wall-clock limit per Machine/Sprite invocation, and any comparison table or pricing-limits doc addressing execution duration.", "evidenceIds": [ "fly-io-docs-4", "fly-io-docs-5", "fly-io-docs-28", "fly-io-docs-14" ] }, { "productId": "fly-io", "storyId": "external-service-bindings", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Fly.io offers secrets management (fly-io-docs-23), org-wide service connections without manual token-pasting (fly-io-docs-11, fly-io-docs-12, fly-io-docs-13), and its own Managed Postgres (fly-io-docs-19), which reduce some credential-wiring boilerplate. However, these are infrastructure-level networking/secrets primitives rather than native code-level bindings/SDKs for external databases, third-party APIs, or storage services akin to Workers-style bindings. Missing for 10: explicit SDK/binding integrations for common third-party services (e.g., S3, Stripe, external APIs), documentation showing reduced application code boilerplate, and independent corroboration that this actually simplifies service wiring in practice.", "evidenceIds": [ "fly-io-docs-11", "fly-io-docs-12", "fly-io-docs-13", "fly-io-docs-19", "fly-io-docs-23" ] }, { "productId": "fly-io", "storyId": "fast-boot-scaling", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Fly.io's own docs explicitly claim Machines 'boot in under a second... and scale out to tens of thousands when the traffic arrives' (fly-io-docs-14), backed by fast-launch API and scale commands (fly-io-docs-18, fly-io-docs-24). However, multiple independent community reports describe real production reliability problems under load — frequent machine failures, stuck/unreachable instances, flaky deploys (70-80% failure rate), and multi-machine outages with no status-page acknowledgment (fly-io-comm-1, fly-io-comm-5, fly-io-comm-7, fly-io-comm-19) — directly undercutting the claim of dependable scaling at volume. missing for 10: independent benchmark data on actual boot latency and large-scale load tests, and resolution/acknowledgment of the reliability complaints.", "evidenceIds": [ "fly-io-docs-14", "fly-io-docs-18", "fly-io-docs-24", "fly-io-comm-1", "fly-io-comm-5", "fly-io-comm-7", "fly-io-comm-19" ] }, { "productId": "fly-io", "storyId": "framework-tool-continuity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Fly.io explicitly supports deploying arbitrary Docker images and offers framework-specific guides, implying existing frameworks and build tooling continue to work rather than requiring a proprietary runtime (fly-io-docs-32, fly-io-docs-2, fly-io-docs-25). However, developers must still adopt the flyctl CLI and fly.toml config to deploy (fly-io-docs-1, fly-io-docs-16), and there is no evidence of native Git-push deploy, first-class CI/CD templates, or editor integrations. missing for 10: explicit CI/CD pipeline integration evidence, git-based deploy workflow, editor/IDE integration, confirmation that no Fly-specific code changes are needed for common frameworks.", "evidenceIds": [ "fly-io-docs-1", "fly-io-docs-2", "fly-io-docs-16", "fly-io-docs-25", "fly-io-docs-32" ] }, { "productId": "fly-io", "storyId": "full-nodejs-runtime-support", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Fly.io's architecture (Fly Machines are hardware-isolated VMs running arbitrary Docker images, not a restricted sandboxed runtime) implies full support for running unmodified Node.js/npm code rather than a limited edge-function subset, and docs mention converting Docker images to fast-booting global VMs. However, the evidence pack never explicitly confirms Node.js/npm support, unmodified code compatibility, or names it as an edge-function alternative. Missing for 10: explicit Node.js/npm runtime documentation, confirmation that npm native modules/unsupported APIs work unmodified, and independent developer reports of running standard Node apps without modification.", "evidenceIds": [ "fly-io-docs-14", "fly-io-docs-18", "fly-io-docs-31", "fly-io-docs-32" ] }, { "productId": "fly-io", "storyId": "full-stack-preview-envs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Fly.io provides general app deployment (fly launch, Machines, scaling) but no dedicated feature or integration for automatic ephemeral preview environments tied to pull requests (e.g., GitHub PR bot, per-PR subdomains, auto-teardown). Missing for 10: PR-triggered environment creation, GitHub Actions/PR integration, automatic teardown on merge/close, full-stack (app+DB) preview provisioning per PR.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "git-push-deploy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Fly.io's docs describe CLI-based deploys (`fly launch`, `fly deploy`, `fly machine run`) but nothing in the evidence pack mentions Git integration, GitHub Actions, or any automatic deploy-on-push workflow. This is a fair axis for a PaaS like Fly.io, but the evidence shows only manual/CLI-driven deploys, missing for 10: any mention of Git-triggered CI/CD, GitHub App integration, or auto-deploy hooks.", "evidenceIds": [ "fly-io-docs-1", "fly-io-docs-16", "fly-io-docs-27" ] }, { "productId": "fly-io", "storyId": "global-edge-network", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Fly.io explicitly documents deploying apps to multiple regions worldwide via a global Anycast network so users connect to the nearest server, backed by fast-booting, hardware-isolated Fly Machines that scale globally (fly-io-docs-14, fly-io-docs-20, fly-io-docs-18). This directly matches the edge-compute story. Missing for 10: independent latency benchmarks confirming real-world edge performance, and community reports of intermittent outages/reliability issues (fly-io-comm-1, fly-io-comm-7, fly-io-comm-19) temper confidence in production robustness even though they don't contradict the core global-deployment capability.", "evidenceIds": [ "fly-io-docs-14", "fly-io-docs-20", "fly-io-docs-18", "fly-io-comm-19" ] }, { "productId": "fly-io", "storyId": "global-static-cdn", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Fly.io provides global compute deployment across regions via Anycast (fly-io-docs-20) and fast-booting VMs, but there is no mention of a CDN, edge caching, or static asset acceleration feature anywhere in the pack.", "evidenceIds": [ "fly-io-docs-20" ] }, { "productId": "fly-io", "storyId": "hard-spending-limits", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence describes hard spending limits, budget caps, or cost-control controls; only pay-as-you-go pricing and cost visibility are mentioned, with community comments criticizing pricing/network costs but no dispute of a spending-limit feature.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "idp-secured-access", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of SSO/SAML/OIDC integration with an external identity provider for securing Fly apps, deployments, or agents. The pack mentions secrets vaults, WireGuard networking, and service-to-service credential rotation, but nothing about enterprise IdP-based authentication or access control.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "in-browser-dev-toolbar", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Fly.io is a cloud/VM infrastructure platform, not a product with an in-browser feedback/feature-flag/performance toolbar for end-user apps; this story is a category error for this kind of product.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "incremental-rollout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows generic deploy commands (`fly launch`, `fly machine run`, `fly scale`) and rollback/checkpoint features for Sprites, but nothing documents incremental rollout mechanisms like canary, rolling, or blue-green deployment strategies for reducing blast radius. Deployment platforms commonly support this, so absence of evidence means the axis is unmet rather than inapplicable.", "evidenceIds": [ "fly-io-docs-16", "fly-io-docs-27", "fly-io-docs-24", "fly-io-docs-26" ] }, { "productId": "fly-io", "storyId": "infra-as-code-blueprint", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Fly.io's `fly launch` workflow creates, configures, and deploys an app from a single project directory (fly-io-docs-16, fly-io-docs-1), and `fly machine run` further shows one-line create+deploy semantics (fly-io-docs-27), implying a single fly.toml-style config drives the whole architecture. However the evidence never explicitly names or documents the fly.toml config schema, nor shows how that file is versioned/rolled back (the 'roll back to any checkpoint' claim, fly-io-docs-9, refers to Sprite/agent checkpoints, not app config versioning), and 'fly scale' (fly-io-docs-24) is a separate CLI action rather than config-driven scaling. Missing for 10: explicit fly.toml schema/documentation, config-file-based versioning or rollback, multi-service/region architecture defined in one file, independent hands-on confirmation.", "evidenceIds": [ "fly-io-docs-16", "fly-io-docs-1", "fly-io-docs-27", "fly-io-docs-24", "fly-io-docs-9", "fly-io-docs-26" ] }, { "productId": "fly-io", "storyId": "instant-folder-deploy", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Fly.io's flyctl/fly launch and fly deploy explicitly work from a local project directory without requiring a Git repository (fly-io-docs-1, fly-io-docs-16), satisfying the 'no Git repo required, folder upload' part of the story. However, there is no evidence of a staging/private mode where a new deployment is kept private until the developer chooses to publish it — Fly apps get a live URL immediately upon deploy. Missing for 10: documentation of a private/draft deployment state, a publish/promote step, or preview-URL gating before public exposure.", "evidenceIds": [ "fly-io-docs-1", "fly-io-docs-16" ] }, { "productId": "fly-io", "storyId": "invisible-bot-protection", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Fly.io is an infrastructure/compute platform (VMs, apps, machines); bot-challenge/CAPTCHA-style traffic protection is a WAF/CDN-layer feature outside its product category, and no evidence pack item discusses such a capability.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "isolated-sandbox-execution", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Fly.io explicitly documents hardware-isolated, disposable Fly Machines/Sprites with 'clean baseline per request', egress-locked, no shared state, and per-agent isolated Linux boxes for untrusted/agent workloads (fly-io-docs-7, fly-io-docs-14, fly-io-docs-4, fly-io-docs-6, fly-io-docs-17). This directly matches the sandboxed untrusted-code execution story. Missing for 10: independent/hands-on verification of the isolation guarantees and no third-party security audit evidence is present.", "evidenceIds": [ "fly-io-docs-7", "fly-io-docs-14", "fly-io-docs-4", "fly-io-docs-6", "fly-io-docs-17" ] }, { "productId": "fly-io", "storyId": "known-overage-pricing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only generic pay-as-you-go pricing claims (compute/storage/bandwidth billed by usage) but no documented overage fee schedule, no specifics on invocation or build-minute overage costs, and no billing rate table. There is also community commentary calling billing UI 'cobbled together' but nothing detailing exact overage fees.", "evidenceIds": [ "fly-io-docs-21", "fly-io-comm-18" ] }, { "productId": "fly-io", "storyId": "load-based-autoscaling", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Fly.io markets that 'Fly Machines ... scale out to tens of thousands when the traffic arrives' and offers a 'fly scale' command for horizontal/vertical scaling, suggesting scaling capability exists, but the evidence never documents concrete load-based autoscaling policies (e.g., CPU/request-triggered auto-scale-up/down) — 'fly scale' reads as a manual operator command rather than automatic burst absorption. Community reports also describe outages and unreliable behavior under production load, undercutting confidence that bursts are absorbed smoothly. Missing for 10: documented automatic autoscaling triggers/policies, hands-on evidence of successful burst handling, and resolution of the reliability complaints during high load.", "evidenceIds": [ "fly-io-docs-14", "fly-io-docs-24", "fly-io-comm-2", "fly-io-comm-7", "fly-io-comm-9" ] }, { "productId": "fly-io", "storyId": "local-dev-simulator", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers flyctl launch/deploy workflow, Fly Machines, Sprites, and MCP server support, but nowhere mentions a local simulator or emulator that replicates the production runtime for local development/testing. This is a reasonable capability to expect from a deploy platform, but no such tool (e.g., local Machines emulator) is documented.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "managed-postgres", "verdict": "disputed", "quality": 3, "confidence": "medium", "rationale": "Fly.io docs mention a 'fully-managed Postgres' service (Managed Postgres) but provide no detail on point-in-time recovery, replica configuration, or HA topology in the evidence pack. Community reports concretely contradict reliability of Postgres on Fly, including databases going down with no info (fly-io-comm-2), explicit complaints about 'postgres machines' unreliability (fly-io-comm-4), and Postgres failing to spin up/link during setup (fly-io-comm-18). missing for 10: explicit documentation of PITR, replica setup, HA failover mechanics, and independent confirmation these work as advertised.", "evidenceIds": [ "fly-io-docs-19", "fly-io-comm-2", "fly-io-comm-4", "fly-io-comm-18" ] }, { "productId": "fly-io", "storyId": "migration-credits", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions migration credits or any financial incentive program for switching from another platform; pricing docs only describe pay-as-you-go billing.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "multi-protocol-native-support", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no documentation or community discussion describing Fly.io's ability to serve HTTP, TCP, gRPC, and WebSocket traffic from a single deployed service without separate protocol configuration; docs cover deployment, scaling, secrets, and Machines but not multi-protocol service configuration.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "object-storage-uploads", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Fly.io offers persistent disks/volumes per Machine (fly-io-docs-10) but no built-in object storage service; a community report explicitly states Fly lacks an S3-equivalent and users had to leave Fly to store user assets (fly-io-comm-13), confirming absence of this capability.", "evidenceIds": [ "fly-io-docs-10", "fly-io-comm-13" ] }, { "productId": "fly-io", "storyId": "on-demand-image-optimization", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Fly.io is a compute/infrastructure platform (VMs, Postgres, deploy tooling); it does not offer an image CDN/transformation product, and this is a wrong-axis question for its category rather than a missing feature.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "one-click-rollback", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Fly.io's marketing copy claims 'Roll back to any checkpoint in seconds' (fly-io-docs-9), which suggests rollback capability, but this appears tied to Sprites (AI-agent sandboxes) checkpointing rather than a documented one-click rollback for standard deployed services/releases. No CLI docs (e.g., 'fly releases rollback') or hands-on/community confirmation of this exact deploy-workflow capability are present. Missing for 10: explicit docs on rolling back a deployed app/service version via CLI or dashboard, a one-click UI flow, and independent verification that it works as described for standard services.", "evidenceIds": [ "fly-io-docs-9", "fly-io-probe-4" ] }, { "productId": "fly-io", "storyId": "one-click-templates", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows `fly launch` scaffolding/deploying from an existing source directory and generic tech-specific docs, but there is no evidence of a curated one-click template/starter gallery for common use cases (e.g., a marketplace of pre-built app templates).", "evidenceIds": [ "fly-io-docs-1", "fly-io-docs-2", "fly-io-docs-16" ] }, { "productId": "fly-io", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Fly.io documents a Machines REST API that can provision and manage Apps, Machines, and Volumes (fly-io-docs-31, fly-io-docs-18), and flyctl itself is built atop this API, suggesting broad API/CLI-UI parity typical of Fly's API-first design. However, no evidence explicitly confirms full feature parity with the web dashboard, and a probe found no discoverable OpenAPI spec (fly-io-probe-3), making it hard to verify completeness. Missing for 10: explicit parity claim/doc mapping UI features to API endpoints, published OpenAPI/swagger spec, and confirmation that dashboard-only features (billing UI, status page, etc.) are also API-accessible.", "evidenceIds": [ "fly-io-docs-31", "fly-io-docs-18", "fly-io-probe-3", "fly-io-probe-4" ] }, { "productId": "fly-io", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence describes data export tooling, open-format export, or an account/data portability process for Fly.io apps, volumes, or databases; the pack covers deployment, scaling, and infrastructure features but nothing about exporting user data to leave the platform.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Fly.io is a proprietary cloud platform; the evidence pack contains no mention of an open-source license or public source repository for the core platform (flyctl is open source per outside knowledge, but no evidence here confirms this), and no docs claim open-source availability of the product's source.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Fly.io is presented purely as a managed public cloud (flyctl + fly launch, hosted Sprites/Machines, hosted secrets, hosted networking) with no evidence anywhere of an open-source or self-hostable core product; all docs and probes describe a proprietary hosted service.", "evidenceIds": [ "fly-io-docs-1", "fly-io-docs-14", "fly-io-probe-1", "fly-io-probe-3" ] }, { "productId": "fly-io", "storyId": "pay-for-active-compute", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Fly.io's docs describe pay-as-you-go pricing tied to actual compute/storage/bandwidth usage rather than fixed dyno-tier billing, plus machines that sleep/suspend when idle and only bill for active compute (e.g., 'Pay only for the tool calls you use', disk billed only for bytes used, Sprites that 'sleep until needed'). This directly matches the devops-lead story of paying for active time, not idle wall-clock.\n\nmissing for 10: explicit per-second billing granularity documentation, and independent/third-party confirmation that idle machines are not billed.", "evidenceIds": [ "fly-io-docs-21", "fly-io-docs-10", "fly-io-docs-6", "fly-io-docs-5", "fly-io-docs-14" ] }, { "productId": "fly-io", "storyId": "pay-per-byte-storage", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Fly.io's own docs explicitly state disk auto-grows with usage up to 100GB per Sprite and billing is only for bytes actually written, matching the story precisely, and general pay-as-you-go storage pricing is reiterated elsewhere. Missing for 10: independent/hands-on verification of this billing behavior, clarity on whether it applies to standard Fly Volumes beyond the newer 'Sprites' product, and confirmation of no minimum/rounding charges.", "evidenceIds": [ "fly-io-docs-10", "fly-io-docs-21" ] }, { "productId": "fly-io", "storyId": "persistent-agent-runtime", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Fly.io's docs explicitly describe 'Sprites' as persistent agent runtimes that checkpoint themselves automatically and can be rolled back to any checkpoint, directly matching the story (fly-io-docs-4, fly-io-docs-5, fly-io-docs-9, fly-io-docs-28). This is a first-party marketing description with no independent hands-on verification of the Sprite checkpointing mechanism specifically, and general community reliability complaints (unrelated to Sprites) raise some doubt about production robustness. Missing for 10: independent/hands-on corroboration of Sprite checkpoint/restore behavior, and detail on failure modes or reliability of the checkpointing feature itself.", "evidenceIds": [ "fly-io-docs-4", "fly-io-docs-5", "fly-io-docs-9", "fly-io-docs-28" ] }, { "productId": "fly-io", "storyId": "polyglot-edge-runtimes", "verdict": "full", "quality": 6, "confidence": "medium", "rationale": "Fly.io runs any Docker container as a Fly Machine deployed globally close to users, which inherently supports any language (Python, Go, Rust, etc.) rather than restricting to JavaScript, and docs highlight global regional deployment via Anycast for low-latency edge-like compute. However, there is no explicit 'edge functions' framework or per-language quickstart tutorials cited, so this is inferred from the general VM/container model rather than dedicated edge-function tooling. Missing for 10: explicit per-language edge-function guides/examples for Python, Go, Rust, and independent corroboration of using Fly for edge-function-style workloads.", "evidenceIds": [ "fly-io-docs-14", "fly-io-docs-20", "fly-io-docs-32", "fly-io-docs-18" ] }, { "productId": "fly-io", "storyId": "pr-preview-urls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack mentions PR-preview environments, GitHub integration, or automatic preview URLs for pull requests; only deployment via flyctl/fly launch is documented.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "preview-collaboration-comments", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Fly.io is an infrastructure/compute platform for deploying apps and VMs, not a collaboration or commenting tool; there is no concept of teammates commenting on environments in its product category.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Fly.io documents global region deployment allowing users to place apps/data near specific regions (e.g. Tokyo, São Paulo, Amsterdam) via its Anycast network, which supports choosing where compute/data runs. However, there's no explicit documentation of data residency guarantees, compliance certifications, or region-locking specifically for data storage/backups (e.g. Postgres/volumes pinned to a region for residency purposes) rather than just latency-driven deployment. Missing for 10: explicit data residency/compliance documentation, volume/database region-pinning guarantees, independent verification of residency enforcement.", "evidenceIds": [ "fly-io-docs-20", "fly-io-docs-19", "fly-io-docs-25" ] }, { "productId": "fly-io", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Fly.io is an infrastructure/compute platform (VMs, deployment, databases) and not an AI model provider that trains on user data or offers an opt-out; this privacy-posture axis about AI training data is a category error for this product type.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation of data retention policies, deletion tooling, or export/erasure controls for Fly.io's platform. The only relevant community evidence (fly-io-comm-8) describes a user whose account-deletion request was ignored, further indicating no working deletion mechanism was demonstrated.", "evidenceIds": [ "fly-io-comm-8" ] }, { "productId": "fly-io", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions telemetry collection, usage tracking, or any opt-out mechanism/settings for Fly.io products or CLI; the docs cover deployment, scaling, and infrastructure features but never address analytics/telemetry controls.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "private-service-networking", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence hints at private networking (WireGuard checks via `fly doctor`, org-wide service connectivity and egress policies for Sprites) but never explicitly documents a general internal DNS/private network (e.g. 6PN/.internal) mechanism for ordinary Fly Apps talking to each other off the public internet. Missing for 10: explicit docs on private networking/internal DNS for app-to-app traffic, VPC-less peering details, and independent confirmation it works reliably at scale.", "evidenceIds": [ "fly-io-docs-22", "fly-io-docs-11", "fly-io-docs-6", "fly-io-docs-7" ] }, { "productId": "fly-io", "storyId": "prod-nonprod-isolation", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence covers Fly Machines, Sprites (AI-agent sandboxes with egress policies), and general platform features, but nothing addresses network segmentation or access control between staging/non-production and production environments for a devops team. Docs-11/12/13 describe org-wide service connectivity and per-Sprite access grants for agent workloads, not environment isolation for prod vs non-prod apps.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "programmatic-multitenant-domains", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no mention of Fly.io's certificate/custom-domain APIs (e.g. `fly certs add`, automated TLS issuance, per-app domain routing) needed for multi-tenant SaaS customer domains — only general Machines/Apps provisioning and networking primitives are described. Missing for 10: any documentation of programmatic custom-domain/cert provisioning, per-tenant routing, or multi-tenant domain API, and any independent confirmation it works reliably.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "public-incident-history", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no vendor/docs-tier claim of a public status page with historical incident postmortems, and the only related mentions are community complaints that Fly's status page shows 'no incidents'/green even during real outages (fly-io-comm-3, fly-io-comm-6, fly-io-comm-9), which undercuts rather than confirms the existence of a trustworthy postmortem history. Since disputed requires a documented vendor claim contradicted by hands-on evidence from a distinct tier, and no such first-party claim is present here, this falls to 'none' rather than 'disputed'.", "evidenceIds": [ "fly-io-comm-3", "fly-io-comm-6", "fly-io-comm-9" ] }, { "productId": "fly-io", "storyId": "published-uptime-sla", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a published SLA document with defined uptime percentage or downtime credits; community evidence even shows Fly's status page failing to reflect actual outages, but this is not a documented SLA dispute, just absence of one.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "redis-compatible-cache", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of Redis, Upstash, or any managed key-value/caching service offered by Fly.io; only generic Machines, Postgres, and Sprite-related docs are present. Missing for 10: any documentation or claim of a Redis-compatible store, caching service, or job-queue integration.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "regional-deployment", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Fly.io explicitly documents deploying apps to specific regions worldwide via its global Anycast network so users connect to the nearest server, and other docs reference region-based multi-server deployment with a single command. missing for 10: independent/hands-on corroboration specifically validating regional deployment behavior (community evidence focuses on outages/reliability, not region selection).", "evidenceIds": [ "fly-io-docs-20", "fly-io-docs-14", "fly-io-docs-25" ] }, { "productId": "fly-io", "storyId": "seat-based-pricing-minimums", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Fly.io's pricing is explicitly usage-based (pay-as-you-go for compute/storage/bandwidth), not per-seat, and no evidence mentions seat counts or minimums; missing for 10: any pricing page or docs referencing seat-based billing or per-user minimums since the model is consumption-based, not applicable in the seat-pricing sense the story asks about.", "evidenceIds": [ "fly-io-docs-21" ] }, { "productId": "fly-io", "storyId": "serverless-functions", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Fly.io's docs describe a workflow (`fly launch`, Fly Machines that boot in under a second and scale to zero, Managed Postgres, global regions, pay-as-you-go pricing) that strongly emphasizes not needing to provision or manage servers for full-stack apps [fly-io-docs-1][fly-io-docs-14][fly-io-docs-19][fly-io-docs-20][fly-io-docs-21]. However, multiple independent hands-on reports describe machines/Postgres going down with no self-healing, requiring manual recovery work, and Fly's own support response recommending users 'run multiple instances to mitigate the impact of single-host failures' — directly undercutting the 'no servers to manage' promise [fly-io-comm-2][fly-io-comm-4][fly-io-comm-9][fly-io-comm-15][fly-io-comm-19]. Missing for 10: true FaaS/serverless-functions primitive (Fly is VM/container-based, not function-as-a-service), and resolution of the reliability complaints that contradict hands-off operation.", "evidenceIds": [ "fly-io-docs-1", "fly-io-docs-14", "fly-io-docs-19", "fly-io-docs-20", "fly-io-docs-21", "fly-io-comm-2", "fly-io-comm-4", "fly-io-comm-9", "fly-io-comm-15", "fly-io-comm-19" ] }, { "productId": "fly-io", "storyId": "serverless-sql-query", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Fly.io offers Managed Postgres, a fully-managed traditional PostgreSQL service (fly-io-docs-19), but there's no evidence of a serverless SQL query interface with no connection management (e.g., HTTP-based query API like Neon's serverless driver); Postgres still requires standard connection handling.", "evidenceIds": [ "fly-io-docs-19" ] }, { "productId": "fly-io", "storyId": "site-password-protection", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of any built-in basic-auth/password-protection feature for deployed apps (e.g., no proxy-level HTTP auth, no dashboard toggle). While secrets management is documented (fly-io-docs-23), that only covers passing credentials to the app, not adding password gating without custom auth code — this axis is plausible for a deployment platform but unevidenced here.", "evidenceIds": [ "fly-io-docs-23" ] }, { "productId": "fly-io", "storyId": "smart-placement", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Fly.io documents global region deployment and Anycast routing so user traffic reaches the nearest server (fly-io-docs-20), and offers a managed Postgres service (fly-io-docs-19), but region placement is manual (fly launch/region flags) rather than an automatic system that senses backend-data location and relocates compute accordingly. Missing for 10: explicit automatic/backend-data-aware placement logic, documentation of latency-optimizing scheduling relative to a database's region, and independent verification that this placement actually reduces latency in production.", "evidenceIds": [ "fly-io-docs-20", "fly-io-docs-19", "fly-io-docs-14" ] }, { "productId": "fly-io", "storyId": "standards-based-runtime-portability", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Fly.io runs standard Docker images and VMs rather than a proprietary edge-runtime API, and docs explicitly state 'the API you start with is the API you keep' with no rearchitecting or migration needed, plus standard Postgres rather than proprietary storage APIs. Missing for 10: independent third-party confirmation that code written for Fly ports unmodified to other platforms, and explicit contrast with proprietary edge-runtime APIs (e.g. Workers/Deno) rather than just general platform-migration claims.", "evidenceIds": [ "fly-io-docs-32", "fly-io-docs-15", "fly-io-docs-18", "fly-io-docs-19", "fly-io-docs-14" ] }, { "productId": "fly-io", "storyId": "support-response-sla", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of documented support-tier SLAs, guaranteed response times, or escalation paths for different plan tiers; docs only vaguely claim support is 'run by Fly.io engineers,' with no timing commitments. Community reports (fly-io-comm-4, -5, -7, -14) even describe unresponsive or non-existent support, but since no vendor claim of guaranteed SLAs exists to contradict, this is an unmet axis rather than a dispute.", "evidenceIds": [ "fly-io-docs-3", "fly-io-comm-4", "fly-io-comm-5", "fly-io-comm-7", "fly-io-comm-14" ] }, { "productId": "fly-io", "storyId": "team-secrets-env-vars", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Fly.io docs show centralized secrets management (encrypted vault) and org-wide connection/credential rotation with granular access grants by Sprite/label, which covers secrets and env-var centralization. However there is no evidence of team-member role management, invites, or permission tiers for a devops-lead to administer team access explicitly. Missing for 10: team/user role management (RBAC), member invite/permission workflows, and independent corroboration of the org-wide credential features actually working at scale.", "evidenceIds": [ "fly-io-docs-23", "fly-io-docs-11", "fly-io-docs-12", "fly-io-docs-13" ] }, { "productId": "fly-io", "storyId": "unified-log-view", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation or mention of a unified logging view for build, deploy, and runtime logs (e.g., no reference to `fly logs`, log streaming, or a logs dashboard). Community evidence even suggests logging/observability gaps ('Fly reports everything healthy, and there's literally no information'), but this doesn't confirm or deny a unified log feature — it's simply absent from the pack.", "evidenceIds": [ "fly-io-comm-1", "fly-io-comm-6" ] }, { "productId": "fly-io", "storyId": "unlimited-team-environments", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Fly.io's Machines/Sprites provide fast-launching, hardware-isolated VMs that can scale out to thousands of instances, and features like per-agent Sprites (docs-4/5/6) and fast `fly machine run`/`fly launch` (docs-16/27) support spinning up many isolated environments for parallel work. However, there's no explicit 'unlimited' guarantee, and multiple independent hands-on reports describe reliability problems (stuck/unreachable machines, flaky deploys, outages affecting staging/production) that would undermine a team's ability to reliably ship simultaneously at scale. Missing for 10: an explicit 'unlimited' or team-collaboration framing, and stronger independent corroboration that concurrent environment creation is reliable at scale.", "evidenceIds": [ "fly-io-docs-4", "fly-io-docs-5", "fly-io-docs-6", "fly-io-docs-7", "fly-io-docs-14", "fly-io-docs-16", "fly-io-docs-27", "fly-io-comm-5", "fly-io-comm-19" ] }, { "productId": "fly-io", "storyId": "visitor-performance-analytics", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Fly.io is an infrastructure/compute platform (VMs, deploys, scaling), not a web-analytics or site-behavior-tracking product; built-in visitor analytics is outside its category (though it offers metrics/logging for ops, not visitor behavior analytics).", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "visual-infra-canvas", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Fly.io is a CLI/API-driven infrastructure platform; no visual canvas or GUI-based infra editing is part of its product category, making this a wrong-axis question.", "evidenceIds": [] }, { "productId": "fly-io", "storyId": "zero-config-repo-deploy", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Fly.io's `fly launch` auto-detects app type and generates config/deploys with minimal manual setup (fly-io-docs-1, fly-io-docs-16), but this is a CLI-driven flow from a local project directory, not a GitHub-repo-connect auto-deploy UX like Vercel/Heroku. No evidence of a 'connect your repo and we build/deploy automatically' web integration. Community reports also cite frequent build/deploy failures undermining the 'no manual configuration' promise (fly-io-comm-5, fly-io-comm-19). Missing for 10: native repo-connect (GitHub App/webhook) auto-deploy flow, evidence of zero manual config across frameworks, and independent confirmation deploys succeed reliably without intervention.", "evidenceIds": [ "fly-io-docs-1", "fly-io-docs-16", "fly-io-comm-5", "fly-io-comm-19" ] }, { "productId": "netlify", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A live probe confirms Netlify hosts an llms.txt at docs.netlify.com/llms.txt returning HTTP 200 with structured LLM-oriented documentation, directly satisfying the story. Missing for 10: no evidence of broader agent-oriented doc formats (e.g., per-page .md variants) or third-party confirmation that agents successfully consume it.", "evidenceIds": [ "netlify-probe-1" ] }, { "productId": "netlify", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Netlify's evidence covers AI Gateway for connecting to AI models, AI agents that fix/ship code, and analytics tracking, but there is no evidence of AI-generated insights or suggestions derived from a user's own data (e.g., analytics interpretation, usage recommendations) surfaced inside the product.", "evidenceIds": [] }, { "productId": "netlify", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Netlify supports event-driven automation (auto-deploy on every push, serverless functions, REST API for automating workflows) which can run without manual intervention, but there is no evidence of scheduled/cron jobs or persistent background agents that operate autonomously beyond deploy/build triggers. Missing for 10: explicit scheduled/cron function support, evidence of long-running or continuously monitoring background automations, and independent confirmation that such automations work reliably.", "evidenceIds": [ "netlify-docs-1", "netlify-docs-6", "netlify-docs-14", "netlify-docs-21" ] }, { "productId": "netlify", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Netlify's docs describe a built-in AI capability that lets users 'prompt AI agents to fix, update, and ship code using your unique project context directly from your Netlify dashboard' and connect AI assistants to the account for deployment workflows, indicating a delegated in-product AI assistant exists. However, evidence is limited to marketing-level docs with no detail on capabilities, limits, or hands-on/independent validation of how well it performs task delegation. Missing for 10: independent/community corroboration of the AI assistant working in practice, technical documentation of its scope and limitations, and concrete examples of delegated tasks being completed.", "evidenceIds": [ "netlify-docs-21", "netlify-docs-22", "netlify-docs-24" ] }, { "productId": "netlify", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Netlify offers a documented CLI and REST API explicitly for automating workflows (deploy, dev servers, project management), plus anonymous/no-login deploy from CLI suitable for CI pipelines, and auto-deploy on push already implies CI-friendly headless operation. Missing for 10: no explicit CI-specific docs (e.g., GitHub Actions recipes) or independent confirmation of headless CI usage beyond docs/probes.", "evidenceIds": [ "netlify-docs-14", "netlify-docs-15", "netlify-docs-3", "netlify-docs-1", "netlify-probe-3" ] }, { "productId": "netlify", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Netlify's docs mention AI Gateway for connecting to AI models and letting AI assistants connect to a Netlify account, but nothing describes Netlify itself acting as an MCP client that plugs in external MCP servers to use their tools. Missing for 10: any mention of MCP protocol/server support, or documentation of Netlify consuming third-party MCP tool servers.", "evidenceIds": [ "netlify-docs-22", "netlify-docs-24", "netlify-docs-7" ] }, { "productId": "netlify", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Docs mention connecting AI assistants and agents to Netlify (netlify-docs-21, netlify-docs-22) but never reference an official MCP server or the Model Context Protocol specifically, so there's no concrete evidence of an MCP server offering.", "evidenceIds": [ "netlify-docs-21", "netlify-docs-22" ] }, { "productId": "netlify", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Netlify offers AI-agent integration features (prompt AI agents to fix/update/ship code from the dashboard, connect AI assistants for deployment workflows) and an llms.txt for LLM-readable docs, suggesting some natural-language/agentic operation surface, but there's no evidence of a broad natural-language command interface (e.g., chat-driven CLI or conversational control of deploys, DNS, env vars). missing for 10: evidence of a general NL command interface across the product (not just AI agent code-fix prompts), independent/hands-on validation of the natural-language workflow actually working.", "evidenceIds": [ "netlify-docs-21", "netlify-docs-22", "netlify-probe-1" ] }, { "productId": "netlify", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Netlify ships a well-documented official CLI (netlify-cli) for deploys, local dev, and project management, confirmed by both docs and a live probe. missing for 10: independent hands-on community validation specifically of the CLI's AI-native/agentic use cases, and no evidence of AI-specific CLI commands beyond general deploy/dev tooling.", "evidenceIds": [ "netlify-docs-3", "netlify-docs-15", "netlify-probe-3" ] }, { "productId": "netlify", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Netlify documents a REST API for managing sites, deploys, DNS, etc. explicitly for 'automating workflows and integrating into your systems,' plus an official CLI wrapping it, and even an llms.txt for AI-native consumption. No formal OpenAPI spec was found at expected locations, slightly weakening machine-readability claims. missing for 10: discoverable OpenAPI/swagger spec, independent developer corroboration of API robustness.", "evidenceIds": [ "netlify-docs-14", "netlify-docs-15", "netlify-probe-1", "netlify-probe-2", "netlify-probe-3" ] }, { "productId": "netlify", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Netlify manages access/secrets/env vars 'by team' and offers a REST API and CLI, but there is no mention of scoped or least-privilege API tokens/credentials specifically for AI agents — no granular permission scopes, no agent-specific credential issuance workflow.", "evidenceIds": [ "netlify-docs-8", "netlify-docs-14", "netlify-docs-22" ] }, { "productId": "netlify", "storyId": "agentic-sdks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Netlify provides an official SDK for building extensions/integrations, a REST API for programmatic control, and a CLI, plus an AI Gateway to connect AI models — all documented as first-party tooling that AI-native builders can target. Missing for 10: independent/hands-on corroboration of SDK usage by AI agents, and multi-language SDK examples beyond the extension SDK.", "evidenceIds": [ "netlify-docs-14", "netlify-docs-15", "netlify-docs-16", "netlify-docs-7", "netlify-probe-1", "netlify-probe-3" ] }, { "productId": "netlify", "storyId": "agentic-webhooks", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The only evidence touching webhooks is a single community comment listing 'forms/webhooks' as a feature alongside other capabilities; there is no first-party doc describing a webhook subscription API, event types, or configuration for AI-native agentic workflows. Missing for 10: official docs on deploy/build event webhooks, subscription/configuration mechanism, and event payload schema.", "evidenceIds": [ "netlify-comm-8" ] }, { "productId": "netlify", "storyId": "ai-global-inference", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Netlify's AI Gateway (netlify-docs-7, netlify-docs-24) lets developers 'connect to popular AI models within your project code... without managing API keys,' which aligns with 'single call, no GPU management' but is essentially a proxy/gateway to third-party model providers rather than Netlify-hosted global inference infrastructure. Missing for 10: details on global edge distribution of inference, capacity/latency guarantees, supported model list, and any independent/hands-on validation of the AI Gateway's performance or reliability.", "evidenceIds": [ "netlify-docs-7", "netlify-docs-24" ] }, { "productId": "netlify", "storyId": "ai-model-gateway", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Netlify documents an AI Gateway that lets code connect to popular AI models without managing API keys, directly matching the unified gateway concept, but there's no evidence of 'hundreds' of models, no model list, and no independent/hands-on corroboration of the gateway's breadth or reliability. missing for 10: documented model count/catalog, hands-on validation, independent confirmation of breadth claims.", "evidenceIds": [ "netlify-docs-7", "netlify-docs-24" ] }, { "productId": "netlify", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "There's a REST API and CLI documentation, but no evidence of an interactive API reference with runnable examples (e.g., API explorer, Swagger/OpenAPI UI, or embedded try-it console); the probe explicitly found no OpenAPI/swagger endpoints (404s across all candidates).", "evidenceIds": [ "netlify-docs-14", "netlify-probe-2" ] }, { "productId": "netlify", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Netlify documents a REST API and CLI but explicit probes for an OpenAPI/swagger spec at standard paths all returned 404, and no evidence pack item shows a downloadable machine-readable API spec.", "evidenceIds": [ "netlify-probe-2", "netlify-docs-14" ] }, { "productId": "netlify", "storyId": "api-sandbox", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Netlify's Deploy Previews and preview environments (per-PR preview URLs, shareable preview environments) provide isolated environments that don't touch production, and Netlify Database claims to let you 'safely iterate' on managed Postgres. However, none of this is framed specifically for AI-agent-driven testing or data isolation guarantees (e.g., branch databases, seeded test data), and there's no independent corroboration of preview data isolation. Missing for 10: explicit AI-agent-oriented sandbox/test-data isolation guarantees, database branching/reset mechanics, and independent verification that preview environments never touch production data.", "evidenceIds": [ "netlify-docs-1", "netlify-docs-19", "netlify-docs-25", "netlify-docs-13" ] }, { "productId": "netlify", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "There is evidence of a REST API and CLI, but no documentation of API versioning scheme or a deprecation policy; the probe explicitly found no OpenAPI spec at expected locations, and no evidence pack item mentions version numbers, sunset timelines, or changelog/deprecation notices.", "evidenceIds": [ "netlify-docs-14", "netlify-probe-2" ] }, { "productId": "netlify", "storyId": "automatic-tls-certs", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Netlify's docs explicitly promise 'Automatic HTTPS' as a core platform feature, and community reviews confirm 'custom domains with SSL' works in practice, supporting the free/auto-renewing TLS claim. However, there is no explicit documentation confirming wildcard domain certificate support, and one community anecdote (netlify-comm-6) mentions a customer hitting cert errors that required manual support intervention rather than fully automatic resolution. missing for 10: explicit wildcard-domain TLS documentation, independent verification of automatic renewal behavior at scale.", "evidenceIds": [ "netlify-docs-9", "netlify-comm-6", "netlify-comm-8" ] }, { "productId": "netlify", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Netlify's REST API and CLI are documented as generally 'ideal for automating workflows' across sites/deploys, which an AI-native user could script to perform bulk actions, but there is no explicit bulk-operation feature (e.g., batch deploy, bulk env-var update, multi-site actions) documented anywhere in the evidence. Missing for 10: any first-party bulk/batch endpoint or CLI command, documented multi-item automation examples, and independent confirmation of bulk workflows.", "evidenceIds": [ "netlify-docs-14", "netlify-docs-15", "netlify-probe-3" ] }, { "productId": "netlify", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Netlify supports some automatic event-triggered actions (auto-deploy on push, PR previews, webhooks/forms, serverless functions triggered by requests, REST API for scripting workflows), but there is no documented general-purpose rules/trigger engine letting users define arbitrary custom event→action automations. Missing for 10: a dedicated automation/rules configuration UI or API for defining custom triggers-to-actions mappings beyond git push and forms, and independent evidence of such a system in use.", "evidenceIds": [ "netlify-docs-1", "netlify-docs-6", "netlify-docs-14", "netlify-comm-8" ] }, { "productId": "netlify", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of scheduled/cron functions or recurring workflow triggers; Netlify's evidence covers deploys, functions, forms, DB, and AI-assist features but nothing about scheduled/recurring job execution. missing for 10: scheduled functions/cron support, recurring workflow trigger docs, any mention of time-based automation.", "evidenceIds": [] }, { "productId": "netlify", "storyId": "automation-versioned-workflows", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Netlify's deploy pipeline provides versioning via git-based deploys, review via Deploy Previews on every PR, and one-click rollback to any prior deploy, directly matching the version/review/rollback story for its core automation (site deploys); community feedback independently praises the PR preview workflow. Missing for 10: independent/hands-on confirmation of rollback specifically, and evidence extending versioning/rollback to other 'automations' like serverless functions or AI agent workflows beyond the site deploy itself.", "evidenceIds": [ "netlify-docs-5", "netlify-docs-25", "netlify-docs-1", "netlify-comm-10" ] }, { "productId": "netlify", "storyId": "cli-deploy", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Netlify CLI docs explicitly support deploying from the terminal without dashboard, including anonymous/no-login deploys via `netlify deploy --allow-anonymous`, plus a full CLI reference for deploys, local dev, and site management. Missing for 10: independent hands-on confirmation of CLI deploy workflow beyond docs.", "evidenceIds": [ "netlify-docs-2", "netlify-docs-3", "netlify-docs-15", "netlify-probe-3" ] }, { "productId": "netlify", "storyId": "cli-scaffold-project", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Netlify CLI supports one-command deploy from a working directory without login (netlify deploy --allow-anonymous) and the CLI docs confirm deploying, running dev servers, and managing projects from the terminal, but there's no evidence of a 'scaffold' (project templating/initialization) step bundled into a single command. missing for 10: evidence of CLI-driven project scaffolding/templating, independent hands-on confirmation of the single-command flow working end-to-end.", "evidenceIds": [ "netlify-docs-2", "netlify-docs-3", "netlify-docs-15", "netlify-probe-3" ] }, { "productId": "netlify", "storyId": "condition-based-alerts", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items mention configurable alerting/notification rules for conditions (e.g., deploy failures, performance thresholds) delivered via chat or email; only analytics tracking and generic Discord community links are mentioned, not an alerting system.", "evidenceIds": [] }, { "productId": "netlify", "storyId": "consistent-api-at-scale", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Netlify's docs explicitly market a spectrum from zero-config prototype deploys (CLI deploy with no login) to enterprise-grade security 'at any scale' and templates claiming to scale to millions of users, implying the same core primitives (CLI, REST API, functions, database) work from small to large projects. However, there is no independent or hands-on evidence confirming large production deployments avoid rearchitecting, and cost-related community complaints about surprise scaling bills hint at friction once traffic grows. Missing for 10: case studies or technical benchmarks of large-scale production use on the same APIs, and independent confirmation that no migration/rearchitecture is needed at massive scale.", "evidenceIds": [ "netlify-docs-2", "netlify-docs-3", "netlify-docs-18", "netlify-intdir-14", "netlify-docs-14", "netlify-docs-15" ] }, { "productId": "netlify", "storyId": "custom-domains", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Netlify docs explicitly describe custom domain management, including adding unique custom domains per project version, plus automatic HTTPS, and community reviews corroborate 'custom domains with SSL' as a delivered feature. Missing for 10: independent hands-on walkthrough of the domain-attachment flow and DNS configuration details beyond a single doc line.", "evidenceIds": [ "netlify-docs-20", "netlify-docs-9", "netlify-comm-8" ] }, { "productId": "netlify", "storyId": "custom-metrics-dashboards", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Netlify provides its own analytics on deploys/traffic (netlify-docs-17) and a REST API for automation, but there is no evidence of custom dashboarding that combines metrics from arbitrary external services (e.g., no Grafana/Datadog-style dashboard builder or multi-source metrics aggregation). This is a fair ask for a devops-facing platform, but nothing in the evidence shows it.", "evidenceIds": [ "netlify-docs-17", "netlify-docs-14" ] }, { "productId": "netlify", "storyId": "default-ddos-waf", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Netlify's docs claim automatic DDoS protection and offer 'Advanced Security' for WAF-like protections, but there's no evidence that a WAF is applied to every request by default (Advanced Security appears to be a separate, presumably paid, tier). Concrete hands-on community evidence directly contradicts the DDoS protection claim: a user was billed ~$104k after a DDoS attack on a simple static site, and Netlify's official response was reportedly 'if you're hosting a small site that gets DDoS'd, don't,' indicating the protection does not shield customers from cost/impact as implied. missing for 10: evidence of a WAF applied by default to all requests, and evidence that DDoS protection actually blocks/absorbs attacks without cost exposure.", "evidenceIds": [ "netlify-docs-9", "netlify-docs-18", "netlify-comm-1", "netlify-comm-2" ] }, { "productId": "netlify", "storyId": "docker-deploy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Netlify's evidence pack covers static site/JAMstack deploys, serverless functions, CLI, database, and integrations, but there is no mention of Docker container deployment or container registry support anywhere in the docs or community evidence.", "evidenceIds": [] }, { "productId": "netlify", "storyId": "documented-rate-limits", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack cites documented rate limits or concurrency caps for Netlify's APIs or serverless functions; billing/DDoS complaints (netlify-comm-1/2/3/15) discuss cost surprises, not documented limits/caps for capacity planning.", "evidenceIds": [] }, { "productId": "netlify", "storyId": "durable-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Netlify's evidence covers serverless functions, edge functions, databases, storage, and CI/CD deploy previews, but there is no mention of a durable/long-running workflow orchestration primitive (e.g., background jobs with built-in retries, queues, or workflow-as-code) comparable to Vercel's workflow/queue offerings. Functions are described as short-lived serverless invocations, not durable workflows.", "evidenceIds": [ "netlify-docs-6", "netlify-docs-23", "netlify-docs-13" ] }, { "productId": "netlify", "storyId": "edge-websocket-support", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Netlify's serverless/edge functions are ephemeral, request-scoped compute (functions, edge functions) with no evidence of persistent WebSocket support; nothing in the docs mentions WebSocket handling, and this is a known architectural limitation of serverless platforms. missing for 10: any mention of WebSocket support, persistent connections, or real-time streaming APIs in edge/serverless functions.", "evidenceIds": [ "netlify-docs-6" ] }, { "productId": "netlify", "storyId": "execution-time-limits", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item specifies actual CPU or wall-clock execution time limits for Netlify Functions; docs only mention 'serverless functions' generically without documenting timeout/CPU constraints.", "evidenceIds": [] }, { "productId": "netlify", "storyId": "external-service-bindings", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Netlify provides native bindings for common services—managed Postgres via Netlify Database, Blob Storage (getStore for file uploads), AI Gateway for connecting to AI models without managing API keys, and an integrations directory with pre-built connectors (Turso, Supabase, Algolia, etc.)—reducing boilerplate for common data/API needs. However, these are curated first-party primitives rather than a general-purpose native binding system for arbitrary external services, and there's no independent/hands-on evidence validating ease-of-use claims. Missing for 10: broader native binding coverage beyond Netlify's own curated services, independent developer corroboration of 'minimal boilerplate' claims, and detail on credential/secret management specifics for third-party bindings.", "evidenceIds": [ "netlify-docs-13", "netlify-docs-12", "netlify-docs-23", "netlify-docs-24", "netlify-intdir-5", "netlify-intdir-12", "netlify-intdir-14" ] }, { "productId": "netlify", "storyId": "fast-boot-scaling", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Netlify's docs mention serverless functions, automatic HTTPS/DDoS protection, and scaling for teams, but there is no evidence of specific boot-time (sub-second cold start) or explicit scale metrics (tens of thousands of concurrent instances) claims. Community evidence even highlights uncontrolled scaling causing massive surprise DDoS bills, which is orthogonal to the performance claim itself. missing for 10: cold-start latency data, explicit scale/throughput benchmarks, autoscaling architecture details.", "evidenceIds": [ "netlify-docs-6", "netlify-docs-9", "netlify-comm-1", "netlify-comm-2" ] }, { "productId": "netlify", "storyId": "framework-tool-continuity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Netlify's evidence shows Git-based auto-deploys, PR previews, a CLI, REST API, and framework-agnostic integrations (Next.js, Gatsby, Remix, Svelte, Angular), supporting continued use of existing Git/framework workflows without lock-in, and community praise for GitHub integration/preview builds. However, there is no explicit evidence about compatibility with arbitrary existing CI pipelines (Netlify's own build system largely replaces CI) or editor-agnosticism, and one community report cites a CLI/dev-server config error suggesting friction. Missing for 10: explicit support for external CI systems, editor-agnostic workflow evidence, and resolution of CLI dev-server friction reports.", "evidenceIds": [ "netlify-docs-1", "netlify-docs-25", "netlify-docs-15", "netlify-docs-14", "netlify-intdir-3", "netlify-intdir-11", "netlify-intdir-13", "netlify-comm-7", "netlify-comm-10", "netlify-comm-12" ] }, { "productId": "netlify", "storyId": "full-nodejs-runtime-support", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence only generically mentions 'Build APIs with serverless functions' (netlify-docs-6) with no detail on Node.js runtime compatibility, npm package support, or contrast with a restricted runtime subset (e.g., edge functions). No documentation or community evidence confirms unmodified Node.js/npm execution in edge or serverless functions.", "evidenceIds": [ "netlify-docs-6" ] }, { "productId": "netlify", "storyId": "full-stack-preview-envs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Netlify's Deploy Previews give every PR a full preview URL with backend features (functions, DB, env vars) tied to the branch, and this is corroborated by independent hands-on community testimony calling the PR preview integration a 'killer feature.' missing for 10: no explicit mention of ephemeral teardown/cleanup behavior or full-stack backend parity (e.g., preview-specific DB branching) in evidence.", "evidenceIds": [ "netlify-docs-1", "netlify-docs-25", "netlify-docs-19", "netlify-comm-10" ] }, { "productId": "netlify", "storyId": "git-push-deploy", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Netlify's core value prop is documented auto-deploy on Git push with PR preview URLs, plus one-click rollback, corroborated by independent community reports praising GitHub integration and automated builds/deploys with PR preview builds working well in practice. Missing for 10: no first-party doc detail on configuring build settings/branch deploy rules, and some community friction with netlify.toml dev config shows minor rough edges.", "evidenceIds": [ "netlify-docs-1", "netlify-docs-5", "netlify-docs-25", "netlify-comm-7", "netlify-comm-10" ] }, { "productId": "netlify", "storyId": "global-edge-network", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Evidence only indirectly implies Netlify Edge Functions exist via a third-party integration blurb ('Turso... brings your data close to your edge functions') rather than first-party documentation describing a global edge-compute network for running developer code. Serverless functions (netlify-docs-6) are mentioned but that's origin/regional compute, not edge. Missing for 10: first-party docs on Edge Functions architecture/global PoPs, performance benchmarks, or independent hands-on confirmation of edge execution.", "evidenceIds": [ "netlify-intdir-5", "netlify-docs-6" ] }, { "productId": "netlify", "storyId": "global-static-cdn", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no explicit claim about a global CDN or edge caching for static assets — mentions are limited to HTTPS/DDoS protection, deploy previews, and edge functions used by third-party integrations, none of which document CDN-based asset delivery speed. Missing for 10: explicit CDN/edge-caching documentation, performance benchmarks, or first-party claims about global asset distribution.", "evidenceIds": [ "netlify-docs-9" ] }, { "productId": "netlify", "storyId": "hard-spending-limits", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No documentation anywhere mentions a hard spend-limit or budget-cap feature; instead community reports (netlify-comm-1, netlify-comm-2, netlify-comm-15) describe customers receiving unexpectedly massive bills after traffic spikes/DDoS with no cap in place, and Netlify's response was to suggest not hosting small sites rather than offering a spending limit.", "evidenceIds": [ "netlify-comm-1", "netlify-comm-2", "netlify-comm-15" ] }, { "productId": "netlify", "storyId": "idp-secured-access", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack mentions team-based access/secrets management and generic 'Advanced Security' or password protection, but contains no mention of SSO, SAML, OIDC, or integration with an external identity provider (e.g., Okta, Azure AD) for securing apps, deployments, or agents. This is a reasonable axis for an enterprise deployment platform, but no supporting evidence exists.", "evidenceIds": [ "netlify-docs-8", "netlify-docs-18" ] }, { "productId": "netlify", "storyId": "in-browser-dev-toolbar", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Netlify is a hosting/deployment platform, not a product with an in-browser dev toolbar for feedback, feature flags, and performance inspection on a live site — this is a wrong-axis story for this product category.", "evidenceIds": [] }, { "productId": "netlify", "storyId": "incremental-rollout", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Netlify offers deploy previews for reviewing changes before going live and one-click rollback to a previous deploy, which reduce risk around releases, but there is no evidence of true incremental/staged rollout mechanisms like percentage-based traffic splitting or canary releases to limit blast radius. Missing for 10: canary/staged rollout support, percentage-based traffic shifting, gradual rollout automation, and any documentation framing deploys as incremental releases.", "evidenceIds": [ "netlify-docs-5", "netlify-docs-25", "netlify-docs-19" ] }, { "productId": "netlify", "storyId": "infra-as-code-blueprint", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack never documents netlify.toml (or any single manifest) as a way to declare an entire architecture (functions, redirects, headers, databases, DNS, envs) in one versioned file; the only reference to netlify.toml is a community complaint about unclear docs for its [dev] section (netlify-comm-12), not a confirmation of full-architecture-as-code capability. CLI and REST API docs (netlify-docs-14, netlify-docs-15) show operational tooling but not a single declarative config for the whole stack.", "evidenceIds": [ "netlify-comm-12", "netlify-docs-14", "netlify-docs-15" ] }, { "productId": "netlify", "storyId": "instant-folder-deploy", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Netlify's docs explicitly describe drag-and-drop folder deploys with no login/Git required (netlify-docs-2, netlify-docs-4) and CLI-based anonymous deploys (netlify-docs-3), plus explicit confirmation that new projects stay private until published (netlify-docs-4). Missing for 10: independent hands-on confirmation of the private-until-publish behavior and more detail on how/when publishing is triggered.", "evidenceIds": [ "netlify-docs-2", "netlify-docs-3", "netlify-docs-4" ] }, { "productId": "netlify", "storyId": "invisible-bot-protection", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence shows generic 'Automatic HTTPS and DDoS protection' and 'Advanced Security' mentions but nothing about invisible challenge-based bot protection (e.g., CAPTCHA-free bot challenges, proof-of-work verification). No documentation or hands-on evidence confirms this specific capability.", "evidenceIds": [ "netlify-docs-9", "netlify-docs-18" ] }, { "productId": "netlify", "storyId": "isolated-sandbox-execution", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence covers deploys, serverless functions, previews, and integrations but contains no mention of hardware-isolated sandboxes, VM-level isolation, or disposable sandbox execution for running untrusted code; this is a legitimate edge-compute question for a platform like Netlify but no supporting evidence exists.", "evidenceIds": [] }, { "productId": "netlify", "storyId": "known-overage-pricing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item documents an explicit overage fee schedule for bandwidth, invocations, or build minutes; the only related evidence is community reports of unexpected surprise bills (e.g., a $104k DDoS bill) rather than any published fee table. missing for 10: a documented overage pricing table, per-unit fee schedule, or billing calculator referenced in official docs.", "evidenceIds": [ "netlify-comm-1", "netlify-comm-2", "netlify-comm-15" ] }, { "productId": "netlify", "storyId": "load-based-autoscaling", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Netlify's docs mention 'Automatic HTTPS and DDoS protection' and serverless functions, implying some infrastructure elasticity, but there is no explicit claim or documentation of load-based autoscaling for traffic bursts. Community reports (e.g., a $104k bill after a DDoS) suggest that while traffic was absorbed, the cost and reliability experience for large bursts was problematic, undermining confidence in seamless autoscaling. missing for 10: explicit autoscaling architecture docs, capacity/burst guarantees, independent benchmark or case study showing successful large-scale traffic absorption.", "evidenceIds": [ "netlify-docs-9", "netlify-comm-1", "netlify-comm-2" ] }, { "productId": "netlify", "storyId": "local-dev-simulator", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Netlify's CLI docs advertise 'run local dev servers' (netlify dev) as part of the CLI (netlify-docs-15, netlify-probe-3), which is the mechanism for simulating the production runtime locally, but a hands-on community report describes a concrete failure ('Unable to determine public folder for the dev server') with unclear config docs (netlify-comm-12), directly contradicting reliable parity with production. Missing for 10: detailed first-party documentation of how netlify dev emulates functions/redirects/edge runtime, and independent corroboration that this works smoothly in practice.", "evidenceIds": [ "netlify-docs-15", "netlify-probe-3", "netlify-comm-12" ] }, { "productId": "netlify", "storyId": "managed-postgres", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Netlify docs mention a managed 'Netlify Database' offering for Postgres, but there is no evidence describing point-in-time recovery, replicas, or high-availability configurations. Missing for 10: explicit documentation of PITR, read replicas, HA/failover architecture, and any independent corroboration of these features.", "evidenceIds": [ "netlify-docs-13", "netlify-docs-23" ] }, { "productId": "netlify", "storyId": "migration-credits", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a migration-credits program or any incentive for moving production infrastructure from another platform; evidence only covers general product features and unrelated billing complaints.", "evidenceIds": [] }, { "productId": "netlify", "storyId": "multi-protocol-native-support", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Netlify is a static site/JAMstack hosting and serverless functions platform, not a general-purpose multi-protocol service runtime; TCP/gRPC/WebSocket multiplexed service hosting is outside its category (it supports HTTP-based deploys and serverless functions only). This is a category mismatch rather than a missing feature.", "evidenceIds": [] }, { "productId": "netlify", "storyId": "object-storage-uploads", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Netlify docs reference built-in storage capability directly relevant to persisting uploads — `getStore(\"file-uploads\")` (Netlify Blobs) and 'Add instant databases and file storage' — showing a code-level API for storing user-generated files without external services. However, evidence is thin: no detail on durability, size limits, access control, or hands-on/independent confirmation of reliability for uploads at scale. missing for 10: deeper docs on Blobs API scope/limits, real-world usage examples, independent/community validation of the storage feature.", "evidenceIds": [ "netlify-docs-12", "netlify-docs-23" ] }, { "productId": "netlify", "storyId": "on-demand-image-optimization", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Netlify's docs explicitly mention image resizing (e.g. 'Resize an image to 800px wide'), indicating a built-in image transformation/CDN capability, but evidence is thin—only one brief doc snippet with no detail on optimization formats, on-demand query params, or independent/hands-on confirmation. Missing for 10: detailed docs on supported transformations (format conversion, quality, cropping), independent verification of performance, and community corroboration of the feature working in practice.", "evidenceIds": [ "netlify-docs-11" ] }, { "productId": "netlify", "storyId": "one-click-rollback", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Netlify's own docs explicitly state one-click rollback to any previous deploy, directly matching the story, and this is corroborated by broader deploy-workflow evidence (deploy previews, auto-deploy per push). missing for 10: independent hands-on confirmation of the rollback feature specifically and details on any limitations (e.g., plan restrictions).", "evidenceIds": [ "netlify-docs-5", "netlify-docs-25", "netlify-docs-1" ] }, { "productId": "netlify", "storyId": "one-click-templates", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Netlify's integrations directory shows starter templates for common use cases (Remix+Supabase, Angular bare-bones project, Clerk B2C auth template), indicating template-based quick deployment exists. However, none of the evidence explicitly describes a 'one-click deploy' button/flow or a dedicated templates gallery UI, and no independent/hands-on confirmation of the one-click experience is present. Missing for 10: explicit one-click 'Deploy to Netlify' button documentation, a dedicated templates marketplace/gallery, and community corroboration of the one-click flow working.", "evidenceIds": [ "netlify-intdir-12", "netlify-intdir-13", "netlify-intdir-14" ] }, { "productId": "netlify", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Netlify documents a REST API for managing sites, deploys, DNS, etc. plus a CLI that wraps much of the same functionality, suggesting broad API/UI parity for core deployment workflows. However, there's no evidence of a public OpenAPI spec (probe returned 404s for all candidates), and no confirmation that newer dashboard-only features (e.g., AI agent prompting from dashboard, Netlify Database management, image resizing, forms/lead collection) are fully exposed via API. missing for 10: OpenAPI/spec discoverability, explicit API coverage for newer UI-only features like AI agent prompting and Database management, independent confirmation of full API-UI parity.", "evidenceIds": [ "netlify-docs-14", "netlify-docs-15", "netlify-probe-2", "netlify-probe-3", "netlify-docs-21", "netlify-docs-13" ] }, { "productId": "netlify", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Netlify's REST API and CLI for managing sites/deploys/DNS, but nothing documents an explicit data-export feature or open-format guarantee for Netlify Database contents, Forms submissions, Blobs/file storage, or env vars that would let a user fully extract their data and leave.", "evidenceIds": [] }, { "productId": "netlify", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Netlify is a proprietary SaaS platform; no evidence indicates its core source code is available under an open license (CLI is separately open-source but the platform itself is not, and no license is cited).", "evidenceIds": [] }, { "productId": "netlify", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Netlify is a proprietary, fully-managed hosting/deploy platform; no evidence of an open-source self-hostable core or self-hosting instructions exists in the evidence pack, and one community comment even frames Netlify as the alternative to self-hosting. This is an applicable axis (many dev-tool platforms could offer OSS self-hosted editions) but no evidence supports it, so it is 'none' not 'na'.", "evidenceIds": [ "netlify-comm-14" ] }, { "productId": "netlify", "storyId": "pay-for-active-compute", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Netlify's serverless Functions (netlify-docs-6) imply pay-per-invocation compute rather than idle wall-clock billing, but there is no explicit pricing documentation stating that compute is billed only for active execution time versus idle time. Community billing complaints focus on bandwidth/DDoS overages, not on compute-vs-idle billing specifically, so they don't directly contradict this claim but do show unpredictable costs. Missing for 10: explicit pricing docs on function/compute billing granularity, independent confirmation that idle time is never charged.", "evidenceIds": [ "netlify-docs-6" ] }, { "productId": "netlify", "storyId": "pay-per-byte-storage", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Netlify does offer storage-adjacent products (Blobs, 'Netlify Database' managed Postgres) that could in principle have auto-grow/pay-per-byte billing, so the axis is not entirely foreign to the product category. However, the evidence pack contains no documentation about disk auto-scaling behavior or a bytes-written billing model for these storage features—only generic marketing mentions ('Add instant databases and file storage', 'Deploy and safely iterate on managed Postgres'). Community evidence instead highlights unpredictable, capacity-unrelated billing surprises (bandwidth/DDoS overage bills), which is off-topic to this specific disk-storage billing claim.", "evidenceIds": [ "netlify-docs-13", "netlify-docs-23" ] }, { "productId": "netlify", "storyId": "persistent-agent-runtime", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Netlify is a static site/serverless deployment platform, not a persistent agent runtime with automatic state checkpointing; the evidence covers deploys, functions, databases, and AI integrations but nothing about a long-lived, checkpointed execution environment for an agent's process state, which is a different product category (sandbox/VM runtime).", "evidenceIds": [] }, { "productId": "netlify", "storyId": "polyglot-edge-runtimes", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions Edge Functions supporting non-JavaScript/TypeScript languages like Python, Go, or Rust; Netlify's edge/serverless function docs cited here focus on JS-based functions only. Missing for 10: any documentation of multi-language edge function runtime support, examples in Go/Rust/Python, or compiler toolchain support for edge compute.", "evidenceIds": [ "netlify-docs-6" ] }, { "productId": "netlify", "storyId": "pr-preview-urls", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Netlify's docs explicitly claim automatic Deploy Previews for every PR/push, and independent community testimony corroborates the GitHub PR preview builds as a standout feature. Missing for 10: no first-party technical deep-dive on preview URL configuration edge cases (e.g., monorepos) in this pack.", "evidenceIds": [ "netlify-docs-1", "netlify-docs-25", "netlify-docs-19", "netlify-comm-10" ] }, { "productId": "netlify", "storyId": "preview-collaboration-comments", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Netlify's docs mention a 'customizable preview environment where you and key collaborators can preview content updates alongside code updates in real-time' and 'Review every change before it goes live with Deploy Previews,' implying some collaborative review workflow, but there is no explicit evidence of an inline commenting feature on preview or production URLs. missing for 10: explicit documentation of a comment/annotation UI on preview or production deploys, independent confirmation of collaborators leaving comments.", "evidenceIds": [ "netlify-docs-19", "netlify-docs-25" ] }, { "productId": "netlify", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, region selection, or storage location controls for Netlify projects/data; the only geographic-adjacent items are about CDN/DDoS protection, not user-selectable data residency. missing for 10: any documentation of region/residency selection, data storage location controls, or compliance certifications (e.g., GDPR data locality) that would let a user choose where data is stored.", "evidenceIds": [] }, { "productId": "netlify", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses AI-training data opt-out or data-usage policies for AI model training; Netlify's docs mention AI Gateway/model connections but nothing about preventing customer data from being used to train models.", "evidenceIds": [] }, { "productId": "netlify", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data retention policies, deletion controls, or AI-specific data handling settings for Netlify; the evidence covers deployment, hosting, integrations, and unrelated billing complaints.", "evidenceIds": [] }, { "productId": "netlify", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry/usage tracking opt-out settings for Netlify's CLI, dashboard, or AI features; nothing documents a privacy control of this kind.", "evidenceIds": [] }, { "productId": "netlify", "storyId": "private-service-networking", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Netlify is a frontend/JAMstack hosting and serverless functions platform, not a networking/infrastructure product that manages private service-to-service traffic or VPC peering. This story concerns internal network architecture (VPC/private networking) which is outside Netlify's product category—a category error, not a gap in evidence.", "evidenceIds": [] }, { "productId": "netlify", "storyId": "prod-nonprod-isolation", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Netlify offers per-team env var/secrets management and separate preview environments for PRs, which provide some separation between non-prod and prod configs, but there is no documented network-level isolation (e.g., VPC/firewall rules preventing preview deploys from reaching production backends). missing for 10: explicit network/service-level isolation controls, documented access restrictions from preview/staging to production APIs, independent verification of isolation guarantees.", "evidenceIds": [ "netlify-docs-8", "netlify-docs-19" ] }, { "productId": "netlify", "storyId": "programmatic-multitenant-domains", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Netlify's REST API explicitly supports managing sites, deploys, and DNS programmatically, and docs mention adding unique custom domains per project version, which could underpin per-tenant domain routing. However, there's no explicit documentation or example showing multi-tenant SaaS customer domain provisioning workflows (e.g., bulk domain API endpoints, automated SSL issuance per tenant, ownership verification flows). Missing for 10: explicit multi-tenant custom domain API docs, tenant-scoped domain automation examples, and independent/hands-on confirmation of this specific use case.", "evidenceIds": [ "netlify-docs-14", "netlify-docs-20", "netlify-docs-9" ] }, { "productId": "netlify", "storyId": "public-incident-history", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item references a public status page, uptime history, or postmortem archive for Netlify; only product docs and unrelated community sentiment are present.", "evidenceIds": [] }, { "productId": "netlify", "storyId": "published-uptime-sla", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item references an SLA document, uptime guarantee, or downtime credit policy; only general reliability/DDoS billing complaints appear, which are off-topic to a published SLA. Missing for 10: a published SLA page, defined uptime percentage, and credit schedule for downtime.", "evidenceIds": [] }, { "productId": "netlify", "storyId": "redis-compatible-cache", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Netlify offers Netlify DB (Postgres), Blobs/file storage, and integrations like Turso, but there is no evidence of a Redis-compatible key-value store for caching or job queues; this is an applicable storage axis given Netlify's data offerings, so absence of evidence yields none.", "evidenceIds": [ "netlify-docs-13", "netlify-docs-23", "netlify-docs-12", "netlify-intdir-5" ] }, { "productId": "netlify", "storyId": "regional-deployment", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of region selection, multi-region deployment, or edge location control for Netlify sites/functions—no docs or community reports address deploying to specific geographic regions closer to users. Missing for 10: any documentation of region-specific deployment configuration, edge function region pinning, or geographic deployment controls.", "evidenceIds": [] }, { "productId": "netlify", "storyId": "seat-based-pricing-minimums", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no official pricing-page or plan documentation describing seat-based tiers or minimum seat counts; the only related signal is a community complaint that Netlify 'tried charging per user on a documentation site repo,' which is anecdotal and not a documented pricing structure. Another community comment (netlify-comm-13) explicitly states pricing information is hard to find at all, reinforcing that no clear evidence exists to confirm or deny a per-seat minimum-seat pricing model.", "evidenceIds": [ "netlify-comm-3", "netlify-comm-13" ] }, { "productId": "netlify", "storyId": "serverless-functions", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Netlify's docs explicitly support serverless functions (netlify-docs-6), instant databases/file storage (netlify-docs-23, netlify-docs-13, netlify-docs-12), auto-deploys with no server management (netlify-docs-1,2,4), plus HTTPS/DDoS protection and rollback, matching the 'no servers to manage' full-stack story. Community evidence corroborates real-world use for SPA and function-based deploys (netlify-comm-7, netlify-comm-8). missing for 10: independent hands-on validation of serverless function scaling/limits and more detail on full-stack (SSR) app support beyond marketing copy.", "evidenceIds": [ "netlify-docs-6", "netlify-docs-23", "netlify-docs-13", "netlify-docs-12", "netlify-docs-1", "netlify-docs-4", "netlify-docs-9", "netlify-comm-7", "netlify-comm-8" ] }, { "productId": "netlify", "storyId": "serverless-sql-query", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Netlify Database offers managed Postgres deployable directly with 'no connection management' framing implied by serverless integration (netlify-docs-13, netlify-docs-23), and Turso edge SQLite integration also supports serverless SQL querying (netlify-intdir-5). Missing for 10: independent hands-on verification of connection pooling/no-connection-management behavior, and detailed API/SDK docs showing query syntax from code.", "evidenceIds": [ "netlify-docs-13", "netlify-docs-23", "netlify-intdir-5" ] }, { "productId": "netlify", "storyId": "site-password-protection", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs explicitly mention 'simple password protection for a personal portfolio site' as a built-in security option distinct from Advanced Security, directly matching the story. However, there's no detailed documentation snippet showing setup steps, no independent/hands-on confirmation of this feature working, and no mention of limitations (e.g., plan tier requirements). Missing for 10: setup documentation detail, independent verification, plan/tier restrictions.", "evidenceIds": [ "netlify-docs-18" ] }, { "productId": "netlify", "storyId": "smart-placement", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence describes automatic compute placement near backend data or latency-optimizing region selection for functions/edge compute; only generic serverless functions and database integration mentions appear (netlify-docs-6, netlify-docs-13, netlify-intdir-5) without any placement/latency logic.", "evidenceIds": [] }, { "productId": "netlify", "storyId": "standards-based-runtime-portability", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows Netlify's runtime features are built on proprietary APIs (e.g., `getStore(\"file-uploads\")` for Blobs, Netlify Database, Netlify Functions) rather than standards-based Web APIs designed for portability across edge platforms; there is no documentation claiming or demonstrating that code written for Netlify Edge/Functions runs unmodified elsewhere.", "evidenceIds": [ "netlify-docs-12", "netlify-docs-13", "netlify-docs-6", "netlify-docs-23" ] }, { "productId": "netlify", "storyId": "support-response-sla", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack documents SLA-backed support response times or escalation paths per plan tier; only anecdotal community reports of ad-hoc support turnaround (e.g., a cert fix within 48 hrs) exist, not guaranteed contractual response times or escalation processes.", "evidenceIds": [ "netlify-comm-6" ] }, { "productId": "netlify", "storyId": "team-secrets-env-vars", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Netlify explicitly claims team-level management of access, secrets, and env vars ([netlify-docs-8]), plus security/access controls at scale ([netlify-docs-18]) and a REST API/CLI for programmatic management ([netlify-docs-14], [netlify-docs-15]). However, there is no detailed documentation or independent/hands-on corroboration of the specifics (e.g., RBAC granularity, secret scoping, audit logs) beyond a single marketing line. Missing for 10: detailed docs on role-based permissions, audit logging, secret rotation, and independent verification of team-management workflows.", "evidenceIds": [ "netlify-docs-8", "netlify-docs-18", "netlify-docs-14", "netlify-docs-15" ] }, { "productId": "netlify", "storyId": "unified-log-view", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack references analytics and performance tracking (netlify-docs-17) but never describes a unified view combining build, deploy, and runtime logs; no docs or community evidence confirm a consolidated logging dashboard. missing for 10: explicit docs on build logs, deploy logs, and function/runtime logs being viewable together, any UI screenshot or dashboard reference, community confirmation of log consolidation.", "evidenceIds": [ "netlify-docs-17" ] }, { "productId": "netlify", "storyId": "unlimited-team-environments", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Netlify supports Deploy Previews for every PR/branch and customizable preview environments for collaborators, which enables isolated environments per change so teams can work in parallel; community feedback corroborates PR preview builds as a standout feature. However, there's no explicit evidence of 'unlimited' environments, per-branch full-stack isolated dev environments (e.g., ephemeral backends/databases per branch), or scaling guarantees for large teams shipping simultaneously. missing for 10: explicit 'unlimited' environment claims, isolated backend/database-per-branch evidence, independent validation of scaling to large teams.", "evidenceIds": [ "netlify-docs-1", "netlify-docs-19", "netlify-docs-25", "netlify-comm-10" ] }, { "productId": "netlify", "storyId": "visitor-performance-analytics", "verdict": "partial", "quality": 6, "confidence": "low", "rationale": "netlify-docs-17 explicitly claims tracking of project performance and visitor/team behavior analytics, directly matching the story, but there is no further first-party detail on dashboards, metrics depth, or independent/hands-on corroboration of the analytics feature itself. Missing for 10: detailed documentation of specific analytics metrics/dashboards, independent hands-on validation, and confirmation of real-time or historical performance tracking capabilities.", "evidenceIds": [ "netlify-docs-17" ] }, { "productId": "netlify", "storyId": "visual-infra-canvas", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Netlify is a web deployment/hosting platform (CLI, dashboard, config files like netlify.toml) with no visual infrastructure-canvas editor evidenced or plausible for its category; this story targets IaC-visualization tools, a different product type.", "evidenceIds": [] }, { "productId": "netlify", "storyId": "zero-config-repo-deploy", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs and community evidence confirm connect-repo auto-deploy with build detection, PR previews, and rollback, matching the zero-config repo-connect deploy workflow (netlify-docs-1, netlify-docs-25, netlify-docs-5). Community reviews corroborate the GitHub integration and automated builds working well in practice (netlify-comm-7, netlify-comm-10). Missing for 10: explicit documentation of automatic build-command/framework detection logic and independent hands-on confirmation that zero manual config is needed for varied frameworks.", "evidenceIds": [ "netlify-docs-1", "netlify-docs-25", "netlify-docs-5", "netlify-comm-7", "netlify-comm-10" ] }, { "productId": "railway", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Railway hosts a live llms.txt confirmed by direct probe (HTTP 200) plus dedicated agent-oriented docs like agents.md and cloud-agents.md pages designed for AI assistants to consume. missing for 10: independent third-party confirmation that agents actually use these files successfully in practice.", "evidenceIds": [ "railway-probe-1", "railway-docs-19", "railway-docs-20", "railway-docs-21" ] }, { "productId": "railway", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "The Railway Agent (dashboard chat assistant) can inspect deployments, diagnose failures, and surface insights like opening PRs to fix broken builds, which is a form of AI-generated insight/suggestion within the product; connectors to Notion/Linear/Sentry extend this. However, this is framed as an operational/agentic assistant rather than data-analytics insights (e.g., no evidence of AI-generated insights over metrics, logs analytics, or cost/usage trends beyond failure diagnosis). Missing for 10: evidence of proactive AI-driven analytics/insights over metrics or usage data, independent user validation of insight quality, and broader scope beyond deployment troubleshooting.", "evidenceIds": [ "railway-docs-25", "railway-docs-26", "railway-docs-27" ] }, { "productId": "railway", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Railway's Cloud Agents keep coding-agent machines running persistently in the background even after disconnecting, and the built-in Railway Agent can autonomously diagnose failures, create/configure services, and open PRs, plus condition-based Slack/Discord/email alerts run without manual triggering. However, evidence shows these are mostly agent-assisted or chat-triggered rather than fully scheduled/cron-style autonomous workflows, and there's no independent corroboration of unattended background automation reliability. Missing for 10: explicit scheduled/triggered automation framework (e.g. cron jobs), independent hands-on verification of autonomous background runs.", "evidenceIds": [ "railway-docs-22", "railway-docs-25", "railway-docs-26", "railway-docs-9" ] }, { "productId": "railway", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Railway ships a built-in chat-based AI assistant (\"Railway Agent\") directly in the dashboard that can create/configure services, inspect deployments, diagnose failures, and open PRs to fix issues, plus Slack integration for delegating tasks conversationally. Missing for 10: independent/hands-on corroboration of the Railway Agent's real-world reliability and depth beyond vendor docs.", "evidenceIds": [ "railway-docs-25", "railway-docs-26", "railway-docs-27", "railway-docs-28" ] }, { "productId": "railway", "storyId": "agentic-headless", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Railway ships an official CLI for local development and deploying from the terminal, which implies scriptable/headless usage in automation contexts, but the evidence pack has no explicit documentation of CI/CD pipeline integration, GitHub Actions examples, or non-interactive auth tokens for CI. missing for 10: explicit CI/CD integration docs, non-interactive auth/token flow for automation, examples of headless CI usage.", "evidenceIds": [ "railway-docs-15", "railway-probe-3" ] }, { "productId": "railway", "storyId": "agentic-mcp-client", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Railway's docs explicitly describe connecting external MCP servers (Notion, Linear, Sentry, or any custom remote MCP server) to the Railway Agent so it can read from and use their tools while operating on projects, directly matching the story. Missing for 10: independent/hands-on corroboration beyond first-party docs, and detail on tool-use scope/limitations when using third-party MCP servers.", "evidenceIds": [ "railway-docs-27", "railway-docs-25", "railway-docs-16" ] }, { "productId": "railway", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Railway ships an official documented MCP server (docs.railway.com/ai/mcp-server.md) that lets AI assistants create projects, deploy templates, manage environments, pull variables, and redeploy services, plus a one-command CLI setup for agent/MCP/auth configuration and an 'use-railway' skill teaching agents to operate the platform. This is first-party, well-documented, and clearly targets AI-native agent connectivity. missing for 10: independent/hands-on third-party corroboration of the MCP server working in practice, and more detail on auth/security model for remote agent connections.", "evidenceIds": [ "railway-docs-16", "railway-docs-19", "railway-docs-20", "railway-docs-21" ] }, { "productId": "railway", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Railway ships a chat-based 'Railway Agent' in-dashboard and via Slack (@Railway) that lets users create/configure services, inspect deployments, diagnose failures, and open PRs using natural language, plus an MCP server and Agent Skills for AI assistants to manage projects, deploy templates, and redeploy services conversationally. This directly matches the ai-native/agenticness story of operating the product via natural-language commands. Missing for 10: independent/hands-on validation of the Railway Agent's reliability and accuracy beyond first-party docs.", "evidenceIds": [ "railway-docs-25", "railway-docs-26", "railway-docs-27", "railway-docs-28", "railway-docs-20", "railway-docs-19", "railway-docs-21" ] }, { "productId": "railway", "storyId": "agentic-official-cli", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Railway ships a well-documented official CLI (docs-15, docs-18, probe-3) and explicitly extends it for AI-native workflows via a one-command agent setup that installs skills, MCP, and authentication (docs-19), making it directly usable by AI coding agents (docs-21, docs-16). missing for 10: independent/hands-on community confirmation of the CLI's AI-native agent setup specifically.", "evidenceIds": [ "railway-docs-15", "railway-docs-18", "railway-docs-19", "railway-docs-16", "railway-docs-21", "railway-probe-3" ] }, { "productId": "railway", "storyId": "agentic-public-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Railway documents an MCP server and CLI that let AI agents create projects, deploy templates, manage environments, and redeploy services (railway-docs-20, railway-docs-15), which functions as a programmatic interface for AI-native drivers. However, probing for a standard public API spec (OpenAPI/Swagger) returned 404 on all candidate paths, indicating no conventional documented REST/GraphQL API surface (railway-probe-2). Missing for 10: a discoverable OpenAPI/GraphQL API reference, independent confirmation of API completeness beyond MCP tool coverage.", "evidenceIds": [ "railway-docs-20", "railway-docs-15", "railway-docs-16", "railway-probe-2" ] }, { "productId": "railway", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Railway offers an MCP server and CLI-based agent authentication (railway-docs-19, railway-docs-20) but nothing describes scoped or least-privilege credential issuance — the agent integration docs only mention broad capabilities (create projects, deploy templates, manage environments, pull variables) with no token-scoping or permission-tiering language. missing for 10: any mention of scoped/limited API tokens, role-based or read-only credentials, or documentation of least-privilege agent authentication.", "evidenceIds": [ "railway-docs-19", "railway-docs-20", "railway-docs-22" ] }, { "productId": "railway", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Railway documents a CLI, MCP server, and agent skills for AI-driven workflows, but no evidence of official SDKs (client libraries) for building against Railway programmatically; a probe for OpenAPI/SDK specs returned 404s. Missing for 10: any first-party SDK (Python, Node, Go, etc.), API client library documentation, or SDK release notes.", "evidenceIds": [ "railway-probe-2", "railway-docs-15", "railway-docs-16" ] }, { "productId": "railway", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence mentions Slack/Discord/email alerts and an MCP server/agent integrations, but nowhere does it document a generic webhook subscription mechanism for external event consumption. No API or docs reference to 'webhooks' as a subscribable event feed is present.", "evidenceIds": [] }, { "productId": "railway", "storyId": "ai-global-inference", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Railway is a general-purpose deployment/hosting platform, not an AI inference/model-serving product; there is no evidence of a managed AI inference API or GPU-backed model-serving product with global edge distribution. This is a category mismatch, not a missing feature.", "evidenceIds": [] }, { "productId": "railway", "storyId": "ai-model-gateway", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Railway is a deployment/infrastructure platform, not an AI model gateway; nothing in the evidence describes a unified endpoint for accessing hundreds of AI models. This story targets a fundamentally different product category (LLM gateway/router), making the axis a category error for Railway.", "evidenceIds": [] }, { "productId": "railway", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of an interactive API reference or runnable examples; the openapi probe explicitly returned 404s for all candidate spec paths, and no docs item mentions an API explorer or try-it-out console.", "evidenceIds": [ "railway-probe-2" ] }, { "productId": "railway", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The probe evidence explicitly shows all standard OpenAPI spec paths (openapi.json, swagger.json, etc.) return 404, and no docs mention a downloadable machine-readable API spec; only an llms.txt file and CLI/MCP docs are found. missing for 10: a hosted OpenAPI/Swagger spec, any documented machine-readable API schema.", "evidenceIds": [ "railway-probe-2", "railway-probe-1" ] }, { "productId": "railway", "storyId": "api-sandbox", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Railway supports isolated per-branch/PR preview environments and 'unlimited environments' for separate dev/staging vs production, which lets a user test changes without touching production (railway-docs-11, railway-docs-12). Cloud agents also run in isolated, ephemeral environments with their own disk (railway-docs-22). However, there is no explicit documentation of sandboxed test data, database seeding/copying, or safeguards preventing accidental production data access. Missing for 10: explicit sandbox/test-data isolation guidance, database cloning/seeding for previews, and independent confirmation that preview environments never touch production data.", "evidenceIds": [ "railway-docs-11", "railway-docs-12", "railway-docs-22" ] }, { "productId": "railway", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of API versioning scheme or a documented deprecation policy; the openapi probe returned 404s on all candidate paths and no docs mention API versioning/deprecation commitments.", "evidenceIds": [ "railway-probe-2" ] }, { "productId": "railway", "storyId": "automatic-tls-certs", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs mention SSL is provided automatically for endpoints from the moment of deployment, implying free TLS certs, but there is no evidence about wildcard domain certificates or explicit auto-renewal guarantees. missing for 10: wildcard certificate support, explicit auto-renewal details, independent confirmation of custom domain TLS coverage.", "evidenceIds": [ "railway-docs-4" ] }, { "productId": "railway", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Railway's docs describe MCP/CLI actions on individual projects, deployments, and variables (e.g., create projects, deploy templates, manage environments, pull variables, redeploy services), but no evidence shows any bulk or batch operation across many items at once. Missing for 10: documentation of batch/bulk API calls, multi-service scripting examples, or CLI flags for operating on multiple resources simultaneously.", "evidenceIds": [ "railway-docs-20", "railway-probe-3" ] }, { "productId": "railway", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Railway supports notification-style alerts ('Slack, Discord, or emails the moment conditions you specify are met') which is a basic trigger-action mechanism, and its Railway Agent/MCP ecosystem allows AI-driven actions like opening PRs on failed deployments, but there's no evidence of a general user-defined rules engine (custom conditions/triggers mapped to arbitrary actions) beyond alerting. missing for 10: a documented rules/automation engine letting users define arbitrary event-condition-action workflows, evidence of custom trigger configuration UI/API, and independent confirmation of reliability of these triggers.", "evidenceIds": [ "railway-docs-9", "railway-docs-25", "railway-docs-26" ] }, { "productId": "railway", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers deployments, scaling, environments, alerts, and AI agent integrations, but contains no mention of cron jobs, scheduled tasks, or recurring workflow execution capabilities. Missing for 10: any documentation of a cron/scheduler feature, recurring job configuration, or workflow automation triggers.", "evidenceIds": [] }, { "productId": "railway", "storyId": "automation-versioned-workflows", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Railway explicitly supports git-based versioning of deployments, per-PR preview reviews before merge, and instant rollback to any previous version, and its Railway Agent can open pull requests for review before changes are merged — covering version/review/rollback of automations end-to-end. missing for 10: independent/hands-on corroboration that rollback and PR-review flows work reliably in practice (community evidence focuses on platform outages, not rollback failures).", "evidenceIds": [ "railway-docs-12", "railway-docs-13", "railway-docs-11", "railway-docs-25", "railway-docs-26" ] }, { "productId": "railway", "storyId": "cli-deploy", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Railway documents a CLI that lets developers develop locally and deploy from the terminal, with a dedicated CLI guide page confirming this is an official, supported workflow. missing for 10: no independent/hands-on corroboration of CLI deploy experience, and no detail on full command coverage (e.g., env management, rollbacks) via CLI alone versus dashboard.", "evidenceIds": [ "railway-docs-15", "railway-docs-18", "railway-probe-3" ] }, { "productId": "railway", "storyId": "cli-scaffold-project", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm a dedicated CLI for local development and terminal deploys, plus one-click templates/starters, but no evidence details a single command that both scaffolds a new project and deploys it end-to-end. missing for 10: explicit CLI command sequence (e.g. init+up) demonstrating one-command scaffold-and-deploy, independent hands-on confirmation of speed/simplicity.", "evidenceIds": [ "railway-docs-15", "railway-docs-17", "railway-probe-3" ] }, { "productId": "railway", "storyId": "condition-based-alerts", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Railway docs explicitly state it sends Slack, Discord, or email notifications the moment specified conditions are met, directly matching the story. missing for 10: no independent/hands-on corroboration of alert configuration UX, no detail on what conditions/metrics can trigger alerts.", "evidenceIds": [ "railway-docs-9", "railway-docs-8" ] }, { "productId": "railway", "storyId": "consistent-api-at-scale", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Railway's docs claim seamless scaling via the same primitives (scale CPU/RAM, add replicas, multi-region deploy) without new tooling, but independent community reports describe a platform-wide outage that took down the entire service and raised doubts about stability and trust at scale, directly undercutting the 'same core APIs from prototype to massive scale' promise. Missing for 10: first-party case studies of large-scale customers not hitting limits, and resolution of the stability concerns raised in the outage thread.", "evidenceIds": [ "railway-docs-6", "railway-docs-7", "railway-docs-18", "railway-comm-1", "railway-comm-4", "railway-comm-5" ] }, { "productId": "railway", "storyId": "custom-domains", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack never explicitly mentions custom domain attachment; docs only reference generic 'public endpoints, SSL' (railway-docs-4) without confirming a custom-domain feature. Missing for 10: any documentation or community proof of attaching/verifying custom domains on deployed services.", "evidenceIds": [ "railway-docs-4" ] }, { "productId": "railway", "storyId": "custom-metrics-dashboards", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Direct first-party claim confirms custom dashboards combining metrics from any service, but the pack offers no independent/hands-on corroboration or detail on dashboard flexibility (e.g., cross-project aggregation, widget types, sharing). missing for 10: independent verification, detail on dashboard customization scope, evidence of multi-service/cross-project metric combination in practice.", "evidenceIds": [ "railway-docs-8" ] }, { "productId": "railway", "storyId": "default-ddos-waf", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers SSL, load balancing, and general networking but no mention of DDoS protection or a web application firewall being applied by default; the outage-related community items discuss general stability, not DDoS/WAF specifically. missing for 10: any mention of DDoS mitigation, WAF, or default request-level security filtering.", "evidenceIds": [ "railway-docs-4" ] }, { "productId": "railway", "storyId": "docker-deploy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes Railway's git-based auto-config, canvas UI, CLI, and AI agent features, but no citation explicitly confirms deploying pre-built Docker container images or Dockerfiles as a supported workflow. missing for 10: explicit docs on Dockerfile/Docker image deployment support, evidence of container registry integration.", "evidenceIds": [] }, { "productId": "railway", "storyId": "documented-rate-limits", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item documents rate limits or concurrency caps for Railway's APIs, MCP server, or functions; OpenAPI probe returned 404s and no rate-limit documentation is cited.", "evidenceIds": [ "railway-probe-2" ] }, { "productId": "railway", "storyId": "durable-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Railway's evidence pack shows it as a deployment/hosting platform (repo-to-deploy, networking, scaling, environments, previews) with no mention of a durable execution engine, workflow-as-code framework, or built-in queue/retry orchestration primitives. The story describes a specific workflow-orchestration capability (like Temporal/Inngest) that Railway does not evidence.", "evidenceIds": [ "railway-docs-18", "railway-docs-6", "railway-docs-15" ] }, { "productId": "railway", "storyId": "edge-websocket-support", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Railway is a persistent-container/service hosting platform, not an edge/serverless-functions runtime (e.g., Cloudflare Workers, Vercel Edge Functions); the story specifically asks about WebSocket handling within short-lived edge/serverless function execution models, which is a different product category than what Railway offers.", "evidenceIds": [] }, { "productId": "railway", "storyId": "execution-time-limits", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Railway's evidence pack describes general CPU/RAM scaling (railway-docs-6) but contains no documentation of specific CPU or wall-clock execution time limits for functions/workloads, which is the core of this story. Missing for 10: any explicit stated time limits, timeout thresholds, or documentation addressing how long-running or agentic workloads are bounded.", "evidenceIds": [ "railway-docs-6" ] }, { "productId": "railway", "storyId": "external-service-bindings", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Railway offers private networking, one-click databases/starters, and infrastructure provisioning with auto-config, which reduces some manual wiring, but the evidence pack lacks specifics on native SDK-free bindings for external third-party services (e.g., managed connectors to Stripe, S3, external APIs) or credential-injection features beyond internal service linking. Missing for 10: concrete documentation of native bindings/connectors to external third-party services and APIs, evidence of automatic credential/env-var injection without manual SDK setup, and independent corroboration of this workflow in practice.", "evidenceIds": [ "railway-docs-1", "railway-docs-4", "railway-docs-17", "railway-docs-18" ] }, { "productId": "railway", "storyId": "fast-boot-scaling", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Railway's docs mention general scaling (CPU/RAM/replicas, load balancing) and running closer to users, but there is no evidence of sub-second instance boot times or scaling to tens of thousands of instances under load, which is the specific edge-compute claim in the story. Community evidence even highlights platform-wide stability issues rather than proven massive-scale elasticity.", "evidenceIds": [ "railway-docs-6", "railway-docs-7", "railway-comm-1", "railway-comm-4" ] }, { "productId": "railway", "storyId": "framework-tool-continuity", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Railway's docs show it connects directly to existing Git repos with auto-config, PR-based preview environments, and a CLI for local development/deploy — i.e., no proprietary editor or framework required (railway-docs-1, railway-docs-12, railway-docs-15, railway-docs-17). However, config is presented as best done via a proprietary visual 'canvas' (YAML optional but canvas-first), and there's no explicit evidence of compatibility with external CI pipelines (e.g., GitHub Actions) or independent developer confirmation of a friction-free existing-workflow experience. missing for 10: evidence of CI-tool interoperability, independent/hands-on confirmation that no Railway-specific tooling is required, editor-specific integration details.", "evidenceIds": [ "railway-docs-1", "railway-docs-12", "railway-docs-15", "railway-docs-17", "railway-docs-3" ] }, { "productId": "railway", "storyId": "full-nodejs-runtime-support", "verdict": "na", "quality": 0, "confidence": "medium", "rationale": "Railway is a general container/service deployment platform (build from repo, deploy full services with CPU/RAM scaling), not an edge/serverless-functions product with a restricted runtime that this story presupposes; no evidence describes an edge-function runtime at all, restricted or otherwise, so the axis is a category mismatch rather than an unmet capability.", "evidenceIds": [ "railway-docs-18", "railway-docs-6", "railway-docs-7" ] }, { "productId": "railway", "storyId": "full-stack-preview-envs", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Railway's docs explicitly state every PR gets its own preview environment with instant, auto-configured full-stack previews and unlimited environments for parallel work. missing for 10: independent/hands-on verification of PR preview behavior at scale and details on environment teardown/ephemerality lifecycle.", "evidenceIds": [ "railway-docs-1", "railway-docs-11", "railway-docs-12" ] }, { "productId": "railway", "storyId": "git-push-deploy", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Railway's docs explicitly describe connecting a repo for auto-config deploys, per-PR preview environments, and unlimited environments so pushes deploy simultaneously, which directly covers git-push-triggered deployment workflows. Missing for 10: independent hands-on confirmation of push-to-deploy behavior and explicit mention of branch-based auto-deploy configuration options.", "evidenceIds": [ "railway-docs-1", "railway-docs-12", "railway-docs-11", "railway-docs-18" ] }, { "productId": "railway", "storyId": "global-edge-network", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Railway's marketing states only a single vague line — 'Run your application closer to where your users are' (railway-docs-7) — with no detail on edge regions, PoPs, or CDN architecture. Community reports of a major outage explicitly attribute it to 'total dependency on google cloud' and note the entire service (including the marketing site) went down together, which contradicts the premise of a distributed, resilient global edge network (railway-comm-4, railway-comm-6, railway-comm-1). missing for 10: documented edge-region list, latency/PoP architecture details, evidence of multi-region failover, and confirmation the claim survives independent scrutiny.", "evidenceIds": [ "railway-docs-7", "railway-comm-4", "railway-comm-6", "railway-comm-1" ] }, { "productId": "railway", "storyId": "global-static-cdn", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a CDN or edge-caching feature for static assets; Railway's evidence covers deployment, scaling, monitoring, and AI agent tooling but nothing about a global CDN or edge caching. 'Run your application closer to where your users are' refers to regional deployment, not CDN/static asset caching.", "evidenceIds": [ "railway-docs-7" ] }, { "productId": "railway", "storyId": "hard-spending-limits", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "A community tweet explicitly praises Railway for offering hard spending limits, confirming the capability exists and is well-received, but there is no first-party documentation detailing configuration options, scope (project vs org-level), or enforcement behavior. Missing for 10: official docs on setting/configuring hard spending limits, independent hands-on verification, and details on limit granularity.", "evidenceIds": [ "railway-docs-14" ] }, { "productId": "railway", "storyId": "idp-secured-access", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions SSO, SAML, OIDC, or any identity-provider integration for securing internal apps, deployments, or agents; coverage is limited to deployment, scaling, monitoring, and AI-agent features unrelated to IdP-based access control.", "evidenceIds": [] }, { "productId": "railway", "storyId": "in-browser-dev-toolbar", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "This story describes an in-browser feedback/feature-flag/performance widget embedded on a live website (a product like a customer feedback/observability toolbar), which is not a category fit for Railway, a cloud infrastructure/deployment platform. Railway's evidence covers deployment, scaling, logs, and dashboards but nothing about an embeddable end-user-facing toolbar for feedback or feature flags on a live site.", "evidenceIds": [] }, { "productId": "railway", "storyId": "incremental-rollout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows rollback, replicas/load balancing, and PR previews, but nothing describing incremental/canary/blue-green rollout strategies or traffic-shifting to limit blast radius during deploys.", "evidenceIds": [] }, { "productId": "railway", "storyId": "infra-as-code-blueprint", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Railway's docs mention that settings can be modified via YAML (\"YAML optional\") alongside a visual canvas, and rollback/versioning of deployments is supported, hinting at config-as-code support, but there is no evidence of a single manifest file that defines and version-controls an entire multi-service architecture (e.g., a documented railway.json/toml schema covering networking, environments, and services in one file). missing for 10: explicit single-file architecture schema/manifest documentation, examples of defining multi-service topology in one file, and confirmation that this file is the primary/versioned source of truth rather than the canvas.", "evidenceIds": [ "railway-docs-3", "railway-docs-13", "railway-docs-11" ] }, { "productId": "railway", "storyId": "instant-folder-deploy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Railway's primary deploy paths are Git-repo connection and CLI/local deploy, but nothing describes a folder-upload deploy flow or a private-until-published toggle for a new deployment.", "evidenceIds": [ "railway-docs-1", "railway-docs-15", "railway-probe-3" ] }, { "productId": "railway", "storyId": "invisible-bot-protection", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions bot protection, challenge pages, CAPTCHA, or invisible bot-mitigation of any kind; Railway's docs focus on deployment, scaling, and networking basics like SSL/load balancing, not bot/WAF-style traffic protection. missing for 10: any mention of bot detection, challenge-based traffic filtering, or WAF/CDN-style bot mitigation features.", "evidenceIds": [] }, { "productId": "railway", "storyId": "isolated-sandbox-execution", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Railway's evidence covers deployment, CI/CD, cloud agents with persistent disks, and AI integrations, but nothing addresses hardware-isolated, disposable sandboxes for running untrusted code — in fact cloud agents are described as persistent ('machine stays running when you disconnect'), the opposite of disposable isolation. Missing for 10: any mention of sandbox isolation technology (VM/hypervisor-level), disposability guarantees, or untrusted-code execution security model.", "evidenceIds": [ "railway-docs-22" ] }, { "productId": "railway", "storyId": "known-overage-pricing", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no documentation of an overage fee schedule for bandwidth, invocations, or build minutes — only a community tweet praising hard spending limits, which is a cap mechanism, not a fee schedule disclosure. missing for 10: published overage rate/pricing table, per-resource overage unit costs, and billing documentation describing what happens when limits are exceeded.", "evidenceIds": [ "railway-docs-14" ] }, { "productId": "railway", "storyId": "load-based-autoscaling", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Docs mention manual scaling (adjusting CPU/RAM or adding replicas with load balancing) but there is no evidence of automatic, load-triggered autoscaling that reacts to traffic bursts without manual intervention.", "evidenceIds": [ "railway-docs-6", "railway-docs-4" ] }, { "productId": "railway", "storyId": "local-dev-simulator", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Railway's CLI lets developers 'develop locally and deploy from your terminal' and inject production-like environment variables, but there's no evidence of an actual local simulator/emulator that mirrors the production runtime (e.g., container parity, network topology, or service mesh simulation). Missing for 10: documentation of a local runtime emulator, evidence of environment parity beyond env-var injection, and any independent confirmation that local dev closely mirrors production behavior.", "evidenceIds": [ "railway-docs-15", "railway-probe-3" ] }, { "productId": "railway", "storyId": "managed-postgres", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation of Railway provisioning managed Postgres with point-in-time recovery, replicas, or high availability — only generic claims about scaling CPU/RAM and adding replicas for services in general (railway-docs-6), with no database-specific backup/PITR/HA feature descriptions. A community comment even suggests backup handling has been criticized (railway-comm-8), but nothing confirms the capability exists as described.", "evidenceIds": [ "railway-docs-6", "railway-comm-8" ] }, { "productId": "railway", "storyId": "migration-credits", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Railway has migration guides (e.g., 'Migrate from Heroku', 'Compare to Render') but no evidence of any migration credit program, discount, or reimbursement offered to customers moving production infrastructure to the platform.", "evidenceIds": [ "railway-docs-29", "railway-docs-30" ] }, { "productId": "railway", "storyId": "multi-protocol-native-support", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Railway's own docs explicitly state 'HTTP, TCP, gRPC, WebSockets handled automatically' alongside private connections, public endpoints, SSL, and load balancing available from deploy time, directly matching the story. Missing for 10: independent/hands-on verification of multi-protocol serving from a single service and technical detail on how protocol detection/config works under the hood.", "evidenceIds": [ "railway-docs-5", "railway-docs-4" ] }, { "productId": "railway", "storyId": "object-storage-uploads", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items mention object storage, file/blob storage, S3-compatible storage, or persistent volumes for user uploads; the evidence covers deployment, scaling, observability, and AI agent tooling only.", "evidenceIds": [] }, { "productId": "railway", "storyId": "on-demand-image-optimization", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Railway is a general-purpose deployment/hosting platform (PaaS), not a media/image service; there's no evidence of any image CDN or image transformation feature, and this is not a core axis for this type of product.", "evidenceIds": [] }, { "productId": "railway", "storyId": "one-click-rollback", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Railway's own docs explicitly claim 'Rollback to any previous version instantly when something breaks,' directly matching the story, but this is a single marketing-style claim with no independent/hands-on corroboration or detail on the one-click UX flow. Community evidence discusses platform stability incidents but does not specifically test or contradict rollback functionality. missing for 10: independent/hands-on verification of the rollback flow, documentation detail on how rollback is triggered (UI steps), confirmation it works across all deploy types.", "evidenceIds": [ "railway-docs-13" ] }, { "productId": "railway", "storyId": "one-click-templates", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly state \"One-click deployable applications and starters for common use cases\" and the platform overall emphasizes quick repo-to-deploy workflows, supporting a template/starter deploy story. Missing for 10: independent hands-on verification of the template gallery experience and details on breadth/quality of available templates.", "evidenceIds": [ "railway-docs-17", "railway-docs-18" ] }, { "productId": "railway", "storyId": "openness-api-parity", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Railway ships a CLI and an MCP server that let AI assistants create projects, deploy templates, manage environments, pull variables, and redeploy services (railway-docs-20, railway-docs-19), showing partial programmatic parity with the UI, but there is no documented OpenAPI/GraphQL spec (probe for openapi paths returned 404s) and no claim that canvas-level configuration, dashboards, or alerting can be fully replicated via API. Missing for 10: explicit full API/UI parity documentation, discoverable OpenAPI/GraphQL schema, and evidence that canvas/dashboard/alert features are API-accessible.", "evidenceIds": [ "railway-docs-19", "railway-docs-20", "railway-docs-25", "railway-probe-2" ] }, { "productId": "railway", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Railway's docs cover migrating INTO Railway (e.g., from Heroku) and CLI/local dev workflows, but there is no evidence of any data export feature, open-format export of configs/logs/variables, or documented offboarding path for leaving the platform with your data intact.", "evidenceIds": [ "railway-docs-15", "railway-docs-29" ] }, { "productId": "railway", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence that Railway's source code is open-licensed or publicly available; Railway is a closed proprietary cloud platform with only docs/CLI/API references, none pointing to an open-source repo or license.", "evidenceIds": [] }, { "productId": "railway", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Railway is a managed cloud PaaS; nothing in the evidence pack mentions an open-source or self-hostable version of the core Railway platform — all docs describe using Railway's own hosted service. missing for 10: any open-source repo/license for the core platform, self-hosting instructions or Docker/K8s deployment guide for running Railway itself.", "evidenceIds": [ "railway-docs-18" ] }, { "productId": "railway", "storyId": "pay-for-active-compute", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no documentation or claims about usage-based/active-compute billing versus idle time; only a community tweet about 'hard spending limits' is present, which addresses spend caps, not billing granularity. Missing for 10: any doc on per-second/active-compute billing model, idle-scale-to-zero mechanics, or pricing page evidence.", "evidenceIds": [ "railway-docs-14" ] }, { "productId": "railway", "storyId": "pay-per-byte-storage", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of persistent volumes, disk auto-scaling behavior, or usage-based storage billing (only general platform, deployment, and AI-agent features). This is a fair question for a cloud infra provider, but nothing in the pack confirms auto-grow disks or byte-metered storage pricing.", "evidenceIds": [] }, { "productId": "railway", "storyId": "persistent-agent-runtime", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Railway's cloud-agents feature explicitly provides a persistent runtime: the agent's machine 'stays running when you disconnect' and files persist on a dedicated disk, with coding-agent credentials carried over (railway-docs-22, 23, 24). However, there is no evidence of automatic state checkpointing/snapshotting of the agent's work — Railway's rollback feature (railway-docs-13) applies to deployment versions, not agent runtime state. Missing for 10: explicit automatic checkpoint/snapshot mechanism for agent state, evidence of restore-from-checkpoint after crash or disconnect, and independent corroboration of persistence reliability.", "evidenceIds": [ "railway-docs-22", "railway-docs-23", "railway-docs-24", "railway-docs-13" ] }, { "productId": "railway", "storyId": "polyglot-edge-runtimes", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Railway is a general-purpose cloud deployment/hosting platform for full applications and services, not an edge-function/edge-compute product with a language-restricted runtime; the story's premise (writing edge functions in specific languages) is a category error for this product type.", "evidenceIds": [] }, { "productId": "railway", "storyId": "pr-preview-urls", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Railway docs explicitly state \"Every pull request gets its own preview. No surprises after merge\" and marketing highlights \"instant previews\" upon connecting a repo, directly matching the story. Missing for 10: independent/hands-on confirmation of PR preview URLs working in practice and details on configuration/limitations.", "evidenceIds": [ "railway-docs-12", "railway-docs-1" ] }, { "productId": "railway", "storyId": "preview-collaboration-comments", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Railway is an infrastructure/deployment platform with PR previews and environments, but no evidence indicates any commenting/collaboration UI on environments — this is a category mismatch (that's a design-review/collab tool feature, not a cloud provider feature).", "evidenceIds": [] }, { "productId": "railway", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Railway docs mention running applications 'closer to where your users are' (railway-docs-7), implying some region choice for deployments, but there is no explicit documentation of selectable regions, data storage location controls, or compliance/residency certifications (e.g., GDPR, SOC2) that would let an AI-native user confirm where their data is stored. Missing for 10: explicit region list/selector for compute and storage, data residency guarantees, and compliance documentation.", "evidenceIds": [ "railway-docs-7" ] }, { "productId": "railway", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items mention data usage for AI model training or any opt-out/privacy controls regarding AI training; this is a cloud infrastructure provider and the axis of preventing AI training on user data is applicable (as with any SaaS handling customer data/code) but wholly unaddressed. Missing for 10: any privacy policy statement on AI training, opt-out settings, or data usage terms.", "evidenceIds": [] }, { "productId": "railway", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses data retention policies, deletion controls, or privacy settings for AI workflows on Railway; the only tangential item (railway-comm-8) is a community jab about backup security, not a documented retention/deletion control. missing for 10: documentation of data retention policy, explicit deletion/opt-out controls, AI-specific data handling disclosures.", "evidenceIds": [ "railway-comm-8" ] }, { "productId": "railway", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item addresses telemetry, usage tracking, or an opt-out mechanism for Railway's CLI, dashboard, or AI features; the axis is applicable (a dev platform could ship telemetry opt-out) but nothing documents it.", "evidenceIds": [] }, { "productId": "railway", "storyId": "private-service-networking", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Railway's docs claim built-in \"Private connections\" alongside public endpoints, SSL, and load balancing \"live from the moment you deploy,\" implying internal service traffic can stay off the public internet without manual VPC setup. However, the evidence pack gives only a single high-level marketing line with no architectural detail, no explicit VPC-avoidance framing, and no independent corroboration of how private networking actually isolates traffic. Missing for 10: detailed technical docs on private networking implementation, explicit no-VPC-needed messaging, and independent/hands-on verification of traffic isolation.", "evidenceIds": [ "railway-docs-4" ] }, { "productId": "railway", "storyId": "prod-nonprod-isolation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence mentions 'unlimited environments' and private networking/endpoints generally, but there is no documentation or claim about network-level isolation preventing non-production environments from reaching production services (e.g., separate private networks, environment-scoped service discovery restrictions). Missing for 10: explicit environment-to-environment network isolation policy, private networking scoped per environment, access-control documentation preventing staging/dev from calling prod services.", "evidenceIds": [ "railway-docs-4", "railway-docs-11" ] }, { "productId": "railway", "storyId": "programmatic-multitenant-domains", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Railway's docs mention custom domains, SSL, and public endpoints being available at deploy time, but there is no evidence of a programmatic API/CLI capability for provisioning and routing custom domains on behalf of a multi-tenant SaaS's own customers, and a probe for an OpenAPI/domains API returned 404s.", "evidenceIds": [ "railway-docs-4", "railway-probe-2" ] }, { "productId": "railway", "storyId": "public-incident-history", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item references a public status page or historical incident postmortems; only community complaints about an outage exist, with no mention of a status.railway.app or postmortem documentation. missing for 10: public status page, incident history, postmortem write-ups.", "evidenceIds": [] }, { "productId": "railway", "storyId": "published-uptime-sla", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack of a published uptime SLA or defined downtime credits; docs focus on features like logs, alerts, and scaling, not contractual guarantees. Community evidence even describes a major outage with no mention of any credit/SLA process, reinforcing the absence.", "evidenceIds": [ "railway-comm-1", "railway-comm-4", "railway-comm-5" ] }, { "productId": "railway", "storyId": "redis-compatible-cache", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no mention of Redis, a key-value store, or caching/queue database offerings on Railway—only generic deployment, templates, and AI-agent features. While Railway as a cloud platform could plausibly offer this (axis applies), there is no documentation or community evidence confirming a Redis-compatible database is available.", "evidenceIds": [] }, { "productId": "railway", "storyId": "regional-deployment", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Railway's marketing docs mention 'Run your application closer to where your users are' (railway-docs-7), implying multi-region deployment, but there is no documentation on selecting specific regions, listing available regions, or configuring region-based deployment, and no independent/hands-on corroboration of this capability. Missing for 10: detailed region list/selection docs, technical configuration guidance, independent verification of region-based deployment working as described.", "evidenceIds": [ "railway-docs-7" ] }, { "productId": "railway", "storyId": "seat-based-pricing-minimums", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items discuss Railway's pricing model, seat-based pricing, or per-seat minimums; all docs focus on features and AI agent tooling. Absence of any pricing/seat-cost evidence for an applicable axis means this cannot be verified.", "evidenceIds": [] }, { "productId": "railway", "storyId": "serverless-functions", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Railway's docs strongly evidence full-stack app deployment with no server management (connect repo, auto-config, scaling, networking, previews), fitting the 'full-stack apps' half of the story. However there is no evidence of a serverless/FaaS functions offering (e.g., function-level deploys, event-triggered execution) — Railway's model is service/container-based rather than classic serverless functions. missing for 10: evidence of serverless function deployment/FaaS primitives, independent corroboration of zero-ops scaling under load.", "evidenceIds": [ "railway-docs-1", "railway-docs-18", "railway-docs-4", "railway-docs-6", "railway-docs-12" ] }, { "productId": "railway", "storyId": "serverless-sql-query", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Railway's evidence pack shows it as a general deployment/hosting platform with infra provisioning, CLI, and observability features, but nothing describes a serverless SQL database with an HTTP/driverless query interface (no connection pooling/management) like Neon or PlanetScale offer. Railway databases are provisioned as long-running services requiring standard connection strings, so the specific 'no connection management' serverless query capability is unevidenced.", "evidenceIds": [] }, { "productId": "railway", "storyId": "site-password-protection", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions password protection, basic auth, or any simple access-gating feature for deployed sites; only general deployment, networking, and AI-agent features are documented.", "evidenceIds": [] }, { "productId": "railway", "storyId": "smart-placement", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Railway's marketing states 'Run your application closer to where your users are' (docs-7), implying region selection for latency, but there is no evidence of automatic compute placement decisions based on backend data location, no documentation of an algorithm or feature that colocates compute with data stores. missing for 10: technical detail on automatic placement logic, evidence of data-aware region selection, and any independent corroboration of latency optimization near data.", "evidenceIds": [ "railway-docs-7" ] }, { "productId": "railway", "storyId": "standards-based-runtime-portability", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Railway is a general-purpose PaaS/cloud provider for deploying containers and standard app runtimes, not an edge-compute platform (like Cloudflare Workers/Deno Deploy) with proprietary storage/runtime APIs that this story is contrasting against. Nothing in the evidence discusses edge-platform portability or standard runtime API compliance across edge providers, so this axis is a category mismatch for Railway's product type.", "evidenceIds": [] }, { "productId": "railway", "storyId": "support-response-sla", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence describes SLA-backed support response times, escalation paths, or plan-tier support guarantees; evidence only covers product features, community outage complaints, and AI tooling.", "evidenceIds": [] }, { "productId": "railway", "storyId": "team-secrets-env-vars", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Evidence only hints at environment management (unlimited environments, 'pull variables' via MCP) but never documents team roles/permissions, secrets vaults, or centralized access control for a devops-lead. Missing for 10: explicit team/role-based access control docs, shared secrets management features, and independent confirmation of these enterprise controls.", "evidenceIds": [ "railway-docs-11", "railway-docs-20" ] }, { "productId": "railway", "storyId": "unified-log-view", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Railway explicitly claims a unified log view ('All logs in one place. Spot issues without switching tools') plus supporting observability features like custom dashboards and metrics, matching the story directly. missing for 10: independent/hands-on verification of unified log UX, and detail on separating build vs deploy vs runtime log views.", "evidenceIds": [ "railway-docs-10", "railway-docs-8" ] }, { "productId": "railway", "storyId": "unlimited-team-environments", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Railway docs explicitly claim unlimited environments enabling simultaneous team shipping, backed by per-PR preview environments and instant rollback, directly matching the story. Missing for 10: independent/hands-on corroboration of environment isolation at scale and no detail on limits/costs for large teams.", "evidenceIds": [ "railway-docs-11", "railway-docs-12", "railway-docs-13" ] }, { "productId": "railway", "storyId": "visitor-performance-analytics", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Railway's docs show infrastructure-level metrics dashboards, logs, and alerting (railway-docs-8, railway-docs-9, railway-docs-10) but nothing about tracking visitor behavior or site-level analytics (page views, user sessions, traffic patterns) as a built-in feature — the evidence only covers service/CPU/RAM/log observability, not visitor analytics.", "evidenceIds": [ "railway-docs-8", "railway-docs-9", "railway-docs-10" ] }, { "productId": "railway", "storyId": "visual-infra-canvas", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Railway's docs explicitly describe a visual canvas showing the entire stack, with settings editable directly on canvas and YAML marked optional, matching the story closely. Missing for 10: independent hands-on review/screenshots demonstrating full editing depth on canvas beyond marketing copy, and no community corroboration of the canvas editing experience specifically.", "evidenceIds": [ "railway-docs-2", "railway-docs-3", "railway-docs-1" ] }, { "productId": "railway", "storyId": "zero-config-repo-deploy", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Railway's docs explicitly state that connecting a repo triggers auto-configuration with no manual setup needed, YAML being optional, and instant PR previews without extra tooling — directly matching the story. Community evidence discusses reliability/outages but does not contradict the repo-connect/auto-config workflow itself. Missing for 10: independent hands-on developer confirmation of zero-config deploys and more detail on edge cases where manual config is still required.", "evidenceIds": [ "railway-docs-1", "railway-docs-3", "railway-docs-12", "railway-docs-18" ] }, { "productId": "render", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Render hosts a live, working llms.txt (HTTP 200) and Markdown-formatted docs (docs.md) confirmed via direct probes, plus per-page .md docs (e.g. mcp-server.md, cli.md) an agent could fetch directly. Missing for 10: no independent/community corroboration that agents are actually being pointed at these files successfully.", "evidenceIds": [ "render-probe-1", "render-probe-2", "render-docs-24", "render-docs-27" ] }, { "productId": "render", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Render doesn't offer a native in-product AI insights/analytics feature, but it does support agent-driven suggestions via its MCP server and Claude/Cursor 'skills' — e.g. an agent can inspect CPU/memory metrics and recommend a compute plan change (render-docs-33), and 20+ official skills give agents guidance for monitoring/scaling (render-docs-34). This is mediated through external coding agents rather than a built-in insights engine. Missing for 10: a first-party AI insights/analytics dashboard inside Render's own UI, independent evidence of these agent recommendations being used in practice.", "evidenceIds": [ "render-docs-33", "render-docs-34", "render-docs-24", "render-docs-13" ] }, { "productId": "render", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Render explicitly supports deploying background workers/agents and processes 'at scale, without wiring up queues, workers, and retry logic,' and its Workflows SDKs (TypeScript/Python) let users define and trigger long-running background tasks autonomously. This directly matches the story of setting up autonomous background automations on the platform. Missing for 10: explicit documentation of scheduled/cron-based triggers and independent hands-on validation of long-running autonomous jobs.", "evidenceIds": [ "render-docs-5", "render-docs-22", "render-docs-23" ] }, { "productId": "render", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Render's evidence shows integration with external AI coding agents (Claude Code, Cursor, Codex) via an MCP server and official 'skills', plus workflow SDKs — but this is external agents connecting to Render, not a built-in AI assistant embedded inside Render's own product for users to delegate tasks to. No evidence of a native in-product assistant/chatbot.", "evidenceIds": [ "render-docs-24", "render-docs-25", "render-docs-34", "render-docs-16" ] }, { "productId": "render", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Render provides an official CLI for scripting/managing deploys and datastores from a terminal, a documented REST API and OpenAPI spec, and blueprint YAML (render.yaml) with CLI validation for defining/deploying architecture as code—together these enable headless automation and CI-driven workflows. Missing for 10: explicit CI/CD pipeline examples (e.g., GitHub Actions integration) and independent hands-on confirmation of CI usage.", "evidenceIds": [ "render-docs-14", "render-docs-15", "render-docs-27", "render-docs-28", "render-docs-29", "render-probe-3", "render-probe-4" ] }, { "productId": "render", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Render is a cloud application/deployment platform, not an AI agent that itself consumes tools; the evidence shows Render *exposes* its own MCP server for external AI apps (Claude, Cursor, Codex) to manage Render infrastructure, which is the opposite role from 'plugging in' third-party MCP servers to gain their tools. This client-side capability is not a natural axis for a PaaS platform, so the story does not apply.", "evidenceIds": [ "render-docs-24", "render-docs-25", "render-docs-26" ] }, { "productId": "render", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Render ships an official MCP server enabling AI apps/agents like Claude Code, Claude Desktop, Codex CLI/Desktop, and Cursor to manage Render infrastructure, with documented tools (e.g., trigger_deploy) and OAuth-based authentication support. Since Render is a cloud platform (not itself an agent), this axis clearly applies and is well-documented with first-party docs and changelog evidence. Missing for 10: independent/hands-on third-party corroboration of the MCP server working in practice.", "evidenceIds": [ "render-docs-24", "render-docs-25", "render-docs-26" ] }, { "productId": "render", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Render ships an MCP server enabling natural-language control from AI apps (Claude Code, Cursor, Codex) with tools like trigger_deploy, plus 20+ official skills for coding agents, indicating agentic/NL operation is supported. However, this operates through third-party AI clients rather than a native NL interface within Render itself, and there's no independent hands-on evidence of reliability. missing for 10: first-party natural-language interface within Render's own UI/CLI, independent/community validation of MCP-driven workflows, evidence of broader command coverage beyond deploy/inspect/scale.", "evidenceIds": [ "render-docs-24", "render-docs-25", "render-docs-26", "render-docs-32", "render-docs-33", "render-docs-34" ] }, { "productId": "render", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Render documents an official CLI for managing services, datastores, and blueprints directly from the terminal, with validation for render.yaml files, corroborated by a dedicated docs page and probe confirmation. missing for 10: independent/hands-on community verification of the CLI itself (community evidence only covers general platform sentiment, not CLI usage) and deeper detail on AI-native-specific CLI workflows beyond blueprint validation.", "evidenceIds": [ "render-docs-14", "render-docs-27", "render-docs-28", "render-probe-4" ] }, { "productId": "render", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Render publishes a documented REST API with a live OpenAPI spec (render-docs-15, render-probe-3), plus a CLI (render-docs-14/27) and SDKs for TypeScript/Python (render-docs-22/23), giving AI-native users multiple documented programmatic entry points. Missing for 10: independent third-party corroboration of API robustness/completeness beyond Render's own docs.", "evidenceIds": [ "render-docs-15", "render-probe-3", "render-docs-14", "render-docs-27", "render-docs-22", "render-docs-23" ] }, { "productId": "render", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Render documents OAuth-based authentication for its MCP server (used by agent tools like Claude Code/Codex/Cursor) and managed OIDC that issues short-lived, auto-rotated identity tokens instead of long-lived API keys, both of which support least-privilege, ephemeral credentialing for agent-driven workflows. However, there is no explicit documentation of fine-grained, scoped API-key permissions (e.g., read-only vs. deploy-only tokens) specifically for AI agents beyond OAuth/OIDC. Missing for 10: explicit scoped/permissioned API key creation UI or docs, granular role-based token scopes, and independent verification of least-privilege enforcement in practice.", "evidenceIds": [ "render-docs-24", "render-docs-26", "render-docs-30" ] }, { "productId": "render", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Render ships official SDKs for TypeScript and Python workflow tasks (render-docs-22, render-docs-23), plus a REST API with a documented OpenAPI spec (render-docs-15, render-probe-3) and an official CLI (render-docs-14, render-probe-4), all of which AI-native developers can build against programmatically. Missing for 10: independent/hands-on developer corroboration of SDK usage quality and broader language SDK coverage beyond TypeScript/Python.", "evidenceIds": [ "render-docs-22", "render-docs-23", "render-docs-15", "render-probe-3", "render-docs-14", "render-probe-4" ] }, { "productId": "render", "storyId": "agentic-webhooks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Render's docs explicitly describe workspace webhooks that notify external systems on service events like deploys or scaling, matching the story of subscribing to events via webhooks. Missing for 10: no independent/hands-on corroboration of webhook reliability or delivery guarantees, and no detail on payload/signature verification depth.", "evidenceIds": [ "render-docs-31" ] }, { "productId": "render", "storyId": "ai-global-inference", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Render is a general cloud application/hosting platform for deploying web services, databases, and background workers; the evidence contains no mention of AI model inference, GPU-backed inference endpoints, or edge/global model serving. This story targets an AI inference/edge-compute platform category, which is a different axis from Render's general PaaS hosting offering — a category mismatch rather than a missing feature.", "evidenceIds": [] }, { "productId": "render", "storyId": "ai-model-gateway", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Render is a cloud application/deployment platform (PaaS), not an AI model gateway; there is no evidence of a unified multi-model inference gateway product. This story is a category error for Render's product type — it's about hosting/deploying apps, not aggregating AI model APIs.", "evidenceIds": [] }, { "productId": "render", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence confirms Render has a REST API and a machine-readable OpenAPI spec (render-docs-15, render-probe-3), but there is no mention of an interactive API reference UI or runnable/try-it-now examples anywhere in the docs or community evidence.", "evidenceIds": [ "render-docs-15", "render-probe-3" ] }, { "productId": "render", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Render exposes a machine-readable OpenAPI spec at a public, well-known endpoint confirmed to contain the 'openapi' key, alongside a documented REST API in first-party docs. Missing for 10: no independent/community corroboration of the OpenAPI file's completeness or version stability.", "evidenceIds": [ "render-probe-3", "render-docs-15" ] }, { "productId": "render", "storyId": "api-sandbox", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Render explicitly ships ephemeral preview environments for full application architectures per change, documents a mechanism to prevent non-production environments from accessing production services, and supports spinning up short-lived instances to inspect runtime or run one-off commands without touching production. Together these give AI-native users a documented, isolated sandbox path distinct from production data. Missing for 10: independent/hands-on verification that preview environments fully isolate data, and explicit guidance on seeding sandbox data for AI agent testing.", "evidenceIds": [ "render-docs-8", "render-docs-18", "render-docs-32" ] }, { "productId": "render", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "There is evidence of a REST API and an OpenAPI spec, but nothing in the pack documents API versioning scheme or a deprecation policy for that API. missing for 10: documented API version numbers/paths, a published deprecation/sunset policy, changelog commitments around breaking changes.", "evidenceIds": [] }, { "productId": "render", "storyId": "automatic-tls-certs", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Render's own docs explicitly state free TLS certificates for every domain including wildcards (render-docs-20), directly matching the story, and custom domains are also documented (render-docs-6). Missing for 10: independent/hands-on corroboration of wildcard cert auto-renewal behavior beyond the vendor claim.", "evidenceIds": [ "render-docs-20", "render-docs-6" ] }, { "productId": "render", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Render's Blueprints (render.yaml) let you define, deploy, and version an entire multi-service architecture from a single file (render-docs-9, render-docs-29), and the REST API/CLI provide programmatic control (render-docs-14, render-docs-15, render-docs-27) that could be scripted for bulk actions. However, there is no explicit documentation of a bulk/batch operation feature (e.g., bulk delete, bulk env-var update, batch service management) beyond blueprint-based multi-service deploys. Missing for 10: explicit bulk/batch API or CLI commands acting across many items, independent confirmation of bulk workflows in practice.", "evidenceIds": [ "render-docs-9", "render-docs-29", "render-docs-14", "render-docs-15", "render-docs-27" ] }, { "productId": "render", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Render supports several event-driven automations — auto-deploy on push (render-docs-1), load-based autoscaling (render-docs-11), and configurable webhooks that notify external systems on service events like deploy start or scale-down (render-docs-31) — which together approximate rule-based triggers. However, there's no first-class 'rules engine' UI for defining custom conditional automations beyond these built-in triggers and webhook notifications. Missing for 10: a dedicated rules/automation builder, support for custom conditional logic (not just fixed event types), and independent evidence of users building automation workflows on these triggers.", "evidenceIds": [ "render-docs-1", "render-docs-11", "render-docs-31" ] }, { "productId": "render", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Render's Workflows SDK for defining and triggering workflow tasks and background/agent processes, but nothing explicitly describes scheduling or recurring/cron-based job execution. Missing for 10: explicit cron/scheduled job feature, recurring-trigger documentation, or interval-based automation evidence.", "evidenceIds": [] }, { "productId": "render", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Render supports git-based deploys with rollbacks (render-docs-1, render-docs-12), infrastructure-as-code versioning via render.yaml blueprints (render-docs-9, render-docs-29), and ephemeral preview environments for reviewing changes before merge (render-docs-8), plus a Workflows SDK for defining/triggering automation tasks (render-docs-22, render-docs-23). However these features target service/infra deploys rather than a dedicated 'automation' review/approval pipeline, and there's no evidence of versioning or rollback specifically for Workflow SDK tasks or agent-driven automations. Missing for 10: explicit version history/rollback UI for Workflows SDK tasks, an approval/review workflow for automations distinct from git PR review, and independent confirmation of rollback reliability.", "evidenceIds": [ "render-docs-1", "render-docs-9", "render-docs-12", "render-docs-29", "render-docs-8", "render-docs-22", "render-docs-23" ] }, { "productId": "render", "storyId": "cli-deploy", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Render documents an official CLI to manage services and datastores directly from the terminal, including validating and deploying via render.yaml blueprints, which covers CLI-based deployment without the dashboard. Missing for 10: independent hands-on confirmation of full deploy parity with the dashboard and detailed CLI deploy command examples in the evidence pack.", "evidenceIds": [ "render-docs-14", "render-docs-27", "render-docs-28", "render-probe-4" ] }, { "productId": "render", "storyId": "cli-scaffold-project", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Render provides a documented CLI ([render-docs-14],[render-docs-27]) and blueprint YAML for defining/deploying full architectures ([render-docs-9],[render-docs-29]) plus render.yaml validation via CLI ([render-docs-28]), suggesting a scaffold-and-deploy workflow is possible, but no evidence shows a single CLI command that scaffolds a new project from scratch (e.g., a 'create' or 'init' command generating boilerplate) — CLI docs focus on managing existing services/datastores rather than project scaffolding. missing for 10: explicit CLI scaffold/init command for new projects, end-to-end single-command example, independent hands-on confirmation of scaffold+deploy in one step.", "evidenceIds": [ "render-docs-14", "render-docs-27", "render-docs-9", "render-docs-29", "render-docs-28", "render-probe-4" ] }, { "productId": "render", "storyId": "condition-based-alerts", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Render supports webhooks to notify other systems on service events (deploy starting, scaling, etc.), which could be routed to chat/email via external integrations, and offers metrics/telemetry streaming to external tools. However, there's no evidence of native alert configuration with custom thresholds/conditions, nor built-in email/chat (Slack) notification channels for alerting. missing for 10: native alerting rules/conditions UI, built-in email/Slack notification channels, documentation of alert-specific configuration beyond generic webhooks and metrics streaming.", "evidenceIds": [ "render-docs-31", "render-docs-13" ] }, { "productId": "render", "storyId": "consistent-api-at-scale", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Render offers consistent primitives (Blueprint YAML, autoscaling to 100x bursts, managed databases, CLI/API) that in principle span prototype to scale without platform migration, and community comments praise its Docker/IaC simplicity versus Heroku. However, evidence lacks concrete case studies of a single app scaling from prototype to massive production scale on unchanged APIs, and community notes cite significant outages, which raises reliability concerns at scale. missing for 10: a documented large-scale production case study showing zero re-architecture, independent verification of scaling claims beyond marketing copy, and resolution of the outage concerns raised by users.", "evidenceIds": [ "render-docs-9", "render-docs-11", "render-docs-29", "render-comm-1", "render-comm-2", "render-comm-3" ] }, { "productId": "render", "storyId": "custom-domains", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Render's docs explicitly list \"Custom domains\" as a feature and also mention free TLS certificates for every domain including wildcards, directly supporting attaching custom domains to services. missing for 10: no independent/hands-on corroboration or detailed walkthrough of the domain-attachment process beyond the doc title reference.", "evidenceIds": [ "render-docs-6", "render-docs-20" ] }, { "productId": "render", "storyId": "custom-metrics-dashboards", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Render exposes infrastructure metrics and can stream telemetry to external tools (render-docs-13), which implies dashboard-building would need to happen in a third-party system rather than natively combining metrics from 'any service' within Render itself. There's no evidence of a built-in custom dashboard builder or multi-service metric aggregation UI. Missing for 10: native custom dashboard UI, ability to pull metrics from non-Render services, evidence of cross-service dashboard composition, independent confirmation of telemetry export working in practice.", "evidenceIds": [ "render-docs-13" ] }, { "productId": "render", "storyId": "default-ddos-waf", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of DDoS protection or a web application firewall (WAF) applied by default or otherwise; Render's docs reference TLS, CDN caching, private networking, and OIDC, but nothing about WAF/DDoS mitigation.", "evidenceIds": [] }, { "productId": "render", "storyId": "docker-deploy", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Render explicitly documents Docker support as a first-class deployment method, corroborated by independent community reports of 'great docker support.' Missing for 10: hands-on walkthrough of a full Docker deploy workflow and more detailed docs on Dockerfile-based build configuration.", "evidenceIds": [ "render-docs-2", "render-comm-1" ] }, { "productId": "render", "storyId": "documented-rate-limits", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack items document specific rate limits or concurrency caps for Render's APIs, functions, or REST API/MCP server; autoscaling is mentioned generically but no numeric limits or quotas are given. Missing for 10: documented API rate limits, function concurrency caps, or any quantified throttling/quota policy.", "evidenceIds": [] }, { "productId": "render", "storyId": "durable-workflows", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Render explicitly advertises deploying 'reliable agents and background processes at scale, without wiring up queues, workers, and retry logic,' and ships Workflows SDKs for both TypeScript and Python that support defining workflow tasks and triggering runs, directly matching the durable-workflow-as-code story. Missing for 10: independent/hands-on validation of durability guarantees under failure, and deeper documentation of retry/backoff semantics beyond marketing copy.", "evidenceIds": [ "render-docs-5", "render-docs-22", "render-docs-23" ] }, { "productId": "render", "storyId": "edge-websocket-support", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Render's evidence pack describes persistent web services, background workers, docker deployments, autoscaling, etc., but contains no mention of edge functions or serverless functions at all, nor any WebSocket-specific capability tied to such a compute model. Without evidence of an edge/serverless function offering supporting persistent WebSockets, this applicable axis is unmet.", "evidenceIds": [] }, { "productId": "render", "storyId": "execution-time-limits", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack documents any CPU or wall-clock execution time limits for Render services or functions—there's no mention of request timeouts, background job duration limits, or serverless-style execution caps. Render is positioned as long-running containers/services rather than time-boxed functions, but nothing confirms or denies specific limits a devops-lead could plan around.", "evidenceIds": [] }, { "productId": "render", "storyId": "external-service-bindings", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Render auto-provisions managed Postgres/Redis and injects env vars/connection strings between services, and offers OIDC-based short-lived credential auth to select external providers (Anthropic, OpenAI) instead of long-lived API keys, reducing manual credential wiring. However, evidence doesn't show broad native SDK bindings for arbitrary third-party APIs/storage beyond Render's own managed services and a couple of AI providers. Missing for 10: evidence of a wide addon/integration marketplace or generic client-library bindings for common external SaaS APIs, independent verification of reduced boilerplate claims.", "evidenceIds": [ "render-docs-3", "render-docs-4", "render-docs-10", "render-docs-30", "render-docs-7" ] }, { "productId": "render", "storyId": "fast-boot-scaling", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Render documents load-based autoscaling handling '100x traffic bursts and beyond' (render-docs-11), which addresses the scaling-to-large-load portion of the story, but there is no evidence of sub-second boot times or edge-compute cold-start performance anywhere in the pack. missing for 10: sub-second instance boot/cold-start benchmarks, edge-compute deployment model, evidence of tens-of-thousands-instance scale in practice.", "evidenceIds": [ "render-docs-11", "render-docs-5" ] }, { "productId": "render", "storyId": "framework-tool-continuity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Render supports connecting existing Git repos, Docker images, and standard CI-triggered deploys (render-docs-1, render-docs-2), plus a CLI and REST API for scripting into existing workflows (render-docs-14, render-docs-15, render-probe-4). However, it also pushes proprietary IaC (render.yaml blueprints) as the primary way to define infrastructure (render-docs-9), which is an additional tool developers must adopt rather than pure compatibility with existing Terraform/Pulumi-style workflows; community notes confirm this is a distinct 'infra-as-code' approach rather than a no-lock-in passthrough (render-comm-2). No specific evidence about editor integrations or framework-agnostic guarantees beyond generic build support. Missing for 10: evidence on editor/IDE integration, confirmation that framework choices are unconstrained, and independent confirmation that CI systems (e.g., GitHub Actions, CircleCI) integrate natively without Render-specific config.", "evidenceIds": [ "render-docs-1", "render-docs-2", "render-docs-9", "render-docs-14", "render-docs-15", "render-comm-2", "render-probe-4" ] }, { "productId": "render", "storyId": "full-nodejs-runtime-support", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Render's evidence describes general Docker-based web services, background workers, and autoscaling, but there is no mention of an edge/serverless functions product or any claim about running unmodified Node.js/npm code where competitors impose a restricted runtime subset. The axis is fair to ask of a cloud platform, but no evidence supports it.", "evidenceIds": [] }, { "productId": "render", "storyId": "full-stack-preview-envs", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Render explicitly advertises 'ephemeral previews of your entire application architecture for every change' tied to blueprint/YAML-defined multi-service architectures, directly matching PR preview environments. However, evidence lacks detail on automatic PR-linking mechanics, teardown behavior, or independent/hands-on confirmation of the feature working end-to-end. missing for 10: hands-on/independent verification of PR-triggered preview creation and teardown, details on preview environment lifecycle and limitations, confirmation of full-stack (DB+services) parity in previews.", "evidenceIds": [ "render-docs-8", "render-docs-9", "render-docs-29" ] }, { "productId": "render", "storyId": "git-push-deploy", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Render's docs explicitly state 'Connect your repo and ship with every push' and support rollbacks, environment variables, and Blueprint YAML deploys, and community feedback corroborates ease of git-based deployment vs. competitors like Heroku. missing for 10: no hands-on trace of the auto-deploy pipeline (e.g. build logs, webhook triggers) beyond the marketing/docs claim, and no independent verification of push-to-deploy latency or reliability.", "evidenceIds": [ "render-docs-1", "render-docs-12", "render-docs-9", "render-comm-1", "render-comm-2" ] }, { "productId": "render", "storyId": "global-edge-network", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Render offers a global CDN only for caching static assets (render-docs-19), not for running arbitrary application code at edge locations; services are deployed to specific regions rather than a distributed edge-compute network. No evidence pack items describe edge functions, edge workers, or code execution distributed globally near end users.", "evidenceIds": [ "render-docs-19" ] }, { "productId": "render", "storyId": "global-static-cdn", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Render explicitly documents serving static assets faster via caching powered by its global CDN, directly matching the story, plus supporting free TLS/custom domains for static sites and community corroboration of free static site hosting. Missing for 10: independent benchmarks of CDN performance and details on edge cache configuration/invalidation controls.", "evidenceIds": [ "render-docs-19", "render-docs-20", "render-comm-3" ] }, { "productId": "render", "storyId": "hard-spending-limits", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers deployment, IaC, datastores, CLI/MCP, and pricing-page usability feedback, but contains no mention of hard spending limits, budget caps, or cost-control alerts on Render. This is a fair axis for a cloud platform, but no evidence supports it being delivered.", "evidenceIds": [] }, { "productId": "render", "storyId": "idp-secured-access", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Render can issue short-lived OIDC tokens for its own services to authenticate outbound to Anthropic/OpenAI (render-docs-30), but nothing documents integrating an organization's own identity provider (SSO/SAML/OIDC) to authenticate users or gate access to internal apps, deployments, or agents on Render. No mention of SSO, SAML, Okta/Azure AD integration, or team-level IdP-based access control appears anywhere in the pack.", "evidenceIds": [ "render-docs-30" ] }, { "productId": "render", "storyId": "in-browser-dev-toolbar", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "This story describes an in-browser toolbar for feedback/feature-flags/performance inspection on a live site — a product-analytics/observability-overlay feature, not something a cloud hosting/deployment platform like Render provides. Render's evidence covers deployment, infra, databases, CLI/MCP tooling, none of which relate to an in-app developer toolbar.", "evidenceIds": [] }, { "productId": "render", "storyId": "incremental-rollout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers deploys, rollbacks, autoscaling, and previews, but nothing describes staged/canary/incremental rollout mechanisms (e.g., percentage-based traffic shifting, blue-green, or gradual rollout to limit blast radius) — only full rollbacks after the fact are mentioned.", "evidenceIds": [ "render-docs-12", "render-docs-11" ] }, { "productId": "render", "storyId": "infra-as-code-blueprint", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Render explicitly documents Blueprints via a single render.yaml file to define, deploy, and version an entire multi-service architecture, with CLI validation support and community corroboration of its docker-compose-like IaC approach. missing for 10: independent hands-on validation of complex multi-service blueprint deployments and versioning workflows at scale, and more detail on drift detection/rollback of the YAML config itself.", "evidenceIds": [ "render-docs-9", "render-docs-29", "render-docs-28", "render-comm-2" ] }, { "productId": "render", "storyId": "instant-folder-deploy", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence emphasizes Git-connected deploys ('Connect your repo and ship with every push') and Docker/CLI/YAML workflows, but nothing in the pack describes uploading a folder of files without a Git repo or a private-until-published deployment flow. missing for 10: any mention of folder/file upload deploy method, no-Git deploy path, or private-draft-then-publish deployment state.", "evidenceIds": [ "render-docs-1" ] }, { "productId": "render", "storyId": "invisible-bot-protection", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions bot protection, CAPTCHA-alternative challenges, or any invisible bot-mitigation feature; Render's documented security features cover TLS, OIDC, and network isolation but not bot traffic challenges.", "evidenceIds": [] }, { "productId": "render", "storyId": "isolated-sandbox-execution", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Render is a PaaS for deploying web services/databases, not an edge-compute sandboxing platform for running untrusted, hardware-isolated disposable code (e.g., WASM/microVM per-request sandboxes). This is a category mismatch, not a missing feature.", "evidenceIds": [] }, { "productId": "render", "storyId": "known-overage-pricing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item documents an overage fee schedule for bandwidth, invocations, or build minutes; community feedback even notes Render's pricing page requires clicking into each component and isn't fully transparent (render-comm-5). The axis applies to a usage-billed cloud platform, but no supporting documentation exists.", "evidenceIds": [ "render-comm-5" ] }, { "productId": "render", "storyId": "load-based-autoscaling", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Render's first-party docs explicitly claim 'load-based autoscaling that handles 100x traffic bursts and beyond,' directly matching the story, and community feedback corroborates autoscaling as a real, working feature (though not stress-tested by 100x claims). However, there's no independent evidence or hands-on benchmark validating the 100x burst claim, and community feedback notes past outages, raising some doubt about reliability under extreme load. Missing for 10: independent load-testing evidence, detailed autoscaling configuration/limits documentation, and confirmation the outages weren't autoscaling-related failures.", "evidenceIds": [ "render-docs-11", "render-comm-1", "render-comm-3" ] }, { "productId": "render", "storyId": "local-dev-simulator", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Render's evidence covers cloud deploys, ephemeral previews, and Docker support, but there is no mention of a local simulator or emulator that replicates the production runtime for local development/testing. Missing for 10: a documented local dev/simulator tool, parity guarantees between local and production runtime, and any independent confirmation of local testing workflows.", "evidenceIds": [] }, { "productId": "render", "storyId": "managed-postgres", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Render's official docs directly state fully-managed Postgres with point-in-time recovery, read replicas, and high availability, plus general datastore recovery features and disk snapshotting for other stores, matching the story closely. Missing for 10: independent/hands-on corroboration of PITR and replica behavior in production, and more detail on HA failover mechanics.", "evidenceIds": [ "render-docs-3", "render-docs-21" ] }, { "productId": "render", "storyId": "migration-credits", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Render explicitly advertises up to $10K in migration credits for moving production infrastructure to their platform, directly matching the story. Missing for 10: details on eligibility criteria/process, independent confirmation from community reports of actually receiving credits.", "evidenceIds": [ "render-docs-17" ] }, { "productId": "render", "storyId": "multi-protocol-native-support", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions Render web services supporting multiple protocols (HTTP, TCP, gRPC, WebSocket) from the same service without extra configuration; docs reference general web/private services, custom domains, TLS, and CDN but never protocol-agnostic multi-protocol serving. Missing for 10: any mention of gRPC support, TCP service type, WebSocket handling, or a single service exposing multiple protocols simultaneously.", "evidenceIds": [] }, { "productId": "render", "storyId": "object-storage-uploads", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Render offering managed databases, in-memory caches, and disk-backed private services (e.g., ClickHouse with disk snapshots), but nowhere mentions a built-in object storage product (S3-like) for arbitrary user uploads. No documentation, blog, or changelog item describes an object storage bucket/service.", "evidenceIds": [ "render-docs-3", "render-docs-4", "render-docs-21", "render-docs-35" ] }, { "productId": "render", "storyId": "on-demand-image-optimization", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Render's evidence covers static asset CDN caching, TLS, deploys, databases, and other infra features, but there is no mention of an image transformation/optimization/resizing CDN capability.", "evidenceIds": [] }, { "productId": "render", "storyId": "one-click-rollback", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Render's docs explicitly list 'Rollbacks' as a feature, confirming the capability exists, but the evidence pack gives only a title reference with no detail on the one-click UX, scope (any previous version vs. just last), or independent/hands-on confirmation. Missing for 10: detailed docs on rollback flow/UI, confirmation of 'any previous version' vs limited history, and community/hands-on verification that rollback works as described.", "evidenceIds": [ "render-docs-12" ] }, { "productId": "render", "storyId": "one-click-templates", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers Blueprints (render.yaml) for defining architecture, but there is no mention of a one-click template gallery or pre-built starters for common use cases (e.g., a 'Deploy to Render' button library or template marketplace).", "evidenceIds": [] }, { "productId": "render", "storyId": "openness-api-parity", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Render documents a full REST API with OpenAPI spec, a CLI, and an MCP server that exposes actions like triggering deploys and managing services/datastores, indicating broad API-level control mirroring UI capabilities. However, there is no explicit vendor claim or independent verification of 100% feature parity between UI and API/CLI, so some UI-only edge cases may remain unconfirmed. Missing for 10: explicit parity statement covering every UI feature, and independent/hands-on confirmation that no UI-only actions exist.", "evidenceIds": [ "render-docs-15", "render-probe-3", "render-docs-14", "render-probe-4", "render-docs-24", "render-docs-25", "render-docs-27" ] }, { "productId": "render", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Render supports YAML-based infra-as-code (render.yaml) and a REST API/CLI, which aids portability, but there is no documented feature for exporting full database/service data in open formats or any explicit 'export and leave' capability. missing for 10: documented data export/backup-download tooling, explicit open-format data portability guarantees, and any mention of migrating data out of Render.", "evidenceIds": [ "render-docs-3", "render-docs-9", "render-docs-27", "render-docs-29" ] }, { "productId": "render", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Render is a closed-source commercial PaaS; no evidence indicates its source code is published under an open license (docs, CLI, SDKs are documented but not shown to be open-source). missing for 10: any evidence of a public open-source repository or license for Render's core platform source.", "evidenceIds": [] }, { "productId": "render", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Render is a proprietary cloud hosting platform with no evidence of an open-source or self-hostable version of its core service; all evidence describes using Render's own managed cloud, not deploying Render itself on one's own infrastructure.", "evidenceIds": [] }, { "productId": "render", "storyId": "pay-for-active-compute", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation of Render's billing granularity (e.g., per-second billing, scale-to-zero for paid services, or idle-time exclusion). Autoscaling (render-docs-11) addresses traffic handling, not billing model, and community comments only discuss pricing clarity, not compute-time-based billing.", "evidenceIds": [] }, { "productId": "render", "storyId": "pay-per-byte-storage", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack only mentions disks in passing (automatic daily snapshots for ClickHouse) but contains no documentation of auto-growing disk capacity or usage-based billing per byte written; Render's disk docs elsewhere describe fixed pre-provisioned sizes, but even setting that aside, this pack has no evidence supporting the story's specific claim.", "evidenceIds": [ "render-docs-35" ] }, { "productId": "render", "storyId": "persistent-agent-runtime", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Render offers persistent disks, private services, and coding-agent integrations (MCP server, CLI, skills) but no evidence describes a persistent runtime environment for an agent that automatically checkpoints its state — the closest hits are disk snapshots for a database service (render-docs-35) and short-lived inspection instances (render-docs-32), neither of which is agent-state checkpointing.", "evidenceIds": [ "render-docs-32", "render-docs-35", "render-docs-16" ] }, { "productId": "render", "storyId": "polyglot-edge-runtimes", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Render is a general-purpose cloud application platform (regional web services, background workers, databases) rather than an edge-compute/edge-functions platform, and nothing in the evidence pack mentions edge functions or an edge network at all. The 'write edge functions in other languages' story is a category mismatch for this product type, not a gap in an applicable capability.", "evidenceIds": [] }, { "productId": "render", "storyId": "pr-preview-urls", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Render docs mention 'ephemeral previews of your entire application architecture for every change' (render-docs-8), which implies PR preview environments, but there is no explicit mention of automatic PR-linked preview URLs, GitHub PR integration, or hands-on confirmation of this workflow. missing for 10: explicit PR-preview-URL documentation, GitHub/GitLab PR status-check integration details, independent/hands-on confirmation of preview links appearing on pull requests.", "evidenceIds": [ "render-docs-8" ] }, { "productId": "render", "storyId": "preview-collaboration-comments", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence describes any commenting or annotation feature on preview/production environments for teammates; Render's evidence covers deploys, previews, CLI, MCP, and infra features but nothing about collaborative commenting on environments.", "evidenceIds": [] }, { "productId": "render", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items mention region selection, data residency, or geographic deployment location options for Render services or data storage. missing for 10: region selection docs, data residency policy, multi-region deployment options.", "evidenceIds": [] }, { "productId": "render", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no privacy policy, data-usage terms, or statements about whether customer code/data is used to train AI models, despite Render offering AI-agent integrations (MCP server, skills, coding agents) where such a concern would be relevant. Since this is a plausible question for a platform with AI features, but no supporting evidence exists, the verdict is none rather than na.", "evidenceIds": [] }, { "productId": "render", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of data retention policies, deletion controls, or privacy/compliance documentation specific to AI usage or data lifecycle management; evidence covers deployment, MCP, CLI, and infra features but nothing on retention/deletion controls.", "evidenceIds": [] }, { "productId": "render", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses telemetry opt-out or usage tracking controls for Render's own product; this is a fair question for a cloud platform but no documentation or community evidence confirms such a setting exists.", "evidenceIds": [] }, { "productId": "render", "storyId": "private-service-networking", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Render's docs explicitly claim private networking that keeps internal service-to-service traffic off the public internet without VPC complexity, matching the story almost verbatim, and related isolation features (blocking non-prod access to prod services) reinforce this. Missing for 10: independent/hands-on verification of private networking behavior and technical detail on how internal DNS/routing works.", "evidenceIds": [ "render-docs-10", "render-docs-18" ] }, { "productId": "render", "storyId": "prod-nonprod-isolation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Render explicitly advertises the capability ('Prevent non-production environments from accessing production services') and separately offers private networking that keeps internal traffic off the public internet, suggesting environment isolation is achievable via private services/network policies. However, there is no detailed documentation of the mechanism (e.g., specific network policies, per-environment isolation controls) or independent/hands-on verification of this feature working as claimed. missing for 10: detailed first-party docs on how isolation is configured/enforced, independent or hands-on confirmation, edge-case handling (e.g., preview environments).", "evidenceIds": [ "render-docs-18", "render-docs-10" ] }, { "productId": "render", "storyId": "programmatic-multitenant-domains", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Render documents custom domains with free TLS (including wildcards) and a general REST API/CLI for managing services, which could in principle be used to provision domains programmatically, but no evidence explicitly confirms a domain-provisioning API endpoint or a documented workflow for multi-tenant SaaS customer domain routing. Missing for 10: explicit API/CLI examples for adding/removing custom domains programmatically, and any mention of multi-tenant domain routing patterns or SaaS-specific guidance.", "evidenceIds": [ "render-docs-6", "render-docs-20", "render-docs-15", "render-docs-14" ] }, { "productId": "render", "storyId": "public-incident-history", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of a public status page or historical incident postmortems for Render; community comments reference past outages but do not reference a status page or postmortem records. This axis applies to any cloud platform a devops-lead would evaluate, but no supporting evidence exists.", "evidenceIds": [] }, { "productId": "render", "storyId": "published-uptime-sla", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item references a published SLA document or downtime credit policy; only general outage anecdotes exist, not a formal SLA with credit terms.", "evidenceIds": [] }, { "productId": "render", "storyId": "redis-compatible-cache", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Render docs explicitly advertise a 'Low-latency in-memory storage ideal for shared caches and job queues' managed datastore offering (Render Key Value, Redis-compatible), alongside general managed database docs with HA and recovery features. Missing for 10: explicit confirmation of Redis protocol compatibility details, independent hands-on benchmarks, and specifics on persistence/eviction configuration.", "evidenceIds": [ "render-docs-4", "render-docs-21", "render-docs-3" ] }, { "productId": "render", "storyId": "regional-deployment", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers many Render features (CDN, TLS, autoscaling, CLI, MCP, blueprints) but never mentions region selection, data-center choice, or geographic deployment options for services. Region-based deployment is a plausible and common capability for a cloud application platform, so the axis applies, but no evidence supports it here.", "evidenceIds": [] }, { "productId": "render", "storyId": "seat-based-pricing-minimums", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items discuss Render's pricing model, seat-based pricing, or minimum seat counts; community comments only mention pricing being 'easier to understand' or 'slightly better' without detail. Missing for 10: any documentation of per-seat pricing structure, minimum seat requirements, or team-based cost scaling.", "evidenceIds": [] }, { "productId": "render", "storyId": "serverless-functions", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Render clearly delivers full-stack app deployment with no server management (git-push deploys, Docker support, managed databases, autoscaling, YAML-based architecture, global CDN/TLS) per render-docs-1,8,9,11,19,20. However, there is no evidence of a dedicated 'serverless functions' product (e.g., FaaS-style short-lived function execution) distinct from its web services/workers model. Missing for 10: explicit serverless functions offering, evidence of per-invocation/scale-to-zero function execution, independent hands-on confirmation of serverless function deploys.", "evidenceIds": [ "render-docs-1", "render-docs-8", "render-docs-9", "render-docs-11", "render-docs-19", "render-docs-20", "render-comm-1", "render-comm-2" ] }, { "productId": "render", "storyId": "serverless-sql-query", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Render's evidence only shows traditional fully-managed Postgres/datastores with point-in-time recovery and replicas (render-docs-3, render-docs-21), which require standard connection management — there is no evidence of a serverless SQL query interface (e.g., HTTP-based driver) that eliminates connection handling.", "evidenceIds": [ "render-docs-3", "render-docs-21" ] }, { "productId": "render", "storyId": "site-password-protection", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions basic auth, password protection, or similar site-access-gating features for deployed services; only unrelated deployment, database, CLI, and MCP capabilities are documented.", "evidenceIds": [] }, { "productId": "render", "storyId": "smart-placement", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence describes automatic compute placement based on proximity to backend data stores or latency-aware scheduling; Render's docs mention region selection and private networking (render-docs-10) but nothing about automatic co-location of compute with data for latency optimization.", "evidenceIds": [] }, { "productId": "render", "storyId": "standards-based-runtime-portability", "verdict": "na", "quality": 0, "confidence": "medium", "rationale": "Render is a general-purpose cloud application/container platform, not an edge computing runtime with proprietary storage/runtime APIs; the story's premise (avoiding lock-in to edge-specific runtime APIs) is a category mismatch for Render's product type.", "evidenceIds": [] }, { "productId": "render", "storyId": "support-response-sla", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions SLA-backed support response times, escalation tiers, or plan-specific support guarantees; only general product docs, community sentiment, and outage anecdotes are present.", "evidenceIds": [] }, { "productId": "render", "storyId": "team-secrets-env-vars", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Render documents environment variables, environment-group isolation between production and non-production, and OIDC-based short-lived credentials in place of long-lived API keys, which supports centralized secrets/config management. However, there is no evidence of team-level role-based access control, member invitation/permission management, or a dedicated secrets manager UI, so the 'team-level access' part of the story is unproven. Missing for 10: team/RBAC management docs, explicit secrets-vault feature, multi-user permission controls.", "evidenceIds": [ "render-docs-7", "render-docs-18", "render-docs-30" ] }, { "productId": "render", "storyId": "unified-log-view", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Render docs mention viewing critical metrics for infrastructure and streaming telemetry to external tools, implying a centralized dashboard for infrastructure observability, but there is no explicit evidence of a unified view combining build, deploy, and runtime logs specifically. Missing for 10: explicit documentation of a unified logs UI covering build/deploy/runtime logs together, independent/hands-on confirmation of this unified logging experience.", "evidenceIds": [ "render-docs-13" ] }, { "productId": "render", "storyId": "unlimited-team-environments", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Render documents ephemeral preview environments of the entire application architecture for every change (render-docs-8) and blueprint-based multi-service definitions (render-docs-9, render-docs-29), which supports spinning up isolated environments for teams. However, there's no explicit claim of 'unlimited' environments, no discussion of per-developer/team concurrency limits or plan-based caps, and no independent evidence validating scale under real team usage. Missing for 10: explicit unlimited-environment guarantees, team-scale concurrency evidence, independent/hands-on confirmation of simultaneous multi-team usage.", "evidenceIds": [ "render-docs-8", "render-docs-9", "render-docs-29", "render-docs-18" ] }, { "productId": "render", "storyId": "visitor-performance-analytics", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Render advertises infrastructure metrics visible 'from day zero' and telemetry streaming to external tools (render-docs-13), which covers basic observability of service performance, but there is no evidence of built-in visitor behavior or web analytics (page views, sessions, user tracking) as the story specifies. Missing for 10: dedicated visitor/behavior analytics dashboard, page-view/session tracking, and any independent corroboration of analytics use.", "evidenceIds": [ "render-docs-13" ] }, { "productId": "render", "storyId": "visual-infra-canvas", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Render's evidence pack consistently emphasizes YAML-based blueprints, CLI, and API-driven infrastructure management (render-docs-9, render-docs-29, render-docs-27) rather than any visual canvas for building or editing infrastructure. No documentation or community evidence mentions a drag-and-drop or visual architecture editor.", "evidenceIds": [] }, { "productId": "render", "storyId": "zero-config-repo-deploy", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Render's docs explicitly advertise connecting a repo and shipping on every push, with Docker support, auto-detected builds, custom domains, TLS, and rollbacks, and community feedback corroborates the ease of setup compared to alternatives like Heroku/Fly.io. Missing for 10: no hands-on step-by-step account of zero-config build detection succeeding for varied stacks, and one community note mentions past outages unrelated to build config but slightly tempering overall reliability confidence.", "evidenceIds": [ "render-docs-1", "render-docs-2", "render-docs-6", "render-docs-12", "render-comm-1", "render-comm-3" ] }, { "productId": "vercel", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Direct probe evidence confirms Vercel serves a live llms.txt at vercel.com/llms.txt (HTTP 200) with a machine-readable product summary, plus a full docs.md markdown mirror of documentation, both explicitly agent-oriented formats. Missing for 10: independent third-party confirmation that agents actually consume and act on these files effectively.", "evidenceIds": [ "vercel-probe-1", "vercel-probe-2" ] }, { "productId": "vercel", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Vercel's docs mention 'framework-aware insights' across apps and AI workflows for monitoring traffic, function performance, and errors (vercel-docs-30), which suggests some automated analysis of the user's own deployment/app data, but there is no clear evidence of AI-generated suggestions or recommendations (e.g., anomaly detection, actionable AI advice) surfaced inside the product. Missing for 10: explicit description of an AI/ML-driven insights engine that generates suggestions from user data, independent corroboration or screenshots of such a feature, and detail on how 'insights' differ from standard dashboards/metrics.", "evidenceIds": [ "vercel-docs-30" ] }, { "productId": "vercel", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vercel's Workflow SDK explicitly supports building 'durable, observable applications and AI agents' on a managed platform, and scheduled ISR regeneration ('on a schedule') shows some background automation capability, but there is no dedicated cron/scheduler product documentation or independent evidence of long-running autonomous agent automations in production. missing for 10: explicit cron/scheduled-job feature docs, evidence of persistent background agent execution beyond ISR, and independent/hands-on confirmation that Workflow SDK automations run unattended reliably.", "evidenceIds": [ "vercel-docs-18", "vercel-docs-21", "vercel-docs-19" ] }, { "productId": "vercel", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows Vercel providing AI infrastructure (AI Gateway, AI SDK, Workflow SDK) and integration hooks for external coding agents (Claude Code, Cursor, etc.) to interact with Vercel, but no evidence of a built-in AI assistant inside the Vercel product itself that a user can delegate tasks to.", "evidenceIds": [] }, { "productId": "vercel", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Vercel ships an official CLI (vercel-docs-4/vercel-gh-1, vercel-probe-4, vercel-gh-2) that supports git push-based and CLI-driven deploys usable in CI pipelines, plus REST/OpenAPI API access (vercel-probe-3) for scripted automation, and CI-friendly features like environment variable management and rollback/promote commands (vercel-docs-23, vercel-docs-29). Missing for 10: no explicit GitHub Actions/CI recipe documentation or independent hands-on report of running the CLI fully headlessly in a third-party CI system.", "evidenceIds": [ "vercel-gh-1", "vercel-gh-2", "vercel-probe-3", "vercel-probe-4", "vercel-docs-23", "vercel-docs-29" ] }, { "productId": "vercel", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows Vercel exposing its own capabilities as tools for AI agents (vercel-docs-4, vercel-docs-15) and building agent infrastructure (Workflow SDK, AI Gateway), but there is no evidence that Vercel itself can act as an MCP client, ingesting external MCP servers so it can use their tools. Missing for 10: any documentation of Vercel connecting to third-party MCP servers as a client and invoking their tools.", "evidenceIds": [ "vercel-docs-4", "vercel-docs-15", "vercel-docs-18" ] }, { "productId": "vercel", "storyId": "agentic-mcp-server", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Vercel's docs mention agent-facing capabilities like 'Create tools for AI agents to interact with your systems' and 'Allow AI agents to deploy a project and let a human take over,' plus agent skill/slash-command support for coding agents (Claude Code, Cursor, etc.), suggesting some official agent-integration surface. However, none of the evidence explicitly names an MCP server, MCP protocol support, or an MCP endpoint. Missing for 10: explicit documentation of an official MCP server/endpoint, MCP protocol naming, or third-party confirmation that agents connect via MCP rather than generic APIs/CLI.", "evidenceIds": [ "vercel-docs-4", "vercel-docs-10", "vercel-docs-15" ] }, { "productId": "vercel", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vercel ships agent-facing 'skills' and slash commands so coding agents (Claude Code, Cursor, Copilot, etc.) can drive deployments and framework tasks via natural language, and allows AI agents to deploy projects and hand off to a human — evidence of AI-native operability rather than a full built-in NL interface. This relies on third-party coding agents/CLIs rather than a native conversational control surface for the whole platform (dashboard, env vars, domains, etc.). Missing for 10: evidence of a first-party natural-language interface covering the full product surface, and independent hands-on validation that these agent skills reliably execute complex operations.", "evidenceIds": [ "vercel-docs-15", "vercel-docs-10", "vercel-docs-4", "vercel-probe-4" ] }, { "productId": "vercel", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Vercel ships an official CLI (vercel-probe-4, vercel-gh-1, vercel-gh-2) and explicitly extends it toward AI-native agent workflows via deployment skills, framework best practices, and slash commands for coding agents like Claude Code, Cursor, and GitHub Copilot (vercel-docs-15), plus AI-agent deployment handoff features (vercel-docs-10). missing for 10: independent/hands-on verification of the CLI's agentic slash-command features beyond first-party docs.", "evidenceIds": [ "vercel-probe-4", "vercel-gh-1", "vercel-gh-2", "vercel-docs-15", "vercel-docs-10" ] }, { "productId": "vercel", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Vercel exposes a documented public OpenAPI spec (vercel-probe-3), an official CLI (vercel-probe-4, vercel-gh-2), and llms.txt/docs.md machine-readable docs (vercel-probe-1, vercel-probe-2), confirming a documented API surface AI-native users can drive programmatically. Additional docs describe AI-agent-oriented tool creation and agent deployment flows (vercel-docs-4, vercel-docs-10, vercel-docs-15). missing for 10: independent/hands-on third-party corroboration of API completeness and no explicit API versioning/rate-limit docs cited.", "evidenceIds": [ "vercel-probe-3", "vercel-probe-4", "vercel-probe-1", "vercel-probe-2", "vercel-docs-4", "vercel-docs-10" ] }, { "productId": "vercel", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Vercel documents securing 'every internal agent, app, and deployment with your identity provider' (vercel-docs-13), implying some access-control mechanism for agents, but there is no concrete documentation of issuing scoped/least-privilege API tokens or credentials specifically for AI agents. Missing for 10: explicit scoped API token/credential creation flow for agents, granularity of permissions, and independent confirmation of least-privilege enforcement.", "evidenceIds": [ "vercel-docs-13", "vercel-docs-4" ] }, { "productId": "vercel", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vercel documents official SDKs like the AI SDK and Workflow SDK, an OpenAI-compatible endpoint, and an OpenAPI spec, giving AI-native builders concrete first-party SDK surfaces to build against. However, evidence is limited to doc summaries and a bare OpenAPI probe with no independent hands-on validation of SDK completeness or developer experience. missing for 10: independent/hands-on corroboration of SDK usage, deeper docs on SDK API surface/versioning, and confirmation of SDK stability beyond marketing copy.", "evidenceIds": [ "vercel-docs-16", "vercel-docs-18", "vercel-probe-3", "vercel-docs-4" ] }, { "productId": "vercel", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item documents a webhooks subscription API or event-driven webhook feature; the closest is a Slack integration for deployment notifications, which is not a webhook mechanism for AI-native subscription. Missing for 10: any docs on a webhooks API, event types, subscription setup, or agent-facing usage of webhooks.", "evidenceIds": [] }, { "productId": "vercel", "storyId": "ai-global-inference", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vercel AI Gateway lets builders call hundreds of models through one endpoint with budgets, fallbacks, and monitoring (a global, single-call inference layer), and Fluid compute scales concurrency for AI workloads without manual GPU management. However, this is a gateway/routing layer to third-party model providers rather than Vercel itself running GPU inference, and there's no evidence of Vercel-hosted GPU capacity planning being abstracted away for actual model hosting. Missing for 10: independent/hands-on verification of global latency and capacity claims, detail on GPU-backed inference infrastructure vs. pure routing, and confirmation this covers self-hosted/custom model inference, not just third-party API aggregation.", "evidenceIds": [ "vercel-docs-2", "vercel-docs-16", "vercel-docs-19", "vercel-docs-30" ] }, { "productId": "vercel", "storyId": "ai-model-gateway", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Vercel's AI Gateway is directly documented as accessing hundreds of models through one unified endpoint, supporting the AI SDK, OpenAI-compatible SDKs, or cURL, with budgets, fallbacks, and usage monitoring. Missing for 10: independent/hands-on corroboration of the gateway's model count and reliability claims beyond first-party docs.", "evidenceIds": [ "vercel-docs-2", "vercel-docs-16" ] }, { "productId": "vercel", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence confirms Vercel exposes a machine-readable OpenAPI spec (openapi.json) and standard docs pages, but nothing in the pack describes an interactive, in-browser API reference with runnable/try-it examples (e.g., a Swagger-style console). Missing for 10: evidence of an interactive API console, 'try it now' runnable request feature, or embedded code execution in the docs UI.", "evidenceIds": [ "vercel-probe-3", "vercel-probe-2" ] }, { "productId": "vercel", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A probe confirms Vercel publishes a machine-readable OpenAPI spec at vercel.com/openapi.json returning valid OpenAPI JSON, directly satisfying the story. Missing for 10: independent third-party corroboration or documentation describing spec versioning/coverage.", "evidenceIds": [ "vercel-probe-3" ] }, { "productId": "vercel", "storyId": "api-sandbox", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vercel provides preview/deployment environments with separate environment variables (docs-1, docs-23, docs-27) and sandboxed ephemeral execution for untrusted/agent-generated code (docs-3, docs-17), which together support testing without touching production data. However, evidence doesn't explicitly confirm data isolation (e.g., separate databases per preview) beyond env var management, and community feedback notes friction connecting databases securely without enterprise plans (vercel-comm-11). missing for 10: explicit documentation of production-data isolation/masking in preview/sandbox environments, independent hands-on confirmation that sandbox testing never touches prod data.", "evidenceIds": [ "vercel-docs-1", "vercel-docs-3", "vercel-docs-17", "vercel-docs-23", "vercel-docs-27", "vercel-comm-11" ] }, { "productId": "vercel", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "There is an OpenAPI spec exposed (vercel-probe-3), showing an API exists, but no evidence of API versioning scheme or a documented deprecation policy anywhere in the pack. Nothing addresses version numbers, sunset timelines, or breaking-change communication practices.", "evidenceIds": [ "vercel-probe-3" ] }, { "productId": "vercel", "storyId": "automatic-tls-certs", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vercel's docs state that adding a custom domain automatically provides SSL certificates and DNS management, implying free, auto-renewing TLS, but the evidence never explicitly confirms wildcard domain certificate support or renewal mechanics. Missing for 10: explicit wildcard-domain TLS support, renewal automation details, and independent confirmation that certs are free/no manual step.", "evidenceIds": [ "vercel-docs-31" ] }, { "productId": "vercel", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While Vercel exposes a CLI and REST/OpenAPI API (vercel-probe-3, vercel-probe-4) that could theoretically be scripted for bulk actions, the evidence pack contains no documentation of a bulk/batch operation feature (e.g., bulk deploy, bulk env-var update, bulk domain management) that an AI-native user could invoke across many items at once.", "evidenceIds": [] }, { "productId": "vercel", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Vercel supports several built-in event-triggered automations (deploy on git push, scheduled/on-demand ISR revalidation, middleware executing before requests, gradual/percentage rollouts, and a Workflow SDK for durable agent workflows), which are automatic actions tied to events. However, there is no evidence of a general user-configurable rules engine where an AI-native user can define arbitrary conditional 'if event X, then action Y' automations across the platform. Missing for 10: a documented rules/automation API or dashboard for custom event-trigger definitions, examples of user-defined conditional logic beyond fixed platform behaviors, and independent confirmation of such a feature working in practice.", "evidenceIds": [ "vercel-docs-1", "vercel-docs-21", "vercel-docs-20", "vercel-docs-18", "vercel-docs-28" ] }, { "productId": "vercel", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Vercel docs mention regenerating content 'on a schedule' (ISR) and a Workflow SDK for building 'durable, observable applications and AI agents,' which together suggest some scheduled/automation capability, but neither explicitly describes a general-purpose cron/recurring-job scheduler for AI-native workflows. missing for 10: explicit dedicated cron/scheduled-jobs feature docs, examples of recurring AI workflow automation, and independent confirmation of scheduling reliability.", "evidenceIds": [ "vercel-docs-21", "vercel-docs-18" ] }, { "productId": "vercel", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vercel provides strong deployment-level version control, review, and rollback (git-based deploys, preview environments, comments, instant rollback, gradual rollout) and explicitly supports AI agents deploying with human takeover (vercel-docs-10), which maps well to versioning/reviewing/rolling back automations triggered by AI. However, evidence doesn't show version history, review, or rollback specifically for Workflow SDK-based agent workflows or tool-call automations themselves, only for deployments. Missing for 10: explicit versioning/rollback UI for AI agent workflows/tool executions (not just code deployments), independent corroboration of this specific AI-automation rollback flow.", "evidenceIds": [ "vercel-docs-5", "vercel-docs-29", "vercel-docs-6", "vercel-docs-28", "vercel-docs-8", "vercel-docs-10", "vercel-docs-18", "vercel-docs-1" ] }, { "productId": "vercel", "storyId": "cli-deploy", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Vercel provides an official CLI (vercel deploy/vc) documented at vercel.com/docs/cli, and the GitHub README confirms users can get started via the CLI to deploy without importing through the dashboard. This directly supports CLI-based deployment as a core, well-documented workflow. Missing for 10: independent hands-on walkthrough of a full CLI deploy session and more detail on CLI command coverage beyond the docs link.", "evidenceIds": [ "vercel-probe-4", "vercel-gh-1", "vercel-gh-2" ] }, { "productId": "vercel", "storyId": "cli-scaffold-project", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "The official GitHub README explicitly describes getting started by importing a project, choosing a template, or using the Vercel CLI, then deploying with git push, and docs confirm a dedicated CLI (vercel-docs-cli) plus zero-config framework deployment (vercel-docs-14). This directly matches scaffold-and-deploy via a single CLI workflow, though the pack lacks a concrete example transcript of a single command producing a scaffolded+deployed app or independent hands-on confirmation. Missing for 10: an explicit worked example of a single CLI invocation (e.g., `vercel init` + `vercel deploy`) and independent/community verification of the scaffold step's simplicity.", "evidenceIds": [ "vercel-gh-1", "vercel-probe-4", "vercel-docs-14" ] }, { "productId": "vercel", "storyId": "condition-based-alerts", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Vercel offers monitoring/observability (vercel-docs-30) and a Slack marketplace integration that notifies on deployment status and comments (vercel-intdir-6), plus third-party logging integrations like Better Stack (vercel-intdir-5), but there is no first-party evidence of configurable alert rules with custom conditions or native email notifications for observability thresholds. Missing for 10: documented native alert-rule/threshold configuration, email notification channel, and evidence these trigger on arbitrary monitoring conditions rather than just deployment events.", "evidenceIds": [ "vercel-docs-30", "vercel-intdir-6", "vercel-intdir-5" ] }, { "productId": "vercel", "storyId": "consistent-api-at-scale", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vercel's docs describe a consistent platform: zero-config framework deploys, Git-based push-to-deploy that works from prototype through production, and 'Fluid compute' explicitly designed to scale concurrency for demanding AI/I/O workloads without changing the deployment model (vercel-docs-14, vercel-docs-19, vercel-docs-27). However, community evidence shows real friction as usage grows — e.g. needing to 'contact sales' about 'enterprise' for basic secure database connections, and surprise large bills from traffic spikes — suggesting the platform isn't fully frictionless at scale and some capabilities require plan/negotiation changes rather than pure API continuity (vercel-comm-11, vercel-comm-12). Missing for 10: independent case studies of apps scaling from prototype to massive production traffic on unchanged APIs, and clearer evidence that enterprise-gated features don't require workflow changes.", "evidenceIds": [ "vercel-docs-14", "vercel-docs-19", "vercel-docs-27", "vercel-comm-11", "vercel-comm-12" ] }, { "productId": "vercel", "storyId": "custom-domains", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Vercel docs explicitly state you can 'Add a custom domain, manage DNS records, and get SSL certificates automatically,' directly matching the story of attaching custom domains to deployed services. This is first-party documentation with no contradicting evidence. Missing for 10: independent/hands-on corroboration of the custom domain workflow beyond vendor docs.", "evidenceIds": [ "vercel-docs-31" ] }, { "productId": "vercel", "storyId": "custom-metrics-dashboards", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Vercel's docs mention its own monitoring/observability dashboard for traffic, function performance, and errors (vercel-docs-30) and marketplace integrations with monitoring tools like Better Stack (vercel-intdir-5), but there is no evidence of a capability to build custom dashboards combining metrics from arbitrary/any external services into one view.", "evidenceIds": [] }, { "productId": "vercel", "storyId": "default-ddos-waf", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Vercel's docs describe a firewall that can 'block attacks, scrapers, and unwanted traffic with custom rules, IP blocking, and managed rulesets' (vercel-docs-24) and bot-protection features (vercel-docs-11, vercel-docs-25), suggesting WAF-like capability, but these are configurable/opt-in rather than clearly stated as applied to every request 'by default'. A community report explicitly states that 'waking up to a 10k vercel bill is pretty common, especially when a DDoS goes undetected' (vercel-comm-12), directly contradicting the idea that DDoS protection is reliably applied automatically. missing for 10: explicit vendor claim of automatic, zero-config DDoS mitigation on every request, independent confirmation that the firewall/bot-protection is enabled by default rather than requiring rule configuration, and resolution of the community-reported undetected-DDoS billing incidents.", "evidenceIds": [ "vercel-docs-24", "vercel-docs-11", "vercel-docs-25", "vercel-comm-12" ] }, { "productId": "vercel", "storyId": "docker-deploy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Vercel's evidence focuses on Git-push deploys, framework support (Next.js/40+ frameworks), functions, and isolated 'container' sandboxes for running untrusted code—none of this describes deploying a developer-supplied Docker image/Dockerfile as the deployment artifact. There is no documentation of Docker container deployment as a supported workflow.", "evidenceIds": [ "vercel-docs-12", "vercel-docs-14", "vercel-docs-27", "vercel-gh-1" ] }, { "productId": "vercel", "storyId": "documented-rate-limits", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes general platform capabilities (Fluid compute for concurrency, Functions, CDN, etc.) but contains no documentation of specific rate limits, concurrency caps, or quota numbers that a developer could use for capacity planning. Community evidence also does not address this topic.", "evidenceIds": [] }, { "productId": "vercel", "storyId": "durable-workflows", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Vercel documents a Workflow SDK explicitly for building 'durable, observable applications and AI agents' on a fully managed platform, directly matching the story of deploying durable workflows as code without manually wiring queues/retries, and this is complemented by Fluid compute for scaling concurrency. Missing for 10: no independent/hands-on evidence of retry semantics or durability guarantees, no detail on how queuing is abstracted, and no community corroboration of real-world reliability.", "evidenceIds": [ "vercel-docs-18", "vercel-docs-19", "vercel-docs-30" ] }, { "productId": "vercel", "storyId": "edge-websocket-support", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item describes WebSocket support in Vercel's serverless/edge functions; the functions described (Edge Middleware, Fluid compute, sandboxes) are documented as request/response or ephemeral execution models, not persistent connections. Absence of evidence for this applicable capability yields 'none'. missing for 10: any documentation of WebSocket/persistent-connection support in serverless or edge functions, any first-party guidance on real-time connection handling.", "evidenceIds": [] }, { "productId": "vercel", "storyId": "execution-time-limits", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item specifies concrete CPU or wall-clock execution time limits for functions (e.g., seconds per plan tier). Docs mention Fluid compute, sandboxes, and workflows but never state numeric duration/CPU limits a devops-lead could use to size agentic workloads. Missing for 10: explicit maxDuration/timeout numbers per plan, CPU allocation limits, and any pricing-page or docs citation stating these limits.", "evidenceIds": [ "vercel-docs-19", "vercel-docs-17", "vercel-docs-12" ] }, { "productId": "vercel", "storyId": "external-service-bindings", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vercel's Marketplace lists native, first-party integrations (Neon Postgres, Pinecone, Shopify, Slack, Jira/Linear, etc.) that plug into a project with managed credentials, reducing manual SDK wiring, and Vercel documents environment/variable management across environments. However, a concrete community report states that connecting to a database with more than a password requires 'contact sales' for enterprise, indicating the native-binding security experience is limited for standard tiers, and there's no first-party technical deep-dive on how the binding/credential injection mechanism works beyond marketplace listings. Missing for 10: detailed docs on the binding mechanism itself, independent hands-on confirmation of minimal-boilerplate setup, and resolution of the enterprise-gating friction noted in community feedback.", "evidenceIds": [ "vercel-docs-32", "vercel-intdir-1", "vercel-intdir-3", "vercel-intdir-6", "vercel-intdir-9", "vercel-docs-23", "vercel-comm-11" ] }, { "productId": "vercel", "storyId": "fast-boot-scaling", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vercel's docs describe serverless/edge functions, Fluid compute for scaling concurrency, and isolated ephemeral execution environments that support rapid, elastic scaling for AI/I/O workloads, aligning with the edge-compute story's spirit. However, no evidence gives concrete sub-second boot benchmarks or explicit tens-of-thousands concurrent instance figures, and community reports of surprise $10k bills during undetected DDoS traffic (vercel-comm-12) hint at real-world scaling being uncontrolled/costly rather than a clean documented guarantee. Missing for 10: quantified cold-start/boot latency data, published large-scale load test results, and independent benchmarks validating claimed scale.", "evidenceIds": [ "vercel-docs-19", "vercel-docs-12", "vercel-docs-17", "vercel-docs-20", "vercel-comm-12" ] }, { "productId": "vercel", "storyId": "framework-tool-continuity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vercel documents git-push deploys with GitHub/GitLab/Bitbucket, support for 40+ frameworks (including zero-config Next.js), and an optional CLI, which lets developers keep their existing Git and editor workflows largely intact (vercel-docs-1, vercel-docs-14, vercel-docs-27, vercel-gh-1, vercel-probe-4). However, community feedback flags real limitations: the platform is seen as effectively locked to the Node/JS/TS/Go ecosystem with 'WebAssembly gimmicks' for other languages, and some users feel it favors its own Next.js stack over full interoperability with arbitrary third-party CI systems (vercel-comm-7, vercel-comm-14). Missing for 10: explicit evidence of compatibility with external CI pipelines (Jenkins/CircleCI/etc.), broader language support beyond JS/TS/Go, and independent confirmation that non-Next.js frameworks get equally first-class treatment.", "evidenceIds": [ "vercel-docs-1", "vercel-docs-14", "vercel-docs-27", "vercel-gh-1", "vercel-probe-4", "vercel-comm-7", "vercel-comm-14" ] }, { "productId": "vercel", "storyId": "full-nodejs-runtime-support", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs reference Fluid compute for running 'server-side code on demand' (vercel-docs-19) and 'production workloads in isolated containers' (vercel-docs-12), implying full Node.js runtime support beyond a restricted Edge subset, but no evidence explicitly confirms unmodified Node.js APIs or full npm package compatibility in edge/serverless functions. Missing for 10: explicit documentation contrasting Node.js runtime vs restricted Edge Runtime, npm package compatibility guarantees, and independent verification of unmodified Node.js execution.", "evidenceIds": [ "vercel-docs-19", "vercel-docs-12", "vercel-docs-3", "vercel-docs-17" ] }, { "productId": "vercel", "storyId": "full-stack-preview-envs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Vercel's docs explicitly describe automatic preview environments for every push/branch/PR, environment variable management across preview/production, preview URL access controls, and comment/collaboration on preview environments—covering the full-stack preview workflow. Missing for 10: independent hands-on verification of preview environment fidelity/performance beyond vendor docs.", "evidenceIds": [ "vercel-docs-1", "vercel-docs-27", "vercel-docs-23", "vercel-docs-8", "vercel-docs-26", "vercel-gh-1" ] }, { "productId": "vercel", "storyId": "git-push-deploy", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Vercel's core value proposition is Git-push deployment: docs explicitly state connecting GitHub/GitLab/Bitbucket triggers automatic deploys with preview URLs per branch, and the official GitHub README confirms the 'git push to deploy' workflow. This is corroborated by both first-party docs and the GitHub repo. Missing for 10: independent hands-on verification beyond vendor docs/GitHub description.", "evidenceIds": [ "vercel-docs-1", "vercel-docs-27", "vercel-gh-1" ] }, { "productId": "vercel", "storyId": "global-edge-network", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Vercel's docs describe a global CDN with edge-executed middleware ('Run code before a request completes to rewrite, redirect, or personalize responses'), automatic image optimization served from the CDN, and Fluid compute scaling for I/O-heavy workloads, all consistent with edge network delivery close to end users. However, the evidence pack lacks explicit independent benchmarks or hands-on confirmation of edge latency/performance, and no community corroboration specifically addresses edge-network proximity. Missing for 10: independent latency benchmarks, explicit documentation of edge region coverage, and hands-on developer confirmation of edge execution behavior.", "evidenceIds": [ "vercel-docs-20", "vercel-docs-22", "vercel-docs-19", "vercel-docs-30" ] }, { "productId": "vercel", "storyId": "global-static-cdn", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Vercel documents serving optimized images 'from the CDN' and offers ISR/edge middleware, implying CDN-backed static asset delivery, but the evidence pack lacks explicit documentation of a global edge network, cache-control/edge caching mechanics, or performance benchmarks/independent corroboration of CDN speed. Missing for 10: explicit global CDN architecture docs, edge cache configuration details, independent performance benchmarks confirming faster asset delivery.", "evidenceIds": [ "vercel-docs-22", "vercel-docs-21", "vercel-docs-14" ] }, { "productId": "vercel", "storyId": "hard-spending-limits", "verdict": "disputed", "quality": 3, "confidence": "medium", "rationale": "Vercel only documents 'budgets' as part of its AI Gateway feature for LLM usage (vercel-docs-2, vercel-docs-16), not a platform-wide hard spending cap for compute/bandwidth costs, and community reports explicitly describe surprise five-figure bills ('Waking up to a 10k vercel bill is pretty common, especially when a DDoS goes undetected') indicating no effective hard limit exists for overall cloud spend. Missing for 10: dedicated hard-cap/kill-switch billing controls for general project usage, first-party docs on enforceable spend limits, and independent confirmation that such limits actually stop overage.", "evidenceIds": [ "vercel-docs-2", "vercel-docs-16", "vercel-comm-12" ] }, { "productId": "vercel", "storyId": "idp-secured-access", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Vercel explicitly claims 'Secure every internal agent, app, and deployment with your identity provider' and separately offers Vercel Authentication/passwords/trusted IPs to gate preview and production URLs, directly matching the devops-lead's IdP-based security story. However, the evidence is a single thin marketing line with no detail on supported IdPs (SAML/OIDC/SSO providers), no docs on configuration steps, and no independent or hands-on corroboration that this works in practice. Missing for 10: detailed IdP integration docs (specific protocols/providers), setup/configuration guidance, and third-party validation of the feature working as described.", "evidenceIds": [ "vercel-docs-13", "vercel-docs-26" ] }, { "productId": "vercel", "storyId": "in-browser-dev-toolbar", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Vercel's docs explicitly describe an in-browser toolbar for leaving feedback, managing feature flags, previewing drafts, and inspecting performance, directly matching the story, and this is reinforced by related features like comments on preview/production environments and framework-aware performance insights. missing for 10: independent/hands-on corroboration of the toolbar beyond first-party docs.", "evidenceIds": [ "vercel-docs-7", "vercel-docs-8", "vercel-docs-30" ] }, { "productId": "vercel", "storyId": "incremental-rollout", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Vercel's docs explicitly describe incremental rollout with percentage-based traffic splitting and promote/rollback controls (vercel-docs-6, vercel-docs-28), plus instant rollback for recovery (vercel-docs-5, vercel-docs-29), directly matching the story of reducing blast radius via incremental deploys. Missing for 10: independent/hands-on corroboration of gradual rollout behavior in practice, and no detail on automated rollout policies (e.g., canary analysis or auto-rollback thresholds).", "evidenceIds": [ "vercel-docs-6", "vercel-docs-28", "vercel-docs-5", "vercel-docs-29" ] }, { "productId": "vercel", "storyId": "infra-as-code-blueprint", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item describes a single configuration file (e.g., vercel.json or IaC-style manifest) that defines and versions an entire architecture; docs mention environment variable management, CLI deploys, and marketplace integrations but nothing about unified architecture-as-code.", "evidenceIds": [] }, { "productId": "vercel", "storyId": "instant-folder-deploy", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence confirms Vercel has a CLI (vercel-probe-4, vercel-gh-1) that can be used as an alternative to Git-based deploys, and preview/production URLs can be locked down via authentication or passwords (vercel-docs-26), which supports the 'private until published' part of the story. However, no evidence explicitly describes a folder-upload (no-Git) deploy flow or confirms new deployments are private-by-default. Missing for 10: explicit documentation of a drag-and-drop or CLI folder-upload deploy without any Git repo, and confirmation that such deployments default to private/unpublished state.", "evidenceIds": [ "vercel-probe-4", "vercel-gh-1", "vercel-docs-26" ] }, { "productId": "vercel", "storyId": "invisible-bot-protection", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Vercel's docs explicitly describe an invisible CAPTCHA/bot check ('An invisible CAPTCHA that protects against sophisticated bots without challenges' and 'Protect sensitive routes from sophisticated bots with an invisible check instead of a CAPTCHA'), directly matching the story. This is complemented by broader firewall/rules-based bot blocking. Missing for 10: independent/hands-on verification of the invisible challenge's effectiveness and no third-party benchmarking of bypass rates.", "evidenceIds": [ "vercel-docs-11", "vercel-docs-25", "vercel-docs-24" ] }, { "productId": "vercel", "storyId": "isolated-sandbox-execution", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Vercel's docs explicitly describe a Sandbox feature for running untrusted or agent-generated code in isolated, ephemeral execution environments, and separately for running production workloads in isolated containers, matching the disposable/isolated sandbox story closely. Missing for 10: explicit confirmation of hardware-level isolation (e.g., microVM/VM-based tech) and independent/hands-on validation beyond first-party docs.", "evidenceIds": [ "vercel-docs-3", "vercel-docs-17", "vercel-docs-12" ] }, { "productId": "vercel", "storyId": "known-overage-pricing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no vendor documentation of a specific overage fee schedule for bandwidth, invocations, or build minutes — only general product/feature docs unrelated to billing granularity. A community comment even suggests unexpected surprise billing (\"Waking up to a 10k vercel bill is pretty common\"), reinforcing that no clear overage schedule evidence exists.", "evidenceIds": [ "vercel-comm-12" ] }, { "productId": "vercel", "storyId": "load-based-autoscaling", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vercel's docs describe Fluid compute scaling concurrency for I/O-heavy workloads and isolated container-based production workloads, implying automatic capacity scaling for traffic bursts, and it offers bot/DDoS mitigation features (invisible CAPTCHA, custom firewall rules) that complement burst handling. However, there's no explicit 'autoscaling SLA' documentation, and community reports (e.g., surprise $10k bills tied to undetected DDoS traffic) suggest that while capacity does scale, cost/traffic-spike management can be unpredictable rather than a clean, bounded autoscaling guarantee. missing for 10: explicit autoscaling architecture docs/SLAs, independent load-test benchmarks, and clearer safeguards against runaway cost during bursts.", "evidenceIds": [ "vercel-docs-19", "vercel-docs-12", "vercel-docs-24", "vercel-docs-11", "vercel-comm-12" ] }, { "productId": "vercel", "storyId": "local-dev-simulator", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence mentions the Vercel CLI and management of local/preview/production environments, but nothing in the pack claims a local simulator that matches the production runtime (e.g., serverless/edge function emulation). Missing for 10: explicit documentation of a local dev runtime (e.g., 'vercel dev') that mirrors production execution, and any hands-on confirmation of runtime parity.", "evidenceIds": [ "vercel-docs-23", "vercel-probe-4" ] }, { "productId": "vercel", "storyId": "managed-postgres", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Vercel offers Postgres only via a third-party Marketplace partner (Neon), described as 'serverless Postgres... with autoscaling and instant branching' rather than a Vercel-native fully-managed service with explicit point-in-time recovery, replicas, or HA guarantees. Missing for 10: first-party documentation of PITR, replica configuration, and HA SLAs, and any confirmation these features are exposed through the Vercel-integrated product rather than left to Neon's own console.", "evidenceIds": [ "vercel-intdir-1", "vercel-docs-32" ] }, { "productId": "vercel", "storyId": "migration-credits", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions migration credits, competitive switching incentives, or pricing offers for moving infrastructure from another platform to Vercel; the pack covers deployment, security, and AI features only.", "evidenceIds": [] }, { "productId": "vercel", "storyId": "multi-protocol-native-support", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Vercel's evidence is entirely about HTTP-based serverless functions, edge middleware, static hosting, and framework deployment; there is no mention of TCP, gRPC, or WebSocket protocol support from the same deployed service. Vercel's serverless model is known to be HTTP-request/response oriented, and no evidence pack item claims multi-protocol serving without separate configuration.", "evidenceIds": [] }, { "productId": "vercel", "storyId": "object-storage-uploads", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack lists Vercel's marketplace integrations (Neon Postgres, Pinecone, Better Stack, etc.) and platform features (deployments, functions, image optimization) but nowhere mentions Vercel Blob or any built-in object storage product for persisting user uploads. Storage-as-a-service is a plausible axis for a hosting platform, but no documentation, marketplace listing, or community evidence confirms this capability.", "evidenceIds": [ "vercel-docs-32", "vercel-intdir-1", "vercel-intdir-10" ] }, { "productId": "vercel", "storyId": "on-demand-image-optimization", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Vercel's docs explicitly describe on-demand image resizing/conversion served from the CDN in modern formats, directly matching the story's requirements. Missing for 10: independent/hands-on corroboration beyond first-party docs and detail on caching/format negotiation specifics.", "evidenceIds": [ "vercel-docs-22" ] }, { "productId": "vercel", "storyId": "one-click-rollback", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Vercel's docs explicitly describe instant rollback to a previous deployment without rebuild for swift incident recovery, matching the one-click rollback story closely (vercel-docs-5, vercel-docs-29). This is corroborated by related gradual-rollout/promote-or-rollback functionality. missing for 10: independent hands-on confirmation of the rollback UX and any edge-case limitations (e.g., config/env var changes not covered by rollback).", "evidenceIds": [ "vercel-docs-5", "vercel-docs-29", "vercel-docs-28" ] }, { "productId": "vercel", "storyId": "one-click-templates", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "vercel-gh-1 confirms users can start a deployment by 'choosing a template' alongside importing a project or using the CLI, which supports one-click templates/starters, but the evidence pack lacks any documentation of a template gallery, specific starter examples, or a dedicated one-click deploy flow beyond this single mention. missing for 10: dedicated template/starter gallery docs, specific example templates, independent hands-on confirmation of one-click deploy UX.", "evidenceIds": [ "vercel-gh-1", "vercel-docs-14" ] }, { "productId": "vercel", "storyId": "openness-api-parity", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Vercel exposes a public REST API (openapi.json) and a CLI covering deployments, domains, env vars, etc., suggesting broad but unconfirmed parity with the UI; no documentation explicitly claims full UI/API feature parity, and features like the in-browser toolbar, comments, and CMS draft preview are described as UI-specific experiences. missing for 10: explicit parity statement, evidence that toolbar/comments/CMS-preview features are API-accessible, independent confirmation of full API coverage.", "evidenceIds": [ "vercel-probe-3", "vercel-probe-4", "vercel-gh-1", "vercel-gh-2", "vercel-docs-7", "vercel-docs-8" ] }, { "productId": "vercel", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item addresses data export, portability, or account/data deletion in open formats for Vercel; nothing about exporting project data, logs, or analytics in a standard/open format for migration. This is a reasonable axis for a PaaS holding deployment/config/data (project settings, env vars, analytics), but no such export capability is documented.", "evidenceIds": [] }, { "productId": "vercel", "storyId": "openness-open-license", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "There's a public GitHub repo (vercel/vercel) referenced for getting started with the CLI, suggesting some source is publicly visible, but no evidence confirms an explicit open-source license or that the core platform (not just the CLI) is open. missing for 10: explicit license documentation, confirmation that the full product/platform source (not just CLI) is publicly available, independent verification of licensing terms.", "evidenceIds": [ "vercel-gh-1" ] }, { "productId": "vercel", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Vercel is a proprietary managed cloud platform; there is no evidence of a self-hostable core product, on-prem package, or open-source server offering—only CLI and SDK tooling that deploy to Vercel's own infrastructure. The evidence even shows community members switching to genuinely self-hostable alternatives like Dokploy specifically to leave Vercel, underscoring the absence of a self-host option.", "evidenceIds": [ "vercel-comm-13", "vercel-probe-4", "vercel-docs-14" ] }, { "productId": "vercel", "storyId": "pay-for-active-compute", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vercel's 'Fluid compute' feature is explicitly marketed as running server-side code 'on demand' with concurrency scaling for I/O and AI workloads, which aligns with paying only for active compute rather than idle time — a core value prop of this pricing model. However, there is no detailed documentation of the actual billing mechanics (e.g., how idle vs active CPU time is measured/charged) and community sentiment includes concerns about unpredictable large bills, suggesting real-world cost behavior isn't fully transparent. Missing for 10: detailed pricing/billing docs on active-vs-idle compute metering, third-party verification of actual cost savings vs traditional serverless, and clarification of how this applies across all function types.", "evidenceIds": [ "vercel-docs-19", "vercel-comm-12" ] }, { "productId": "vercel", "storyId": "pay-per-byte-storage", "verdict": "na", "quality": 0, "confidence": "medium", "rationale": "Vercel is a serverless/PaaS deployment platform with ephemeral functions and edge/static hosting rather than a provider of attached block storage volumes for compute instances; the evidence pack shows no concept of 'attached disks' at all, only marketplace-integrated databases (e.g., Neon) which are a separate axis. This 'auto-growing disk with per-byte billing' story is a category mismatch for Vercel's architecture.", "evidenceIds": [] }, { "productId": "vercel", "storyId": "persistent-agent-runtime", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Vercel's execution environments are explicitly described as 'ephemeral' (vercel-docs-3, vercel-docs-17), the opposite of a persistent, auto-checkpointing runtime; no evidence describes state persistence or automatic checkpointing for agent workloads.", "evidenceIds": [ "vercel-docs-3", "vercel-docs-17", "vercel-docs-12" ] }, { "productId": "vercel", "storyId": "polyglot-edge-runtimes", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no vendor documentation claiming edge functions can be written in Python, Go, or Rust; all edge/runtime references are JS/TS-focused. A community comment explicitly complains that Vercel (and Netlify) only offer 'JS/TS/Go, or WebAssembly gimmicks,' reinforcing that non-JS edge language support is not a real, documented capability.", "evidenceIds": [ "vercel-comm-14" ] }, { "productId": "vercel", "storyId": "pr-preview-urls", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Vercel's docs explicitly state that connecting a Git repo (GitHub, GitLab, Bitbucket) triggers automatic preview environments with a unique preview URL for every branch/push, which directly matches the PR-preview workflow story, and the GitHub repo confirms the git push-to-deploy flow. Missing for 10: independent hands-on confirmation beyond vendor docs of PR-specific (vs branch-only) preview URLs.", "evidenceIds": [ "vercel-docs-1", "vercel-docs-27", "vercel-gh-1" ] }, { "productId": "vercel", "storyId": "preview-collaboration-comments", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Vercel docs explicitly state teams and invited collaborators can comment on preview and production environments, corroborated by marketplace integrations that convert those comments into GitHub/Jira/Linear issues, showing the feature is real and integrated into workflows. Missing for 10: independent/hands-on user testimonials specifically about the commenting feature's usability.", "evidenceIds": [ "vercel-docs-8", "vercel-docs-7", "vercel-intdir-7", "vercel-intdir-8", "vercel-intdir-9" ] }, { "productId": "vercel", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item mentions data residency, region selection, or storage location controls for Vercel deployments or data; the closest items are about environments, domains, and functions, not geographic data residency.", "evidenceIds": [] }, { "productId": "vercel", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses AI training data opt-out or data usage policies for AI model training; Vercel's docs cover deployment, AI Gateway, sandboxes, and infra but never mention data-training controls.", "evidenceIds": [] }, { "productId": "vercel", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of data retention policies, deletion controls, data lifecycle management, or privacy/GDPR-style deletion tooling for Vercel. While security-adjacent topics (identity, environment variables, incident disclosure) appear, none address user-controllable data retention/deletion.", "evidenceIds": [] }, { "productId": "vercel", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions telemetry, usage tracking, or an opt-out mechanism for Vercel's CLI, platform, or AI tools; nothing in the docs, GitHub, or community sources addresses this capability.", "evidenceIds": [] }, { "productId": "vercel", "storyId": "private-service-networking", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation of private networking, VPC peering, or mechanisms to keep service-to-service traffic off the public internet; a community comment even suggests secure database connectivity beyond a password requires an undocumented 'enterprise' sales conversation rather than a built-in feature. This is a fair axis for a cloud/deployment platform, but no capability is evidenced.", "evidenceIds": [ "vercel-comm-11" ] }, { "productId": "vercel", "storyId": "prod-nonprod-isolation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Vercel lets teams manage separate environment variables per environment and gate preview/production URL access via authentication, passwords, or trusted IPs, which supports some separation between non-prod and prod environments. However, there's no explicit evidence of network-level isolation preventing a preview deployment from reaching production databases/services, and a community report notes that secure database connectivity beyond a password requires an enterprise sales conversation, undercutting the case for out-of-box isolation. Missing for 10: documented network segmentation or firewall rules between environments, default prevention of preview-to-production service calls, and independent verification that isolation works without enterprise upsell.", "evidenceIds": [ "vercel-docs-23", "vercel-docs-26", "vercel-docs-13", "vercel-comm-11" ] }, { "productId": "vercel", "storyId": "programmatic-multitenant-domains", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Vercel docs mention adding custom domains with automatic DNS/SSL (vercel-docs-31) and an OpenAPI surface exists (vercel-probe-3), suggesting some programmatic domain management is possible, but there is no explicit documentation of a multi-tenant SaaS domain-provisioning API or workflow for routing customer-owned domains. Missing for 10: explicit Domains API docs for multi-tenant use cases, per-tenant provisioning examples, and independent confirmation of this workflow working at scale.", "evidenceIds": [ "vercel-docs-31", "vercel-probe-3" ] }, { "productId": "vercel", "storyId": "public-incident-history", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item references a public status page or historical incident postmortem repository; the only related material is community criticism that Vercel's 2026 security incident disclosure was vague and lacked specifics, which does not constitute a status/postmortem page.", "evidenceIds": [] }, { "productId": "vercel", "storyId": "published-uptime-sla", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions an uptime SLA, uptime percentage guarantee, or service credits for downtime; only general platform features and community commentary on a security incident are present.", "evidenceIds": [] }, { "productId": "vercel", "storyId": "redis-compatible-cache", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack lists many Vercel marketplace integrations (Neon Postgres, Shopify, Perplexity, Pinecone, etc.) but none mention a Redis-compatible key-value store, caching layer, or job-queue product like Upstash/Vercel KV. Storage/caching is a plausible axis for a deployment platform, but no evidence here demonstrates this capability.", "evidenceIds": [] }, { "productId": "vercel", "storyId": "regional-deployment", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack never mentions region selection, multi-region deployment, or edge network configuration for compute; middleware/CDN mentions (vercel-docs-20, vercel-docs-22) refer to on-demand code execution and image optimization, not choosing deployment regions. Since region control is a plausible and common capability for a deployment platform, absence of evidence yields 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "vercel", "storyId": "seat-based-pricing-minimums", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses Vercel's pricing model, seat-based billing, or minimum seat counts; all citations concern product features, integrations, or unrelated community sentiment/security incidents. This axis applies to any SaaS platform but is unaddressed here.", "evidenceIds": [] }, { "productId": "vercel", "storyId": "serverless-functions", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Vercel's docs demonstrate zero-config framework deployment, git-based deploys, serverless/fluid compute, isolated execution sandboxes, and full-stack support for 40+ frameworks with automatic scaling and no server management (vercel-docs-14, vercel-docs-19, vercel-docs-12, vercel-docs-17, vercel-gh-1). This is corroborated by independent commentary describing Vercel's smooth 'code and go' onboarding as a leading serverless platform experience (vercel-comm-3). Missing for 10: deeper independent hands-on benchmarks of serverless function cold-starts/scaling limits beyond community anecdotes.", "evidenceIds": [ "vercel-docs-14", "vercel-docs-19", "vercel-docs-12", "vercel-docs-17", "vercel-gh-1", "vercel-comm-3" ] }, { "productId": "vercel", "storyId": "serverless-sql-query", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Vercel's marketplace lists Neon as a serverless Postgres partner offering autoscaling and instant branching, which implies serverless SQL access without manual connection pooling, but this is a third-party integration rather than a first-party Vercel database product with explicit 'no connection management' messaging. A community comment also notes friction: securely connecting Vercel to a database beyond a password reportedly requires an enterprise sales conversation, undercutting the 'no connection management' simplicity claim. Missing for 10: first-party Vercel docs describing a native serverless SQL/Postgres offering with pooling handled transparently, code examples showing zero-config querying, and independent hands-on confirmation that connection management is truly abstracted.", "evidenceIds": [ "vercel-intdir-1", "vercel-comm-11" ] }, { "productId": "vercel", "storyId": "site-password-protection", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Vercel docs explicitly state you can 'Control who can open your preview and production URLs with Vercel Authentication, passwords, or trusted IPs' (vercel-docs-26), directly matching the story of simple password protection without building custom auth. Missing for 10: independent/hands-on corroboration of the password-protection feature working as described and details on any plan restrictions.", "evidenceIds": [ "vercel-docs-26" ] }, { "productId": "vercel", "storyId": "smart-placement", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Vercel's edge/compute features (Edge Middleware, Fluid compute, CDN, ISR) but contains no mention of automatic compute placement based on backend/data location, region-aware routing, or latency-optimizing placement near a data store — this specific edge-compute capability is unevidenced. missing for 10: any documentation of automatic region/compute placement tied to backend data location, region-selection configuration, or latency-based routing near data stores.", "evidenceIds": [ "vercel-docs-19", "vercel-docs-20", "vercel-docs-22" ] }, { "productId": "vercel", "storyId": "standards-based-runtime-portability", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation of Vercel supporting standard, portable runtime/storage APIs (e.g., Web APIs, WinterCG standards) that would let code run unmodified on other edge platforms; all functions (Edge Middleware, Functions, KV/Blob storage, ISR) are described as Vercel-specific proprietary features. Community commentary even explicitly flags lock-in ('locked into the NextJs ecosystem') as a reason to avoid Vercel, reinforcing the absence of cross-platform portability. missing for 10: evidence of standard Web/Edge runtime API compliance, documentation of avoiding proprietary storage/runtime lock-in, cross-platform interoperability claims or benchmarks.", "evidenceIds": [ "vercel-comm-7", "vercel-docs-19", "vercel-docs-20" ] }, { "productId": "vercel", "storyId": "support-response-sla", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions support SLAs, guaranteed response times, or escalation paths for any plan tier; only product features, community sentiment, and unrelated docs are provided.", "evidenceIds": [] }, { "productId": "vercel", "storyId": "team-secrets-env-vars", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm centralized environment variable management across local/preview/production/custom environments and identity-provider based security for internal agents/apps/deployments, which supports centralized secrets/access control. However, there is no explicit documentation of team roles/permissions (RBAC) or audit logging, and a community report notes secure DB connections beyond a password require an 'enterprise' sales conversation, suggesting some access-control features are gated. Missing for 10: dedicated team roles/permissions docs, audit-log evidence, and confirmation that secret-sharing controls are available outside enterprise tiers.", "evidenceIds": [ "vercel-docs-23", "vercel-docs-13", "vercel-comm-11" ] }, { "productId": "vercel", "storyId": "unified-log-view", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vercel's docs describe an observability feature that monitors traffic, function performance, and errors across apps and AI workflows (vercel-docs-30), and the platform inherently surfaces deployment/build status through its Git-integrated deploy flow (vercel-docs-1, vercel-docs-27). However, no evidence explicitly confirms a single unified view combining build logs, deploy logs, and runtime logs together. missing for 10: explicit documentation of a unified log viewer spanning build, deploy, and runtime stages, and independent/hands-on confirmation of this consolidated experience.", "evidenceIds": [ "vercel-docs-30", "vercel-docs-1", "vercel-docs-27" ] }, { "productId": "vercel", "storyId": "unlimited-team-environments", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Vercel's preview environments automatically spin up an isolated deployment for every git push/branch, enabling parallel work by any team member without conflict, and environment/variable management supports local, preview, production, and custom environments concurrently. This directly matches the story of unlimited isolated environments for simultaneous shipping. Missing for 10: explicit confirmation of 'unlimited' concurrency limits/pricing tiers and independent hands-on validation of scale under heavy team usage.", "evidenceIds": [ "vercel-docs-1", "vercel-docs-27", "vercel-docs-23", "vercel-gh-1" ] }, { "productId": "vercel", "storyId": "visitor-performance-analytics", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vercel docs claim built-in monitoring of traffic, function performance, and errors with framework-aware insights (vercel-docs-30), which covers site performance observability; the in-browser toolbar also lets users inspect performance (vercel-docs-7). However, there's no explicit mention of visitor behavior analytics (e.g., page views, sessions, geography breakdowns) beyond generic 'traffic' monitoring, and no independent/hands-on corroboration of the analytics product's depth or dashboards. missing for 10: dedicated visitor-behavior analytics detail (e.g. Web Analytics feature specifics), independent verification of analytics accuracy/usefulness, and any devops-lead-specific alerting/dashboard evidence.", "evidenceIds": [ "vercel-docs-30", "vercel-docs-7" ] }, { "productId": "vercel", "storyId": "visual-infra-canvas", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Vercel is a code-first deployment/hosting platform with a CLI and config files (vercel.json, git-based deploys); there is no evidence of a visual canvas for designing or editing infrastructure without config. This story targets an IaC-visualization product category, which is a wrong axis for a deployment platform like Vercel.", "evidenceIds": [] }, { "productId": "vercel", "storyId": "zero-config-repo-deploy", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Vercel's docs and GitHub repo describe connecting a Git repo (GitHub/GitLab/Bitbucket) to auto-deploy on push with preview URLs, plus zero-config deployment for Next.js and 40+ frameworks, directly matching the story. This is corroborated by both docs and the official GitHub README describing the 'git push to deploy' flow. Missing for 10: independent hands-on verification beyond vendor docs/README confirming zero manual config in practice.", "evidenceIds": [ "vercel-docs-1", "vercel-docs-27", "vercel-docs-14", "vercel-gh-1" ] } ]