[ { "productId": "blaxel", "storyId": "agent-self-provisions", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Blaxel docs describe agents provisioning sandboxes via a skill/API with zero code, exposing each sandbox as an MCP server so agents can execute code via tool calls, and scale-to-zero/teardown behavior after inactivity — covering provision, execute, read results, and teardown end-to-end without human intervention. missing for 10: independent hands-on verification of a full agentic loop and explicit documentation of an agent reading back execution results programmatically.", "evidenceIds": [ "blaxel-docs-1", "blaxel-docs-5", "blaxel-docs-2", "blaxel-docs-7" ] }, { "productId": "blaxel", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A probe directly confirms llms.txt is live and returns HTTP 200 with a structured documentation summary at docs.blaxel.ai/llms.txt, and Blaxel also ships agent-oriented skills/MCP docs for agents to consume. Missing for 10: no independent third-party confirmation that agents actually consume/parse this llms.txt successfully in practice.", "evidenceIds": [ "blaxel-probe-1", "blaxel-docs-1", "blaxel-docs-4" ] }, { "productId": "blaxel", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Blaxel is a developer infrastructure platform for deploying agents, sandboxes, and MCP servers—not a data/analytics product that surfaces AI-generated insights from a user's own data. This axis is a category mismatch for its product type.", "evidenceIds": [] }, { "productId": "blaxel", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Blaxel supports background-running infrastructure like Jobs for batch processing, scale-to-zero agents/sandboxes, and MCP-server hosting, which can enable autonomous background workloads, but there's no explicit documentation of a scheduling/trigger/automation-workflow feature (e.g., cron-like triggers or event-based automation) for setting up self-running automations. missing for 10: explicit scheduler/trigger mechanism for autonomous automations, independent/hands-on evidence of autonomous background execution, and clear docs describing 'automation' as a first-class concept rather than inferred from batch jobs and hosting.", "evidenceIds": [ "blaxel-docs-10", "blaxel-docs-11", "blaxel-docs-2", "blaxel-docs-9" ] }, { "productId": "blaxel", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Blaxel is an infrastructure platform for deploying/hosting AI agents, sandboxes, and MCP servers — it does not itself ship a built-in assistant that end-users converse with to delegate tasks; the 'skill' referenced is for external agents (e.g., Claude) to control Blaxel infra, not a chat assistant embedded in Blaxel's own UI.", "evidenceIds": [] }, { "productId": "blaxel", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Blaxel documents CLI with API keys usable in CI, and explicitly points to an official GitHub Action for CI pipeline integration, plus batch Jobs for automation-style workloads. However, there's no evidence of a full headless CI walkthrough, no independent corroboration, and no explicit CI examples for deploying/testing agents beyond the brief GitHub Action mention. Missing for 10: detailed CI workflow examples/docs, independent hands-on confirmation of CI usage, broader automation guidance beyond batch jobs.", "evidenceIds": [ "blaxel-docs-12", "blaxel-docs-13", "blaxel-docs-10", "blaxel-probe-4" ] }, { "productId": "blaxel", "storyId": "agentic-mcp-client", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Blaxel documents hosting MCP servers (docs-11) and exposing sandboxes as MCP servers (docs-5), and its skill can 'host MCP servers' for agents (docs-1), suggesting agents built on Blaxel can be wired to MCP tool servers. However there is no explicit documentation of connecting arbitrary third-party/external MCP servers into an agent so it can call their tools as a client — the evidence is server-hosting/exposure oriented rather than consumption-oriented. Missing for 10: explicit client-side MCP integration docs, examples of an agent consuming an external MCP server's tools, and independent confirmation this works in practice.", "evidenceIds": [ "blaxel-docs-1", "blaxel-docs-5", "blaxel-docs-11", "blaxel-probe-3" ] }, { "productId": "blaxel", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Blaxel documents that every sandbox is exposed via an official MCP server enabling agents to operate it via tool calls, and it offers a serverless 'MCP Server Hosting' service for hosting remote MCP servers, with a dedicated docs page confirmed by probe. This is a platform (not itself an agent), so the connect-via-official-MCP-server axis clearly applies and is met by first-party evidence. missing for 10: independent/hands-on confirmation of connecting an external agent to these MCP servers, and more detail on the protocol/setup beyond the docs summary.", "evidenceIds": [ "blaxel-docs-5", "blaxel-docs-11", "blaxel-probe-3" ] }, { "productId": "blaxel", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Blaxel offers an installable 'skill' that lets an agent deploy agents, create sandboxes, host MCP servers, and run jobs via simple prompts with zero code, directly supporting natural-language operation. However, this is scoped to agent-orchestration skills rather than a full natural-language interface across the entire product surface (e.g., CLI/dashboard operations still require explicit commands). Missing for 10: independent/hands-on verification of the skill's natural-language reliability, and evidence that all major product functions (not just deploy/sandbox/mcp/jobs) are controllable via natural language.", "evidenceIds": [ "blaxel-docs-1", "blaxel-docs-4", "blaxel-probe-3" ] }, { "productId": "blaxel", "storyId": "agentic-official-cli", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Blaxel has an official CLI documented at a dedicated CLI reference, integrable into CI via a GitHub Action, and used for API key auth alongside SDK/API — consistent with AI-native, agentic workflows (deploying agents, sandboxes, MCP servers, jobs). Missing for 10: no independent/hands-on corroboration of CLI usage, no detailed command list or examples in the evidence pack beyond the reference page's existence.", "evidenceIds": [ "blaxel-probe-4", "blaxel-docs-12", "blaxel-docs-13" ] }, { "productId": "blaxel", "storyId": "agentic-public-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm a public API exists and that API keys can drive it (used across CLI, SDK, and API) and CLI reference documentation is available, but a dedicated public API reference/OpenAPI spec could not be located (all openapi.json/swagger.json paths returned 404), so the story is only partially evidenced. missing for 10: a discoverable OpenAPI/API reference doc, hands-on example of calling the API directly, independent confirmation of API completeness.", "evidenceIds": [ "blaxel-docs-12", "blaxel-probe-2", "blaxel-probe-4" ] }, { "productId": "blaxel", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Blaxel issues generic API keys usable across its APIs, CLI, and SDK (blaxel-docs-12), and preview URLs can require an access token (blaxel-docs-8), but there is no documentation of scoping, role-based permissions, or least-privilege credential configuration for agents specifically.", "evidenceIds": [ "blaxel-docs-12", "blaxel-docs-8" ] }, { "productId": "blaxel", "storyId": "agentic-sdks", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Only indirect evidence exists that an SDK exists (API keys usable in \"Blaxel APIs, CLI and SDK\"), but there is no dedicated SDK documentation, language coverage, install instructions, or examples in the pack. missing for 10: dedicated SDK docs/reference, supported languages, code examples, independent corroboration of SDK usage.", "evidenceIds": [ "blaxel-docs-12" ] }, { "productId": "blaxel", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions webhooks, event subscriptions, or push notifications from Blaxel; documentation covers sandboxes, MCP hosting, jobs, and CLI/API but nothing about webhook subscription mechanisms.", "evidenceIds": [] }, { "productId": "blaxel", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The probe explicitly found no OpenAPI/Swagger spec at any candidate path (404s), and no evidence exists of an interactive API reference with runnable examples—only static markdown docs and CLI/skills references are documented.", "evidenceIds": [ "blaxel-probe-2" ] }, { "productId": "blaxel", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The probe explicitly checked common OpenAPI spec locations and found all returned 404, and no documentation item references a downloadable machine-readable API spec; only an llms.txt for documentation is present.", "evidenceIds": [ "blaxel-probe-2", "blaxel-probe-1" ] }, { "productId": "blaxel", "storyId": "api-sandbox", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Blaxel's core product is isolated sandbox environments for code execution, separate from production infra, with dedicated MCP-controlled sandbox instances, custom templates, and preview URLs, clearly enabling safe testing without touching production data. missing for 10: no explicit documentation contrasting sandbox vs production data isolation guarantees, and no independent/hands-on verification of data isolation beyond vendor docs.", "evidenceIds": [ "blaxel-docs-5", "blaxel-docs-7", "blaxel-docs-9", "blaxel-docs-8", "blaxel-docs-1" ] }, { "productId": "blaxel", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of API versioning scheme or a documented deprecation policy; the OpenAPI spec probe even returned 404s, and no docs mention version history or deprecation timelines.", "evidenceIds": [ "blaxel-probe-2" ] }, { "productId": "blaxel", "storyId": "auto-teardown-timeouts", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm automatic scale-to-zero after 5s of inactivity with fast resume, which directly supports idle shutdown and implies billing stops when inactive (blaxel-docs-2). However, there is no evidence of a developer-configurable timeout parameter (e.g., custom idle duration) or explicit confirmation that billing halts during scale-to-zero, and no independent/hands-on corroboration of this behavior. Missing for 10: explicit configurable timeout setting, billing-stop confirmation, independent verification.", "evidenceIds": [ "blaxel-docs-2" ] }, { "productId": "blaxel", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Blaxel's Jobs feature explicitly supports running many AI tasks in parallel via batch processing, and the Blaxel skill can invoke batch jobs via simple prompts, showing bulk-operation capability for AI-native users. However, the evidence gives no detail on limits, monitoring, or how a user would trigger operations across a large item set (e.g., bulk API endpoints), nor any independent/hands-on validation of batch performance. Missing for 10: concrete batch API/CLI examples, scale limits, and independent corroboration of bulk job execution.", "evidenceIds": [ "blaxel-docs-10", "blaxel-docs-1" ] }, { "productId": "blaxel", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Blaxel's evidence covers deployment, sandboxes, MCP hosting, batch jobs, and scale-to-zero infrastructure, but nothing describes a rules/trigger engine where users define conditions that automatically fire actions on events (e.g., webhooks, event-based automation rules). Since this is a plausible capability for an agent-hosting platform, absence of evidence means 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "blaxel", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Blaxel documents one-off batch 'Jobs' for parallel AI task processing (blaxel-docs-10) but no evidence shows support for recurring/scheduled triggers, cron-like scheduling, or workflow orchestration over time. missing for 10: scheduling/cron mechanism, recurring trigger configuration, workflow automation docs.", "evidenceIds": [ "blaxel-docs-10" ] }, { "productId": "blaxel", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of versioning, review workflows, or rollback capabilities for automations/agents/jobs; documentation covers deployment, sandboxes, scaling, and infra regions but nothing about tracking automation history or reverting changes.", "evidenceIds": [] }, { "productId": "blaxel", "storyId": "browser-in-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence describes sandboxes for code execution, MCP server exposure, filesystem mounting, and batch jobs, but no mention of headless browser support or full desktop/GUI environment capability inside a sandbox.", "evidenceIds": [] }, { "productId": "blaxel", "storyId": "concurrent-fleets", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs show sandboxes can run in parallel (Jobs batch processing, scale-to-zero/resume, regional pinning) suggesting fleet-scale operation, but no evidence anywhere states explicit concurrency limits, quotas, or maximum fleet sizes a platform-engineer could plan against. missing for 10: documented per-account/per-region concurrency limits, quota API or dashboard, load-test or benchmark data showing fleet scale in practice.", "evidenceIds": [ "blaxel-docs-2", "blaxel-docs-10", "blaxel-docs-6", "blaxel-docs-9" ] }, { "productId": "blaxel", "storyId": "custom-images-templates", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs explicitly reference 'Sandbox images' allowing creation of customized & reusable sandbox environments (blaxel-docs-7), which is direct evidence of custom template/image support. However, no evidence details how to bring an arbitrary Dockerfile/OCI image, base image customization workflow, or CLI/SDK commands for building/pushing custom images. missing for 10: concrete docs/examples on bring-your-own-container-image workflow, supported base image formats, and CLI commands for building/publishing custom sandbox templates.", "evidenceIds": [ "blaxel-docs-7" ] }, { "productId": "blaxel", "storyId": "expose-public-url", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs confirm sandboxes support preview URLs, including custom domains and private/token-protected URLs, directly enabling exposing a port to reach services running inside the sandbox via a public URL. Missing for 10: no explicit hands-on example showing the port-to-URL mapping mechanics or independent third-party confirmation.", "evidenceIds": [ "blaxel-docs-3", "blaxel-docs-8" ] }, { "productId": "blaxel", "storyId": "fast-cold-starts", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs claim resume from standby under 25ms even after weeks of inactivity, implying sub-second warm start, and scale-to-zero after 5s inactivity, which is documented performance behavior specific to sandboxes. However, this is vendor-claimed only with no independent benchmark or hands-on cold-start timing (first-boot from image) reported, and no numbers for true 'cold start' (first creation) vs resume-from-standby. Missing for 10: independent benchmarks/hands-on cold-start measurements, explicit distinction between first-time cold start and standby resume times.", "evidenceIds": [ "blaxel-docs-2", "blaxel-docs-7" ] }, { "productId": "blaxel", "storyId": "filesystem-io", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes sandboxes, MCP-based tool operation, and a separate 'Agent Drive' distributed filesystem, but nothing documents SDK methods for reading, writing, uploading, or downloading files within a sandbox. Missing for 10: SDK file I/O API reference, upload/download examples, hands-on confirmation of file operations in sandbox.", "evidenceIds": [ "blaxel-docs-5", "blaxel-docs-9", "blaxel-docs-7" ] }, { "productId": "blaxel", "storyId": "free-tier-start", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items mention pricing, free tier, included credits, or self-service signup without sales contact; all evidence covers technical features (sandboxes, MCP, jobs, regions).", "evidenceIds": [] }, { "productId": "blaxel", "storyId": "gpu-access", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item mentions GPU support, GPU instance types, or hardware acceleration for sandboxes; documentation covers scaling, preview URLs, filesystems, jobs, and MCP hosting but never GPU attachment. missing for 10: any mention of GPU-enabled sandbox instances, GPU pricing/config, or GPU-specific workload examples.", "evidenceIds": [] }, { "productId": "blaxel", "storyId": "hard-isolation-boundary", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes Blaxel's sandbox features (scale-to-zero, MCP exposure, preview URLs, templates) but never documents the underlying isolation mechanism (e.g., microVM, gVisor, container hardening) or any explicit security boundary guarantee between sandboxes and host systems. Missing for 10: any mention of microVM/hypervisor-based isolation, a security/architecture whitepaper describing the isolation boundary, or third-party/independent verification of sandbox isolation.", "evidenceIds": [ "blaxel-docs-5", "blaxel-docs-7", "blaxel-docs-8" ] }, { "productId": "blaxel", "storyId": "long-running-sessions", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs describe scale-to-zero after 5s inactivity with sub-25ms resume even after weeks, implying persistent state across long periods, but this describes a serverless suspend/resume model rather than an explicit 'keep running continuously for hours/days' capability, and there is no documentation of session duration limits or long-running execution during active agent work. missing for 10: explicit session/timeout limits, evidence of continuous multi-hour/day execution without suspension, independent confirmation of long-running task support.", "evidenceIds": [ "blaxel-docs-2" ] }, { "productId": "blaxel", "storyId": "multi-language-runtimes", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Blaxel's docs confirm sandboxes exist for code execution (blaxel-docs-1, blaxel-docs-7) and can be customized via templates, but there is no explicit evidence of specific language runtime support (Python, JavaScript, etc.) or of structured/rich result output formats from executed code. Missing for 10: explicit list of supported language runtimes, documentation of output/result format (e.g., stdout, images, rich objects), and any hands-on demonstration of multi-language execution.", "evidenceIds": [ "blaxel-docs-1", "blaxel-docs-5", "blaxel-docs-7" ] }, { "productId": "blaxel", "storyId": "network-egress-policy", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence describes any network egress control, firewall, or allow/deny-list policy for sandboxes—docs cover scaling, preview URLs, filesystem mounting, and regions but nothing about network access restriction. missing for 10: any mention of egress policy configuration, allow/deny lists, network isolation controls, or firewall rules for sandboxes.", "evidenceIds": [] }, { "productId": "blaxel", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Blaxel offers API keys usable across API, CLI, and SDK, and documents CLI/SDK for core resources (sandboxes, agents, jobs, MCP servers), suggesting broad programmatic control. However, no OpenAPI/spec was found (all candidate paths 404), and there is no explicit documentation claiming full UI/API feature parity. Missing for 10: a public OpenAPI reference, an explicit parity statement, and independent confirmation that all UI actions are exposed via API.", "evidenceIds": [ "blaxel-docs-12", "blaxel-probe-4", "blaxel-probe-2", "blaxel-docs-13" ] }, { "productId": "blaxel", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items address data export or portability in open formats, or any means to leave the platform with one's data; the closest is a migration skill for importing sandbox code from other providers, not exporting data out of Blaxel.", "evidenceIds": [] }, { "productId": "blaxel", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence indicates Blaxel's core source code is published under an open license; only one skill (migration skill) is mentioned as open source, but the product itself (platform/CLI/SDK) has no cited open-license repository or licensing statement.", "evidenceIds": [ "blaxel-docs-4" ] }, { "productId": "blaxel", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Blaxel is presented entirely as a serverless/managed cloud platform (scale-to-zero, regions, hosted MCP servers, hosted sandboxes) with no mention of a self-hostable core product, on-prem deployment, or open-source server component that a user could run themselves; only a migration skill and CLI/SDK for interacting with the hosted service are documented.", "evidenceIds": [ "blaxel-docs-2", "blaxel-docs-6", "blaxel-docs-11", "blaxel-docs-14" ] }, { "productId": "blaxel", "storyId": "pause-resume-state", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs mention automatic scale-to-zero after inactivity with resume from standby in under 25ms 'even after weeks' (blaxel-docs-2), implying state is preserved across pause/resume cycles, and Agent Drive provides a persistent distributed filesystem mountable to sandboxes (blaxel-docs-9). However, there is no explicit developer-triggered pause/resume API, and no confirmation that in-memory process state (not just filesystem) is preserved. missing for 10: explicit manual pause/resume command, explicit statement that memory/process state (not just filesystem) survives resume, independent verification of the claim.", "evidenceIds": [ "blaxel-docs-2", "blaxel-docs-9" ] }, { "productId": "blaxel", "storyId": "per-second-billing", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs describe scale-to-zero after 5s of inactivity and fast resume (blaxel-docs-2), implying sandboxes only consume compute while active, which is suggestive of fine-grained usage-based billing, but no evidence explicitly states a per-second pricing/billing model or shows pricing pages confirming this granularity. missing for 10: explicit pricing/billing documentation stating per-second metering, evidence of actual invoicing granularity, independent confirmation of cost savings from scale-to-zero.", "evidenceIds": [ "blaxel-docs-2" ] }, { "productId": "blaxel", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm users can pin agents/MCP servers to a specific region via blaxel.toml and that deployment region choice is sometimes available, which directly supports data residency control. Missing for 10: a full list of available regions, compliance/certification details (e.g., GDPR, SOC2 residency guarantees), and independent confirmation that this pinning is enforced for all data at rest (not just compute location).", "evidenceIds": [ "blaxel-docs-6", "blaxel-docs-14" ] }, { "productId": "blaxel", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item addresses data usage for AI model training, opt-out policies, or privacy commitments regarding training data; this is an infrastructure/agent-hosting product where such a policy could exist but is undocumented here. Missing for 10: any privacy policy or ToS statement on training data usage, opt-out mechanism, or data retention controls.", "evidenceIds": [] }, { "productId": "blaxel", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data retention policies, deletion controls, or data lifecycle management for user data/sandboxes; the docs focus on infrastructure, scaling, and access tokens rather than privacy/retention controls.", "evidenceIds": [] }, { "productId": "blaxel", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry, usage tracking, or any opt-out mechanism; the docs cover deployment, sandboxes, MCP hosting, and security tokens but nothing about data collection settings. missing for 10: any documentation or setting related to telemetry collection or opt-out, privacy policy mention, or usage-tracking configuration.", "evidenceIds": [] }, { "productId": "blaxel", "storyId": "run-coding-agents-inside", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes Blaxel sandboxes, MCP exposure, skills, and job/batch features, but nowhere mentions a documented recipe or guide for running Claude Code or Codex specifically inside a Blaxel sandbox. Missing for 10: any vendor recipe/tutorial naming Claude Code or Codex, or a hands-on report of doing so.", "evidenceIds": [ "blaxel-docs-5", "blaxel-docs-7", "blaxel-docs-1" ] }, { "productId": "blaxel", "storyId": "run-untrusted-code", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Blaxel provides sandboxes explicitly for isolated code execution, exposed via MCP tool calls, with scale-to-zero/resume infrastructure meaning execution happens off the developer's own infra, plus custom sandbox images and private preview URLs with tokens for security. Missing for 10: independent/hands-on evidence of isolation guarantees (e.g., security audits, escape testing) and explicit detail on resource/network sandboxing limits beyond docs claims.", "evidenceIds": [ "blaxel-docs-5", "blaxel-docs-7", "blaxel-docs-2", "blaxel-docs-8", "blaxel-docs-1" ] }, { "productId": "blaxel", "storyId": "secrets-stay-out", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Blaxel provides isolated sandboxes for code execution with MCP-based control and access tokens for API/CLI/SDK auth, implying separation between host credentials and sandbox execution, but there's no explicit documentation stating host secrets/credentials are unreachable by sandbox code or describing secret-injection isolation guarantees. missing for 10: explicit secret-isolation architecture docs, third-party security audit or penetration-test evidence, and confirmation that host credentials cannot leak into sandbox runtime.", "evidenceIds": [ "blaxel-docs-1", "blaxel-docs-5", "blaxel-docs-7", "blaxel-docs-12" ] }, { "productId": "blaxel", "storyId": "shell-and-packages", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Blaxel's sandboxes are documented as code-execution environments exposed via MCP tool calls, and customizable sandbox images imply support for installing packages/dependencies, but no evidence explicitly shows arbitrary shell command execution or package installation examples/APIs. missing for 10: explicit shell/exec command documentation, package manager install examples, hands-on demonstration of arbitrary command execution.", "evidenceIds": [ "blaxel-docs-1", "blaxel-docs-5", "blaxel-docs-7" ] }, { "productId": "blaxel", "storyId": "snapshot-restore-fork", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence covers sandbox creation, scale-to-zero/resume, preview URLs, templates, and filesystem mounting, but nowhere mentions snapshotting a sandbox's state or forking new sandboxes from a saved snapshot. Sandbox images/templates ([blaxel-docs-7]) are pre-built environments, not runtime snapshots of a live sandbox's state.", "evidenceIds": [ "blaxel-docs-2", "blaxel-docs-7", "blaxel-docs-9" ] }, { "productId": "blaxel", "storyId": "spinup-via-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm sandboxes are a core primitive with fast resume (<25ms from standby), customizable/reusable images, API/CLI/SDK access via API keys, and MCP-server exposure for tool control, all pointing to programmatic creation. However, there's no explicit example or evidence of a single one-call SDK/API invocation creating a sandbox with a concrete latency claim for cold creation (only resume-from-standby timing is given), and no independent/hands-on confirmation of 'seconds to live environment' for a fresh (non-cached) sandbox. missing for 10: explicit one-call code sample creating a sandbox, documented cold-start provisioning latency, independent verification of speed claims.", "evidenceIds": [ "blaxel-docs-2", "blaxel-docs-7", "blaxel-docs-12", "blaxel-docs-5", "blaxel-probe-1" ] }, { "productId": "cloudflare-sandbox", "storyId": "agent-self-provisions", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs clearly show an agent-driven lifecycle: getSandbox() to provision/reconnect, exec()/interpreter code execution with stdout/stderr capture, and readFile to retrieve results — all callable programmatically without a human in the loop. However, there is no documented explicit 'destroy/teardown' API, and community feedback confirms there's no built-in idle-timeout auto-destroy, forcing developers to implement their own cleanup logic. Missing for 10: a documented sandbox.destroy()/terminate call or auto-expiry mechanism, and independent hands-on confirmation of a full provision-execute-teardown loop working end-to-end.", "evidenceIds": [ "cloudflare-sandbox-docs-37", "cloudflare-sandbox-docs-1", "cloudflare-sandbox-docs-3", "cloudflare-sandbox-docs-2", "cloudflare-sandbox-docs-11", "cloudflare-sandbox-comm-7" ] }, { "productId": "cloudflare-sandbox", "storyId": "agentic-agent-docs", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "A probe confirms Cloudflare's developer docs expose a working llms.txt (HTTP 200) covering the whole docs site including Sandbox, giving agents a structured entry point. However, a probe for the markdown version of the specific Sandbox get-started page returned 404, showing per-page agent-friendly docs aren't uniformly available. Missing for 10: dedicated llms.txt or .md endpoints scoped to the Sandbox product docs, and no community/hands-on confirmation of agents actually consuming it successfully.", "evidenceIds": [ "cloudflare-sandbox-probe-1", "cloudflare-sandbox-probe-2" ] }, { "productId": "cloudflare-sandbox", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Cloudflare Sandboxes is a code-execution/infrastructure primitive (VM-isolated containers, file I/O, command exec) for building agents or apps, not a product with its own dataset or built-in AI insight/suggestion features over user data. The evidence pack shows only execution/session/file/tunnel APIs — no AI-generated insights or analysis output is a first-party product feature, making this axis a category mismatch rather than a gap.", "evidenceIds": [ "cloudflare-sandbox-docs-24", "cloudflare-sandbox-docs-34" ] }, { "productId": "cloudflare-sandbox", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show sandboxes can run background processes, maintain persistent sessions/state, and reconnect via stable IDs (docs-43, docs-9, docs-37, docs-32), which supports basic autonomous background execution. However, there's no documented native scheduling/trigger mechanism for kicking off automations, and community feedback notes the lack of auto-idle cleanup means developers must manually manage sandbox lifecycle rather than it running fully autonomously (comm-7). Missing for 10: built-in cron/trigger scheduling, evidence of true unattended long-running automation, and confirmation that background processes persist reliably without manual intervention.", "evidenceIds": [ "cloudflare-sandbox-docs-43", "cloudflare-sandbox-docs-9", "cloudflare-sandbox-docs-37", "cloudflare-sandbox-docs-32", "cloudflare-sandbox-comm-7" ] }, { "productId": "cloudflare-sandbox", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Cloudflare Sandboxes is an infrastructure/execution-environment product (isolated containers/VMs for running code, files, commands) used by developers to build agents, not a product that itself ships a built-in AI assistant UI for end users to delegate tasks to. The 'delegate tasks to a built-in AI assistant' story is a category error for this kind of infra tool.", "evidenceIds": [ "cloudflare-sandbox-docs-24", "cloudflare-sandbox-docs-34" ] }, { "productId": "cloudflare-sandbox", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Cloudflare Sandboxes are SDK/Workers-driven (exec, code execution, git operations, bindings access) which is inherently headless/programmatic and well-suited to CI/CD automation, and docs explicitly mention CI/CD as a target use case. However there's no dedicated CLI, no CI integration examples (GitHub Actions, etc.), and no evidence of auto-cleanup/idle-timeout controls needed for reliable CI runs — community notes explicitly flag the lack of auto-destroy on idle, requiring manual cleanup logic. missing for 10: explicit CI/CD pipeline examples or integrations, a standalone CLI for headless invocation, documented idle-timeout/auto-teardown controls, and independent verification of headless automation at scale.", "evidenceIds": [ "cloudflare-sandbox-docs-24", "cloudflare-sandbox-docs-1", "cloudflare-sandbox-docs-22", "cloudflare-sandbox-docs-20", "cloudflare-sandbox-comm-7" ] }, { "productId": "cloudflare-sandbox", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "cloudflare-sandbox", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Cloudflare Sandboxes is an execution/infrastructure SDK (not itself an agent), so an official MCP server for agent connectivity is a fair axis to ask about—but the evidence pack contains no mention of MCP, an MCP server, or any protocol for connecting AI agents beyond raw SDK APIs (exec, files, sessions, tunnels, etc.).", "evidenceIds": [] }, { "productId": "cloudflare-sandbox", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Cloudflare Sandboxes is a programmatic SDK (sandbox.exec(), writeFile(), createSession(), etc.) intended to be called from Worker code, not a product with a natural-language command interface; while it markets itself for building AI agents (docs-24), there is no evidence of a chat/NL layer for operating the sandbox itself. missing for 10: any NL command parsing/interface, evidence of a conversational control layer, examples of natural-language-driven sandbox operations.", "evidenceIds": [ "cloudflare-sandbox-docs-1", "cloudflare-sandbox-docs-24", "cloudflare-sandbox-docs-43" ] }, { "productId": "cloudflare-sandbox", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack shows the Sandbox SDK is an API/library used from Workers code (exec, files, sessions, etc.) but nowhere mentions an official CLI tool for AI-native workflows; interaction is entirely via SDK calls or the general Wrangler CLI, not a dedicated Sandbox CLI.", "evidenceIds": [] }, { "productId": "cloudflare-sandbox", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Cloudflare publishes extensive, structured API documentation for the Sandbox SDK (exec, file I/O, sessions, code interpreter, ports/tunnels, watch, snapshots, git, Docker, bindings) explicitly marketed for building AI agents that execute code (docs-24, docs-34). The docs are machine-readable (llms.txt, openapi.json probes return 200) and cover the full surface an AI-native developer would need to drive.\nmissing for 10: independent hands-on confirmation that agents successfully integrate via this API in production, and a dedicated REST/OpenAPI spec for the sandbox endpoints themselves (only the SDK is documented, not a standalone HTTP API schema).", "evidenceIds": [ "cloudflare-sandbox-docs-1", "cloudflare-sandbox-docs-24", "cloudflare-sandbox-docs-34", "cloudflare-sandbox-docs-9", "cloudflare-sandbox-docs-42", "cloudflare-sandbox-probe-1", "cloudflare-sandbox-probe-3" ] }, { "productId": "cloudflare-sandbox", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents sandbox execution, filesystem, networking, and session features but contains no mention of scoped or least-privilege API credential/token issuance for agents accessing the sandbox APIs; in fact, community feedback notes egress control is 'all or nothing' rather than fine-grained, suggesting limited scoping capability overall.", "evidenceIds": [ "cloudflare-sandbox-comm-2" ] }, { "productId": "cloudflare-sandbox", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Cloudflare publishes an extensive official SDK (TypeScript/JavaScript) with detailed docs covering exec, file I/O, sessions, code interpreter for Python/JS, port exposure, snapshots, and bindings, explicitly positioned for building AI agents that execute code. Community commentary discusses pricing/latency but does not dispute the SDK's functionality or documentation quality. Missing for 10: independent hands-on developer reviews of the SDK's ergonomics/reliability, and confirmation of multi-language SDK parity beyond JS/TS/Python.", "evidenceIds": [ "cloudflare-sandbox-docs-1", "cloudflare-sandbox-docs-3", "cloudflare-sandbox-docs-9", "cloudflare-sandbox-docs-24", "cloudflare-sandbox-docs-42", "cloudflare-sandbox-docs-37" ] }, { "productId": "cloudflare-sandbox", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack of a webhook subscription mechanism for sandbox events; the SDK exposes file watching, SSE streams, and callback-based output but nothing about registering external webhook endpoints for event notifications.", "evidenceIds": [] }, { "productId": "cloudflare-sandbox", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows extensive static code snippets across Cloudflare's Sandbox docs (exec, file I/O, sessions, interpreter, etc.) but nothing indicates an interactive, in-browser 'try it now' API reference or runnable playground — the openapi.json probe is for Cloudflare's general API, not a Sandbox-specific interactive explorer. Missing for 10: an in-page runnable code sandbox/playground, an interactive API explorer (e.g., Swagger/Redoc try-it-out), or any embedded live execution environment tied to the docs.", "evidenceIds": [ "cloudflare-sandbox-docs-1", "cloudflare-sandbox-docs-27", "cloudflare-sandbox-probe-3", "cloudflare-sandbox-probe-2" ] }, { "productId": "cloudflare-sandbox", "storyId": "api-machine-spec", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Cloudflare's developer platform exposes a general openapi.json (HTTP 200, containing an 'openapi' key), but there's no evidence this spec specifically documents the Sandbox SDK's methods (exec, writeFile, sessions, etc.), which is a code-level SDK rather than a REST API; a dedicated machine-readable doc for the sandbox pages itself returned 404. Missing for 10: confirmation that the openapi.json actually covers Sandbox-specific endpoints/methods, and a dedicated llms.txt/OpenAPI export scoped to the Sandbox SDK docs.", "evidenceIds": [ "cloudflare-sandbox-probe-3", "cloudflare-sandbox-probe-2", "cloudflare-sandbox-probe-1" ] }, { "productId": "cloudflare-sandbox", "storyId": "api-sandbox", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Cloudflare Sandboxes are explicitly designed as isolated VM/container environments for running untrusted code, separate from production infra, with sessions, snapshots, and opt-in bindings to access KV/R2/Durable Objects only when explicitly wired in — matching the AI-native 'test without touching production' need well (docs-34, docs-38, docs-50, docs-20, docs-24). Community feedback doesn't contradict the isolation guarantee itself, though it flags coarse outbound network controls and unclear persistence semantics as caveats (cloudflare-sandbox-comm-2, cloudflare-sandbox-comm-4). Missing for 10: independent hands-on validation that no production data leakage occurs in practice, and finer-grained egress/network isolation controls.", "evidenceIds": [ "cloudflare-sandbox-docs-34", "cloudflare-sandbox-docs-38", "cloudflare-sandbox-docs-50", "cloudflare-sandbox-docs-20", "cloudflare-sandbox-docs-24", "cloudflare-sandbox-comm-2", "cloudflare-sandbox-comm-4" ] }, { "productId": "cloudflare-sandbox", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains extensive API documentation for the Sandbox SDK's features (exec, files, sessions, snapshots, etc.) but nowhere mentions API versioning schemes, version numbers, or a documented deprecation policy for the SDK/API surface. No evidence of semver commitments, changelogs with deprecation notices, or migration guides for breaking changes.", "evidenceIds": [] }, { "productId": "cloudflare-sandbox", "storyId": "auto-teardown-timeouts", "verdict": "disputed", "quality": 3, "confidence": "low", "rationale": "Docs only describe a per-command timeout within sessions (cloudflare-sandbox-docs-19), not an idle-shutdown/auto-destroy mechanism for the whole sandbox tied to billing; one community comment claims the underlying Container platform auto-shuts down after no requests (cloudflare-sandbox-comm-5), but a hands-on developer explicitly states it does not seem possible to auto-destroy a sandbox after an idle period, forcing manual cleanup (cloudflare-sandbox-comm-7). This is a direct contradiction between vendor-adjacent claims and hands-on experience. Missing for 10: explicit first-party docs on idle-timeout auto-shutdown/destroy tied to billing, and independent confirmation it works as described.", "evidenceIds": [ "cloudflare-sandbox-docs-19", "cloudflare-sandbox-comm-5", "cloudflare-sandbox-comm-7" ] }, { "productId": "cloudflare-sandbox", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Cloudflare Sandboxes supports concurrent/parallel operations via a multiplexed connection that 'avoids subrequest limits when making many concurrent operations' (docs-17) and allows multiple isolated sessions per sandbox for parallel workflows (docs-9, docs-18, docs-39), plus getSandbox() to manage many sandbox instances by ID. However there's no documented batch/bulk API primitive for iterating over large item sets, and community feedback flags unclear resource limits and high per-vCPU pricing that would constrain large-scale bulk workloads (comm-3, comm-6, comm-11). Missing for 10: an explicit bulk/batch operation API or example, documented concurrency/resource ceilings, and independent evidence of successful large-scale parallel execution.", "evidenceIds": [ "cloudflare-sandbox-docs-17", "cloudflare-sandbox-docs-9", "cloudflare-sandbox-docs-18", "cloudflare-sandbox-docs-39", "cloudflare-sandbox-docs-37", "cloudflare-sandbox-comm-3", "cloudflare-sandbox-comm-6", "cloudflare-sandbox-comm-11" ] }, { "productId": "cloudflare-sandbox", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "The SDK provides a file-system watch capability with an `onEvent` callback (inotify-based) that lets developers trigger custom actions when files change, which is a form of event-driven automation, but this is a narrow, code-level primitive rather than a declarative rules/automation engine. Missing for 10: no declarative rule-definition mechanism, no support for triggering on non-filesystem events (e.g., scheduled/cron, external webhooks, custom event types), and no evidence of a rules/condition-action system beyond raw watcher callbacks.", "evidenceIds": [ "cloudflare-sandbox-docs-4", "cloudflare-sandbox-docs-26", "cloudflare-sandbox-docs-33" ] }, { "productId": "cloudflare-sandbox", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes on-demand sandbox execution, sessions, and lifecycle management, but nowhere mentions cron-style scheduling, recurring triggers, or workflow orchestration for repeated runs; community threads even highlight the lack of automatic idle/cleanup logic, implying no built-in scheduling primitives.", "evidenceIds": [ "cloudflare-sandbox-docs-9", "cloudflare-sandbox-docs-18", "cloudflare-sandbox-comm-7" ] }, { "productId": "cloudflare-sandbox", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Sandboxes offer git clone/branch operations for code and R2-based directory snapshot/restore, which give raw building blocks for versioning and rolling back sandbox state, but there is no dedicated 'automation' versioning, review, or rollback UI/workflow — it's up to the developer to wire this together. Missing for 10: no built-in review/diff tooling for automations, no automatic rollback trigger, no evidence of a workflow-specific version history feature, and no independent confirmation these primitives are actually used for automation rollback in practice.", "evidenceIds": [ "cloudflare-sandbox-docs-22", "cloudflare-sandbox-docs-48", "cloudflare-sandbox-docs-8", "cloudflare-sandbox-docs-7" ] }, { "productId": "cloudflare-sandbox", "storyId": "browser-in-sandbox", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack details shell execution, file I/O, sessions, tunnels, snapshots, and code interpretation, but nowhere mentions running a headless browser (e.g., Playwright/Puppeteer) or a full desktop/GUI environment inside the sandbox. This is a reasonable capability to ask of a code-execution sandbox product, but no docs or community evidence confirm it exists.", "evidenceIds": [] }, { "productId": "cloudflare-sandbox", "storyId": "concurrent-fleets", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Docs describe per-sandbox features (sessions, exec, files) and one architectural note that multiplexing avoids subrequest limits, but there is no documentation of concurrency limits, fleet-level scaling guidance, or how many sandboxes/instances can run concurrently. Community comments explicitly flag unanswered questions about resource limits and lack of auto-cleanup, reinforcing the absence of concrete concurrency documentation.", "evidenceIds": [ "cloudflare-sandbox-docs-17", "cloudflare-sandbox-comm-11", "cloudflare-sandbox-comm-7" ] }, { "productId": "cloudflare-sandbox", "storyId": "custom-images-templates", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes running commands, files, sessions, Python/JS execution, tunnels, and Docker-command execution inside a sandbox, but nothing documents defining a custom sandbox template or supplying a custom base container image for the sandbox environment itself. Since bring-your-own-image is a standard, expected axis for a container/sandbox product, the lack of any supporting evidence yields 'none' rather than 'na'.", "evidenceIds": [ "cloudflare-sandbox-docs-21", "cloudflare-sandbox-docs-24", "cloudflare-sandbox-docs-38", "cloudflare-sandbox-docs-50" ] }, { "productId": "cloudflare-sandbox", "storyId": "expose-public-url", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly describe exposing a sandbox port and getting a public preview URL, including zero-config *.trycloudflare.com URLs via sandbox.tunnels.get(port) and Worker-side routing of HTTP/WebSocket requests to the correct sandbox container, directly matching the story. Missing for 10: independent/hands-on confirmation of preview URL reliability beyond first-party docs.", "evidenceIds": [ "cloudflare-sandbox-docs-5", "cloudflare-sandbox-docs-6", "cloudflare-sandbox-docs-31", "cloudflare-sandbox-docs-36", "cloudflare-sandbox-docs-46", "cloudflare-sandbox-docs-15" ] }, { "productId": "cloudflare-sandbox", "storyId": "fast-cold-starts", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No first-party Cloudflare documentation in the evidence pack states specific cold-start timing (sub-second to few-second) for sandboxes; the community threads discuss competitors' or ambiguous Firecracker boot times without confirming Cloudflare's own documented cold-start figures. missing for 10: official docs quantifying cold-start latency, benchmark data, or a first-party performance page addressing startup time.", "evidenceIds": [ "cloudflare-sandbox-comm-1", "cloudflare-sandbox-comm-8" ] }, { "productId": "cloudflare-sandbox", "storyId": "filesystem-io", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly cover readFile/writeFile/mkdir with absolute paths, streaming/binary uploads >32MiB via ReadableStream, and directory watching, giving comprehensive first-party coverage of file read/write/upload/download via the SDK. Missing for 10: independent hands-on developer confirmation of file upload/download behavior (community discussion focuses on persistence questions, not concrete failures).", "evidenceIds": [ "cloudflare-sandbox-docs-2", "cloudflare-sandbox-docs-14", "cloudflare-sandbox-docs-30", "cloudflare-sandbox-docs-44", "cloudflare-sandbox-docs-45", "cloudflare-sandbox-docs-25" ] }, { "productId": "cloudflare-sandbox", "storyId": "free-tier-start", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents extensive product capabilities and self-serve pricing figures (e.g., per-vCPU costs debated in community threads) but contains no mention of a free tier, free trial credits, or explicit self-serve signup flow that lets a developer start without contacting sales. missing for 10: explicit free-tier/credit offer, signup flow documentation, confirmation no sales contact is required.", "evidenceIds": [ "cloudflare-sandbox-comm-3", "cloudflare-sandbox-comm-6" ] }, { "productId": "cloudflare-sandbox", "storyId": "gpu-access", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the docs or community discussion mentions GPU support, GPU attachment, or GPU-accelerated workloads for Cloudflare Sandboxes; the product is described purely in terms of CPU-based containers/VMs for code execution.", "evidenceIds": [] }, { "productId": "cloudflare-sandbox", "storyId": "hard-isolation-boundary", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Cloudflare's own docs explicitly state 'VM-based isolation - Each sandbox runs in its own VM' and that each sandbox 'runs in its own isolated container with a full Linux environment, providing strong security boundaries' for running untrusted code (docs-38, docs-50, docs-34). This gives a documented isolation boundary, though the docs mix 'VM' and 'container' language rather than naming a specific microVM technology, and community discussion raises open questions about network egress granularity rather than disputing the isolation claim itself. Missing for 10: a clearer, consistent technical description of the isolation mechanism (e.g., naming Firecracker or similar) and independent third-party security audit/corroboration.", "evidenceIds": [ "cloudflare-sandbox-docs-38", "cloudflare-sandbox-docs-50", "cloudflare-sandbox-docs-34", "cloudflare-sandbox-comm-2" ] }, { "productId": "cloudflare-sandbox", "storyId": "long-running-sessions", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Docs describe stable-ID reconnection (getSandbox) and sessions with persistent state/env, suggesting sandboxes could be resumed across long tasks, but community reports explicitly say containers auto-shutdown after idle periods, that there's no built-in idle-timeout control (forcing manual cleanup), and that persistence semantics during idle states are unclear — directly undercutting reliable multi-hour/day continuous operation for unattended agent tasks. Missing for 10: first-party guidance on running sessions continuously for hours/days without disconnection, clear idle-timeout/keep-alive controls, and independent confirmation of long-duration reliability.", "evidenceIds": [ "cloudflare-sandbox-docs-9", "cloudflare-sandbox-docs-18", "cloudflare-sandbox-docs-37", "cloudflare-sandbox-docs-19", "cloudflare-sandbox-comm-4", "cloudflare-sandbox-comm-5", "cloudflare-sandbox-comm-7" ] }, { "productId": "cloudflare-sandbox", "storyId": "multi-language-runtimes", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs clearly document a code interpreter API executing Python, JavaScript, and TypeScript with rich outputs (charts, tables, dataframes, stateful contexts), plus shell exec with stdout/stderr capture, matching the story directly. Missing for 10: independent/hands-on verification of the rich-output rendering quality and broader language support beyond the three named runtimes.", "evidenceIds": [ "cloudflare-sandbox-docs-3", "cloudflare-sandbox-docs-42", "cloudflare-sandbox-docs-27", "cloudflare-sandbox-docs-11", "cloudflare-sandbox-docs-1", "cloudflare-sandbox-docs-41" ] }, { "productId": "cloudflare-sandbox", "storyId": "network-egress-policy", "verdict": "disputed", "quality": 3, "confidence": "medium", "rationale": "Docs mention only a mechanism to 'Intercept and handle outbound HTTP from sandboxes using Workers' (docs-23), which is not the same as a declarative allow/deny egress policy, and no docs describe configurable network egress rules. A named community commenter explicitly states outbound network control is 'all or nothing right now' with no fine-grained control available, directly contradicting the story's requirement for explicit egress policy. missing for 10: documented egress allow/deny policy API, first-party confirmation of granular network restriction, and any independent verification that such a policy exists.", "evidenceIds": [ "cloudflare-sandbox-docs-23", "cloudflare-sandbox-comm-2" ] }, { "productId": "cloudflare-sandbox", "storyId": "openness-api-parity", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Cloudflare Sandboxes is an API/SDK-first product with no separate management UI described in the evidence; every capability (exec, file I/O, sessions, git, Docker, port exposure, snapshots, code interpreter, filesystem watch, even the browser terminal) is exposed and built entirely through the SDK/API, and the terminal UI itself is just a WebSocket client wired to the same API. Missing for 10: no explicit statement contrasting a Cloudflare dashboard GUI with the API, and no independent/third-party confirmation that no UI-exclusive controls exist (e.g., resource limits raised as unclear in community threads).", "evidenceIds": [ "cloudflare-sandbox-docs-1", "cloudflare-sandbox-docs-10", "cloudflare-sandbox-docs-21", "cloudflare-sandbox-docs-22", "cloudflare-sandbox-docs-48", "cloudflare-sandbox-comm-11" ] }, { "productId": "cloudflare-sandbox", "storyId": "openness-full-export", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "The SDK exposes standard file I/O (readFile/writeFile), S3-compatible bucket mounting (R2/S3/GCS), and snapshot/restore to R2, which are open, non-proprietary formats a user could use to extract their data. However, there is no documented bulk 'export all my data' feature, no account-level portability tooling, and no independent confirmation that a user can fully migrate off the platform with all state intact. missing for 10: explicit data-export/account-portability feature, independent verification of full data extraction, documentation on non-file state (e.g., session/process state) portability.", "evidenceIds": [ "cloudflare-sandbox-docs-2", "cloudflare-sandbox-docs-7", "cloudflare-sandbox-docs-8", "cloudflare-sandbox-docs-44", "cloudflare-sandbox-docs-48" ] }, { "productId": "cloudflare-sandbox", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions an open-source license or a public source repository for the Cloudflare Sandbox SDK; all citations are product docs, pricing/feature discussions, or probes, none addressing source availability or licensing.", "evidenceIds": [] }, { "productId": "cloudflare-sandbox", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Cloudflare Sandboxes is built entirely on Cloudflare's proprietary Workers/Durable Objects/container infrastructure, and no evidence in the pack mentions any open-source release, self-hosted deployment option, or ability to run the core product outside Cloudflare's platform. Community discussion focuses on pricing, cold-start times, and resource limits, not self-hosting.", "evidenceIds": [ "cloudflare-sandbox-docs-16", "cloudflare-sandbox-docs-38", "cloudflare-sandbox-comm-6" ] }, { "productId": "cloudflare-sandbox", "storyId": "pause-resume-state", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm directory-level snapshot/restore to R2 ('Create point-in-time snapshots of directories and restore them from R2') and idle-state persistence claims, giving filesystem-level pause/resume support, but there's no documented mechanism for preserving in-memory/process execution state across a pause, and community discussion notes ambiguity about exactly how idle-state persistence works. missing for 10: explicit memory/process-state preservation on pause, clear docs on what 'idle state' persistence actually preserves, independent verification that restored sandboxes resume mid-execution rather than just filesystem contents.", "evidenceIds": [ "cloudflare-sandbox-docs-8", "cloudflare-sandbox-docs-48", "cloudflare-sandbox-docs-7", "cloudflare-sandbox-comm-4" ] }, { "productId": "cloudflare-sandbox", "storyId": "per-second-billing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No documentation in the evidence pack describes per-second or usage-based billing for sandbox compute; the only pricing-related evidence is community commentary describing flat, expensive per-vCPU pricing (comm-3, comm-6) and the absence of automatic idle shutdown, meaning engineers must build their own cleanup to avoid being billed for idle sandboxes (comm-7). This contradicts rather than supports the story of paying only for actual per-second compute use.", "evidenceIds": [ "cloudflare-sandbox-comm-3", "cloudflare-sandbox-comm-6", "cloudflare-sandbox-comm-7" ] }, { "productId": "cloudflare-sandbox", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack discusses data residency, region selection, or storage location controls for Cloudflare Sandboxes; docs focus on execution, files, sessions, tunnels, and R2 snapshots without any region/residency configuration options mentioned.", "evidenceIds": [] }, { "productId": "cloudflare-sandbox", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Cloudflare Sandboxes is a developer infrastructure product for running isolated code execution environments, not a consumer-facing AI assistant/chat product with a data-training opt-out policy; the evidence covers execution, filesystem, networking, and architecture features but nothing about AI model training data usage or opt-out controls. This axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "cloudflare-sandbox", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The docs describe file read/write, snapshot/restore to R2, and bucket mounting, but there is no documented mechanism for users to control data retention windows or guarantee deletion of sandbox data. Community evidence explicitly states 'it does not seem to be possible to auto destroy a sandbox after certain amount of idle time... forces whoever is implementing this to do their own cleanup,' and persistence semantics are described as unclear, indicating the product does not provide the retention/deletion controls an AI-native user would need.", "evidenceIds": [ "cloudflare-sandbox-comm-7", "cloudflare-sandbox-comm-4", "cloudflare-sandbox-docs-8", "cloudflare-sandbox-docs-7" ] }, { "productId": "cloudflare-sandbox", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses telemetry, usage tracking, or opt-out settings for Cloudflare Sandboxes; docs focus purely on execution, filesystem, sessions, and networking features. This is a fair privacy-posture question for a developer platform product, so absence of evidence yields 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "cloudflare-sandbox", "storyId": "run-coding-agents-inside", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The docs describe Sandboxes as ideal for building AI agents and provide generic exec/git/docker primitives, but there is no vendor-published recipe or guide specifically for running Claude Code or Codex inside a Sandbox. missing for 10: an explicit vendor tutorial/recipe naming Claude Code or Codex, any first-party integration guide, and community confirmation that such agents run inside the sandbox.", "evidenceIds": [ "cloudflare-sandbox-docs-24", "cloudflare-sandbox-docs-21", "cloudflare-sandbox-docs-22" ] }, { "productId": "cloudflare-sandbox", "storyId": "run-untrusted-code", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Cloudflare's docs explicitly market Sandboxes for running untrusted, AI-generated code with VM/container-based isolation, file/exec APIs, and strong security boundaries (cloudflare-sandbox-docs-34, -38, -50, -24). However, hands-on community feedback flags real gaps for this exact use case: outbound network access is 'all or nothing' with no fine-grained egress control, and there's no built-in auto-destroy for idle sandboxes, forcing manual cleanup. Missing for 10: granular network egress control, automatic idle sandbox teardown, independent security audit of the VM isolation claims.", "evidenceIds": [ "cloudflare-sandbox-docs-34", "cloudflare-sandbox-docs-38", "cloudflare-sandbox-docs-50", "cloudflare-sandbox-docs-24", "cloudflare-sandbox-comm-2", "cloudflare-sandbox-comm-7" ] }, { "productId": "cloudflare-sandbox", "storyId": "secrets-stay-out", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Cloudflare documents strong process/VM isolation ('VM-based isolation', 'isolated container with a full Linux environment', 'run untrusted code securely in isolated environments') which implies the host's runtime and secrets aren't directly reachable by sandboxed code, and bindings (KV/R2/DO) are explicitly scoped and passed in rather than ambient. However, there is no explicit documentation describing how host secrets/credentials are excluded or scrubbed from the sandbox environment, and community feedback notes only coarse-grained outbound network control ('all or nothing') rather than fine-grained secret/credential access restrictions, leaving open questions about exfiltration vectors. Missing for 10: explicit documentation on secret/credential isolation guarantees, independent security audit or hands-on test confirming host credentials are unreachable, and finer-grained egress/network controls to prevent secret exfiltration.", "evidenceIds": [ "cloudflare-sandbox-docs-34", "cloudflare-sandbox-docs-38", "cloudflare-sandbox-docs-50", "cloudflare-sandbox-docs-20", "cloudflare-sandbox-comm-2" ] }, { "productId": "cloudflare-sandbox", "storyId": "shell-and-packages", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly document sandbox.exec() for running arbitrary shell commands with stdout/stderr/exit codes, stdin support, streaming/SSE output, background processes, plus Docker command execution and Git operations inside the sandbox, giving developers full shell/package-install capability. Community threads corroborate real-world usage of the sandbox for running containerized workloads, though none specifically contradict shell/package execution. missing for 10: independent hands-on verification of installing arbitrary packages (e.g., apt/pip) beyond docs examples.", "evidenceIds": [ "cloudflare-sandbox-docs-1", "cloudflare-sandbox-docs-12", "cloudflare-sandbox-docs-13", "cloudflare-sandbox-docs-21", "cloudflare-sandbox-docs-22", "cloudflare-sandbox-docs-28", "cloudflare-sandbox-docs-29", "cloudflare-sandbox-docs-43", "cloudflare-sandbox-docs-50" ] }, { "productId": "cloudflare-sandbox", "storyId": "snapshot-restore-fork", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm directory-level snapshot/restore via R2 (docs-8, docs-48), but there's no evidence of forking new independent sandbox instances from a snapshot — only restoring a directory's state, and community reports flag ambiguity in how persistence/restore actually behaves (comm-4). missing for 10: explicit 'fork sandbox from snapshot' API, clarity on full sandbox-state (not just directory) restoration, and independent confirmation that restore reliably recreates a working sandbox.", "evidenceIds": [ "cloudflare-sandbox-docs-8", "cloudflare-sandbox-docs-48", "cloudflare-sandbox-comm-4" ] }, { "productId": "cloudflare-sandbox", "storyId": "spinup-via-api", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs show a single call (getSandbox()) creates or reconnects to an isolated VM/container sandbox with immediate access to exec, file, and session APIs (docs-37, docs-38, docs-50), and community discussion corroborates boot times in the 1-5 second range consistent with 'seconds' provisioning (comm-1, comm-8). Missing for 10: an independent benchmark or first-party latency SLA confirming consistent sub-few-second cold starts across load, since community notes it's slower than some Firecracker-based competitors.", "evidenceIds": [ "cloudflare-sandbox-docs-37", "cloudflare-sandbox-docs-38", "cloudflare-sandbox-docs-50", "cloudflare-sandbox-docs-1", "cloudflare-sandbox-comm-1", "cloudflare-sandbox-comm-8" ] }, { "productId": "daytona", "storyId": "agent-self-provisions", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Daytona provides SDKs (Python/TS), CLI, API, and an MCP server explicitly designed for AI agents to programmatically create sandboxes, execute code (stateless/stateful), read results, and destroy sandboxes without human intervention — with docs showing exact code like sandbox.create() and process.exec(). This is corroborated by an OpenAPI spec, agent-skill docs, and llms.txt describing Daytona as infrastructure for AI-generated code execution. Missing for 10: independent hands-on validation of a full agentic create-execute-teardown loop (current community evidence is older/unrelated to this workflow).", "evidenceIds": [ "daytona-docs-1", "daytona-docs-6", "daytona-docs-16", "daytona-docs-33", "daytona-docs-35", "daytona-probe-2", "daytona-probe-3", "daytona-probe-4" ] }, { "productId": "daytona", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Daytona serves a live llms.txt (200 OK) and .md-suffixed docs pages for agent consumption, plus a dedicated agent-skills page listing API/CLI/SDK patterns for agents. This directly satisfies pointing an agent at llms.txt/agent-oriented docs. Missing for 10: independent third-party confirmation that agents actually consume these successfully in practice.", "evidenceIds": [ "daytona-probe-1", "daytona-probe-2", "daytona-docs-35" ] }, { "productId": "daytona", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Daytona is sandbox infrastructure for running AI-generated code, not a data product that surfaces AI-generated insights/suggestions from a user's own data; this story is a category error for its product type.", "evidenceIds": [] }, { "productId": "daytona", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Daytona documents programmatic sandbox creation, SDKs, CLI, and an MCP server for agents to control sandboxes, but there is no evidence of a built-in scheduler, cron, trigger, or background automation runner that would let a user set up autonomous background automations natively within Daytona itself. Missing for 10: any documented scheduling/trigger/webhook mechanism, evidence of persistent autonomous background jobs, or first-party automation orchestration beyond on-demand sandbox execution.", "evidenceIds": [ "daytona-docs-1", "daytona-docs-10", "daytona-docs-11", "daytona-docs-12", "daytona-docs-16" ] }, { "productId": "daytona", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Daytona is sandbox/execution infrastructure for AI agents, not itself an AI assistant product; it exposes MCP servers and SDKs so external agents can use it, but there is no built-in assistant delegated to within Daytona itself.", "evidenceIds": [] }, { "productId": "daytona", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Daytona is designed for programmatic, headless use: SDKs (Python/TypeScript), a CLI, and a REST API (OpenAPI spec) let sandboxes be created, managed, and code executed entirely without a UI, and snapshots/warm pools support fast, repeatable automation suitable for CI pipelines. Docs show example code creating sandboxes and running commands programmatically, and network/firewall controls support automated untrusted-code execution scenarios common in CI. Missing for 10: no explicit CI integration guide (e.g., GitHub Actions example) or third-party report of using Daytona inside a CI pipeline.", "evidenceIds": [ "daytona-docs-10", "daytona-docs-11", "daytona-docs-12", "daytona-docs-33", "daytona-probe-3", "daytona-docs-23", "daytona-docs-14" ] }, { "productId": "daytona", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "daytona", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Daytona is a sandbox/infrastructure platform, not itself an agent, so publishing an MCP server is on-axis, and docs explicitly confirm a first-party MCP server letting agents like Claude, Cursor, and Windsurf interact with sandboxes programmatically. Missing for 10: independent/hands-on corroboration of the MCP server working in practice beyond vendor docs.", "evidenceIds": [ "daytona-docs-5", "daytona-docs-16", "daytona-probe-4" ] }, { "productId": "daytona", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Daytona ships an official MCP server that lets AI agents (Claude, Cursor, Windsurf) drive sandbox creation/management via natural language, and an 'agent skill' doc packages API/CLI/SDK patterns for agent use, so natural-language operation is possible when mediated through a connected AI agent. However Daytona itself has no native NL command interface — direct operation is via SDKs, CLI, API, or web terminal, all imperative/code-based. Missing for 10: evidence of a first-party conversational/NL interface built into Daytona (not just via third-party agent + MCP), and independent confirmation of this workflow in practice.", "evidenceIds": [ "daytona-docs-5", "daytona-docs-16", "daytona-docs-35", "daytona-probe-4" ] }, { "productId": "daytona", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Daytona documents an official CLI providing command-line access to sandbox lifecycle, snapshots, and other core features, and this CLI is also integrated into an agent-skill workflow alongside SDKs and API. Community evidence corroborates hands-on CLI usage (favorable comparison to DevPod for command-line users), though one report flags install-script quality issues unrelated to CLI functionality itself. Missing for 10: deeper independent third-party validation of CLI feature completeness and no explicit AI-native command examples beyond agent-skill reference.", "evidenceIds": [ "daytona-docs-12", "daytona-probe-5", "daytona-docs-35", "daytona-comm-5" ] }, { "productId": "daytona", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Daytona exposes a documented OpenAPI spec, Python/TypeScript SDKs, a CLI, and an MCP server, all explicitly aimed at programmatic sandbox control by AI agents, with code examples (e.g., daytona.create(), sandbox.process.exec). This directly satisfies API-driven, agentic usage; missing for 10: independent third-party validation of API completeness/versioning stability beyond vendor docs.", "evidenceIds": [ "daytona-probe-3", "daytona-docs-10", "daytona-docs-11", "daytona-docs-12", "daytona-probe-4", "daytona-docs-33", "daytona-docs-35" ] }, { "productId": "daytona", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows API keys/SSH tokens exist and network/firewall controls limit egress, but there is no documentation of scoped or least-privilege credential issuance (e.g., per-agent permission scopes, role-based API keys) for agents. Missing for 10: any docs on scoped/limited-permission API key creation, credential scoping to specific sandboxes/actions, or least-privilege agent auth model.", "evidenceIds": [ "daytona-docs-8", "daytona-docs-19", "daytona-docs-26", "daytona-docs-32" ] }, { "productId": "daytona", "storyId": "agentic-sdks", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Daytona ships official Python and TypeScript SDKs with documented programmatic interfaces and code samples, plus a CLI, OpenAPI spec, and MCP server for agent integration—strong evidence of first-party, agent-friendly SDK support. Missing for 10: independent hands-on developer corroboration specifically of SDK usage (community evidence is about unrelated install/SSH issues, not SDK quality).", "evidenceIds": [ "daytona-docs-10", "daytona-docs-11", "daytona-docs-33", "daytona-docs-12", "daytona-probe-3", "daytona-probe-5", "daytona-docs-35" ] }, { "productId": "daytona", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack mentions webhooks or event subscription mechanisms; Daytona's documented capabilities cover sandboxes, snapshots, SDKs, CLI, MCP, and SSH, but nothing about publishing or subscribing to events via webhooks.", "evidenceIds": [] }, { "productId": "daytona", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Daytona publishes a machine-readable OpenAPI spec and SDK docs with code snippets (e.g., Python example), suggesting an API reference exists, but there is no evidence of an interactive, in-browser 'try it now' explorer or runnable examples embedded in the docs themselves. missing for 10: evidence of a Swagger/ReadMe-style interactive API explorer, in-docs code execution, or runnable example sandbox tied to the API reference.", "evidenceIds": [ "daytona-probe-3", "daytona-docs-33", "daytona-docs-10", "daytona-docs-11" ] }, { "productId": "daytona", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Daytona hosts a machine-readable OpenAPI spec directly at /openapi.json (verified HTTP 200 with valid 'openapi' key), confirming a downloadable API spec exists. Missing for 10: no independent third-party corroboration of spec completeness or versioning.", "evidenceIds": [ "daytona-probe-3" ] }, { "productId": "daytona", "storyId": "api-sandbox", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Daytona sandboxes are isolated, ephemeral runtime environments with dedicated kernel/filesystem/network stack, firewalled outbound access, and can be spun up/torn down in milliseconds from snapshots—purpose-built for AI agents to run and test code without touching production systems. SDKs, CLI, and MCP server make it straightforward for AI-native workflows to programmatically create disposable sandboxes for testing. Missing for 10: no explicit documentation or case study framing sandboxes as a 'staging vs production data' workflow, and no independent/hands-on verification of isolation guarantees beyond vendor docs.", "evidenceIds": [ "daytona-docs-1", "daytona-docs-28", "daytona-docs-19", "daytona-docs-26", "daytona-docs-14", "daytona-docs-16", "daytona-probe-2", "daytona-docs-23" ] }, { "productId": "daytona", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "There is an OpenAPI spec and SDKs, but no evidence of API versioning scheme or a documented deprecation policy anywhere in the docs pack. Missing for 10: explicit API version scheme (e.g., v1/v2 paths), changelog of breaking changes, documented deprecation/sunset policy or timelines.", "evidenceIds": [ "daytona-probe-3" ] }, { "productId": "daytona", "storyId": "auto-teardown-timeouts", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows fast sandbox creation/teardown and general lifecycle management (docs-14, docs-12) but contains no mention of configurable idle/auto-stop timeouts or billing-tied automatic shutdown settings. missing for 10: explicit auto-stop/idle-timeout configuration option, documentation on billing behavior when idle, confirmation that timeout can be set per-sandbox.", "evidenceIds": [ "daytona-docs-14", "daytona-docs-12" ] }, { "productId": "daytona", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Daytona's SDKs/CLI/API allow programmatic creation and management of sandboxes, and features like snapshot-based warm pools (docs-23/30) imply scaling to many pre-created instances, which could support scripted bulk workflows. However, there is no documented explicit bulk/batch operation primitive (e.g., bulk delete, bulk snapshot, batch API endpoint) for acting on many items in a single call. missing for 10: explicit bulk/batch API or CLI commands, independent evidence of bulk operations at scale, documentation of concurrency limits or bulk performance.", "evidenceIds": [ "daytona-docs-11", "daytona-docs-12", "daytona-docs-23", "daytona-docs-30", "daytona-docs-33" ] }, { "productId": "daytona", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Daytona provides sandbox infrastructure and SDKs for programmatic control, but there is no evidence of an event/trigger-based rules engine (e.g., webhooks, event listeners, or conditional automation triggers) that fires actions automatically on events.", "evidenceIds": [] }, { "productId": "daytona", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Daytona provides sandbox infrastructure, SDKs, CLI, and MCP server for on-demand code execution, but there is no evidence of any scheduling, cron-like, or recurring job/workflow orchestration feature. missing for 10: scheduler/cron API, recurring job definitions, workflow orchestration, trigger-based automation.", "evidenceIds": [] }, { "productId": "daytona", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "daytona", "storyId": "browser-in-sandbox", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Daytona has a documented Computer Use feature providing mouse, keyboard, screenshot, screen recording, and display operations for GUI automation within sandboxes, plus Windows VM sandboxes for running desktop applications and tooling. This directly supports running headless browser/desktop environments inside the sandbox. Missing for 10: independent/hands-on verification of Computer Use in production and explicit mention of a headless browser package or Chrome/Chromium support out-of-the-box.", "evidenceIds": [ "daytona-docs-7", "daytona-docs-18", "daytona-docs-34", "daytona-docs-13", "daytona-docs-15" ] }, { "productId": "daytona", "storyId": "concurrent-fleets", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers sandbox lifecycle, snapshots, warm pools, GPU/VM options, and SDKs, but no documented concurrency limits, fleet-scale benchmarks, or per-account/org sandbox quotas are given anywhere in the pack. Missing for 10: documented concurrency/rate limits, fleet-scale benchmarks or case studies, quota/scaling guidance for platform engineers.", "evidenceIds": [ "daytona-docs-14", "daytona-docs-23", "daytona-docs-30" ] }, { "productId": "daytona", "storyId": "custom-images-templates", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs clearly show custom snapshot creation from a Dockerfile/image, defining base OS, runtimes, packages, and project-level setup, which Daytona builds into a snapshot used to create sandboxes — this directly satisfies bringing a custom container image/template. missing for 10: independent/hands-on verification of custom image builds and no explicit mention of registry-hosted image import or advanced build customization options.", "evidenceIds": [ "daytona-docs-21", "daytona-docs-2", "daytona-docs-22", "daytona-docs-23" ] }, { "productId": "daytona", "storyId": "expose-public-url", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers sandboxes, snapshots, volumes, network limits, SSH access, web terminal, and SDKs, but contains no mention of preview URLs or exposing a running service's port publicly from a sandbox. No documentation item addresses port forwarding or preview link generation.", "evidenceIds": [] }, { "productId": "daytona", "storyId": "fast-cold-starts", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Daytona's docs claim sandboxes spin up 'in milliseconds' and that warm pools let sandboxes be claimed 'instantly' from pre-created pools, directly supporting fast cold-start claims. However, there are no concrete benchmark numbers (e.g., specific p50/p95 latency stats) and no independent/hands-on corroboration of actual startup times in the evidence pack. Missing for 10: quantified latency benchmarks, independent/third-party timing verification, and details on cold vs warm start differences beyond marketing copy.", "evidenceIds": [ "daytona-docs-14", "daytona-docs-23", "daytona-docs-30", "daytona-docs-27" ] }, { "productId": "daytona", "storyId": "filesystem-io", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Daytona's SDK docs (Python/TypeScript) are described as a general interface for programmatically interacting with sandboxes, and volume docs mention shared file access ('read from large files instantly - no need to upload files manually'), implying some file I/O capability, but the evidence pack never explicitly documents dedicated SDK methods for reading, writing, uploading, or downloading individual files in the sandbox filesystem. missing for 10: explicit SDK fs.read_file/write_file/upload_file/download_file API documentation, code examples showing file upload/download via SDK, independent confirmation of these operations working.", "evidenceIds": [ "daytona-docs-10", "daytona-docs-11", "daytona-docs-4", "daytona-docs-24", "daytona-docs-31", "daytona-docs-33" ] }, { "productId": "daytona", "storyId": "free-tier-start", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Daytona's pricing page explicitly offers $200 in free compute included and lets developers start immediately ('spin up sandboxes in milliseconds... use what you need, when you need it'), plus the GitHub repo is public/free to use without needing sales contact. Missing for 10: independent confirmation of the free-tier signup flow/limits and no evidence of any sales-gated onboarding step.", "evidenceIds": [ "daytona-docs-27", "daytona-docs-14", "daytona-gh-1" ] }, { "productId": "daytona", "storyId": "gpu-access", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Daytona docs explicitly describe GPU sandboxes with NVIDIA and AMD GPU acceleration for model inference, fine-tuning, and CUDA/GPU-accelerated compute, directly matching the story. Missing for 10: no independent/hands-on confirmation of GPU attachment working in practice, and no detail on how GPU type/quantity is specified in SDK/CLI calls.", "evidenceIds": [ "daytona-docs-3", "daytona-docs-29" ] }, { "productId": "daytona", "storyId": "hard-isolation-boundary", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Daytona documents each sandbox running with a dedicated kernel, filesystem, network stack, and VM sandboxes (dedicated Linux/Windows VM), plus firewall/network isolation, indicating some hard-isolation architecture, but the docs never explicitly name a specific microVM technology (e.g., Firecracker/gVisor) or provide a security whitepaper/threat model for platform engineers. missing for 10: explicit microVM/hypervisor technology name, formal security/isolation whitepaper or third-party audit, independent verification of isolation guarantees.", "evidenceIds": [ "daytona-docs-28", "daytona-docs-15", "daytona-docs-19", "daytona-docs-26", "daytona-docs-22" ] }, { "productId": "daytona", "storyId": "long-running-sessions", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Docs describe fast sandbox creation/teardown, snapshots, and warm pools, but there is no evidence of explicit support for sustaining a single sandbox session continuously for hours or days (e.g., keep-alive settings, max runtime limits, or session persistence guarantees) — the messaging even emphasizes ephemeral 'spin up in milliseconds, shut down just as fast' usage. missing for 10: explicit documentation of long-running session support, auto-stop/keep-alive controls, or customer evidence of multi-hour/day agent tasks sustained in a single sandbox.", "evidenceIds": [ "daytona-docs-14", "daytona-docs-1", "daytona-docs-23" ] }, { "productId": "daytona", "storyId": "multi-language-runtimes", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Daytona docs explicitly document code execution across multiple languages with both stateless and stateful/persistent interpreter contexts, backed by Python/TypeScript SDK examples and support for images/Dockerfiles defining custom language runtimes. Rich results are implied via persistent state, process exec responses, and structured sandbox tooling, though no explicit mention of rich output formats (e.g., images/plots/HTML) is shown. missing for 10: explicit documentation of rich/structured output formats (charts, images, HTML) and independent/hands-on confirmation of multi-language execution results.", "evidenceIds": [ "daytona-docs-6", "daytona-docs-17", "daytona-docs-25", "daytona-docs-33", "daytona-docs-21", "daytona-docs-11", "daytona-docs-10" ] }, { "productId": "daytona", "storyId": "network-egress-policy", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Daytona documents explicit network egress policy controls via firewall parameters set at sandbox creation or updated at runtime, restricting which IPs/domains a sandbox can reach to prevent data exfiltration—directly matching the platform-engineer's egress policy story. Missing for 10: no independent/hands-on verification of the firewall enforcement or examples of complex allow/deny rule syntax.", "evidenceIds": [ "daytona-docs-8", "daytona-docs-19", "daytona-docs-26" ] }, { "productId": "daytona", "storyId": "openness-api-parity", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Daytona exposes a full OpenAPI spec, Python/TypeScript SDKs, and a CLI covering sandbox lifecycle, snapshots, volumes, execution, and network limits, indicating strong API-first design consistent with programmatic parity to UI actions. However, there is no explicit documentation enumerating every UI feature and confirming 1:1 API coverage, so full parity is implied rather than proven. Missing for 10: explicit UI-to-API feature parity statement, independent confirmation that all dashboard actions (e.g., billing, org management) are also API-exposed.", "evidenceIds": [ "daytona-probe-3", "daytona-docs-10", "daytona-docs-11", "daytona-docs-12", "daytona-docs-1", "daytona-docs-2", "daytona-docs-4" ] }, { "productId": "daytona", "storyId": "openness-full-export", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Daytona's core repo is open source and free to fork/self-host (daytona-gh-1), and its CLI/SDK/SSH access let users programmatically pull files, snapshots, and volumes out of sandboxes (daytona-docs-12, daytona-docs-31, daytona-docs-32), giving a de facto path to extract data. However there is no documented bulk 'export all data' feature, no stated open data format for exports, and no explicit anti-lock-in/portability commitment. Missing for 10: an explicit data-export tool or guarantee, documented open format for exported snapshots/volumes, and any statement addressing account/data portability upon leaving.", "evidenceIds": [ "daytona-gh-1", "daytona-docs-12", "daytona-docs-31", "daytona-docs-32", "daytona-docs-22" ] }, { "productId": "daytona", "storyId": "openness-open-license", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "The GitHub repo evidence states Daytona 'remains public and free to use, fork, and build on under the LICENSE' (daytona-gh-1), suggesting an open-license posture, but community evidence directly reports that Daytona went closed source, with users criticizing it as 'security by obscurity' (daytona-comm-6) — a concrete contradiction rather than mere skepticism. Missing for 10: a specific named OSI license, clarity on current licensing status post-controversy, and independent confirmation of what is actually open today.", "evidenceIds": [ "daytona-gh-1", "daytona-comm-6" ] }, { "productId": "daytona", "storyId": "openness-self-host", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "The Daytona GitHub repo states the project 'remains public and free to use, fork, and build on under the LICENSE,' implying self-hosting is possible, and docs show programmatic sandbox management that could run locally. However, community evidence reports Daytona later 'going closed source,' with users explicitly skeptical of the change, directly contradicting the self-hostable/forkable claim. Missing for 10: clear current licensing status, self-hosting setup/deployment docs, and confirmation the core sandbox runtime (not just SDKs/CLI) can run outside Daytona's hosted infrastructure.", "evidenceIds": [ "daytona-gh-1", "daytona-comm-6" ] }, { "productId": "daytona", "storyId": "pause-resume-state", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Daytona docs confirm snapshots capture filesystem and, for VM sandboxes, memory state ('hot snapshots'), enabling pause/resume with state intact, while container sandboxes only get filesystem-only 'cold snapshots'. There's no independent/hands-on evidence confirming actual pause/resume behavior in practice or performance at scale. missing for 10: hands-on/independent verification of resume fidelity, explicit pause/resume API documentation (vs. snapshot creation), and confirmation that memory state persistence works reliably outside VM-only sandboxes.", "evidenceIds": [ "daytona-docs-2", "daytona-docs-22", "daytona-docs-21" ] }, { "productId": "daytona", "storyId": "per-second-billing", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Daytona's pricing page emphasizes millisecond spin-up/teardown and 'use what you need, when you need it' with $200 free compute credit, implying usage-based billing, but the evidence never states an explicit per-second billing granularity or rate card. Missing for 10: explicit per-second billing rate documentation, independent confirmation of billing granularity, and detail on how partial-second usage is metered/rounded.", "evidenceIds": [ "daytona-docs-14", "daytona-docs-27" ] }, { "productId": "daytona", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, region selection, or geographic storage controls for sandboxes/snapshots/volumes; missing for 10: any documentation of selectable regions/data centers, compliance/residency guarantees, or admin controls for data location.", "evidenceIds": [] }, { "productId": "daytona", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses AI training data usage, data retention policies, or opt-out mechanisms for model training; the evidence covers sandboxing, SDKs, and infrastructure but not privacy/data-training posture.", "evidenceIds": [] }, { "productId": "daytona", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items address data retention policies, deletion controls, or data lifecycle management for sandboxes/volumes/snapshots; the docs cover sandbox creation, execution, networking and SDKs but nothing on retention/deletion controls a user can configure.", "evidenceIds": [] }, { "productId": "daytona", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items mention telemetry, usage tracking, analytics collection, or an opt-out mechanism for Daytona; the axis applies (any cloud/SaaS dev platform could document such settings) but no such documentation or capability is evidenced.", "evidenceIds": [] }, { "productId": "daytona", "storyId": "run-coding-agents-inside", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Daytona sandboxes can execute arbitrary code, host an MCP server for external AI agents (Claude, Cursor, Windsurf) to control sandboxes, and includes an 'agent skill' for creating/running sandboxes — but none of this documents a vendor recipe for actually running Claude Code or Codex *inside* a sandbox as a coding agent. Missing for 10: an explicit vendor tutorial/recipe demonstrating installation and execution of Claude Code/Codex within a sandbox environment.", "evidenceIds": [ "daytona-docs-5", "daytona-docs-16", "daytona-docs-35" ] }, { "productId": "daytona", "storyId": "run-untrusted-code", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Daytona's core product is isolated sandboxes with dedicated kernel/filesystem/network stack, network firewalling to prevent exfiltration, SDKs/CLI for programmatic code execution, and explicit positioning as infrastructure for running AI-generated code safely. This directly matches the story of executing untrusted AI-generated code without risking one's own infrastructure. Missing for 10: no independent third-party security audit or hands-on penetration test confirming isolation guarantees hold under adversarial conditions.", "evidenceIds": [ "daytona-docs-1", "daytona-docs-19", "daytona-docs-26", "daytona-docs-28", "daytona-probe-1", "daytona-probe-2", "daytona-docs-33" ] }, { "productId": "daytona", "storyId": "secrets-stay-out", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Daytona sandboxes run in fully isolated environments with dedicated kernel, filesystem, and network stack separate from the host (docs-28), and network egress is firewalled per-sandbox to prevent exfiltration (docs-19, docs-26), meaning code executed via the SDK/MCP server never has access to the host machine or its credentials by construction. missing for 10: explicit documentation of secret/credential injection or scoping mechanisms, and independent/hands-on confirmation that host secrets specifically cannot leak (only architectural isolation claims).", "evidenceIds": [ "daytona-docs-28", "daytona-docs-19", "daytona-docs-26", "daytona-docs-1", "daytona-probe-2" ] }, { "productId": "daytona", "storyId": "shell-and-packages", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Daytona sandboxes support running arbitrary commands via SDK (sandbox.process.exec), web terminal, SSH access, and CLI, plus snapshot/image building for installing packages and dependencies (Dockerfile/image-based setup). This directly covers shell command execution and package installation with strong first-party documentation across multiple interfaces. Missing for 10: independent hands-on verification specifically of arbitrary shell/package install workflows (community evidence covers other aspects like SSH glitches, not this exact capability).", "evidenceIds": [ "daytona-docs-33", "daytona-docs-21", "daytona-docs-9", "daytona-docs-32", "daytona-docs-12", "daytona-docs-6" ] }, { "productId": "daytona", "storyId": "snapshot-restore-fork", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Daytona docs explicitly describe snapshots as persistent point-in-time captures of sandbox filesystem/state (docs-2, docs-22), created from images/Dockerfiles or existing sandboxes (docs-21), and used to create new sandboxes/pools (docs-23/30), directly matching the snapshot-then-restore/fork workflow, with SDK/CLI support for managing snapshots (docs-11, docs-12). Missing for 10: independent/hands-on verification of fork/restore behavior beyond docs, and no explicit example code showing a restore-from-snapshot call.", "evidenceIds": [ "daytona-docs-2", "daytona-docs-21", "daytona-docs-22", "daytona-docs-23", "daytona-docs-30", "daytona-docs-12" ] }, { "productId": "daytona", "storyId": "spinup-via-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Daytona's docs and SDK examples show one-call sandbox creation (daytona.create()), millisecond-scale spin-up/teardown, and isolated per-sandbox runtime with dedicated kernel/filesystem/network, backed by Python/TS SDKs, CLI, and OpenAPI-documented REST API. Community evidence (comm-4, comm-5) independently corroborates ease-of-use versus alternatives like DevPod. Missing for 10: no independent hands-on benchmark verifying actual 'seconds' provisioning latency claim beyond vendor marketing copy.", "evidenceIds": [ "daytona-docs-33", "daytona-docs-14", "daytona-docs-28", "daytona-docs-10", "daytona-docs-11", "daytona-docs-12", "daytona-probe-3", "daytona-comm-5" ] }, { "productId": "e2b", "storyId": "agent-self-provisions", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "E2B's SDK API directly supports programmatic sandbox creation, code execution, filesystem read/write, and teardown/kill — all callable by an agent without human intervention (e2b-docs-1, e2b-docs-6, e2b-docs-19, e2b-docs-8). Docs explicitly describe running AI coding agents like Claude Code/Codex end-to-end in sandboxes with full terminal/filesystem access (e2b-docs-28, e2b-docs-42, e2b-docs-43), and timeouts/kill provide automated teardown. Missing for 10: no independent hands-on benchmark of a fully autonomous end-to-end agent loop (only vendor docs and one community note about agent friction, not a failure of the core provision/execute/teardown flow).", "evidenceIds": [ "e2b-docs-1", "e2b-docs-6", "e2b-docs-19", "e2b-docs-8", "e2b-docs-28", "e2b-docs-42", "e2b-docs-43", "e2b-docs-5" ] }, { "productId": "e2b", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "E2B serves a live llms.txt index (HTTP 200) and .md-suffixed doc pages explicitly designed for agent consumption, with the quickstart page itself pointing agents to the llms.txt index. missing for 10: no independent/community confirmation that agents actually consume llms.txt successfully in practice.", "evidenceIds": [ "e2b-probe-1", "e2b-probe-2" ] }, { "productId": "e2b", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "E2B is developer sandbox infrastructure for running AI-generated code, not an end-user data product with its own analytics/insights surface; the analyze-data-with-ai use case describes enabling third-party AI agents to run code, not E2B itself surfacing insights to a user inside a product.", "evidenceIds": [] }, { "productId": "e2b", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "E2B provides persistent, resumable sandboxes, background command execution (--background flag), webhooks for lifecycle events, and long-running session support (up to 24h, pause/resume), which are building blocks for autonomous background automation. However, there's no evidence of a native scheduler/cron, no first-party 'automation' orchestration layer, and no documented trigger system beyond webhooks — the user must build the autonomy logic themselves using an external agent/LLM. missing for 10: native scheduling/cron or trigger orchestration, evidence of fully autonomous multi-step agent loops running unattended, independent case studies of long-running unattended automations.", "evidenceIds": [ "e2b-docs-3", "e2b-docs-4", "e2b-docs-22", "e2b-docs-27", "e2b-docs-38", "e2b-docs-28" ] }, { "productId": "e2b", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "E2B is sandbox/infrastructure for running AI-generated code and hosting external agents (Claude Code, Codex, etc.) — it is not itself a product with an embedded conversational assistant to delegate tasks to. The evidence consistently shows E2B as a backend that other LLMs/agents connect to, not a UI with a built-in assistant, so this axis is a category mismatch for this product type.", "evidenceIds": [ "e2b-docs-28", "e2b-docs-32", "e2b-docs-42", "e2b-gh-3" ] }, { "productId": "e2b", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "E2B has a documented CLI, SDK, and non-interactive auth mode explicitly recommended for CI/CD pipelines, servers, and containers, plus scriptable sandbox lifecycle (create, exec with --background, shutdown, snapshot) and webhooks for lifecycle events — all consistent with headless/CI automation. missing for 10: no independent third-party report of a real CI pipeline using E2B, and no explicit CI provider integration examples (e.g., GitHub Actions workflow snippet).", "evidenceIds": [ "e2b-docs-35", "e2b-docs-9", "e2b-docs-38", "e2b-docs-25", "e2b-docs-27", "e2b-docs-30", "e2b-docs-34" ] }, { "productId": "e2b", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "E2B is sandbox/code-execution infrastructure that agents and LLMs call as a tool (e.g., docs describe connecting LLMs to E2B, or running agents like Claude Code inside E2B sandboxes), not an AI agent/assistant that itself would consume external tools via MCP client integration. The evidence never shows E2B acting as an MCP client plugging in external tool servers, consistent with this being a category mismatch rather than a missing feature.", "evidenceIds": [ "e2b-docs-32", "e2b-docs-28", "e2b-docs-42" ] }, { "productId": "e2b", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "E2B is a sandbox/code-execution infrastructure product (not itself an agent), so an official MCP server exposing its sandbox tools to agents is a fair, applicable axis. The evidence pack documents SDKs, CLI, templates, and LLM tool-use integration (e2b-docs-32) but never mentions an MCP server, official MCP integration, or MCP protocol support anywhere.", "evidenceIds": [ "e2b-docs-32", "e2b-docs-9", "e2b-gh-3" ] }, { "productId": "e2b", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "E2B's own interface is programmatic (SDK calls like Sandbox.create(), files.read(), commands.run()) and a CLI (e2b-docs-9), not a natural-language command surface; however docs explicitly describe connecting LLMs via tool-use so an AI agent can drive E2B through natural-language-triggered function calls (e2b-docs-32, e2b-docs-28, e2b-docs-43). This shows AI-native operation is possible only indirectly through an LLM/agent layer, not as a built-in NL command mode of the product itself. missing for 10: no first-party NL chat/command interface for E2B itself, no evidence of a natural-language CLI or prompt-driven control plane, only indirect LLM tool-use integration documented.", "evidenceIds": [ "e2b-docs-32", "e2b-docs-28", "e2b-docs-43", "e2b-docs-9" ] }, { "productId": "e2b", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "E2B ships an official CLI documented extensively (create/connect/exec/list/shutdown/snapshot/auth/metrics commands) covering sandbox lifecycle management, well-suited for AI-native agentic workflows. missing for 10: independent/hands-on community validation of the CLI itself (community evidence only covers SDK usage, not CLI) and no evidence of CLI-specific agent integration examples.", "evidenceIds": [ "e2b-docs-9", "e2b-docs-23", "e2b-docs-24", "e2b-docs-25", "e2b-docs-26", "e2b-docs-31", "e2b-docs-35", "e2b-docs-36", "e2b-docs-37", "e2b-docs-38", "e2b-docs-39", "e2b-docs-40", "e2b-probe-4" ] }, { "productId": "e2b", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "E2B ships extensive documented SDK/CLI APIs (Sandbox.create, files.read/write, commands.run, timeouts, metrics, snapshots, webhooks) with quickstart and reference docs, and explicit LLM/agent integration guides (connect-llms, coding-agents), demonstrating a fully documented public API surface for AI-native use. Missing for 10: no discoverable formal OpenAPI/swagger spec (probe found 404s across candidate paths) and no independent third-party audit of API completeness beyond vendor docs/community anecdotes.", "evidenceIds": [ "e2b-docs-1", "e2b-docs-7", "e2b-docs-8", "e2b-docs-9", "e2b-docs-30", "e2b-docs-32", "e2b-docs-28", "e2b-probe-3" ] }, { "productId": "e2b", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows E2B supports multiple API keys per team/project (e2b-docs-30, e2b-docs-31, e2b-docs-34, e2b-docs-35) but no documentation of scoped or least-privilege permissions (e.g., read-only, sandbox-restricted, rate-limited keys) that would let an AI-native user issue a restricted credential to an agent.", "evidenceIds": [ "e2b-docs-30", "e2b-docs-31", "e2b-docs-34", "e2b-docs-35" ] }, { "productId": "e2b", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "E2B ships official SDKs (JS/Python) with extensive first-party docs covering sandbox creation, code execution, filesystem, commands, metrics, persistence, and explicit guidance for connecting LLMs/agent frameworks (e2b-docs-1,7,8,32,42-44), plus a CLI (e2b-docs-9, e2b-probe-4) and community confirmation of Python/TS SDK availability (e2b-comm-1). missing for 10: broader independent/hands-on validation beyond one HN thread, and no discoverable OpenAPI spec for programmatic SDK generation (e2b-probe-3).", "evidenceIds": [ "e2b-docs-1", "e2b-docs-7", "e2b-docs-8", "e2b-docs-9", "e2b-docs-32", "e2b-docs-42", "e2b-docs-43", "e2b-docs-44", "e2b-comm-1", "e2b-probe-4" ] }, { "productId": "e2b", "storyId": "agentic-webhooks", "verdict": "full", "quality": 6, "confidence": "medium", "rationale": "E2B docs explicitly describe webhooks that deliver notifications to an external server on sandbox lifecycle events, directly matching the story. Missing for 10: detailed docs on event types/payload schema, setup/configuration steps, and independent/community corroboration of webhook usage.", "evidenceIds": [ "e2b-docs-27" ] }, { "productId": "e2b", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows extensive static documentation with code snippets (e2b-docs-1, e2b-docs-2, etc.) but no evidence of an interactive, runnable API reference — the probe explicitly found no OpenAPI/Swagger spec (e2b-probe-3), and no playground or 'try it' console is mentioned anywhere in the pack.", "evidenceIds": [ "e2b-probe-3", "e2b-docs-1", "e2b-docs-21" ] }, { "productId": "e2b", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "E2B exposes SDKs and a CLI, but active probing found no OpenAPI/swagger spec at any standard location (all candidate paths returned 404), and no documentation page references a machine-readable API spec.", "evidenceIds": [ "e2b-probe-3" ] }, { "productId": "e2b", "storyId": "api-sandbox", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "E2B provides fully isolated, ephemeral cloud sandboxes (Firecracker microVMs) for running AI-generated code, with no connection to production systems — sandboxes are created fresh from templates, have their own filesystem/network, and can be killed or auto-expire, ensuring test isolation. Persistence/pause-resume and metrics further support safe iterative testing without touching real data. Missing for 10: explicit documentation or case study contrasting sandbox vs production data separation, and independent/hands-on verification beyond vendor docs.", "evidenceIds": [ "e2b-docs-1", "e2b-docs-3", "e2b-docs-6", "e2b-docs-10", "e2b-docs-17", "e2b-gh-3", "e2b-docs-44" ] }, { "productId": "e2b", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers SDK usage, CLI commands, sandbox lifecycle, and self-hosting, but contains no mention of API versioning scheme, changelog, or a documented deprecation policy; the openapi probe even returned 404s. This is a fair axis for an API-first product, but no evidence supports it.", "evidenceIds": [ "e2b-probe-3" ] }, { "productId": "e2b", "storyId": "auto-teardown-timeouts", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "E2B documents explicit timeout controls (setTimeout/set_timeout), default and configurable session lengths (24h Pro/1h Base), automatic kill on timeout, and manual kill/shutdown via SDK and CLI, directly enabling idle sandboxes to auto-terminate. missing for 10: explicit first-party confirmation that billing stops precisely at timeout/kill (only inferred from lifecycle docs), and independent/hands-on verification of billing behavior.", "evidenceIds": [ "e2b-docs-5", "e2b-docs-6", "e2b-docs-20", "e2b-docs-22", "e2b-docs-25", "e2b-docs-39" ] }, { "productId": "e2b", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "E2B supports several bulk-adjacent capabilities: up to 100 concurrent sandboxes, writing multiple files at once, listing all sandboxes, and shutting down single or all running sandboxes via CLI. However there is no unified batch/bulk API (e.g., no evidence of submitting a list of code executions or files to process as one bulk call) — bulk behavior is assembled from separate CLI/SDK primitives rather than a first-class bulk-operations feature. Missing for 10: a documented batch execution or bulk-processing API, evidence of bulk operations on non-sandbox resources (e.g., bulk template management), and independent/hands-on validation of bulk workflows at scale.", "evidenceIds": [ "e2b-docs-14", "e2b-docs-16", "e2b-docs-24", "e2b-docs-25", "e2b-docs-39", "e2b-docs-38" ] }, { "productId": "e2b", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "E2B offers webhooks that deliver notifications for sandbox lifecycle events, which is a minimal building block for event-triggered automation, but there is no evidence of an actual rule-engine (defining conditional actions that execute automatically) beyond forwarding notifications to an external server. missing for 10: built-in conditional rule definitions, in-product automatic action execution, and any hands-on evidence of the webhook mechanism working end-to-end.", "evidenceIds": [ "e2b-docs-27" ] }, { "productId": "e2b", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "E2B provides sandbox creation, execution, persistence (pause/resume), and lifecycle webhooks, but nothing in the evidence pack describes a native scheduler, cron-like trigger, or recurring workflow mechanism — scheduling would have to be built by the user via external tooling. This axis is fair for an automation-focused sandbox platform, so absence of evidence means 'none' rather than 'na'.", "evidenceIds": [ "e2b-docs-3", "e2b-docs-4", "e2b-docs-27", "e2b-docs-5" ] }, { "productId": "e2b", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "E2B is sandboxed code-execution infrastructure for running AI-generated code/agents, not a workflow/automation builder with a concept of 'automations' that can be versioned, reviewed, and rolled back; its templates/snapshots version sandbox environments, not user-authored automations, so this axis is a category mismatch.", "evidenceIds": [] }, { "productId": "e2b", "storyId": "browser-in-sandbox", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "E2B's GitHub README confirms a separate 'Desktop SDK' providing mouse, keyboard, screenshot, application, and desktop streaming APIs for full desktop environments, but this is an add-on SDK rather than a built-in feature of the core Sandbox class documented elsewhere. There's no first-party doc page (e.g., under docs.e2b.dev) detailing headless browser support or code examples for it. missing for 10: dedicated docs/quickstart for the Desktop SDK, headless browser (e.g., Chromium/Playwright) usage examples, independent hands-on confirmation of desktop streaming reliability.", "evidenceIds": [ "e2b-gh-1" ] }, { "productId": "e2b", "storyId": "concurrent-fleets", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "E2B publishes concrete documented concurrency limits (up to 100 concurrently running sandboxes, up to 24-hour session length on Pro plans) and metrics/CLI tooling to manage fleets, giving platform engineers real numbers to plan around. However missing for 10: no documentation of how limits scale for enterprise/self-hosted deployments, no guidance on requesting higher concurrency, no independent/hands-on benchmarks validating fleet behavior at scale, and no discussion of failure modes or queuing when limits are hit.", "evidenceIds": [ "e2b-docs-14", "e2b-docs-22", "e2b-docs-12", "e2b-docs-29", "e2b-gh-2" ] }, { "productId": "e2b", "storyId": "custom-images-templates", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "E2B templates explicitly let developers define a custom base image, environment variables, files, commands, and start command, and the CLI supports building/naming/snapshotting templates for reuse; this is well-documented first-party functionality with CLI tooling support. Missing for 10: no independent/hands-on third-party confirmation of custom template builds succeeding in practice.", "evidenceIds": [ "e2b-docs-17", "e2b-docs-11", "e2b-docs-40", "e2b-docs-26", "e2b-docs-36", "e2b-docs-9" ] }, { "productId": "e2b", "storyId": "expose-public-url", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers sandbox lifecycle (create, pause/resume, timeouts, files, commands, metrics, CLI) but contains no mention of exposing ports or generating public preview URLs to reach services running inside a sandbox. This is a fair capability question for a sandbox execution platform, so the axis applies, but nothing in the evidence supports it.", "evidenceIds": [] }, { "productId": "e2b", "storyId": "fast-cold-starts", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes sandbox creation, persistence, pausing/resuming, and templates with 'zero wait time,' but contains no documented benchmark or specific claim of sub-second-to-few-second cold start times. Missing for 10: any explicit startup latency figures, benchmark data, or documentation stating cold-start speed in seconds.", "evidenceIds": [ "e2b-docs-1", "e2b-docs-11", "e2b-docs-17" ] }, { "productId": "e2b", "storyId": "filesystem-io", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "E2B SDK docs clearly document files.write() and files.read() for single and multiple files, plus files.list() for directory listing, covering read/write/list operations via the SDK. Upload/download is implicit in write/read of file content but not explicitly named as 'upload/download' with byte-stream examples. missing for 10: explicit upload/download terminology or examples (e.g., binary file transfer, streaming), independent/hands-on corroboration beyond docs.", "evidenceIds": [ "e2b-docs-7", "e2b-docs-19", "e2b-docs-16", "e2b-docs-2" ] }, { "productId": "e2b", "storyId": "free-tier-start", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "E2B explicitly gives every new account $100 in credits with no sales contact required (e2b-docs-41), and the pricing page details a self-serve tier with concrete limits (24-hour sessions, 100 concurrent sandboxes, e2b-docs-14) alongside quickstart docs showing immediate SDK usage without any account-approval step. missing for 10: no independent/community confirmation of the signup flow or credit amount, and no explicit mention of a free tier distinct from trial credits.", "evidenceIds": [ "e2b-docs-41", "e2b-docs-14", "e2b-docs-13", "e2b-docs-1" ] }, { "productId": "e2b", "storyId": "gpu-access", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack mentions GPU support, GPU attachment, or GPU-enabled sandboxes; docs only reference customizable CPU & RAM, disk, and memory metrics. This is a fair question for a sandbox/compute infrastructure product, but nothing indicates GPU capability is offered.", "evidenceIds": [ "e2b-docs-13", "e2b-docs-14", "e2b-docs-12" ] }, { "productId": "e2b", "storyId": "hard-isolation-boundary", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "E2B docs explicitly state that code runs 'in an isolated Firecracker microVM instead of your server' with network controls and pause/resume, directly documenting the hard isolation boundary platform engineers need. Missing for 10: independent third-party security audit or pentest report, and more detail on the isolation boundary's guarantees beyond a single doc line.", "evidenceIds": [ "e2b-docs-44", "e2b-docs-43", "e2b-docs-10", "e2b-gh-3" ] }, { "productId": "e2b", "storyId": "long-running-sessions", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "E2B sandboxes can run continuously for hours (up to 24h on Pro plans, with configurable/resettable timeouts via setTimeout), covering the 'hours' part of the story fully, and for multi-day tasks E2B explicitly recommends pause/resume to preserve full memory and filesystem state indefinitely rather than true continuous execution. This satisfies long-running agent tasks but with a documented architectural limitation (base tier capped at 1h, Pro at 24h continuous) requiring the pause/resume workaround for day-scale persistence. Missing for 10: evidence of truly continuous multi-day execution without pause/resume, and independent/hands-on confirmation of stability over such long sessions.", "evidenceIds": [ "e2b-docs-22", "e2b-docs-3", "e2b-docs-4", "e2b-docs-5", "e2b-docs-20", "e2b-docs-14" ] }, { "productId": "e2b", "storyId": "multi-language-runtimes", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "E2B's docs show runCode execution (e2b-docs-1, e2b-docs-21) with sandboxes supporting Python and JavaScript SDKs, plus file I/O and terminal command execution for rich interaction with results (e2b-docs-7, e2b-docs-8, e2b-docs-19). It's explicitly positioned for running AI-generated code across languages with tool-use integration (e2b-docs-32, e2b-docs-33). missing for 10: explicit enumeration of all supported language runtimes beyond Python/JS/CLI, and independent hands-on confirmation of 'rich results' (e.g., charts/images) beyond docs claims.", "evidenceIds": [ "e2b-docs-1", "e2b-docs-21", "e2b-docs-7", "e2b-docs-8", "e2b-docs-19", "e2b-docs-32", "e2b-docs-33" ] }, { "productId": "e2b", "storyId": "network-egress-policy", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "E2B docs explicitly document network egress control: outbound internet access is on by default and can be restricted via 'a simple on/off switch or fine-grained allow and deny lists,' directly matching the platform-engineer's need for explicit egress policy. missing for 10: independent/hands-on verification of allow/deny list behavior, and detailed policy syntax/API reference beyond the summary.", "evidenceIds": [ "e2b-docs-10", "e2b-docs-44" ] }, { "productId": "e2b", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "E2B is API/SDK-first, so most core operations (sandbox create, exec, filesystem, commands, metrics, templates, snapshots, webhooks) are documented as API/CLI actions rather than gated behind a UI, and the CLI mirrors sandbox lifecycle management. However, some UI-surfaced features like Desktop streaming (mouse/keyboard/screenshot) require a separate Desktop SDK, and no public OpenAPI spec was found, making full parity hard to verify. Missing for 10: a confirmed OpenAPI/full REST spec, explicit confirmation that dashboard-only features (e.g., billing, team management) are also API-accessible, independent verification of full UI/API parity.", "evidenceIds": [ "e2b-docs-1", "e2b-docs-9", "e2b-docs-17", "e2b-docs-26", "e2b-docs-27", "e2b-gh-1", "e2b-probe-3", "e2b-docs-30" ] }, { "productId": "e2b", "storyId": "openness-full-export", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "E2B is open-source and self-hostable (Terraform-based), and sandbox files can be read/downloaded via files.read()/CLI, giving a technical path to extract data and avoid lock-in, but there is no documented 'export all account data' feature or guarantee of open data formats for account-level artifacts like snapshots/templates. missing for 10: explicit data-export tooling, documented open format guarantees for exported data, account-level export across all sandboxes/snapshots.", "evidenceIds": [ "e2b-gh-2", "e2b-gh-3", "e2b-docs-19", "e2b-docs-26" ] }, { "productId": "e2b", "storyId": "openness-open-license", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "GitHub repo description explicitly states 'E2B is an open-source infrastructure' and provides a self-hosting guide with Terraform, corroborating that the source is publicly available and usable, not just marketing docs. Missing for 10: explicit citation of the license name (e.g., Apache/MIT) and independent confirmation of license terms.", "evidenceIds": [ "e2b-gh-3", "e2b-gh-2" ] }, { "productId": "e2b", "storyId": "openness-self-host", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "E2B is explicitly open-source and provides a documented self-hosting guide using Terraform to deploy the infrastructure on your own, plus BYOC/self-hosted deployment options referenced in client docs. However, the evidence lacks depth on self-hosting: no details on required components, hardware/scale limits, maintenance burden, or independent reports of someone successfully self-hosting. missing for 10: detailed self-hosting setup docs/tutorial, independent verification of a working self-hosted deployment, and clarity on feature parity between self-hosted and cloud versions.", "evidenceIds": [ "e2b-gh-2", "e2b-gh-3", "e2b-docs-30", "e2b-docs-34" ] }, { "productId": "e2b", "storyId": "pause-resume-state", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "E2B's docs explicitly document pausing a sandbox and resuming it later 'from the exact state it was in, including the filesystem and the memory, so running processes and loaded variables survive,' with paused sandboxes kept indefinitely, plus CLI support for snapshot/pause management (e2b sandbox snapshot, list --state paused). This directly matches the story's core claim. missing for 10: independent/hands-on third-party verification of state fidelity after resume, and more detail on any limitations (e.g., GPU/network state, long-running process edge cases) beyond first-party docs.", "evidenceIds": [ "e2b-docs-3", "e2b-docs-4", "e2b-docs-22", "e2b-docs-26", "e2b-docs-24", "e2b-docs-44" ] }, { "productId": "e2b", "storyId": "per-second-billing", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "The pricing page confirms usage-based configuration (customizable CPU/RAM, session-length/concurrency tiers) and the SDK/CLI let you kill a sandbox or reset its timeout at any moment, plus real-time CPU/memory/disk metrics — all consistent with paying only for compute actually consumed. However, no evidence pack item explicitly states a per-second billing granularity or shows a billing/invoice example. Missing for 10: explicit per-second billing rate documentation, an invoice/billing dashboard example, or a stated billing unit (e.g., 'billed per second of sandbox uptime').", "evidenceIds": [ "e2b-docs-13", "e2b-docs-14", "e2b-docs-5", "e2b-docs-6", "e2b-docs-12", "e2b-docs-29" ] }, { "productId": "e2b", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "E2B docs mention BYOC and self-hosted deployment options (Terraform-based) which would let an organization control where infrastructure/data physically resides, but there is no evidence of an explicit region/residency selector or documented list of hosting regions for the standard managed service. missing for 10: explicit region-selection UI/API, documented list of available data-residency regions, compliance certifications tied to specific regions.", "evidenceIds": [ "e2b-docs-30", "e2b-docs-34", "e2b-gh-2" ] }, { "productId": "e2b", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of E2B's data-training policies, opt-out controls, or data-use terms for AI model training; all evidence concerns sandbox execution, CLI, persistence, and networking features. This axis is a fair question for a product processing user code/data, but nothing in the pack shows E2B addressing it.", "evidenceIds": [] }, { "productId": "e2b", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "E2B gives concrete lifecycle controls over sandbox data: sandboxes can be killed/shutdown anytime, timeouts set/reset, and paused sandboxes are 'kept indefinitely with no automatic deletion' until explicitly resumed or removed via CLI shutdown/kill commands, giving users direct control over when sandbox state (filesystem, memory) persists or is destroyed. However, there's no evidence of broader data-retention policy documentation (e.g., logs, telemetry, backups) or explicit deletion/export guarantees at the account or compliance level (GDPR/SOC2-style retention statements). Missing for 10: explicit data-retention/deletion policy documentation beyond sandbox kill/pause mechanics, account-level data deletion guarantees, and compliance/regulatory retention statements.", "evidenceIds": [ "e2b-docs-4", "e2b-docs-6", "e2b-docs-20", "e2b-docs-25", "e2b-docs-39", "e2b-docs-3" ] }, { "productId": "e2b", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item mentions telemetry, analytics collection, or an opt-out/opt-in setting for usage tracking; E2B's self-hosting option (e2b-gh-2) implies infrastructure control but does not document any telemetry toggle for the hosted product.", "evidenceIds": [] }, { "productId": "e2b", "storyId": "run-coding-agents-inside", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "E2B provides a dedicated vendor recipe ('Run AI coding agents like Claude Code, Codex, and Amp in secure E2B sandboxes with full terminal, filesystem, and git access') and a pre-built `claude` template with Claude Code already installed, plus similar guides for other agent frameworks. This is a first-party documented recipe with concrete setup instructions. Missing for 10: independent/hands-on confirmation that Codex specifically runs smoothly (community evidence only discusses a different agent framework's rough edges, not Claude Code/Codex directly), and no third-party benchmark of the recipe's reliability.", "evidenceIds": [ "e2b-docs-28", "e2b-docs-42", "e2b-docs-43", "e2b-docs-44" ] }, { "productId": "e2b", "storyId": "run-untrusted-code", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "E2B is explicitly designed as isolated sandbox infrastructure (Firecracker microVMs) to run AI-generated/untrusted code without touching the developer's own machine, with persistence, network controls, resource limits, and self-hosting/BYOC options for stricter isolation needs (e2b-gh-3, e2b-docs-44, e2b-docs-10, e2b-gh-2). Docs also show direct integration for coding agents (Claude Code, Codex, Amp) executing in these sandboxes with full filesystem/terminal access (e2b-docs-28, e2b-docs-42).\nMissing for 10: independent third-party security audit or hands-on penetration-test evidence beyond vendor docs/GitHub description.", "evidenceIds": [ "e2b-gh-3", "e2b-docs-44", "e2b-docs-10", "e2b-gh-2", "e2b-docs-28", "e2b-docs-42", "e2b-docs-3" ] }, { "productId": "e2b", "storyId": "secrets-stay-out", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "E2B's docs and repo repeatedly describe sandboxes as isolated Firecracker microVMs separate from the host/server ('your agent's code runs in an isolated Firecracker microVM instead of your server', 'isolated sandbox instead of your machine'), and network access to the sandbox can be restricted via allow/deny lists, all supporting the claim that a sandboxed agent cannot reach host resources. However, there is no explicit documentation or hands-on test specifically confirming host secrets/credentials (e.g., env vars, cloud IAM creds) are unreachable from sandboxed code, nor independent security audit evidence. Missing for 10: explicit secret/credential-isolation guarantees, third-party security audit or hands-on penetration test confirming host secrets are unreachable.", "evidenceIds": [ "e2b-gh-3", "e2b-docs-44", "e2b-docs-43", "e2b-docs-10" ] }, { "productId": "e2b", "storyId": "shell-and-packages", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "E2B docs explicitly document commands.run() for executing arbitrary shell commands inside sandboxes, plus CLI exec-command support with background flag, and templates allow custom package installation via build commands. Combined with file read/write and persistent sandbox sessions, this directly delivers arbitrary shell execution and package installation. Missing for 10: independent hands-on benchmark specifically verifying package installation success/failure rates.", "evidenceIds": [ "e2b-docs-8", "e2b-docs-38", "e2b-docs-17", "e2b-docs-23", "e2b-docs-36" ] }, { "productId": "e2b", "storyId": "snapshot-restore-fork", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "E2B explicitly supports pause/resume of a sandbox preserving filesystem and memory, and exposes CLI/SDK snapshot management (`e2b sandbox snapshot`), naming snapshot templates that can be reused to create new sandboxes, effectively enabling fork-like behavior from a saved state. missing for 10: explicit documentation/example of forking multiple independent sandboxes from a single snapshot simultaneously, and independent hands-on verification of restore/fork fidelity beyond vendor docs.", "evidenceIds": [ "e2b-docs-3", "e2b-docs-4", "e2b-docs-22", "e2b-docs-26", "e2b-docs-40" ] }, { "productId": "e2b", "storyId": "spinup-via-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs clearly show a single SDK call (Sandbox.create()) provisions an isolated sandbox with near-instant readiness, plus templates offering 'zero wait time' startup, CLI equivalents, and evidence of the core create/run/kill/timeout lifecycle. missing for 10: independent hands-on benchmarking of actual boot latency beyond vendor claims.", "evidenceIds": [ "e2b-docs-1", "e2b-docs-11", "e2b-docs-15", "e2b-docs-21", "e2b-docs-36", "e2b-gh-3" ] }, { "productId": "maritime", "storyId": "agent-self-provisions", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Maritime's CLI and SDK let an agent provision (idempotent) itself, run in a sandboxed micro-VM/container, and be managed via authenticated API/CLI calls with JSON output designed for machine consumption, and agents can be torn down/put to sleep automatically when idle. However, the evidence is docs-only with no hands-on trace of an agent actually reading back code-execution results or explicit 'delete/destroy' teardown call, and no independent verification of the full autonomous loop. missing for 10: concrete evidence of an agent reading execution output programmatically, an explicit teardown/delete API example, and independent/hands-on confirmation of the full provision→execute→read→teardown loop.", "evidenceIds": [ "maritime-docs-2", "maritime-docs-9", "maritime-docs-22", "maritime-docs-23", "maritime-docs-28", "maritime-docs-45", "maritime-docs-19" ] }, { "productId": "maritime", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Maritime hosts a live llms.txt (confirmed by a 200 probe) plus a dedicated CLI llms.txt and an ai-agents docs page explicitly designed for agents (Claude Code, Cursor, etc.) to consume, authenticate, and act on — directly matching the story. Missing for 10: no independent third-party confirmation that agents actually navigate these docs successfully in practice.", "evidenceIds": [ "maritime-probe-1", "maritime-docs-7", "maritime-docs-9", "maritime-docs-22", "maritime-docs-42" ] }, { "productId": "maritime", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Maritime's evidence describes agent deployment, hosting, webhooks, CLI/SDK tooling, and templates, but nothing shows the platform itself surfacing AI-generated insights or suggestions from a user's own data within the product UI — the docs describe running agents, not an analytics/insights layer on user data.", "evidenceIds": [] }, { "productId": "maritime", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Maritime provides cron triggers with standard five-field expressions that wake agents on schedule and deliver prompts, plus agents can self-schedule their own wake triggers (e.g., 'send me a digest every weekday at 9'), and agents run persistently on always-on micro-VMs with sleep/wake economics, enabling true background autonomous automation. Missing for 10: independent/hands-on verification of trigger reliability and no mention of monitoring/alerting on automation failures.", "evidenceIds": [ "maritime-docs-10", "maritime-docs-32", "maritime-docs-48", "maritime-docs-12", "maritime-docs-28", "maritime-docs-37" ] }, { "productId": "maritime", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Maritime is presented as infrastructure for hosting/deploying user-built AI agents, not as a product with its own built-in assistant for task delegation. Evidence shows external assistants (Claude, Cursor) can drive the Maritime CLI on the user's behalf, but nothing indicates Maritime itself ships an embedded assistant the user delegates tasks to inside the product.", "evidenceIds": [ "maritime-docs-42", "maritime-docs-22" ] }, { "productId": "maritime", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Maritime provides a CLI with --json flag on every command, machine-readable output, stable exit codes, and env-var (non-interactive) authentication via MARITIME_TOKEN, explicitly designed for automation/CI use and control by other AI agents rather than humans, plus a RESTful API and idempotent provisioning for scripted workflows. This directly supports headless/CI operation across CLI and SDK/API paths. Missing for 10: no explicit CI pipeline example (e.g., GitHub Actions/GitLab CI snippet) or independent third-party confirmation of CI usage.", "evidenceIds": [ "maritime-docs-9", "maritime-docs-22", "maritime-docs-23", "maritime-docs-43", "maritime-docs-36", "maritime-docs-45", "maritime-probe-4" ] }, { "productId": "maritime", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Maritime is a platform for deploying and hosting agents built with frameworks (CrewAI, LangGraph, etc.), but the evidence pack shows no mention of MCP server integration or a mechanism for agents to consume external MCP tool servers. Nothing describes plugging in MCP servers to extend tool use.", "evidenceIds": [] }, { "productId": "maritime", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Maritime is a platform for deploying/hosting agents (not itself an agent), so an official MCP server is a fair axis to expect, but no evidence pack item mentions MCP, an MCP server, or MCP-compatible endpoints — only REST API, CLI, SDKs, and webhooks are documented.", "evidenceIds": [ "maritime-docs-36", "maritime-docs-35", "maritime-docs-8" ] }, { "productId": "maritime", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Maritime documents natural-language operation at the agent level (chat with agents, cron-triggered prompts set via natural language like 'send me a digest every weekday at 9') and CLI/API designed to be driven by AI agents via prompts (paste a prompt into Claude/Cursor to install and operate). However, the core Maritime platform itself (dashboard, agent management) is not shown to be operable purely via natural-language commands by the end user — the NL interface is mediated through the CLI/API being agent-friendly rather than a native NL command surface for platform operations. missing for 10: evidence of a native NL interface for platform-level operations (not just agent chat/prompts), independent/hands-on confirmation of this workflow succeeding.", "evidenceIds": [ "maritime-docs-32", "maritime-docs-42", "maritime-docs-48", "maritime-docs-22", "maritime-docs-16" ] }, { "productId": "maritime", "storyId": "agentic-official-cli", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Maritime ships an official npm-installable CLI explicitly designed for AI-native use: machine-readable/--json output on every command, stable exit codes, a self-describing contract, and env-var (non-interactive) authentication via MARITIME_TOKEN, all documented at /docs/cli and /docs/ai-agents. It also supports agent-driven install/auth/deploy flows via prompted assistants. Missing for 10: independent/hands-on verification beyond vendor docs.", "evidenceIds": [ "maritime-docs-35", "maritime-docs-41", "maritime-docs-22", "maritime-docs-23", "maritime-docs-9", "maritime-docs-42", "maritime-probe-4" ] }, { "productId": "maritime", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Maritime documents a full public API (REST endpoints, TypeScript/Python SDKs, idempotent provisioning, webhooks, scoped bearer-token auth) plus a CLI explicitly built for AI agents with machine-readable JSON output and stable exit codes, directly enabling AI-native driving of the platform. missing for 10: a discoverable machine-readable spec (openapi.json/swagger.json returned 404) and independent third-party corroboration of the API's stability.", "evidenceIds": [ "maritime-docs-36", "maritime-docs-8", "maritime-docs-2", "maritime-docs-45", "maritime-docs-3", "maritime-docs-22", "maritime-docs-23", "maritime-docs-9", "maritime-probe-3" ] }, { "productId": "maritime", "storyId": "agentic-scoped-keys", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Maritime documents scoped API keys explicitly: every key carries scopes, users should hand subsystems the narrowest key they need, and a dedicated example shows a Computers-only key being refused on all other endpoints — a concrete least-privilege mechanism. This pairs with CLI/env-based key minting (mk_ tokens) suited to agent automation. Missing for 10: an exhaustive scope catalog/reference and independent (non-vendor) verification of scope enforcement in practice.", "evidenceIds": [ "maritime-docs-4", "maritime-docs-25", "maritime-docs-26", "maritime-docs-44", "maritime-docs-46", "maritime-docs-23", "maritime-docs-43" ] }, { "productId": "maritime", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Maritime documents official TypeScript and Python SDKs with idempotent agent provisioning, scoped authentication, webhooks, messaging, and a Files API, plus an npm-installable CLI and RESTful API, all clearly aimed at building AI-native integrations. missing for 10: no public OpenAPI/swagger spec found (404s) and no independent/hands-on corroboration beyond first-party docs.", "evidenceIds": [ "maritime-docs-2", "maritime-docs-8", "maritime-docs-18", "maritime-docs-28", "maritime-docs-36", "maritime-docs-40", "maritime-docs-45", "maritime-probe-3" ] }, { "productId": "maritime", "storyId": "agentic-webhooks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs explicitly describe subscribing a URL to receive signed JSON POSTs on every agent lifecycle event, replacing polling, which directly matches the story. missing for 10: independent/hands-on confirmation of webhook delivery reliability, payload schema details, and signature verification specifics.", "evidenceIds": [ "maritime-docs-3", "maritime-docs-4" ] }, { "productId": "maritime", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Docs describe a RESTful API and CLI but there's no evidence of an interactive API reference (e.g., Swagger/OpenAPI explorer) with runnable examples; probes explicitly show no openapi.json/swagger.json spec found at any candidate path. missing for 10: interactive API console/playground, OpenAPI spec, runnable code examples in docs.", "evidenceIds": [ "maritime-docs-36", "maritime-docs-8", "maritime-probe-3" ] }, { "productId": "maritime", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Maritime documents a RESTful API and CLI with --json output, but there is no evidence of a downloadable machine-readable spec (OpenAPI/Swagger); active probes for openapi.json, swagger.json, and similar paths all returned 404.", "evidenceIds": [ "maritime-probe-3", "maritime-docs-36", "maritime-docs-8" ] }, { "productId": "maritime", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Maritime's docs cover deployment, keys/scopes, templates, triggers, and agent lifecycle, but nothing describes a sandbox/staging environment isolated from production data for testing purposes. Missing for 10: any mention of a sandbox/test mode, staging environment, or data isolation guarantees between test and production agents.", "evidenceIds": [] }, { "productId": "maritime", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of API versioning scheme (e.g., v1/v2 paths) or any documented deprecation policy; OpenAPI spec probes all 404, and no docs mention version support windows or breaking-change policy.", "evidenceIds": [ "maritime-probe-3", "maritime-docs-36" ] }, { "productId": "maritime", "storyId": "auto-teardown-timeouts", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The docs describe Maritime as serverless agent hosting and mention an 'always-on' paid add-on (implying default agents are not always running), but no evidence describes a developer-configurable idle timeout or automatic shutdown-to-stop-billing mechanism. Missing for 10: any documented timeout setting, idle-detection policy, or billing-pause behavior tied to inactivity.", "evidenceIds": [ "maritime-docs-12" ] }, { "productId": "maritime", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "There is explicit support for bulk import of config/env vars (maritime-docs-11) and a scriptable CLI/API with JSON output (maritime-docs-9, maritime-docs-36) that an AI-native user could loop over to perform bulk actions, but there is no documented batch/bulk endpoint for operating on many agents or resources at once. missing for 10: a dedicated bulk-operation API/CLI command (e.g. bulk create/delete/update agents), evidence of scaling to many items reliably, and independent confirmation of bulk workflows in practice.", "evidenceIds": [ "maritime-docs-11", "maritime-docs-9", "maritime-docs-36", "maritime-docs-22" ] }, { "productId": "maritime", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Maritime supports triggers (cron with five-field expressions, natural-language schedule setting, webhooks for lifecycle events) that wake agents and fire actions automatically, which covers rule-based automation on events. However, this is event/schedule-based triggering rather than a general-purpose conditional rules engine (e.g., no evidence of arbitrary if/then logic across multiple event types or complex conditions). missing for 10: evidence of a general conditional rules/policy engine beyond cron and webhook triggers, and independent/hands-on confirmation of trigger reliability.", "evidenceIds": [ "maritime-docs-10", "maritime-docs-32", "maritime-docs-48", "maritime-docs-3", "maritime-docs-12" ] }, { "productId": "maritime", "storyId": "automation-scheduled-jobs", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Maritime explicitly supports cron triggers with standard five-field expressions that wake agents and deliver prompts on schedule, plus a natural-language flow where an agent sets its own recurring schedule ('send me a digest every weekday at 9') that syncs into real wake triggers. This directly covers recurring job/workflow scheduling for AI-native users. Missing for 10: independent/hands-on verification beyond vendor docs, and no detail on managing/editing/cancelling existing scheduled triggers via CLI/API.", "evidenceIds": [ "maritime-docs-10", "maritime-docs-32", "maritime-docs-48", "maritime-docs-12" ] }, { "productId": "maritime", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of version history, review workflows, or rollback for automations/agent configs; docs cover deployment, triggers, builds, and CLI but nothing about versioning or reverting agent configuration/automation changes.", "evidenceIds": [] }, { "productId": "maritime", "storyId": "browser-in-sandbox", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Maritime docs explicitly describe 'Computers': a persistent Linux desktop (XFCE, Chromium, LibreOffice) per user driven by a model, and the changelog confirms agents get a real Linux desktop with screenshot/mouse/keyboard control and live takeover — this covers full desktop environments including a browser (Chromium) inside the sandbox. missing for 10: no independent/hands-on corroboration of performance or headless-only browser mode, and no detail on resource limits or concurrency.", "evidenceIds": [ "maritime-docs-5", "maritime-docs-50", "maritime-docs-25" ] }, { "productId": "maritime", "storyId": "concurrent-fleets", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of documented concurrency limits or guidance for running large fleets of sandboxes at scale; docs cover provisioning, idempotency, and billing per-agent but never state concurrency ceilings, throughput benchmarks, or fleet-scale operational guidance.", "evidenceIds": [] }, { "productId": "maritime", "storyId": "custom-images-templates", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm bring-your-own-image support via a documented three-endpoint contract that works with any Docker image (CrewAI, LangGraph, AutoGen, custom scripts), plus a build log and template system for packaging custom agents. However, there's no detailed spec of the contract itself, no example custom Dockerfile, and no independent/hands-on confirmation of successfully running a fully custom image end-to-end. missing for 10: full technical spec of the three-endpoint contract, a worked example of a custom image deployment, independent verification of custom image support beyond docs claims.", "evidenceIds": [ "maritime-docs-1", "maritime-docs-7", "maritime-docs-6", "maritime-docs-13" ] }, { "productId": "maritime", "storyId": "expose-public-url", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Maritime's docs describe deploying any Dockerfile-based repo to a public, no-login HTTPS URL that reaches the running container's service, matching the core story of exposing an internal service via a public preview URL (maritime-docs-29, maritime-docs-47). However, the docs never explicitly describe port-selection mechanics (e.g., multiple ports, custom port binding) and all evidence is first-party with no independent/hands-on corroboration. missing for 10: explicit port-selection/multi-port docs, independent verification that the public URL reaches an arbitrary internal port.", "evidenceIds": [ "maritime-docs-29", "maritime-docs-47" ] }, { "productId": "maritime", "storyId": "fast-cold-starts", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Maritime's docs describe sleep/wake behavior (e.g., 'Sleeping agents wake automatically', 'puts it to sleep when idle, and wakes it on the next visit') and an 'always-on' add-on for millisecond reaction, but no evidence anywhere documents actual cold-start latency numbers (sub-second to few-second) for waking a sandboxed agent from sleep.", "evidenceIds": [ "maritime-docs-28", "maritime-docs-29", "maritime-docs-12" ] }, { "productId": "maritime", "storyId": "filesystem-io", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions SDK-based file read/write/upload/download operations against a sandbox filesystem; the docs cover agent provisioning, webhooks, auth scoping, triggers, computers (desktop), CLI, and migration but nothing about filesystem file I/O via SDK. Missing for full: no filesystem API/SDK method docs, no upload/download endpoint references, no code samples showing file operations.", "evidenceIds": [] }, { "productId": "maritime", "storyId": "free-tier-start", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly state the free plan includes 3 agents with no card required, and quickstart/CLI docs show a self-serve path (npm install, deploy in under 5 minutes) with no sales contact implied anywhere. missing for 10: independent/third-party confirmation of the free tier's real-world limits and no hands-on verification of the signup flow.", "evidenceIds": [ "maritime-docs-20", "maritime-docs-14", "maritime-docs-41", "maritime-docs-27" ] }, { "productId": "maritime", "storyId": "gpu-access", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack mentions GPUs, GPU-accelerated instances, or hardware acceleration for sandboxed agent workloads; all compute descriptions reference micro-VMs, sleep/wake, and CPU-based containers only. Missing for 10: any mention of GPU attachment, GPU instance types, or hardware accelerator support.", "evidenceIds": [ "maritime-docs-37", "maritime-docs-19" ] }, { "productId": "maritime", "storyId": "hard-isolation-boundary", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item documents a specific isolation mechanism (microVM, gVisor, container hardening, VM boundary) between sandboxes and host systems; docs mention Docker images, serverless infra, and scoped API keys but not the underlying execution isolation boundary. Missing for 10: any explicit statement of sandbox isolation technology, security model documentation, or third-party audit/confirmation of hard isolation.", "evidenceIds": [] }, { "productId": "maritime", "storyId": "long-running-sessions", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Maritime advertises serverless infra by default with an 'always-on' add-on for agents that must run continuously, and a persistent Linux desktop model for end users, suggesting long-lived sessions are possible but only via a paid add-on rather than a default guarantee for hours/days-long sandbox sessions. missing for 10: explicit documentation of session/session-timeout limits, confirmation that a sandbox session can run unattended for hours or days without the always-on add-on, and independent verification of long-running task stability.", "evidenceIds": [ "maritime-docs-12", "maritime-docs-5", "maritime-probe-1" ] }, { "productId": "maritime", "storyId": "multi-language-runtimes", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Maritime's evidence describes deploying persistent Docker-based agents (any Docker image with sleep/wake/channels/triggers), a Files API, and a Linux desktop 'Computers' feature, but nothing describes a code-execution/sandbox API that runs snippets in multiple language runtimes (Python, JS, etc.) and returns structured/rich results (e.g., stdout, images, dataframes) the way a code-interpreter tool would.", "evidenceIds": [ "maritime-docs-19", "maritime-docs-5", "maritime-docs-40", "maritime-docs-1" ] }, { "productId": "maritime", "storyId": "network-egress-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions network egress controls, firewall rules, or any explicit allow/deny policy for outbound traffic from the sandbox/agent VM; the closest material covers API key scoping and secret management, not network policy.", "evidenceIds": [] }, { "productId": "maritime", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show a broad API/SDK/CLI surface (provisioning, keys, webhooks, triggers, computers, templates, builds) and CLI --json mode implying the CLI itself is just an API wrapper, suggesting an API-first architecture. However there's no explicit statement of full UI/API parity and no discoverable OpenAPI spec (probes return 404s for all schema locations), so completeness of API coverage relative to every UI feature can't be confirmed. missing for 10: explicit parity statement, published OpenAPI/API reference, independent confirmation that every UI action has an API equivalent.", "evidenceIds": [ "maritime-docs-8", "maritime-docs-9", "maritime-docs-2", "maritime-docs-4", "maritime-docs-5", "maritime-docs-6", "maritime-docs-7", "maritime-probe-3" ] }, { "productId": "maritime", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows import/migration INTO Maritime (from AWS/Heroku/VPS, .env bulk import) and a CLI with --json output, but nothing documents exporting agent configs, data, secrets, or state OUT in open formats to leave the platform. missing for 10: an export command or documented data-portability feature, open-format export of agent state/configs, any migration-out guide.", "evidenceIds": [ "maritime-docs-11", "maritime-docs-15", "maritime-docs-9" ] }, { "productId": "maritime", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Maritime is a closed commercial SaaS platform; there is no evidence of an open-source license or public source repository anywhere in the docs or probes. missing for 10: an open-source license, a public source code repository, any mention of licensing terms for the platform's own code.", "evidenceIds": [] }, { "productId": "maritime", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Maritime is explicitly described as a cloud platform with agents running on serverless infrastructure, and no documentation offers a self-hosted deployment option for the core platform itself (Docker images are for agent frameworks running atop Maritime's cloud, not for self-hosting Maritime).", "evidenceIds": [ "maritime-probe-1", "maritime-docs-15" ] }, { "productId": "maritime", "storyId": "pause-resume-state", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions pausing/suspending a sandbox or preserving filesystem and memory state across a pause/resume cycle; docs cover provisioning, webhooks, triggers, templates, and CLI/API auth but nothing about snapshot or suspend-resume persistence.", "evidenceIds": [] }, { "productId": "maritime", "storyId": "per-second-billing", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Maritime's own billing docs explicitly state the opposite of per-second usage billing: 'One subscription, a set number of agents, flat prices for extras. No meters, no hourly rates: how much your agents run never changes the bill' and 'Always-on is a flat add-on.' There is no evidence of any per-second or usage-metered compute pricing for sandboxes.", "evidenceIds": [ "maritime-docs-27", "maritime-docs-12" ] }, { "productId": "maritime", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, region selection, or geographic storage controls anywhere in Maritime's docs or probes; nothing addresses where agent/user data is physically stored.", "evidenceIds": [] }, { "productId": "maritime", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data-training opt-out or AI model training policies for Maritime; the docs cover deployment, hosting, keys, CLI, etc., but nothing about preventing customer data from being used to train AI models.", "evidenceIds": [] }, { "productId": "maritime", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence describes data retention policies, deletion controls, or export/purge mechanisms for agent data, files, or session logs; docs mention encrypted-at-rest secrets and append-only session logs but nothing about user-controlled retention or deletion.", "evidenceIds": [] }, { "productId": "maritime", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions telemetry, usage tracking, analytics collection, or an opt-out mechanism anywhere in Maritime's docs; the closest topics (scoped API keys, secret variables) address access control, not telemetry preferences.", "evidenceIds": [] }, { "productId": "maritime", "storyId": "run-coding-agents-inside", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Maritime documents dedicated template recipes for running Claude Code and Codex inside its sandboxed micro-VMs (persistent storage, sleep/wake, metered LLM access by default, chat channels), directly matching the vendor's own recipe framing for these named coding agents. missing for 10: independent/hands-on verification beyond vendor docs, and no direct evidence of a step-by-step 'follow this exact recipe' walkthrough output for a live run.", "evidenceIds": [ "maritime-docs-37", "maritime-docs-38", "maritime-docs-13", "maritime-docs-16", "maritime-docs-42" ] }, { "productId": "maritime", "storyId": "run-untrusted-code", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Maritime hosts agents as Docker images on 'serverless infrastructure' (maritime-docs-1, maritime-probe-1) and provides persistent isolated Linux desktops per end user (maritime-docs-5), which implies agent code runs off the developer's own machines/infra. However, there is no explicit documentation of sandboxing guarantees, resource/network isolation, or security boundaries specifically for running untrusted or AI-generated code safely. Missing for 10: explicit sandbox/isolation security model, resource-limit guarantees, and any independent verification that untrusted code cannot escape or affect other tenants/infra.", "evidenceIds": [ "maritime-docs-1", "maritime-docs-5", "maritime-probe-1" ] }, { "productId": "maritime", "storyId": "secrets-stay-out", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Maritime's docs describe per-agent micro-VM isolation with secrets 'injected only into that agent's VM' and encrypted at rest, plus scoped API keys so a leaked key 'can only do what its scope allows,' which together imply host-level credentials aren't reachable by agent code. However, there's no explicit documentation of sandbox mechanics like blocking cloud metadata endpoints, filesystem isolation from the host, or independent security testing confirming secrets are truly unreachable. missing for 10: explicit host-credential isolation guarantees (e.g., no metadata service access, filesystem boundary details), independent/hands-on security verification.", "evidenceIds": [ "maritime-docs-33", "maritime-docs-4", "maritime-docs-25", "maritime-docs-44", "maritime-docs-26" ] }, { "productId": "maritime", "storyId": "shell-and-packages", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Maritime's docs describe deploying agents via Docker images, templates, and a custom framework contract, but there is no evidence of an interactive shell/terminal access or arbitrary package installation inside a running sandbox for a developer to use directly. The 'Computers' feature offers a persistent desktop for end-users, not a shell for arbitrary command execution by the developer.", "evidenceIds": [] }, { "productId": "maritime", "storyId": "snapshot-restore-fork", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack mentions snapshotting a sandbox/agent state or restoring/forking new instances from such a snapshot; templates (maritime-docs-7) allow cloning a pre-configured agent package but not capturing/restoring runtime state.", "evidenceIds": [] }, { "productId": "maritime", "storyId": "spinup-via-api", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Maritime's SDK exposes an idempotent `provision` call that creates/returns a live agent environment, and docs claim full agent deployment (image pull → live, chat-ready) in under 5 minutes, with CLI/API equivalents (`maritime keys create`, RESTful API). This matches the 'one call, live environment fast' story well, though the fastest documented time is minutes not literal seconds and there's no independent/hands-on latency benchmark or public OpenAPI spec confirming the exact call shape. Missing for 10: independent timing/benchmark evidence corroborating 'seconds', and a verifiable OpenAPI/API schema (probe shows openapi.json 404).", "evidenceIds": [ "maritime-docs-2", "maritime-docs-45", "maritime-docs-14", "maritime-docs-13", "maritime-docs-36", "maritime-docs-35", "maritime-probe-3" ] }, { "productId": "modal", "storyId": "agent-self-provisions", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Modal Sandboxes are explicitly designed for programmatic, API-driven provisioning, code execution, log/result retrieval, and teardown, with SDK examples (Python/JS/Go) showing sandbox creation, exec, file access, and logs, plus a documented agent-code-execution example. missing for 10: independent hands-on report showing a fully autonomous agent loop (provision->execute->read->teardown) with no human intervention, and clearer documentation of teardown/termination API specifics.", "evidenceIds": [ "modal-docs-2", "modal-docs-3", "modal-docs-13", "modal-docs-19", "modal-docs-21", "modal-docs-22", "modal-docs-23", "modal-probe-2", "modal-comm-2" ] }, { "productId": "modal", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Modal has a working llms.txt (HTTP 200) with a documentation summary, and docs pages are also served as markdown (.md suffix) confirming agent-oriented doc access. missing for 10: no independent confirmation that agents actually consume this llms.txt successfully in practice, and no dedicated agent-docs index beyond the generic llms.txt.", "evidenceIds": [ "modal-probe-1", "modal-probe-2" ] }, { "productId": "modal", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Modal Sandboxes is compute/sandbox infrastructure for running untrusted or agent code, not a data product that ingests user data and surfaces AI-generated insights/suggestions from it — this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "modal", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Modal Sandboxes explicitly supports running autonomous agent code in isolated environments at scale (docs-3, docs-4, docs-22), and community evidence shows real-world use for background automation (self-hosted CI runners on modal-comm-1, agent runtimes on modal-comm-2). However, the evidence pack lacks explicit documentation of scheduling/cron triggers or persistent background job orchestration—sandboxes are framed as on-demand execution environments rather than a native 'set it and forget it' automation scheduler. Missing for 10: native scheduling/cron trigger docs, first-party persistent orchestration for long-running background automations.", "evidenceIds": [ "modal-docs-3", "modal-docs-4", "modal-docs-22", "modal-comm-1", "modal-comm-2" ] }, { "productId": "modal", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Modal Sandboxes is an infrastructure platform for running untrusted/agent code, not itself an AI assistant or agent product; it provides no built-in assistant to delegate tasks to — this is a wrong-axis question for this product category.", "evidenceIds": [] }, { "productId": "modal", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Modal Sandboxes is built for programmatic/headless use via Python, JS/Go SDKs and a CLI, with sandboxes created and controlled entirely through code (modal-docs-2, modal-docs-19, modal-docs-20, modal-gh-1). Community evidence confirms real-world CI/automation usage — a developer built GitHub Actions self-hosted runners on Modal Sandboxes explicitly for scalable, pay-per-use automation (modal-comm-1), and another wired gRPC control for an agent runtime (modal-comm-2). Missing for 10: no first-party CI/CD integration guide (e.g., official GitHub Actions/Jenkins docs) or dedicated 'headless CI' documentation section beyond community projects.", "evidenceIds": [ "modal-docs-2", "modal-docs-19", "modal-docs-20", "modal-gh-1", "modal-comm-1", "modal-comm-2", "modal-docs-15", "modal-probe-4" ] }, { "productId": "modal", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "modal", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "modal", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Modal Sandboxes is controlled exclusively via SDK (Python/JS/Go) calls and a traditional CLI (modal run, modal shell, etc.); there is no evidence of a natural-language command interface or chat-driven control layer for operating sandboxes. Evidence shows programmatic/API-first design (modal-docs-2, modal-docs-15, modal-docs-19) rather than any NL-command capability.", "evidenceIds": [ "modal-docs-2", "modal-docs-15", "modal-docs-19", "modal-probe-4" ] }, { "productId": "modal", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Modal ships an official CLI (documented at modal.com/docs/reference/cli) supporting sandbox-related operations like running commands/shells and managing/connecting to containers, confirmed by probe access to the CLI reference page. This CLI is core to Modal's agentic/AI-native workflows (creating and interacting with sandboxes for untrusted/agent code). missing for 10: independent hands-on user reports specifically praising/using the CLI (vs. SDK), and more detail on CLI coverage of full sandbox lifecycle (snapshots, networking) beyond the two documented subcommands.", "evidenceIds": [ "modal-docs-15", "modal-docs-18", "modal-probe-4", "modal-docs-13" ] }, { "productId": "modal", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Modal exposes documented Python, JS, and Go SDKs plus a CLI for programmatically creating, controlling, and interacting with Sandboxes (exec, filesystem, snapshots, networking, logs), and community developers confirm driving it via gRPC/SDK for agent runtimes. missing for 10: no public OpenAPI/REST spec (probe found only 404s), so the 'documented public API' is SDK/CLI-based rather than a formal API contract, and there's no independent audit of API stability/versioning.", "evidenceIds": [ "modal-docs-19", "modal-docs-20", "modal-docs-21", "modal-gh-1", "modal-docs-15", "modal-docs-18", "modal-docs-23", "modal-comm-2", "modal-probe-2", "modal-probe-3" ] }, { "productId": "modal", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes secure sandboxed execution environments and general platform features, but there is no concrete evidence of a mechanism for issuing scoped/least-privilege API credentials specifically to agents—only a vague marketing line about 'credentials already in place' (modal-docs-3) with no documentation of credential scoping, token permissions, or per-agent access control. Missing for 10: any documented API/credential-scoping mechanism, permission granularity docs, or examples of issuing least-privilege tokens to agents.", "evidenceIds": [ "modal-docs-3" ] }, { "productId": "modal", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Modal ships official Python SDK plus JS/Go SDKs for Sandboxes with documented APIs (exec, filesystem, snapshots, networking, logs) and community developers confirm building against them (GitHub Actions runners, gRPC-based JS agent runtime). missing for 10: independent third-party SDK reference/benchmarks beyond Modal's own docs and one community mention, and no evidence of broader language SDK coverage (e.g. Rust, Java).", "evidenceIds": [ "modal-gh-1", "modal-docs-19", "modal-docs-20", "modal-docs-21", "modal-docs-23", "modal-comm-2", "modal-comm-1" ] }, { "productId": "modal", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack of a webhook subscription mechanism or event-driven notification system for Modal Sandboxes; the docs focus on programmatic spin-up, execution, logging, and filesystem access, not outbound event/webhook subscriptions.", "evidenceIds": [] }, { "productId": "modal", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows standard docs pages, CLI reference, and SDK code snippets, but no interactive API reference or runnable-example playground; the openapi probe returned 404s and no docs mention an in-browser code runner or live API explorer.", "evidenceIds": [ "modal-probe-3", "modal-docs-15", "modal-docs-21" ] }, { "productId": "modal", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The probe explicitly found all candidate OpenAPI/swagger endpoints return 404, and no other evidence pack item mentions a machine-readable API spec being published; Modal's public interface is SDK/CLI-based rather than a REST API with a spec.", "evidenceIds": [ "modal-probe-3" ] }, { "productId": "modal", "storyId": "api-sandbox", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Modal Sandboxes are explicitly designed as ephemeral, isolated containers for running untrusted or agent code, with filesystem, networking/security, and snapshot controls documented (modal-docs-2, modal-docs-8, modal-docs-9, modal-docs-10, modal-probe-2), and community examples show real usage isolating workloads from production infra (modal-comm-1, modal-comm-2). This satisfies the 'test without touching production' need since each sandbox is a fresh, isolated environment rather than shared prod state. Missing for 10: explicit documentation or case study framing isolation specifically in terms of 'production data' separation (e.g., no prod DB/staging parity guidance), and no independent audit confirming true isolation guarantees.", "evidenceIds": [ "modal-docs-2", "modal-docs-8", "modal-docs-9", "modal-docs-10", "modal-docs-19", "modal-docs-22", "modal-comm-1", "modal-comm-2", "modal-probe-2" ] }, { "productId": "modal", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows release notes and beta/alpha labels for some sandbox features, but no documentation of API versioning scheme or a deprecation policy; OpenAPI spec probes also 404. missing for 10: explicit versioning scheme documentation, deprecation policy page, changelog with deprecation notices, stability guarantees for GA vs beta APIs.", "evidenceIds": [ "modal-docs-11", "modal-docs-12", "modal-docs-23", "modal-docs-24", "modal-probe-3" ] }, { "productId": "modal", "storyId": "auto-teardown-timeouts", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack only mentions general pay-per-second billing and bursty scaling, but never documents an explicit sandbox timeout or idle-shutdown parameter/config. No citation confirms configurable timeouts that auto-terminate sandboxes when idle or complete.", "evidenceIds": [] }, { "productId": "modal", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Modal supports spinning up hundreds of thousands of concurrent sandboxes/rollout environments programmatically, which is strong evidence of bulk/parallel scaling (modal-docs-4, modal-docs-14, modal-docs-5), and this is corroborated by community builds using it for scaled runner fleets (modal-comm-1). However, this is infrastructure-level bulk provisioning, not a documented 'bulk operations' API/pattern for operating across many existing items (e.g., batch file ops, batch snapshot management, or a dedicated batch API) — missing for 10: a first-party batch/bulk-operation API or workflow example (vs just scaling raw sandbox count), independent benchmarks confirming claimed concurrency at scale, and documentation of bulk management operations (e.g., bulk terminate/list/update) rather than just creation.", "evidenceIds": [ "modal-docs-4", "modal-docs-14", "modal-docs-5", "modal-comm-1", "modal-docs-6" ] }, { "productId": "modal", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes Modal Sandboxes as a programmatic, on-demand execution environment for untrusted/agent code, but nothing in the pack shows a rules/trigger engine that fires actions automatically on external events (e.g., webhooks, event-driven policies). Absent such evidence, this applicable automation-depth axis is unmet.", "evidenceIds": [] }, { "productId": "modal", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers ephemeral sandbox creation, execution, networking, snapshots, and SDKs, but there is no mention of cron-like scheduling, recurring triggers, or persistent workflow orchestration for Modal Sandboxes. missing for 10: any scheduling/cron API, recurring job trigger mechanism, or workflow orchestration documentation tied to Sandboxes.", "evidenceIds": [ "modal-docs-11", "modal-docs-13", "modal-docs-9" ] }, { "productId": "modal", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "modal", "storyId": "browser-in-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence describes Modal Sandboxes as isolated containers/VMs for running arbitrary code, agents, filesystem access, networking, and command execution, but nowhere mentions headless browser or full desktop/GUI environment support (e.g., Xvfb, VNC, browser automation tooling). No docs or community reports reference browser or desktop environments running inside sandboxes.", "evidenceIds": [] }, { "productId": "modal", "storyId": "concurrent-fleets", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Modal markets very high concurrency ('hundreds of thousands of concurrent rollout environments', 'scale to thousands of concurrent runs') and even documents a backend upgrade for higher creation rates/concurrency, plus community reports of building scalable runner fleets on Sandboxes, showing the capability is real and used at scale. However there is no evidence of documented hard limits, quotas, or per-account/per-plan concurrency ceilings that a platform engineer could plan capacity against. Missing for 10: explicit documented concurrency/rate limits or quotas, benchmark/load-test data, and any per-tier scaling caps.", "evidenceIds": [ "modal-docs-4", "modal-docs-5", "modal-docs-14", "modal-docs-16", "modal-docs-24", "modal-comm-1" ] }, { "productId": "modal", "storyId": "custom-images-templates", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Modal explicitly supports creating Sandboxes with custom Images ('Custom images, any dependency' and 'Create a Sandbox with the specified Image, and mount a Volume'), and community projects (GitHub Actions runners, MCP-JS) confirm developers configuring custom sandbox environments in practice. Missing for 10: explicit documentation of importing arbitrary Docker/OCI registry images or a dedicated 'bring your own container' guide beyond the Image API reference.", "evidenceIds": [ "modal-docs-2", "modal-docs-20", "modal-docs-19", "modal-comm-1", "modal-comm-2" ] }, { "productId": "modal", "storyId": "expose-public-url", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The docs pack includes a 'Networking and security' guide for sandboxes (modal-docs-10) which is the likely location for tunnel/port-exposure features, but no evidence text actually describes exposing a port to a public preview URL, tunnel URLs, or reachability details. missing for 10: explicit documentation or example showing sandbox.tunnels()/port forwarding, a public URL format, and independent confirmation of accessing services inside a sandbox via that URL.", "evidenceIds": [ "modal-docs-10", "modal-docs-11", "modal-probe-2" ] }, { "productId": "modal", "storyId": "fast-cold-starts", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence shows general speed/scale claims (\"spin up hundreds of thousands of concurrent rollout environments in seconds,\" faster backend for higher creation rates) but no explicit documented cold-start latency benchmark (e.g., sub-second or few-second startup time) for sandboxes specifically. Missing for 10: a concrete cold-start time benchmark/number, independent performance verification, and comparison across sandbox sizes/images.", "evidenceIds": [ "modal-docs-4", "modal-docs-14", "modal-docs-24", "modal-docs-2" ] }, { "productId": "modal", "storyId": "filesystem-io", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Modal explicitly documents a dedicated 'Filesystem Access' guide for sandboxes (modal-docs-8) alongside SDK examples showing file read/write via mounted volumes and exec commands (modal-docs-20, modal-docs-21), confirming read/write/upload/download-style operations through the SDK. missing for 10: no independent/hands-on community confirmation specifically of file upload/download methods (only exec/volume examples), and no explicit code snippet showing a dedicated upload/download API method distinct from volume mounting.", "evidenceIds": [ "modal-docs-8", "modal-docs-19", "modal-docs-20", "modal-docs-21", "modal-probe-2" ] }, { "productId": "modal", "storyId": "free-tier-start", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack includes pricing-related docs (pay-per-use, burst capacity) but no mention of a free tier, free credits, or self-serve signup without contacting sales; nothing confirms developers can start building without sales contact.", "evidenceIds": [] }, { "productId": "modal", "storyId": "gpu-access", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Modal docs explicitly state GPUs (H100s, A100s, A10Gs) can be attached to any sandbox on demand with per-second billing, and community evidence (HN runner project) confirms real-world GPU-backed sandbox usage. missing for 10: independent hands-on benchmark of GPU attach performance/reliability, and explicit code example showing GPU parameter in sandbox creation API.", "evidenceIds": [ "modal-docs-5", "modal-docs-16", "modal-comm-1" ] }, { "productId": "modal", "storyId": "hard-isolation-boundary", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Modal repeatedly describes Sandboxes as 'secure,' 'isolated' containers for untrusted code, and references a 'VM Sandboxes Beta' page implying microVM-based isolation, but no evidence pack item actually documents the underlying isolation technology (e.g., Firecracker/gVisor specifics, threat model, or security boundary guarantees) that a platform engineer would need to rely on. Missing for 10: explicit technical documentation of the isolation mechanism (hypervisor/kernel boundary), any third-party security audit or independent verification of the boundary's robustness.", "evidenceIds": [ "modal-docs-11", "modal-probe-2", "modal-docs-19", "modal-docs-22" ] }, { "productId": "modal", "storyId": "long-running-sessions", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Modal docs describe sandboxes as isolated, scalable execution environments with snapshotting, filesystem, and networking support, and a community project (self-hosted GitHub Actions runners) demonstrates sandboxes being kept alive for extended, real workloads. However, no evidence pack item states explicit maximum session duration, timeout configuration, or guarantees for multi-day persistence, so long-running-session support is inferred rather than directly documented. Missing for 10: explicit docs on configurable/no timeout limits, guidance for multi-day sandbox lifecycles, and independent confirmation of hours/days-long sessions beyond one community anecdote.", "evidenceIds": [ "modal-docs-9", "modal-docs-11", "modal-comm-1", "modal-docs-6", "modal-docs-23" ] }, { "productId": "modal", "storyId": "multi-language-runtimes", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Modal Sandboxes let you run arbitrary commands/code in isolated containers with custom images (any dependency, any language stack), including SDK examples for JS/Go exec and Python examples of running agent code (modal-docs-2, modal-docs-13, modal-docs-19-21). However, evidence never shows explicit multi-language runtime presets (Python vs JS vs others) or structured/rich result payloads (e.g., images, dataframes, mime-typed outputs) — outputs shown are plain stdout/exec results. missing for 10: explicit built-in multi-language runtime templates, documented rich/structured result formats beyond raw stdout.", "evidenceIds": [ "modal-docs-2", "modal-docs-13", "modal-docs-19", "modal-docs-20", "modal-docs-21", "modal-docs-22" ] }, { "productId": "modal", "storyId": "network-egress-policy", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Modal has a dedicated 'Networking and security' guide for Sandboxes, confirming the platform documents network controls for sandboxes, but the evidence pack only shows the page title without excerpts detailing how egress allow/deny policies are configured. missing for 10: actual policy syntax/examples (e.g. allowlist/blocklist configuration), confirmation of default-deny behavior, independent/hands-on verification that egress restrictions work as documented.", "evidenceIds": [ "modal-docs-10", "modal-probe-2" ] }, { "productId": "modal", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Modal Sandboxes is fundamentally API/SDK-driven (Python, JS/Go) with a full CLI for running commands and managing containers, and docs describe programmatic control over image building, execution, filesystem, networking, snapshots, and logs — suggesting the API is the primary and most complete interface. However, there's no explicit documentation stating full UI/API parity or listing any UI-exclusive features, and no OpenAPI spec is discoverable (probe found 404s), making it hard to confirm completeness of a public REST API surface beyond the SDKs/CLI. Missing for 10: explicit parity statement between dashboard UI and API/CLI capabilities, and a public API reference/spec confirming no UI-only actions.", "evidenceIds": [ "modal-docs-8", "modal-docs-9", "modal-docs-10", "modal-docs-13", "modal-docs-15", "modal-docs-18", "modal-gh-1", "modal-docs-19", "modal-probe-3", "modal-probe-4" ] }, { "productId": "modal", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "modal", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows a GitHub repo for the client SDK (modal-client) used to invoke Modal Sandboxes, not the underlying sandbox/runtime platform itself, and no license terms are mentioned. There is no evidence that Modal Sandboxes' core product source is published under an open license.", "evidenceIds": [ "modal-gh-1" ] }, { "productId": "modal", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Modal is a fully-managed cloud platform for running Sandboxes; there is no evidence of any self-hostable/on-prem version of the core product, and everything described (pay-per-second billing, GPU access, hosted CLI) implies a proprietary SaaS deployment model only.", "evidenceIds": [] }, { "productId": "modal", "storyId": "pause-resume-state", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Modal has a dedicated 'Snapshots' doc for sandboxes suggesting some pause/resume-with-state capability, but the evidence pack only gives the page title with no detail on what state (filesystem/memory) is preserved or how resume works. Missing for 10: documented mechanics of snapshot/resume (filesystem+memory fidelity), explicit confirmation memory state is preserved, independent/hands-on validation of the feature working as described.", "evidenceIds": [ "modal-docs-9", "modal-docs-8" ] }, { "productId": "modal", "storyId": "per-second-billing", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Modal's pricing docs and marketing explicitly state per-second billing with no reserved capacity ('pay by the second with no reserved capacity', 'only pay for what you use... burst up to what you need without over-allocating'), directly matching the story, and a community user corroborates paying only for resources actually consumed when building GPU runners on Sandboxes. Missing for 10: independent verification of exact billing granularity/metering mechanics and no first-party pricing breakdown showing per-second line items in the evidence pack.", "evidenceIds": [ "modal-docs-7", "modal-docs-16", "modal-docs-17", "modal-comm-1" ] }, { "productId": "modal", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions data residency, region selection, or geographic data storage controls for Modal Sandboxes; the pack covers execution, scaling, GPUs, filesystem, and networking but nothing about choosing storage region/residency.", "evidenceIds": [] }, { "productId": "modal", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Modal Sandboxes is an infrastructure/execution platform for running untrusted code, not an AI model or AI product with a training-data policy; there's no concept of 'training on user data' applicable to a sandbox execution service. This axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "modal", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers sandbox ephemerality, filesystem access, snapshots, and networking/security, but nothing describes data retention policies, deletion controls, or compliance/data-handling settings that a user could configure. missing for 10: explicit data retention policy documentation, user-controllable deletion/purge mechanisms, compliance certifications (e.g., SOC2/GDPR data handling), retention period configuration.", "evidenceIds": [] }, { "productId": "modal", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions telemetry, usage tracking, analytics collection, or an opt-out mechanism for Modal Sandboxes; the docs cover sandboxing, GPUs, filesystem, and logging but not data-collection/privacy controls.", "evidenceIds": [] }, { "productId": "modal", "storyId": "run-coding-agents-inside", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Modal's docs show general support for running 'agents' and 'untrusted code' in Sandboxes (modal-docs-3, modal-docs-22, modal-docs-19), but no evidence pack item mentions a vendor recipe, tutorial, or example specifically for running Claude Code or Codex inside a Sandbox.", "evidenceIds": [] }, { "productId": "modal", "storyId": "run-untrusted-code", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Modal Sandboxes is explicitly built for running untrusted/AI-generated code in isolated, ephemeral containers with networking/filesystem controls, resource limits, and logging, and community examples show real-world use (self-hosted CI runners, agent runtimes) confirming the isolation model works in practice. Missing for 10: independent security audit or adversarial escape-testing evidence, and clearer documentation of resource/network isolation guarantees beyond marketing copy.", "evidenceIds": [ "modal-docs-1", "modal-docs-2", "modal-docs-10", "modal-docs-11", "modal-docs-19", "modal-docs-22", "modal-probe-2", "modal-comm-1", "modal-comm-2" ] }, { "productId": "modal", "storyId": "secrets-stay-out", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Modal Sandboxes are documented as isolated, ephemeral containers for running untrusted/agent code with dedicated networking-and-security guides, supporting the general isolation story (modal-docs-1, modal-docs-2, modal-docs-10, modal-probe-2, modal-docs-19, modal-docs-22). However, no evidence explicitly states that host secrets/credentials are unreachable by default or describes the secrets-injection model that would confirm this — one doc even implies credentials are provisioned into the agent environment (modal-docs-3), which cuts against a strong 'unreachable unless granted' guarantee. Missing for 10: explicit secrets-isolation documentation, details of the opt-in secrets mechanism, and independent/hands-on confirmation that host credentials cannot leak into sandboxed code.", "evidenceIds": [ "modal-docs-1", "modal-docs-2", "modal-docs-3", "modal-docs-10", "modal-docs-19", "modal-docs-22", "modal-probe-2" ] }, { "productId": "modal", "storyId": "shell-and-packages", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Modal Sandboxes documentation explicitly covers running arbitrary shell commands (sb.exec, CLI 'run a command or interactive shell inside a Modal container'), custom images with any dependency installation, filesystem access, and community reports (HN self-hosted runners, JS agent runtime) corroborate real-world use for arbitrary command execution and untrusted code. Missing for 10: no independent benchmark of package installation limits or explicit documented apt/pip install examples beyond generic 'any dependency' claim.", "evidenceIds": [ "modal-docs-13", "modal-docs-15", "modal-docs-19", "modal-docs-21", "modal-docs-2", "modal-comm-1", "modal-comm-2", "modal-probe-2" ] }, { "productId": "modal", "storyId": "snapshot-restore-fork", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Modal explicitly documents a 'Sandbox Snapshots' feature (modal-docs-9), confirming snapshot capability exists, but the evidence pack gives no detail on the actual snapshot/restore/fork API mechanics, no confirmation that new independent sandboxes can be forked from a single snapshot, and no community/hands-on corroboration of this specific workflow. missing for 10: documented API details on restoring/forking multiple sandboxes from one snapshot, independent/hands-on confirmation of snapshot-restore-fork workflow, and clarity on any beta/experimental limitations.", "evidenceIds": [ "modal-docs-9", "modal-docs-11" ] }, { "productId": "modal", "storyId": "spinup-via-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs and SDK references (Python, JS/Go) show a single call (e.g. modal.Sandbox.create) spins up an isolated, ephemeral container/VM in seconds, with fast creation rates highlighted (hundreds of thousands of concurrent environments in seconds, faster backend for higher creation rates). Community builders (HN posts) independently confirm using Modal Sandboxes programmatically for real workloads like CI runners and agent runtimes. Missing for 10: independent benchmark of actual cold-start latency and a public OpenAPI spec (probe found 404s), so exact 'seconds' claim isn't third-party verified.", "evidenceIds": [ "modal-docs-2", "modal-docs-4", "modal-docs-19", "modal-docs-20", "modal-docs-24", "modal-comm-1", "modal-comm-2", "modal-probe-2" ] }, { "productId": "runloop", "storyId": "agent-self-provisions", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Runloop's core product is programmatic sandbox (Devbox) provisioning, execution, and teardown via API/SDK/CLI, explicitly designed for AI agents — including an Agents API for creating/managing/deploying agents, blueprints for reusable environments, snapshots for state capture, and suspend/resume lifecycle, all achievable without human intervention (docs-1, docs-13, docs-22, docs-23, docs-2, docs-4, docs-27, docs-12). The quickstart shows a devbox spun up in ~1 minute and CLI/SDK access covers execution and result retrieval end-to-end.\nmissing for 10: independent/hands-on third-party verification of a full agent-driven provision→execute→teardown loop, and a concrete code example showing an agent reading execution results programmatically.", "evidenceIds": [ "runloop-docs-1", "runloop-docs-13", "runloop-docs-22", "runloop-docs-23", "runloop-docs-2", "runloop-docs-4", "runloop-docs-27", "runloop-docs-12", "runloop-docs-26" ] }, { "productId": "runloop", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Direct probe evidence confirms a working llms.txt file (HTTP 200) with a documentation index, plus docs-md endpoints explicitly pointing agents to fetch llms.txt for full documentation, and Cursor rules files further supporting agent-oriented consumption of docs. missing for 10: independent third-party confirmation that agents successfully use these docs in practice.", "evidenceIds": [ "runloop-probe-1", "runloop-probe-2", "runloop-docs-29" ] }, { "productId": "runloop", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Runloop is sandbox/infrastructure tooling for running AI coding agents (Devboxes, blueprints, benchmarks, agent APIs) — it is not a data platform that ingests a user's data and surfaces AI-generated insights or suggestions. This story is a category error for this product type.", "evidenceIds": [] }, { "productId": "runloop", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Runloop's Axons feature gives agents a persistent event stream letting them suspend when idle, wake on demand, recover state, and hand off between agents 'without you building the plumbing' (runloop-docs-17, runloop-docs-18), and devboxes support suspend/resume to preserve state across runs (runloop-docs-27). The GitHub Action for agent deployment (runloop-docs-24) supports triggering agent runs from CI workflows, enabling background automation setups. Missing for 10: no explicit cron/scheduler primitive, no independent/hands-on account of long-running unattended automations actually working in production, and no example of a full always-on autonomous pipeline beyond docs claims.", "evidenceIds": [ "runloop-docs-17", "runloop-docs-18", "runloop-docs-27", "runloop-docs-24" ] }, { "productId": "runloop", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Runloop is a sandbox/infrastructure platform for running and orchestrating AI coding agents (Devboxes, Agents API, benchmarks), not an end-user product with its own embedded AI assistant to delegate tasks to. The evidence describes tools for building/hosting agents (and integrating with external assistants like Cursor), not a built-in assistant persona within Runloop itself.", "evidenceIds": [] }, { "productId": "runloop", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Runloop provides a CLI (rli), SDKs, an Agents API, GitHub Actions integration for deploying agents, and benchmark jobs runnable via CLI—all supporting headless/CI automation of sandboxes and agents. missing for 10: no independent/hands-on CI case study, and no explicit CI provider-specific guide beyond GitHub Actions example.", "evidenceIds": [ "runloop-docs-6", "runloop-docs-8", "runloop-docs-22", "runloop-docs-23", "runloop-docs-24" ] }, { "productId": "runloop", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Runloop publishes its own MCP server so external agents can call Runloop's tools (runloop-probe-4), but there is no evidence that Runloop itself can act as an MCP client and plug in third-party MCP servers to use their tools. missing for 10: any documentation of an MCP-client integration point, config for adding external MCP servers, or examples of Devboxes/agents consuming external MCP tool servers.", "evidenceIds": [ "runloop-probe-4" ] }, { "productId": "runloop", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Runloop is a sandbox/devbox platform (not itself an agent), so shipping an official MCP server is a valid axis, and the docs explicitly reference one at docs.runloop.ai/docs/tools/ai-tools. This confirms the capability exists, though the evidence pack lacks detail on the specific tools/resources exposed or independent hands-on confirmation. Missing for 10: detailed documentation of MCP server tool surface, independent/community verification of usage.", "evidenceIds": [ "runloop-probe-4" ] }, { "productId": "runloop", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Runloop exposes an official MCP server and ships Cursor '.mdc' rule files that give AI assistants context on the SDK (devbox lifecycle, file ops, blueprints, snapshots, tunnels), enabling natural-language operation through AI coding assistants rather than a bespoke NL interface. There's no first-party chat/NL command surface of Runloop's own, and no independent evidence of how well the MCP/Cursor integration performs in practice. Missing for 10: dedicated natural-language interface docs, hands-on/independent validation of the MCP or Cursor integration actually working end-to-end.", "evidenceIds": [ "runloop-probe-4", "runloop-docs-29", "runloop-docs-8" ] }, { "productId": "runloop", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Runloop ships an official CLI (`rli`) with both interactive TUI and traditional commands for managing devboxes, blueprints, SSH sessions, and benchmarks, documented directly in first-party docs and confirmed by probe. This is clearly built for AI-native/agentic workflows (blueprint creation, benchmark submission, devbox lifecycle management). Missing for 10: independent/hands-on community corroboration beyond vendor docs, and no changelog/version history showing active CLI maintenance cadence.", "evidenceIds": [ "runloop-docs-8", "runloop-docs-9", "runloop-docs-2", "runloop-docs-6", "runloop-probe-5" ] }, { "productId": "runloop", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Runloop documents a full public API surface: Devboxes API, Agents API, SDKs in common languages, CLI (rli), and GitHub Action integration, all callable programmatically for agent-driven workflows. This is strong first-party documentation of a public API, though probes show no discoverable OpenAPI spec, slightly weakening independent verification. Missing for 10: a published OpenAPI/swagger spec and independent third-party corroboration of API usage.", "evidenceIds": [ "runloop-docs-22", "runloop-docs-23", "runloop-docs-8", "runloop-docs-24", "runloop-probe-3", "runloop-probe-5" ] }, { "productId": "runloop", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Runloop's Agent Gateways let agents call LLM APIs like Anthropic/OpenAI 'without ever seeing your API keys,' which is a direct least-privilege credentialing mechanism for agents. However, the docs give only a single brief mention with no detail on fine-grained scopes, per-agent permission levels, revocation, or audit logging. Missing for 10: detailed scoping/permission model documentation, credential rotation/revocation controls, and independent verification of the gateway's isolation guarantees.", "evidenceIds": [ "runloop-docs-10", "runloop-docs-21" ] }, { "productId": "runloop", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Runloop explicitly documents official SDKs in common languages for interacting with its API, plus tutorials integrating with SDKs like OpenAI Agents SDK, supporting an AI-native builder workflow. However, evidence lacks specifics on which languages are covered, versioning/maturity, or independent developer corroboration. Missing for 10: enumerated language list/maturity details, independent hands-on confirmation, changelog/versioning evidence.", "evidenceIds": [ "runloop-docs-23", "runloop-docs-27", "runloop-docs-29" ] }, { "productId": "runloop", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence mentions webhooks or event subscription mechanisms; Runloop's docs describe devbox lifecycle events via Axons event streams but nothing about webhook subscriptions for external systems. Missing for 10: any documentation of a webhooks API, subscription/callback URLs, or event delivery mechanism to external endpoints.", "evidenceIds": [] }, { "productId": "runloop", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of an interactive API reference with runnable examples; the explicit probe for OpenAPI/swagger specs returned 404s across all candidate paths, and docs mention SDKs/CLI but not a runnable API explorer.", "evidenceIds": [ "runloop-probe-3" ] }, { "productId": "runloop", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Runloop offers SDKs and a CLI but there is no evidence of a downloadable OpenAPI/Swagger spec; explicit probes to common OpenAPI paths (openapi.json, swagger.json, etc.) all returned 404, confirming absence rather than just lack of mention.", "evidenceIds": [ "runloop-probe-3", "runloop-docs-23" ] }, { "productId": "runloop", "storyId": "api-sandbox", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Runloop's Devboxes are isolated sandbox execution environments (secure, snapshot/blueprint-based, no production data by default) explicitly designed for testing AI agents, fully separate from any production system. missing for 10: no independent/hands-on corroboration beyond vendor docs, no explicit statement contrasting sandbox vs production data isolation guarantees.", "evidenceIds": [ "runloop-docs-1", "runloop-docs-13", "runloop-docs-4", "runloop-docs-15", "runloop-docs-12", "runloop-docs-26" ] }, { "productId": "runloop", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of API versioning scheme or a documented deprecation policy; OpenAPI spec probe returned 404s and no changelog/versioning docs are cited.", "evidenceIds": [ "runloop-probe-3" ] }, { "productId": "runloop", "storyId": "auto-teardown-timeouts", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Runloop documents suspend/resume to \"save state and costs\" and Axons that let agents \"suspend when idle, wake on demand,\" indicating some idle-triggered lifecycle management, but there is no explicit documentation of a developer-configurable timeout parameter that auto-shuts-down a Devbox and stops billing after inactivity or completion. missing for 10: explicit timeout/idle-shutdown configuration API or CLI flag, confirmation that suspension halts billing automatically without agent action.", "evidenceIds": [ "runloop-docs-17", "runloop-docs-27", "runloop-docs-18" ] }, { "productId": "runloop", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Runloop supports running 10k+ parallel sandboxes, benchmark jobs that provision devboxes/run agents/aggregate outputs 'with a single command,' and snapshots for fan-out across multiple approaches — evidence of bulk/parallel operation across many devboxes. However, there's no explicit documentation of a generic bulk-operation API/CLI verb (e.g., bulk delete, bulk update, batch create) for arbitrary resources beyond devboxes/benchmarks. missing for 10: explicit bulk CRUD operations across resources like snapshots/storage objects/secrets, documented batch API endpoints, and independent evidence of large-scale bulk usage beyond vendor docs.", "evidenceIds": [ "runloop-docs-12", "runloop-docs-6", "runloop-docs-15", "runloop-docs-7" ] }, { "productId": "runloop", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Runloop's Axons give agents a persistent event stream that can suspend on idle, wake on demand, recover state, and hand off between agents, which is a form of automatic event-driven behavior, and the Broker component sits between Axon and agent process to manage this. However, there's no documentation of a general user-facing rules engine where arbitrary conditions map to arbitrary actions across the platform (e.g., webhooks, triggers on Devbox/snapshot events, custom rule definitions). Missing for 10: a documented rules/trigger definition interface, examples of custom event-to-action mappings beyond agent suspend/wake, and evidence of triggers on other platform events (snapshots, storage, secrets).", "evidenceIds": [ "runloop-docs-17", "runloop-docs-18" ] }, { "productId": "runloop", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes Devboxes, snapshots, benchmarks, and Axons for event-driven suspend/resume, but nothing about scheduling recurring jobs or cron-like recurring workflows. Absence of evidence for this applicable automation capability yields 'none'.", "evidenceIds": [] }, { "productId": "runloop", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Runloop's Snapshots explicitly support rolling back Devbox state to a known good point, and Blueprints let you version/reuse startup configurations, giving a rudimentary rollback mechanism for agent environments. However, there is no evidence of a review workflow (diffs, approvals, audit trail) or of 'automations' as a first-class versioned object distinct from raw disk/environment snapshots. Missing for 10: explicit review/diff tooling for automations, first-class automation versioning (not just devbox/environment snapshots), audit/history UI for changes.", "evidenceIds": [ "runloop-docs-15", "runloop-docs-14", "runloop-docs-4" ] }, { "productId": "runloop", "storyId": "browser-in-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence Runloop supports running a headless browser or desktop GUI inside its Devbox sandboxes; the only browser-related tutorial explicitly states the architecture avoids this, running Chromium externally via Browserbase and connecting over CDP so 'no Chromium ever runs in the devbox.' Blueprints/docs never mention GUI, X11, VNC, or bundled browser binaries as a supported environment type.", "evidenceIds": [ "runloop-docs-28" ] }, { "productId": "runloop", "storyId": "concurrent-fleets", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Runloop's marketing site claims support for 10k+ parallel sandboxes and sub-2s startup for 10GB images, showing scale is a design goal, but there is no documented concurrency limit, quota tiering, or rate-limit table a platform engineer could plan capacity around. Missing for 10: explicit documented per-account/org concurrency limits, guidance on requesting increases, and independent benchmarks validating the 10k+ claim at sustained load.", "evidenceIds": [ "runloop-docs-12" ] }, { "productId": "runloop", "storyId": "custom-images-templates", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Runloop's Blueprints support building custom sandbox images from a Dockerfile (`rli blueprint from-dockerfile`), plus reusable prebuilt blueprints and snapshotting of custom states, directly enabling developers to bring their own container image or define custom templates. Missing for 10: independent/hands-on verification of Dockerfile-based blueprint builds and more detail on supported image registries or base image constraints.", "evidenceIds": [ "runloop-docs-2", "runloop-docs-3", "runloop-docs-14", "runloop-docs-4", "runloop-docs-15" ] }, { "productId": "runloop", "storyId": "expose-public-url", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Runloop explicitly documents devbox tunnels for securely accessing ports on a devbox over a simple URL, directly matching the story's requirement to expose a service port publicly. Missing for 10: independent/hands-on confirmation of tunnel reliability and details on URL persistence/security controls beyond the doc snippet.", "evidenceIds": [ "runloop-docs-5" ] }, { "productId": "runloop", "storyId": "fast-cold-starts", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Runloop's marketing site documents '10GB image startup time in <2s' and a 1-minute quickstart, supporting sub-second-to-few-second cold starts, but this is a single first-party marketing claim without detailed benchmarks, methodology, or independent corroboration. missing for 10: independent/third-party benchmark confirmation, documentation of cold start times across varying image sizes/configurations, and reproducible performance data beyond the single homepage claim.", "evidenceIds": [ "runloop-docs-12", "runloop-docs-26" ] }, { "productId": "runloop", "storyId": "filesystem-io", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs confirm the SDK/CLI supports 'file operations' as part of Devbox management (mentioned alongside lifecycle, command execution, etc. in the Cursor rules integration), implying read/write file capability, but no dedicated documentation page or code sample specifically shows upload/download or read/write file APIs. Missing for 10: explicit SDK method docs for file read/write, and explicit upload/download examples or independent corroboration.", "evidenceIds": [ "runloop-docs-29", "runloop-docs-13", "runloop-docs-23" ] }, { "productId": "runloop", "storyId": "free-tier-start", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly state Runloop's trial includes $50 free credits with no credit card required, and a quickstart lets a developer spin up a first devbox in about a minute, both without any sales contact. Missing for 10: independent/community confirmation of the free-tier signup experience and clarity on long-term free-tier limits beyond trial credits.", "evidenceIds": [ "runloop-docs-25", "runloop-docs-26" ] }, { "productId": "runloop", "storyId": "gpu-access", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes Devboxes, blueprints, snapshots, mounts, storage, and networking features but never mentions GPU support or GPU-attached compute for sandboxed workloads; nothing indicates hardware acceleration is available.", "evidenceIds": [] }, { "productId": "runloop", "storyId": "hard-isolation-boundary", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Runloop repeatedly describes Devboxes as \"secure sandboxed execution environments\" and \"isolated Linux sandbox environments,\" implying some isolation boundary, but the evidence never names or documents the actual isolation mechanism (e.g., microVM, Firecracker, gVisor, container-only) or any architecture diagram/security whitepaper explaining the boundary. missing for 10: explicit documentation of the underlying isolation technology (microVM/hypervisor-level or equivalent), any security architecture doc or third-party audit confirming hard isolation guarantees.", "evidenceIds": [ "runloop-docs-1", "runloop-docs-13", "runloop-docs-16", "runloop-probe-1" ] }, { "productId": "runloop", "storyId": "long-running-sessions", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Runloop Devboxes explicitly support long-running persistence via snapshots (save/restore disk state), suspend/resume to pause and continue exactly where left off, and Axons providing a persistent event stream that lets agents suspend when idle and wake on demand/recover state — directly addressing multi-hour/multi-day agent sessions without losing progress. Combined with SSH/tunnel access and mounts/secrets for long tasks, this covers the core story well. Missing for 10: independent/hands-on evidence (e.g., a third-party report) confirming multi-day session stability at scale, and explicit documented max session duration/limits.", "evidenceIds": [ "runloop-docs-4", "runloop-docs-15", "runloop-docs-27", "runloop-docs-17", "runloop-docs-18" ] }, { "productId": "runloop", "storyId": "multi-language-runtimes", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Runloop's Devboxes are described as full-featured secure sandboxed execution environments that can run arbitrary commands (via blueprints, SSH, CLI) which implies multi-language support, but there is no explicit documentation of a code-execution API returning 'rich results' (e.g., structured outputs, images, dataframes) as in dedicated code-interpreter products. missing for 10: explicit multi-language runtime examples (Python/JS/etc.), rich/structured result formatting (e.g., stdout+images+tables), and independent confirmation of these capabilities.", "evidenceIds": [ "runloop-docs-1", "runloop-docs-13", "runloop-docs-2", "runloop-docs-8", "runloop-docs-23" ] }, { "productId": "runloop", "storyId": "network-egress-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item describes network egress controls, firewall rules, or allow/deny policies for Devbox sandboxes; tunnels and secrets features are documented but network egress policy configuration is never mentioned.", "evidenceIds": [] }, { "productId": "runloop", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Runloop is API-first: the CLI (rli) wraps SDKs/API for devboxes, blueprints, snapshots, benchmarks, agents, secrets, etc., suggesting broad API/CLI parity with any dashboard functionality. However there is no explicit statement claiming full UI/API feature parity, no documented OpenAPI spec (404s on all probed paths), and no direct dashboard-vs-API comparison. Missing for 10: explicit parity claim between web dashboard and API, published OpenAPI/reference spec, confirmation that all dashboard-only actions (e.g., billing, org settings) are also API-accessible.", "evidenceIds": [ "runloop-docs-8", "runloop-docs-22", "runloop-docs-23", "runloop-docs-6", "runloop-probe-3" ] }, { "productId": "runloop", "storyId": "openness-full-export", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Runloop offers Snapshots to save Devbox disk state and export/re-create Devboxes, and Dockerfile-based blueprints imply portability of environments, but there is no documented feature for exporting all account/user data (benchmarks, configs, agent history) in open formats or a clear data-export/account-deletion workflow. missing for 10: explicit full account data export tool, open-format guarantees for all data types, data portability/deletion documentation.", "evidenceIds": [ "runloop-docs-4", "runloop-docs-2", "runloop-docs-3" ] }, { "productId": "runloop", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence indicates Runloop's core source (the sandbox platform, Devboxes, Axons, etc.) is released under an open license; the evidence pack shows only docs, CLI, and SDK references without any license or open-source repository disclosure.", "evidenceIds": [] }, { "productId": "runloop", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Runloop is presented entirely as a hosted cloud platform (Devboxes, Agents API, benchmarks) with no evidence of an on-premises or self-hostable deployment option, open-source core, or self-hosting documentation.", "evidenceIds": [] }, { "productId": "runloop", "storyId": "pause-resume-state", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Runloop explicitly documents 'Suspend and resume' functionality that pauses sandboxes to save state and resume exactly where left off, plus snapshot capability that saves disk state and can recreate Devboxes from that state. This directly matches the story's pause/resume with filesystem+memory state intact. Missing for 10: no independent/hands-on corroboration of memory-state fidelity across suspend/resume, and no detailed technical explanation of how memory state (not just disk) is preserved.", "evidenceIds": [ "runloop-docs-27", "runloop-docs-4", "runloop-docs-15" ] }, { "productId": "runloop", "storyId": "per-second-billing", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item states an explicit per-second or usage-based billing model; the only tangential mention is that suspend/resume can 'save costs' (runloop-docs-27), which doesn't establish a per-second pricing mechanism.", "evidenceIds": [] }, { "productId": "runloop", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, region selection, or geographic storage controls anywhere in Runloop's docs; the pack covers sandboxes, blueprints, snapshots, secrets, and tooling but never data-location choice.", "evidenceIds": [] }, { "productId": "runloop", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack addresses AI training data usage, opt-out policies, or data-retention commitments for model training; the docs focus on sandbox infrastructure (Devboxes, secrets, gateways) but never mention training-data policy.", "evidenceIds": [] }, { "productId": "runloop", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers sandboxing, blueprints, snapshots, secrets encryption, and tooling, but contains no documentation of data retention policies, data deletion controls, or user-facing privacy/compliance settings for Devboxes, snapshots, or storage objects. missing for 10: explicit data retention policy docs, user-initiated deletion/purge APIs, compliance/privacy documentation.", "evidenceIds": [ "runloop-docs-21", "runloop-docs-4", "runloop-docs-20" ] }, { "productId": "runloop", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions telemetry, analytics opt-out, or usage-tracking controls for Runloop; the docs cover sandboxing, blueprints, snapshots, secrets, etc., but nothing about a privacy/telemetry toggle.", "evidenceIds": [] }, { "productId": "runloop", "storyId": "run-coding-agents-inside", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Runloop documents running AI coding agents generally inside Devboxes (blueprints, benchmarks, Agents API, OpenAI Agents SDK tutorial) but no specific vendor recipe or tutorial naming Claude Code or Codex is present in the evidence. missing for 10: explicit Claude Code/Codex quickstart or blueprint, named integration guide for these specific agents.", "evidenceIds": [ "runloop-docs-6", "runloop-docs-7", "runloop-docs-22", "runloop-docs-27", "runloop-docs-2", "runloop-docs-3" ] }, { "productId": "runloop", "storyId": "run-untrusted-code", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Runloop's core product is Devboxes, secure sandboxed execution environments purpose-built to run AI agent/code workloads isolated from user infrastructure, with snapshots, tunnels, secrets isolation, and scaling to 10k+ parallel sandboxes. This directly matches the story of executing untrusted AI-generated code without risking one's own infra. Missing for 10: independent/third-party hands-on validation of isolation guarantees, and explicit documentation of the underlying sandboxing/virtualization technology or security boundary details.", "evidenceIds": [ "runloop-docs-1", "runloop-docs-13", "runloop-docs-12", "runloop-docs-4", "runloop-docs-5", "runloop-docs-21", "runloop-probe-1" ] }, { "productId": "runloop", "storyId": "secrets-stay-out", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Runloop's Devboxes are documented as sandboxed execution environments with encrypted secrets and, notably, Agent Gateways that let agents call LLM APIs 'without ever seeing your API keys' — directly supporting the isolation-of-credentials use case. However, secrets are also stated to be 'automatically made available as environment variables' inside Devboxes, which is somewhat in tension with 'unreachable by code' and isn't clarified as scoped/restricted; there's no independent/hands-on verification of secret unreachability or a security audit. missing for 10: independent verification of secret isolation, clarity on env-var secret exposure vs true unreachability, and details on network/filesystem-level credential isolation guarantees.", "evidenceIds": [ "runloop-docs-10", "runloop-docs-21", "runloop-docs-1", "runloop-docs-13" ] }, { "productId": "runloop", "storyId": "shell-and-packages", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Runloop Devboxes are full Linux execution environments with SSH access via the CLI (runloop-docs-9) and blueprints built from Dockerfiles that install packages/dependencies at startup (runloop-docs-2, runloop-docs-14), implying arbitrary shell command execution and package installation. However, the evidence pack lacks a direct hands-on example of running an explicit shell command or apt/pip install inside a Devbox. missing for 10: explicit example/output of running an arbitrary shell command and installing a package inside a Devbox, independent third-party confirmation.", "evidenceIds": [ "runloop-docs-9", "runloop-docs-2", "runloop-docs-14", "runloop-docs-13" ] }, { "productId": "runloop", "storyId": "snapshot-restore-fork", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Runloop's docs explicitly describe snapshotting a Devbox's disk state and creating new Devboxes from that saved state, including fan-out use cases (multiple approaches from one snapshot) and rollback scenarios, directly matching the snapshot/restore/fork story. missing for 10: independent/hands-on corroboration beyond first-party docs, and no explicit code example showing the fork API call in the evidence pack.", "evidenceIds": [ "runloop-docs-4", "runloop-docs-15" ] }, { "productId": "runloop", "storyId": "spinup-via-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs describe Devbox creation via SDK/CLI/API, quickstart claiming ~1 minute to a running devbox, blueprints/prebuilt images for fast startup, and marketing claims of 10k+ parallel sandboxes with <2s startup for a 10GB image, matching the one-call, seconds-to-live-environment story. missing for 10: independent/hands-on benchmark confirming real-world provisioning latency beyond vendor claims.", "evidenceIds": [ "runloop-docs-1", "runloop-docs-12", "runloop-docs-13", "runloop-docs-26", "runloop-docs-23", "runloop-docs-3" ] }, { "productId": "vercel-sandbox", "storyId": "agent-self-provisions", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Vercel Sandbox provides a full programmatic SDK/CLI to create sandboxes (with names, custom images, regions), run blocking or detached commands, stream logs/output, and stop/tear down sandboxes, with API-based authentication (OIDC/tokens) suitable for unattended agent use — no human-in-the-loop required. missing for 10: no independent/hands-on third-party report confirming an agent autonomously completing the full provision-execute-teardown loop, and no explicit example showing an agent reading command output/results programmatically end-to-end.", "evidenceIds": [ "vercel-sandbox-docs-1", "vercel-sandbox-docs-2", "vercel-sandbox-docs-19", "vercel-sandbox-docs-20", "vercel-sandbox-docs-21", "vercel-sandbox-docs-32", "vercel-sandbox-docs-33", "vercel-sandbox-docs-36", "vercel-sandbox-docs-39" ] }, { "productId": "vercel-sandbox", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Probes confirm both a working llms.txt at the root domain (vercel.com/llms.txt, HTTP 200) and a markdown-formatted docs endpoint for Sandbox specifically (sandbox.md, HTTP 200), which are exactly the agent-oriented docs formats this story asks for. Missing for 10: no explicit evidence llms.txt itself deep-links to sandbox-specific docs, and no independent/community confirmation of an agent successfully consuming these files.", "evidenceIds": [ "vercel-sandbox-probe-1", "vercel-sandbox-probe-2", "vercel-sandbox-docs-43" ] }, { "productId": "vercel-sandbox", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Vercel Sandbox is an infrastructure/execution product for running untrusted or agent-generated code in isolated microVMs — it is not a data product that surfaces its own AI-generated insights/suggestions to end users; that capability is out of scope for this axis.", "evidenceIds": [] }, { "productId": "vercel-sandbox", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Vercel Sandbox supports running agent-generated code, detached/background commands with log streaming, timeouts/extension, and persistence/resume, which enable building autonomous background automations. However, there's no built-in scheduler, trigger system, or native 'automation' orchestration layer — users must build their own scheduling/trigger logic around the SDK/CLI. missing for 10: native scheduling/cron or event-trigger mechanism, independent evidence of long-running unattended automations in production, and explicit 'autonomous background job' framing beyond detached commands.", "evidenceIds": [ "vercel-sandbox-docs-19", "vercel-sandbox-docs-20", "vercel-sandbox-docs-22", "vercel-sandbox-docs-11", "vercel-sandbox-docs-28", "vercel-sandbox-docs-10" ] }, { "productId": "vercel-sandbox", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Vercel Sandbox is an isolated execution environment (microVMs) for running agent-generated or untrusted code, not itself an AI assistant that a user delegates tasks to — it's infrastructure other agents/tools call into, so this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "vercel-sandbox", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Vercel Sandbox is designed for headless/programmatic use via SDKs (@vercel/sandbox) and a CLI, with access tokens explicitly documented for external CI/CD systems and non-Vercel environments, plus OpenAPI-backed REST access confirmed by probe. missing for 10: no explicit third-party/independent CI pipeline case study or GitHub Actions example demonstrating real-world headless automation.", "evidenceIds": [ "vercel-sandbox-docs-5", "vercel-sandbox-docs-7", "vercel-sandbox-docs-32", "vercel-sandbox-docs-33", "vercel-sandbox-probe-3" ] }, { "productId": "vercel-sandbox", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "vercel-sandbox", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Vercel Sandbox is an execution/infrastructure product (not itself an agent), so an official MCP server exposing it to agents is a fair, applicable axis; however no evidence in the pack mentions MCP, an MCP server, or any agent-protocol integration—only SDKs, CLI, and REST/OpenAPI docs are described.", "evidenceIds": [] }, { "productId": "vercel-sandbox", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Vercel Sandbox is designed to run agent-generated code and is explicitly marketed for 'agent workflows,' but the evidence only describes CLI commands (`sandbox create`, `sandbox run`) and SDK calls, not a natural-language interface for operating the product itself. No documentation shows a chat/NL command layer for creating or managing sandboxes. Missing for 10: any NL command parser, chat-driven CLI, or natural-language API for controlling Sandbox operations.", "evidenceIds": [ "vercel-sandbox-docs-27", "vercel-sandbox-docs-36", "vercel-sandbox-docs-39" ] }, { "productId": "vercel-sandbox", "storyId": "agentic-official-cli", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Vercel Sandbox ships an official `sandbox` CLI explicitly positioned for manual testing, agentic workflows, debugging, and one-off operations, with documented commands like `sandbox create --name` and `sandbox run`. missing for 10: independent/hands-on corroboration beyond vendor docs, and no CLI reference/full command list shown.", "evidenceIds": [ "vercel-sandbox-docs-27", "vercel-sandbox-docs-36", "vercel-sandbox-docs-39", "vercel-sandbox-docs-1" ] }, { "productId": "vercel-sandbox", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Vercel Sandbox is explicitly built for agentic/AI use cases and exposes a documented SDK (@vercel/sandbox), CLI, and REST API (openapi.json) for programmatic control—creating sandboxes, running commands, streaming logs, managing files, images, and persistence. This directly satisfies driving the product via a documented public API for AI-native workflows. Missing for 10: independent third-party corroboration beyond vendor docs/probe checks.", "evidenceIds": [ "vercel-sandbox-docs-1", "vercel-sandbox-docs-5", "vercel-sandbox-docs-27", "vercel-sandbox-docs-32", "vercel-sandbox-probe-2", "vercel-sandbox-probe-3" ] }, { "productId": "vercel-sandbox", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only describes authentication mechanisms (automatic OIDC token in production, access tokens for CI/CD) but nowhere documents scoped or least-privilege permission models for these credentials — no mention of granular scopes, role restrictions, or per-agent credential limits.", "evidenceIds": [ "vercel-sandbox-docs-6", "vercel-sandbox-docs-7", "vercel-sandbox-docs-33" ] }, { "productId": "vercel-sandbox", "storyId": "agentic-sdks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Vercel Sandbox offers an official `@vercel/sandbox` SDK for JS/TS plus a CLI, with docs on authentication, commands, and sandbox lifecycle management — evidence that AI-native builders have a first-party SDK to integrate against. missing for 10: no mention of official SDKs for other languages (e.g., Python), no independent/hands-on corroboration of SDK usage beyond vendor docs.", "evidenceIds": [ "vercel-sandbox-docs-5", "vercel-sandbox-docs-32", "vercel-sandbox-docs-27", "vercel-sandbox-docs-33", "vercel-sandbox-docs-19" ] }, { "productId": "vercel-sandbox", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers sandbox creation, execution modes, persistence, and streaming logs from detached commands, but there is no mention of webhooks or event subscription mechanisms for external systems to receive Sandbox events.", "evidenceIds": [] }, { "productId": "vercel-sandbox", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Vercel has an OpenAPI spec and Sandbox docs, but nothing describes an interactive API reference with runnable/live examples for Sandbox itself — no docs playground, no 'try it' console, no runnable code snippets in-page. missing for 10: interactive API reference UI, runnable example execution in docs, evidence of a docs playground feature.", "evidenceIds": [ "vercel-sandbox-probe-3", "vercel-sandbox-docs-32" ] }, { "productId": "vercel-sandbox", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A probe confirms Vercel publishes a machine-readable OpenAPI spec at https://vercel.com/openapi.json which validly contains the 'openapi' key, and Sandbox is part of Vercel's documented API/SDK surface. Missing for 10: no explicit confirmation that Sandbox-specific endpoints are included in that spec versus general Vercel API.", "evidenceIds": [ "vercel-sandbox-probe-3", "vercel-sandbox-docs-32" ] }, { "productId": "vercel-sandbox", "storyId": "api-sandbox", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Vercel Sandbox is explicitly designed to run untrusted/agent-generated code in isolated Linux microVMs 'without exposing your production systems,' with per-agent isolation, ephemeral or persistent state, and no shared access to production infrastructure. This directly matches the AI-native story of testing agent code safely away from production data.\n\nmissing for 10: independent/hands-on evidence of real-world isolation guarantees, and explicit documentation on data-boundary controls preventing production data leakage into sandboxes.", "evidenceIds": [ "vercel-sandbox-docs-1", "vercel-sandbox-docs-2", "vercel-sandbox-docs-10", "vercel-sandbox-docs-26", "vercel-sandbox-docs-38" ] }, { "productId": "vercel-sandbox", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows SDK/CLI docs, features, and an OpenAPI spec, but no mention of API versioning scheme or a documented deprecation policy for Vercel Sandbox's APIs. Some features are explicitly marked 'beta' (docs-14, docs-44), suggesting instability without a stated deprecation process. missing for 10: explicit API version numbers, a deprecation/sunset policy, changelog entries documenting breaking-change handling.", "evidenceIds": [] }, { "productId": "vercel-sandbox", "storyId": "auto-teardown-timeouts", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly state sandboxes have a default 5-minute timeout, configurable via the `timeout` option at creation, extendable via `sandbox.extendTimeout()`, and can be manually stopped from the dashboard, directly supporting automatic shutdown to stop billing when idle or done. Missing for 10: independent/hands-on confirmation that billing actually stops immediately upon timeout, and documentation of any maximum timeout limits or edge cases.", "evidenceIds": [ "vercel-sandbox-docs-22", "vercel-sandbox-docs-37", "vercel-sandbox-docs-21", "vercel-sandbox-docs-42" ] }, { "productId": "vercel-sandbox", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Vercel Sandbox docs describe creating/running individual sandboxes, multi-agent isolation, and SDK-driven command execution, but there is no evidence of a bulk/batch operation feature for processing many items at once (e.g., batch job APIs, fan-out orchestration, or bulk file/task processing).", "evidenceIds": [] }, { "productId": "vercel-sandbox", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "vercel-sandbox", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Vercel Sandbox provides on-demand ephemeral/persistent execution environments with manual creation, timeouts, and resumption, but no evidence of a scheduler, cron-like trigger, or recurring job orchestration feature; scheduling would need to be built externally (e.g., Vercel Cron) which isn't documented here as part of Sandbox itself.", "evidenceIds": [ "vercel-sandbox-docs-22", "vercel-sandbox-docs-37", "vercel-sandbox-docs-18" ] }, { "productId": "vercel-sandbox", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "vercel-sandbox", "storyId": "browser-in-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence describes Linux microVMs, custom OCI images, and general code execution, but nothing mentions headless browser support, display/VNC, GUI desktop environments, or browser automation tooling. missing for 10: any mention of headless browser (e.g., Chromium/Playwright) support, GPU/display capabilities, or desktop environment provisioning.", "evidenceIds": [] }, { "productId": "vercel-sandbox", "storyId": "concurrent-fleets", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs mention that Sandbox usage is metered and that quotas apply per plan, plus region availability and spend management for cost control, but no concrete documented concurrency limits (e.g., max simultaneous sandboxes) are shown in the evidence. missing for 10: explicit concurrency limit numbers, guidance on running large fleets of sandboxes concurrently, and any performance/scale benchmarks.", "evidenceIds": [ "vercel-sandbox-docs-42", "vercel-sandbox-docs-23", "vercel-sandbox-docs-24" ] }, { "productId": "vercel-sandbox", "storyId": "custom-images-templates", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly support custom sandbox templates and BYO container images: users can build custom OCI images stored in Vercel Container Registry, start sandboxes from managed, shared/public, or custom images, and Docker-image-based workflows are directly supported. Also supports snapshotting for reusable custom environments. missing for 10: independent/hands-on corroboration of custom image builds beyond first-party docs.", "evidenceIds": [ "vercel-sandbox-docs-16", "vercel-sandbox-docs-17", "vercel-sandbox-docs-30", "vercel-sandbox-docs-35", "vercel-sandbox-docs-34" ] }, { "productId": "vercel-sandbox", "storyId": "expose-public-url", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs mention running dev servers with 'live previews' and testing agent-generated code with access to 'live previews', implying some port-exposure/public-URL capability, but no evidence pack item describes the actual API/CLI mechanism (e.g., a domain/getUrl method, port argument, or URL format) for exposing a sandboxed port publicly. missing for 10: explicit documentation of the port-exposure API or CLI command, example of the resulting public URL, and confirmation of access control/authentication on that URL.", "evidenceIds": [ "vercel-sandbox-docs-4", "vercel-sandbox-docs-26", "vercel-sandbox-docs-38" ] }, { "productId": "vercel-sandbox", "storyId": "fast-cold-starts", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The docs describe microVM isolation, snapshotting for faster restarts, and persistence, but no evidence gives concrete cold-start latency figures (e.g., sub-second or few-second boot times) for starting a fresh sandbox. Missing for 10: documented benchmark numbers or explicit cold-start time claims, any independent performance measurement.", "evidenceIds": [ "vercel-sandbox-docs-31", "vercel-sandbox-docs-40" ] }, { "productId": "vercel-sandbox", "storyId": "filesystem-io", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm the SDK provides a programmatic interface to manage files (mount remote storage, drives, filesystem persistence via snapshots) and general file access is implied through 'file edits' and FUSE-mounted storage, but no explicit documentation of discrete SDK methods like sandbox.writeFiles(), readFile(), or upload/download APIs is present. missing for 10: explicit SDK method documentation/examples for read, write, upload, and download file operations, and independent/hands-on confirmation of these specific API calls.", "evidenceIds": [ "vercel-sandbox-docs-32", "vercel-sandbox-docs-29", "vercel-sandbox-docs-14", "vercel-sandbox-docs-26", "vercel-sandbox-docs-40" ] }, { "productId": "vercel-sandbox", "storyId": "free-tier-start", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack mentions billing, plans, and quotas (e.g., vercel-sandbox-docs-42, docs-24) but never explicitly states a free tier or included credits usable without contacting sales. Missing for 10: explicit free-tier/plan pricing details, evidence of self-serve signup without sales contact.", "evidenceIds": [ "vercel-sandbox-docs-42", "vercel-sandbox-docs-24" ] }, { "productId": "vercel-sandbox", "storyId": "gpu-access", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the docs mentions GPU support or GPU attachment for Vercel Sandbox; sandboxes are described only as CPU-based Linux microVMs with system-privileged process support, storage/drives, and networking. missing for 10: any mention of GPU hardware, GPU-enabled images, or GPU pricing/quota.", "evidenceIds": [ "vercel-sandbox-docs-8", "vercel-sandbox-docs-9", "vercel-sandbox-docs-42" ] }, { "productId": "vercel-sandbox", "storyId": "hard-isolation-boundary", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vercel's first-party docs repeatedly and explicitly claim microVM-based isolation for running untrusted/agent-generated code without exposing production systems, plus additional isolation features (per-agent Linux users, system-privileged process support). However, all evidence is vendor documentation with no independent security audit, third-party penetration test, or technical deep-dive into the underlying microVM technology (e.g. Firecracker) confirming the boundary in practice. missing for 10: independent/third-party verification of the isolation boundary, technical detail on the hypervisor/microVM implementation, and any hands-on security assessment.", "evidenceIds": [ "vercel-sandbox-docs-1", "vercel-sandbox-docs-2", "vercel-sandbox-docs-25", "vercel-sandbox-docs-9", "vercel-sandbox-docs-10" ] }, { "productId": "vercel-sandbox", "storyId": "long-running-sessions", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show sandboxes have a default 5-minute timeout that can be explicitly extended via `timeout` option or `sandbox.extendTimeout()`, and persistent sandboxes auto-snapshot on stop so you can quickly resume a long agent task across sessions rather than losing state. However, there's no evidence of an unbounded or multi-day continuous session—developers must actively manage/extend timeouts or rely on stop/resume snapshotting rather than a single always-on session. missing for 10: documented maximum extendable duration or explicit support for a single continuous multi-hour/day session without developer-managed timeout extensions, independent corroboration of long-running stability.", "evidenceIds": [ "vercel-sandbox-docs-22", "vercel-sandbox-docs-37", "vercel-sandbox-docs-11", "vercel-sandbox-docs-28", "vercel-sandbox-docs-40", "vercel-sandbox-docs-31" ] }, { "productId": "vercel-sandbox", "storyId": "multi-language-runtimes", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs confirm sandboxes run Linux images with Python 3.14 and Node.js LTS pre-installed by default, plus support for any other Linux distro via custom OCI images, enabling multi-language execution (JS/TS, Python, and more) (docs-5, docs-8, docs-34, docs-30). Execution results are surfaced via logs, streamed output, and file access (docs-19, docs-20, docs-26), but there's no explicit mention of structured/rich result formats (e.g., images, dataframes, plots) beyond logs and files. Missing for 10: explicit support for rich/structured output types (charts, tables, media) and independent hands-on confirmation of multi-language fidelity.", "evidenceIds": [ "vercel-sandbox-docs-5", "vercel-sandbox-docs-8", "vercel-sandbox-docs-34", "vercel-sandbox-docs-30", "vercel-sandbox-docs-19", "vercel-sandbox-docs-20", "vercel-sandbox-docs-26" ] }, { "productId": "vercel-sandbox", "storyId": "network-egress-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack mentions network egress controls, firewall rules, or outbound traffic policy for Vercel Sandbox—coverage focuses on isolation via microVMs, filesystem, images, and multi-agent user separation but never explicit egress allow/deny configuration.", "evidenceIds": [] }, { "productId": "vercel-sandbox", "storyId": "openness-api-parity", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Vercel Sandbox is fundamentally API/SDK-driven (creation, commands, snapshots, tags, drives, custom images all documented via SDK/CLI), and the dashboard only offers a subset (viewing/stopping sandboxes) that mirrors API capability rather than adding unique UI-only features, and an OpenAPI spec exists confirming programmatic access. Missing for 10: an explicit vendor statement confirming full UI/API parity, and independent confirmation that every dashboard action (e.g., tag/drive management) is scriptable via API with no UI-exclusive features.", "evidenceIds": [ "vercel-sandbox-docs-32", "vercel-sandbox-docs-27", "vercel-sandbox-docs-21", "vercel-sandbox-docs-43", "vercel-sandbox-probe-3" ] }, { "productId": "vercel-sandbox", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Vercel Sandbox stores state via drives, snapshots, and container images, so data portability is a fair question, but no evidence describes an explicit data-export feature or guarantee of open, exportable formats for that stored state. Missing for 10: any documented export/download-all-data capability, explicit open-format guarantees for drives/snapshots, or migration tooling to leave the platform.", "evidenceIds": [ "vercel-sandbox-docs-14", "vercel-sandbox-docs-12", "vercel-sandbox-docs-17" ] }, { "productId": "vercel-sandbox", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence that Vercel Sandbox's source code is published under an open license; the evidence pack only covers docs, features, pricing, and API surface, with no mention of a public repo or license.", "evidenceIds": [] }, { "productId": "vercel-sandbox", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Vercel Sandbox is a fully managed, hosted service running on Vercel's own microVM infrastructure (iad1/sfo1/cle1/cdg1 regions); there is no evidence of any self-hostable core, open-source runtime, or on-prem deployment option — it is inherently tied to Vercel's cloud.", "evidenceIds": [ "vercel-sandbox-docs-1", "vercel-sandbox-docs-23", "vercel-sandbox-docs-42" ] }, { "productId": "vercel-sandbox", "storyId": "pause-resume-state", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly describe persistent sandboxes that auto-save filesystem state on stop and resume where left off by default, with configuration preserved across sessions, plus explicit snapshotting to skip dependency reinstallation and resuming an existing sandbox by name. This directly matches the pause/resume-with-state story via first-party documentation. Missing for 10: independent hands-on verification that memory/process state (not just filesystem) is preserved across pause/resume, and no third-party corroboration beyond vendor docs.", "evidenceIds": [ "vercel-sandbox-docs-11", "vercel-sandbox-docs-12", "vercel-sandbox-docs-18", "vercel-sandbox-docs-28", "vercel-sandbox-docs-31", "vercel-sandbox-docs-40" ] }, { "productId": "vercel-sandbox", "storyId": "per-second-billing", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs confirm Sandbox usage is metered on a pricing page with configurable timeouts and spend-management alerts to control costs, implying granular, usage-based billing, but no evidence explicitly states billing is charged 'per second' of compute used. missing for 10: explicit per-second billing granularity documentation, real-world billing statement/invoice examples showing sub-minute metering.", "evidenceIds": [ "vercel-sandbox-docs-42", "vercel-sandbox-docs-22", "vercel-sandbox-docs-37", "vercel-sandbox-docs-24" ] }, { "productId": "vercel-sandbox", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm region selection (iad1, sfo1, cle1, cdg1) can be set per sandbox or as a project default, giving users some control over where sandboxes execute, and one changelog note claims availability 'in all regions.' However, there is no explicit data residency/compliance guarantee, no mention of where persistent Drives/snapshots/data-at-rest are stored, and no discussion of data sovereignty controls beyond compute region choice. missing for 10: explicit data-at-rest residency guarantees, compliance/certification details tied to region choice, documentation on storage location for Drives/snapshots, and independent confirmation of region enforcement.", "evidenceIds": [ "vercel-sandbox-docs-23", "vercel-sandbox-docs-45" ] }, { "productId": "vercel-sandbox", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Vercel Sandbox is a code-execution/microVM infrastructure product, not an AI model or AI-training data product; 'preventing data from being used to train AI models' is a data/privacy-policy axis for AI model providers, not for a sandbox execution service. This story is a category error for this product type.", "evidenceIds": [] }, { "productId": "vercel-sandbox", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Docs describe sandbox lifecycle (timeouts, stopping, persistence, snapshots) but there is no explicit statement about data retention policies, deletion guarantees, or privacy controls for stored sandbox data/drives. missing for 10: explicit data retention policy, explicit deletion/purge controls, privacy/compliance documentation for stored data.", "evidenceIds": [ "vercel-sandbox-docs-21", "vercel-sandbox-docs-22", "vercel-sandbox-docs-28", "vercel-sandbox-docs-40" ] }, { "productId": "vercel-sandbox", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry/usage-tracking opt-out for Vercel Sandbox; documentation covers execution, isolation, images, and pricing but nothing about telemetry controls.", "evidenceIds": [] }, { "productId": "vercel-sandbox", "storyId": "run-coding-agents-inside", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm the default 'vercel/sandbox/universal' image ships with coding agents preinstalled and highlight agent workflows (e.g. Claude Code/Codex-style use), but the evidence pack lacks an explicit vendor 'recipe' walkthrough naming Claude Code or Codex step-by-step. missing for 10: a dedicated vendor tutorial/recipe explicitly running Claude Code or Codex inside the sandbox, and independent hands-on confirmation of this exact workflow.", "evidenceIds": [ "vercel-sandbox-docs-34", "vercel-sandbox-docs-1", "vercel-sandbox-docs-25", "vercel-sandbox-docs-10", "vercel-sandbox-docs-27" ] }, { "productId": "vercel-sandbox", "storyId": "run-untrusted-code", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Vercel Sandbox's docs directly describe running untrusted or AI agent-generated code in isolated Linux microVMs without exposing production systems, with per-agent isolation, configurable timeouts, and managed images—precisely matching the story. Missing for 10: independent/hands-on verification beyond first-party docs.", "evidenceIds": [ "vercel-sandbox-docs-1", "vercel-sandbox-docs-2", "vercel-sandbox-docs-10", "vercel-sandbox-docs-25", "vercel-sandbox-docs-26" ] }, { "productId": "vercel-sandbox", "storyId": "secrets-stay-out", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Vercel Sandbox docs explicitly describe running agent-generated/untrusted code in isolated Linux microVMs 'without exposing your production systems,' and support per-agent Linux user isolation, which directly implies host secrets/credentials are not reachable by sandboxed code. missing for 10: explicit documentation of how host secrets/env vars are excluded or scoped from the sandbox environment, and independent/hands-on verification of this isolation guarantee.", "evidenceIds": [ "vercel-sandbox-docs-1", "vercel-sandbox-docs-2", "vercel-sandbox-docs-10", "vercel-sandbox-docs-25" ] }, { "productId": "vercel-sandbox", "storyId": "shell-and-packages", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs clearly show running arbitrary commands (blocking/detached modes, `sandbox run echo ...`, CLI and SDK) inside Linux microVMs, with custom images and full Linux distros allowing package installs (e.g., Ubuntu/Arch, custom OCI images, system-privileged processes). missing for 10: no independent/hands-on report confirming package installation workflows beyond first-party docs.", "evidenceIds": [ "vercel-sandbox-docs-1", "vercel-sandbox-docs-8", "vercel-sandbox-docs-9", "vercel-sandbox-docs-19", "vercel-sandbox-docs-27", "vercel-sandbox-docs-34", "vercel-sandbox-docs-39", "vercel-sandbox-docs-30" ] }, { "productId": "vercel-sandbox", "storyId": "snapshot-restore-fork", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vercel Sandbox docs clearly describe automatic snapshotting of a sandbox's filesystem/state on stop and restoring it on resume, with named sandboxes resumable and snapshots speeding up subsequent runs by skipping setup. However, the evidence only shows resuming the *same* sandbox from its own snapshot, not explicitly forking multiple independent new sandboxes from one saved snapshot. Missing for 10: explicit support/documentation for forking multiple new sandboxes off a single snapshot, and any hands-on/independent confirmation of this fork workflow.", "evidenceIds": [ "vercel-sandbox-docs-12", "vercel-sandbox-docs-28", "vercel-sandbox-docs-40", "vercel-sandbox-docs-18", "vercel-sandbox-docs-31" ] }, { "productId": "vercel-sandbox", "storyId": "spinup-via-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs show a clear SDK/CLI path (`@vercel/sandbox`, `sandbox create`/`sandbox run`) to spin up isolated Linux microVMs, with fast startup via managed images or snapshots that skip dependency installation, matching the 'one call, live environment in seconds' story. Missing for 10: independent/hands-on timing benchmarks confirming 'seconds' latency and third-party corroboration beyond first-party docs.", "evidenceIds": [ "vercel-sandbox-docs-1", "vercel-sandbox-docs-5", "vercel-sandbox-docs-27", "vercel-sandbox-docs-32", "vercel-sandbox-docs-39", "vercel-sandbox-docs-31", "vercel-sandbox-docs-30" ] } ]