[ { "productId": "coreweave", "storyId": "agent-provisions-gpu", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "CoreWeave documents a REST API (verified live and Bearer-token-gated in probe) and Terraform provider for creating/listing/updating/deleting CKS clusters (GPU node pools), plus Kubernetes-native monitoring, autoscaling, and quota/billing visibility, all of which an agent could drive without a human touching the Console. However, the MCP endpoint found is only for docs search/retrieval, not for provisioning or lifecycle actions, and there's no CLI or agent-specific tooling documented beyond kubectl/Terraform/API — full automated teardown and monitoring loop is implied but not shown end-to-end in a single agent-facing workflow. missing for 10: an agent-oriented CLI, an MCP server exposing actual provisioning/monitor/teardown actions (not just doc search), and a documented single end-to-end agent workflow example tying create→monitor→run→delete together.", "evidenceIds": [ "coreweave-docs-5", "coreweave-docs-6", "coreweave-docs-7", "coreweave-docs-8", "coreweave-docs-3", "coreweave-docs-12", "coreweave-docs-13", "coreweave-probe-rt-1", "coreweave-probe-rt-2" ] }, { "productId": "coreweave", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Direct probe confirms an llms.txt file is live at docs.coreweave.com/llms.txt with structured agent-oriented summary and links, and a companion MCP endpoint further confirms agent-reachable documentation. Missing for 10: independent third-party confirmation beyond the vendor's own probe/docs.", "evidenceIds": [ "coreweave-probe-1", "coreweave-probe-rt-2" ] }, { "productId": "coreweave", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "CoreWeave is GPU cloud infrastructure (Kubernetes, storage, inference API, compute provisioning) — it is not a data product with an interface where end-users get AI-generated insights/suggestions on their own data; this is a wrong-axis question for an infrastructure/IaaS platform rather than an analytics or SaaS application.", "evidenceIds": [] }, { "productId": "coreweave", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "coreweave", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "CoreWeave is GPU cloud/Kubernetes infrastructure; it does not ship a built-in AI assistant persona for users to delegate tasks to. This is a wrong-axis question for an infrastructure provider — a docs MCP endpoint for retrieval is not a built-in AI assistant.", "evidenceIds": [] }, { "productId": "coreweave", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "CoreWeave provides a full REST API (Bearer-token gated, confirmed live via probe) for cluster/resource management, a Terraform provider for IaC, and kubeconfig/kubectl access, all of which support headless CI/automation workflows. missing for 10: no dedicated CI/CD pipeline examples (e.g., GitHub Actions integration) or first-party automation SDKs beyond generated API clients, and no independent hands-on report confirming a full CI pipeline running against CoreWeave.", "evidenceIds": [ "coreweave-docs-5", "coreweave-docs-6", "coreweave-docs-7", "coreweave-docs-8", "coreweave-docs-28", "coreweave-probe-rt-1" ] }, { "productId": "coreweave", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "CoreWeave is a GPU cloud/infrastructure platform for training and inference workloads, not an agentic assistant or IDE-like product that itself consumes external tools via MCP; the evidence shows CoreWeave publishing an MCP server for its own docs (server-side), not any agent-like feature that plugs in third-party MCP servers as tools. This client-side 'consume MCP tools' story is a category error for an infra/platform product of this kind.", "evidenceIds": [ "coreweave-probe-rt-2" ] }, { "productId": "coreweave", "storyId": "agentic-mcp-server", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "A probe confirms CoreWeave operates a live, keyless MCP server at docs.coreweave.com/mcp that completes a JSON-RPC initialize handshake and exposes search/retrieval tools over its documentation, so an agent can officially connect via MCP. However, this MCP server only covers documentation search rather than actual platform management (clusters, node pools, inference), and there is no first-party doc page describing it or independent community verification beyond the probe. Missing for 10: a documented/announced MCP server (not just a discovered endpoint), MCP tools that let an agent actually operate CoreWeave resources (not just search docs), and independent community corroboration.", "evidenceIds": [ "coreweave-probe-rt-2", "coreweave-probe-1" ] }, { "productId": "coreweave", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "CoreWeave's documented interfaces are structured (Cloud Console, Terraform/OpenTofu, REST/gRPC API, kubectl) with no evidence of a natural-language command interface for operating clusters, node pools, or inference deployments. The MCP endpoint only exposes documentation search/retrieval tools, not natural-language operation of the platform itself.", "evidenceIds": [ "coreweave-docs-1", "coreweave-docs-6", "coreweave-docs-7", "coreweave-probe-rt-2" ] }, { "productId": "coreweave", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "CoreWeave's docs describe API access tokens, a Terraform/OpenTofu provider, generated gRPC/Connect clients, and kubectl via kubeconfig, but no evidence of a dedicated official CoreWeave CLI tool exists in the pack.", "evidenceIds": [] }, { "productId": "coreweave", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "CoreWeave documents a public REST/gRPC API for CKS clusters and Inference (create/list/update/delete), Bearer-token auth, generated clients in multiple languages, and a Terraform provider for IaC control — and a live runtime probe confirms the CKS API endpoint is reachable and Bearer-gated exactly as documented. missing for 10: independent third-party hands-on API usage reports/tutorials beyond CoreWeave's own docs.", "evidenceIds": [ "coreweave-docs-5", "coreweave-docs-8", "coreweave-docs-9", "coreweave-docs-19", "coreweave-docs-25", "coreweave-docs-21", "coreweave-docs-6", "coreweave-probe-rt-1" ] }, { "productId": "coreweave", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "CoreWeave documents API Access Tokens for authenticating to clusters/VPCs (coreweave-docs-8, coreweave-docs-17, coreweave-docs-26), but there is no evidence of scoped, least-privilege, or role-based permission configuration for these tokens — nothing describing granular scopes, IAM-style policies, or agent-specific credential issuance. missing for 10: documentation of configurable token scopes/permissions, role-based access control, or any mechanism to restrict a credential to a minimal set of actions for an autonomous agent.", "evidenceIds": [ "coreweave-docs-8", "coreweave-docs-17", "coreweave-docs-26" ] }, { "productId": "coreweave", "storyId": "agentic-sdks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "CoreWeave documents official generated clients for its Inference API across Connect/gRPC/Protobuf ecosystems in Go, Python, TypeScript, Java, Kotlin, Rust, and Swift, plus a first-party Terraform/OpenTofu provider and token-gated REST APIs for CKS and inference, all directly supporting AI-native programmatic/SDK access. Missing for 10: independent/hands-on developer corroboration of SDK quality, and explicit links to public SDK repos or version/release info.", "evidenceIds": [ "coreweave-docs-9", "coreweave-docs-21", "coreweave-docs-25", "coreweave-docs-6", "coreweave-docs-28", "coreweave-docs-19", "coreweave-probe-rt-1" ] }, { "productId": "coreweave", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack of a webhook subscription mechanism or event-driven notification system in CoreWeave's API or platform; the evidence covers cluster management, storage, inference API, and Terraform but nothing about webhooks or event subscriptions.", "evidenceIds": [] }, { "productId": "coreweave", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows static API reference docs (CKS API, Inference API) and client libraries, but nothing describes an interactive, runnable API explorer (e.g., a 'try it' console or embedded runnable code samples). Absence of such evidence for an applicable capability (API reference documentation) yields none.", "evidenceIds": [ "coreweave-docs-5", "coreweave-docs-19", "coreweave-docs-25", "coreweave-docs-21" ] }, { "productId": "coreweave", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents REST/gRPC/Protobuf APIs (CKS API, Inference API) and generated client libraries, but nowhere mentions an OpenAPI/Swagger spec or any downloadable machine-readable schema file that an AI agent could ingest directly.", "evidenceIds": [ "coreweave-docs-5", "coreweave-docs-9", "coreweave-docs-19", "coreweave-docs-21", "coreweave-docs-25" ] }, { "productId": "coreweave", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "CoreWeave's evidence covers cluster/node-pool creation, autoscaling, storage, and Terraform/API management, but nothing describes a dedicated sandbox/test environment or a mechanism to isolate test workloads from production data. While a customer could theoretically stand up a separate cluster, no docs, tutorials, or examples describe this as a supported 'sandbox vs production' workflow.", "evidenceIds": [ "coreweave-docs-1", "coreweave-docs-2", "coreweave-docs-15" ] }, { "productId": "coreweave", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack documents CoreWeave's various APIs (CKS API, Inference API, Terraform provider) but contains no mention of API versioning scheme or a documented deprecation policy anywhere in the docs or community sources.", "evidenceIds": [] }, { "productId": "coreweave", "storyId": "auto-shutdown-spend-guards", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows quota management and billing usage dashboards (coreweave-docs-12, coreweave-docs-13), but nothing about auto-shutdown timers, idle-instance termination, or configurable spend limits/budget alerts that would stop a forgotten instance from running up costs.", "evidenceIds": [ "coreweave-docs-12", "coreweave-docs-13" ] }, { "productId": "coreweave", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "CoreWeave's API and Terraform provider let you create/list/update/delete clusters, node pools, and other infra as code, which supports scripted bulk provisioning across many resources, and autoscaling lets node pools scale in bulk with demand. However, there is no explicit 'bulk operation' endpoint, batch API, or documented way to perform multi-item operations (e.g., bulk delete/update across many storage objects or inference deployments) in a single call — missing for 10: dedicated batch/bulk API endpoints, documented bulk operations for storage/inference resources, and independent evidence of large-scale bulk usage.", "evidenceIds": [ "coreweave-docs-5", "coreweave-docs-6", "coreweave-docs-3", "coreweave-docs-15", "coreweave-docs-28" ] }, { "productId": "coreweave", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "CoreWeave's docs only describe fixed Kubernetes Cluster Autoscaler behavior reacting to resource demand, not a general-purpose rules/automation engine where users define custom event-trigger-action logic; no evidence of webhooks, alert-based actions, or configurable automation rules.", "evidenceIds": [ "coreweave-docs-3", "coreweave-docs-16", "coreweave-docs-30" ] }, { "productId": "coreweave", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence describes cluster provisioning, node pools, autoscaling, Slurm-on-Kubernetes (SUNK) for training/inference jobs, and Terraform/API management, but nothing documents recurring job scheduling, cron-style triggers, or workflow orchestration primitives for automating repeated runs.", "evidenceIds": [] }, { "productId": "coreweave", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "CoreWeave is a GPU cloud/infrastructure platform (Kubernetes clusters, storage, inference API, Terraform); it has no automation/workflow-building feature to which versioning, review, and rollback of 'automations' would apply. This story targets no-code/agentic automation builders, not an IaaS provider.", "evidenceIds": [] }, { "productId": "coreweave", "storyId": "billing-usage-api", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs mention a Billing Insights view for consumption breakdowns by resource type (coreweave-docs-13), but there is no evidence of a programmatic API/export for pulling usage or billing data, nor any mention of attributing spend by team, project, or workload tags. missing for 10: documented billing/usage API or export endpoint, evidence of team/workload-level cost attribution or tagging, and any programmatic (non-console) access to billing data.", "evidenceIds": [ "coreweave-docs-13" ] }, { "productId": "coreweave", "storyId": "custom-docker-images", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "CKS is described as a managed Kubernetes service running directly on bare metal, which implies workloads (including custom Docker containers) can be deployed, but the evidence never explicitly documents how to submit a custom Docker image or create a custom machine/VM template with a specific environment. missing for 10: explicit docs on deploying custom container images to CKS, and any mention of custom machine templates/VM image support.", "evidenceIds": [ "coreweave-docs-14", "coreweave-docs-27", "coreweave-docs-15" ] }, { "productId": "coreweave", "storyId": "data-transfer-cloud-sync", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "CoreWeave documents S3-compatible Object Storage for moving training data, checkpoints, and model weights directly to GPU compute (coreweave-docs-10), plus a Terraform provider for managing storage buckets/policies as code (coreweave-docs-6). However, there is no documentation of cloud-storage sync tooling (e.g., rsync-like utilities, cross-cloud migration tools) or a dedicated CLI/SDK specifically for bulk data transfer beyond generic S3 API compatibility. missing for 10: dedicated data-migration/sync tooling, third-party or first-party CLI examples for bulk transfer, independent hands-on verification of transfer performance/throughput claims.", "evidenceIds": [ "coreweave-docs-10", "coreweave-docs-6", "coreweave-docs-8" ] }, { "productId": "coreweave", "storyId": "exposed-ports-networking", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence covers Kubernetes cluster management, node pools, storage, and Terraform/API access, but there is no mention of exposing ports/ingress for serving applications or of private networking/VPC connectivity between instances. Missing for 10: documentation on ingress/port exposure for serving apps, VPC or private networking setup between instances, and any hands-on confirmation of connectivity features.", "evidenceIds": [] }, { "productId": "coreweave", "storyId": "gpu-availability-transparency", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "CoreWeave documents a 'Capacity Finder' tool that lets users compare placement availability across zones for a requested instance type and node count before creating a Spot Node Pool, plus quota visibility/error reporting in the Cloud Console — this directly supports pre-provisioning availability checks. However, evidence is limited to Spot Node Pools and doesn't clearly show real-time GPU availability by type/region across all provisioning paths (e.g., on-demand reserved instances), and there's no independent/hands-on confirmation of accuracy or granularity. missing for 10: broader coverage beyond Spot pools (on-demand/reserved capacity visibility), independent verification that Capacity Finder prevents stockouts in practice, API/programmatic access to availability data, and region-level (not just zone-level) granularity confirmation.", "evidenceIds": [ "coreweave-docs-23", "coreweave-docs-31", "coreweave-docs-12", "coreweave-docs-4" ] }, { "productId": "coreweave", "storyId": "gpu-breadth-latest-hardware", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers CKS node pools, autoscaling, spot/on-demand pricing, and capacity finder across 'instance types' and 'Zones,' but never names specific GPU models or generations (H100/H200/B200 or older SKUs) nor confirms a menu of current- vs previous-generation GPU choices. Without any explicit mention of GPU SKU/generation selection, this applicable capacity-availability axis has no supporting evidence.", "evidenceIds": [] }, { "productId": "coreweave", "storyId": "instance-lifecycle-management", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "CoreWeave's CKS API supports programmatic create/list/update/delete of clusters and Node Pools, autoscaling to grow/shrink capacity, and Spot Node Pools are explicitly pay-as-you-go with no commitment, and billing dashboards show usage-based consumption — a live runtime probe even confirms the create/list/delete API is reachable and token-gated. However, the story asks specifically about start/stop/restart of individual instances, and evidence only documents cluster/node-pool-level create, delete, and autoscale operations rather than explicit stop/start/restart semantics for a single running instance. Missing for 10: explicit instance-level start/stop/restart API or docs (only pool-level scale/create/delete and Spot on-demand billing are evidenced).", "evidenceIds": [ "coreweave-docs-5", "coreweave-docs-3", "coreweave-docs-4", "coreweave-docs-13", "coreweave-docs-20", "coreweave-probe-rt-1" ] }, { "productId": "coreweave", "storyId": "jupyter-ide-access", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows CoreWeave provisions bare-metal Kubernetes clusters, kubeconfig access, and Terraform/API management, but there is no mention of Jupyter notebooks, VS Code/Cursor remote-connect integration, or any one-step IDE/notebook connection workflow; developers would need to manually deploy and configure such tooling themselves via generic Kubernetes primitives.", "evidenceIds": [ "coreweave-docs-7", "coreweave-docs-27", "coreweave-docs-14" ] }, { "productId": "coreweave", "storyId": "keyless-catalog-pricing-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows a public pricing page and Capacity Finder UI for availability, but no documented/public API endpoint that returns live GPU catalog pricing and availability programmatically for an agent to query before committing spend.", "evidenceIds": [ "coreweave-docs-20", "coreweave-docs-23", "coreweave-docs-31", "coreweave-docs-12" ] }, { "productId": "coreweave", "storyId": "managed-slurm-kubernetes", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "CoreWeave offers CKS as managed Kubernetes on bare metal with autoscaling node pools, and SUNK lets platform engineers run managed Slurm jobs inside that same Kubernetes cluster, directly delivering scheduling without building custom schedulers on raw nodes. missing for 10: independent/hands-on validation of Slurm-on-K8s (SUNK) at scale beyond CoreWeave's own docs, and details on job-queue features (priorities, preemption) comparable to a full HPC scheduler.", "evidenceIds": [ "coreweave-docs-11", "coreweave-docs-18", "coreweave-docs-24", "coreweave-docs-27", "coreweave-docs-3", "coreweave-docs-16", "coreweave-docs-14" ] }, { "productId": "coreweave", "storyId": "multi-node-clusters", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "CoreWeave documents self-service provisioning of multi-node GPU clusters via Console, Terraform, and API (docs-1,5,6,28), with Node Pools spanning many nodes, autoscaling, and Spot capacity available 'without long-term commitments or reservations' (docs-4,15,16,20,23) — all consistent with no-sales-cycle self-service. However, the evidence pack never explicitly mentions fast interconnect (e.g., InfiniBand/NVLink) specs, and quota pages imply some capacity increases require a request process (docs-12) which could reintroduce a sales-like step. missing for 10: explicit fast-interconnect/networking specs for multi-node training, and clearer confirmation that quota/capacity requests bypass sales entirely.", "evidenceIds": [ "coreweave-docs-1", "coreweave-docs-4", "coreweave-docs-5", "coreweave-docs-6", "coreweave-docs-15", "coreweave-docs-16", "coreweave-docs-20", "coreweave-docs-23", "coreweave-docs-12", "coreweave-probe-rt-1" ] }, { "productId": "coreweave", "storyId": "on-demand-gpu-provisioning", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "CoreWeave documents Console/Terraform/API provisioning of Spot and On-Demand Node Pools with kubeconfig-based access to run kubectl/code, and the API is confirmed live and Bearer-token-gated by a runtime probe. However, provisioning is framed around Kubernetes clusters/Node Pools rather than a single quick 'instance' spin-up, and there is no first-party or independent evidence of actual end-to-end timing (minutes) or a simple single-VM/instance API akin to typical cloud on-demand GPU flows. missing for 10: evidence of a simple single-instance (non-cluster) on-demand GPU provisioning path, documented/observed time-to-running-code, and independent hands-on confirmation of the 'minutes' claim.", "evidenceIds": [ "coreweave-docs-1", "coreweave-docs-4", "coreweave-docs-5", "coreweave-docs-7", "coreweave-docs-20", "coreweave-docs-23", "coreweave-probe-rt-1" ] }, { "productId": "coreweave", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "CoreWeave documents a broad API (CKS cluster CRUD, Terraform provider covering CKS/VPC/storage/inference, Inference API for gateways/deployments/capacity) confirmed live via a runtime probe (401 Bearer-gated), and notes some UI-only helpers like Capacity Finder that lack a documented API equivalent. Not all Cloud Console features (e.g., billing insights, quota views, Capacity Finder) are explicitly confirmed as API-accessible, so full UI/API parity isn't demonstrated. missing for 10: explicit API endpoints for billing insights/quota viewing, explicit API equivalent for Capacity Finder, independent third-party confirmation of full UI/API parity.", "evidenceIds": [ "coreweave-docs-5", "coreweave-docs-6", "coreweave-docs-19", "coreweave-docs-12", "coreweave-docs-13", "coreweave-docs-23", "coreweave-docs-31", "coreweave-probe-rt-1" ] }, { "productId": "coreweave", "storyId": "openness-full-export", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "CoreWeave's object storage is S3-compatible (open standard) and infrastructure is managed via open Terraform/OpenTofu configs and standard kubeconfig/kubectl access, which support some data/infra portability, but there is no explicit documented 'export all your data' tool or account-exit workflow. missing for 10: a dedicated bulk data-export feature, documentation of exporting model weights/configs/billing history in open formats, and any statement about facilitating full account migration/leave.", "evidenceIds": [ "coreweave-docs-10", "coreweave-docs-6", "coreweave-docs-28", "coreweave-docs-7" ] }, { "productId": "coreweave", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "CoreWeave is a closed, proprietary GPU cloud platform, not open-source software; there is no evidence of an open-license source code repository, and this is a category error for an infrastructure-as-a-service product.", "evidenceIds": [] }, { "productId": "coreweave", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "CoreWeave is itself a managed GPU cloud/infrastructure provider (bare-metal Kubernetes, storage, inference services) — there is no 'core product' artifact that a customer could instead self-host on their own hardware; the entire value proposition is CoreWeave-operated bare-metal infrastructure. Self-hosting is a category error for this kind of product, not a missing feature.", "evidenceIds": [ "coreweave-docs-14", "coreweave-docs-27" ] }, { "productId": "coreweave", "storyId": "per-second-billing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Docs mention pay-as-you-go, on-demand/spot capacity, and billing insights showing usage breakdowns, but no evidence specifies per-second or per-minute billing granularity or confirms billing only occurs while instances are actively running.", "evidenceIds": [ "coreweave-docs-13", "coreweave-docs-20", "coreweave-docs-4" ] }, { "productId": "coreweave", "storyId": "persistent-network-storage", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm S3-compatible object storage for persisting training data, checkpoints, and model weights independent of GPU compute (coreweave-docs-10), which satisfies the 'outlives any single GPU rental' need, but there's no explicit evidence of attachable persistent block/network volumes (e.g., Kubernetes PersistentVolumes or NFS-style storage) for CKS nodes specifically surviving instance teardown. missing for 10: documentation of block/network-attached persistent volumes for CKS nodes, PVC/storage-class details, and independent confirmation of data survival across instance teardown.", "evidenceIds": [ "coreweave-docs-10" ] }, { "productId": "coreweave", "storyId": "prebuilt-ml-templates", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers CKS cluster/node-pool management, Terraform, storage, Slurm-on-K8s, and API access tokens, but contains no mention of pre-built ML environment templates or container images for PyTorch, CUDA, vLLM, or ComfyUI that a developer could launch directly. This is a fair axis for a GPU cloud platform, but no supporting evidence exists.", "evidenceIds": [] }, { "productId": "coreweave", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence describes CoreWeave's compute 'Zones' for GPU capacity placement (docs-23, docs-31) but never documents a customer-facing region/residency selection control for data storage (e.g., choosing a region for AI Object Storage or CKS clusters) or any data-residency compliance guarantees.", "evidenceIds": [] }, { "productId": "coreweave", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "CoreWeave is an infrastructure/GPU cloud provider; the evidence pack contains no privacy policy, data usage terms, or opt-out mechanism regarding AI model training on customer data. This is a fair question for an AI infrastructure vendor (buyers may ask whether their training data or workloads are used to improve the provider's own models), but no evidence addresses it.", "evidenceIds": [] }, { "productId": "coreweave", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows CoreWeave APIs can delete clusters/resources (coreweave-docs-5) but there's no documentation of data retention policies, data deletion guarantees for stored training data/checkpoints, or privacy controls governing customer data lifecycle — the core of the story is unaddressed.", "evidenceIds": [ "coreweave-docs-5", "coreweave-docs-10" ] }, { "productId": "coreweave", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry opt-out or usage-tracking controls; the docs cover infrastructure, clusters, storage, and billing but nothing about user-level telemetry preferences. Missing for 10: any documentation of a telemetry/analytics opt-out setting, privacy controls dashboard, or usage-tracking disclosure.", "evidenceIds": [] }, { "productId": "coreweave", "storyId": "quota-limit-transparency", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "CoreWeave docs explicitly cover viewing quotas in the Cloud Console, interpreting quota-exceeded errors, and requesting capacity increases, plus a Capacity Finder tool to check placement availability before requesting Spot Node Pools. Missing for 10: independent/hands-on corroboration of the request process turnaround or SLA, and no detail on approval workflow specifics.", "evidenceIds": [ "coreweave-docs-12", "coreweave-docs-23", "coreweave-docs-31", "coreweave-docs-22" ] }, { "productId": "coreweave", "storyId": "reserved-committed-discounts", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence mentions on-demand and spot capacity (no long-term commitment) and references a 'capacity-plans' section, but no doc excerpt describes reserved or committed-use discount pricing, contract terms, or commitment tiers for platform engineers to lock in.", "evidenceIds": [ "coreweave-docs-4", "coreweave-docs-20", "coreweave-docs-23" ] }, { "productId": "coreweave", "storyId": "security-compliance-posture", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of SOC 2 certification, compliance attestations, data handling/privacy policies, or datacenter tier certifications; it only covers infrastructure/API features (CKS, Terraform, autoscaling, tokens) and unrelated financial/community discussion. Missing for 10: SOC 2 or ISO certifications, data handling/privacy documentation, physical datacenter tier/uptime certifications, any compliance trust page or audit report.", "evidenceIds": [] }, { "productId": "coreweave", "storyId": "serverless-gpu-endpoints", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "CoreWeave's documented autoscaling is Kubernetes Cluster Autoscaler scaling Node Pools/bare-metal instances up/down with demand (coreweave-docs-3, coreweave-docs-16, coreweave-docs-30), and Spot/On-Demand node pools for burst capacity (coreweave-docs-4, coreweave-docs-20) — this is infrastructure-level cluster scaling, not a serverless 'deploy code and it scales to zero' abstraction. No evidence describes a serverless function/endpoint product, a scale-to-zero guarantee, or a developer simply pushing code without managing nodes/pools.", "evidenceIds": [ "coreweave-docs-3", "coreweave-docs-4", "coreweave-docs-16", "coreweave-docs-20", "coreweave-docs-30" ] }, { "productId": "coreweave", "storyId": "spot-interruptible-pricing", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "CoreWeave docs confirm Spot Node Pools exist as pay-as-you-go, no-commitment bare-metal capacity with a Capacity Finder tool to check availability, but the evidence never documents actual preemption semantics (notice period, eviction behavior, discount percentage vs on-demand) that an ML engineer would need to plan around interruptions. Missing for 10: documented eviction/notice mechanics, explicit discount pricing tied to spot vs on-demand, and any hands-on or independent confirmation of how preemption actually behaves.", "evidenceIds": [ "coreweave-docs-4", "coreweave-docs-23", "coreweave-docs-31", "coreweave-docs-20" ] }, { "productId": "coreweave", "storyId": "ssh-root-access", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "CoreWeave's documented access model is Kubernetes-native (kubeconfig + API tokens via CKS) rather than traditional SSH-with-your-own-keys into a GPU instance; no docs mention SSH key injection, root shell access, or instance-level SSH at all — access is described purely in terms of kubectl/API authentication to clusters running on bare metal without VMs.", "evidenceIds": [ "coreweave-docs-7", "coreweave-docs-14", "coreweave-docs-26", "coreweave-docs-17" ] }, { "productId": "coreweave", "storyId": "team-access-controls", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "CoreWeave documents API Access Tokens that authenticate and scope access to specific resources (CKS clusters, VPCs) and billing insight views, showing some credential and spend visibility, but there is no evidence of team-member/user role management (RBAC for humans, org roles, or permission tiers) or spend controls/limits tied to specific keys. missing for 10: team member/role management (invite users, assign roles), scoped-key permission granularity beyond resource type, and spend-limiting or budget-control features tied to API keys.", "evidenceIds": [ "coreweave-docs-17", "coreweave-docs-26", "coreweave-docs-7", "coreweave-docs-13" ] }, { "productId": "coreweave", "storyId": "transparent-gpu-pricing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The only pricing-related evidence is a marketing snippet from coreweave.com/pricing touting 'flexibility of great pricing' for on-demand/spot GPUs, but no evidence shows an actual published per-GPU-hour price table or rate card visible without contacting sales.", "evidenceIds": [ "coreweave-docs-20" ] }, { "productId": "lambda-labs", "storyId": "agent-provisions-gpu", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Lambda documents a full REST API for provisioning, launching, and managing on-demand instances/clusters (with cloud-init for workload setup and a live, key-gated OpenAPI endpoint confirmed by probe), plus usage/monitoring pages and GPU dashboards — enabling non-human, API-driven lifecycle management. However, there is no official CLI and no first-party MCP server; the only MCP integration found is a third-party/unofficial community-built CLI+MCP wrapper, and monitoring is oriented toward billing/usage dashboards rather than a documented job-status API for agents. missing for 10: official CLI, first-party MCP server, and a documented workload-monitoring/job-status API distinct from usage billing.", "evidenceIds": [ "lambda-labs-docs-5", "lambda-labs-docs-23", "lambda-labs-docs-31", "lambda-labs-docs-39", "lambda-labs-probe-rt-1", "lambda-labs-docs-6", "lambda-labs-docs-20", "lambda-labs-comm-3" ] }, { "productId": "lambda-labs", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe of https://docs.lambda.ai/llms.txt returned HTTP 404, and there is no other evidence of an llms.txt file or agent-oriented documentation format anywhere in the pack; the OpenAPI spec is a REST API description, not agent-native docs guidance.", "evidenceIds": [ "lambda-labs-probe-1" ] }, { "productId": "lambda-labs", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Lambda is a GPU cloud infrastructure provider (compute instances, clusters, Slurm, Kubernetes) — it is not a data/analytics product that generates AI-driven insights or suggestions from a user's data. This story asks about an application-layer AI-insights feature, which is a category mismatch for an infrastructure/IaaS product; the only AI-related community item is an unofficial third-party MCP/CLI wrapper for provisioning instances, not data insights.", "evidenceIds": [] }, { "productId": "lambda-labs", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Lambda is GPU IaaS with an API, cloud-init for launch-time config, and webhooks for support tickets, but there is no native scheduler/automation service that runs workflows autonomously in the background; the only agentic automation (CLI/MCP server to launch/terminate instances via AI agents) is an unofficial third-party project, not a Lambda-native feature.", "evidenceIds": [ "lambda-labs-docs-31", "lambda-labs-docs-40", "lambda-labs-docs-24", "lambda-labs-comm-3" ] }, { "productId": "lambda-labs", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Lambda is a GPU cloud/infrastructure provider (on-demand instances, clusters, Slurm, Kubernetes), not a product with a built-in AI assistant for task delegation. This axis is a category error for an IaaS GPU platform; the community mention of an unofficial MCP/CLI wrapper built by a third party does not constitute a first-party built-in assistant.", "evidenceIds": [] }, { "productId": "lambda-labs", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Lambda's On-Demand Cloud exposes a full REST API for programmatically launching/terminating GPU instances, with cloud-init for launch-time automation, confirmed live and key-gated by a runtime probe — enabling headless/CI-driven provisioning of GPU workloads without any UI interaction. Missing for 10: an official first-party CLI/SDK or CI/CD templates (only an unofficial community-built CLI/MCP server exists) and documented CI examples (e.g., GitHub Actions) from Lambda itself.", "evidenceIds": [ "lambda-labs-docs-5", "lambda-labs-docs-23", "lambda-labs-docs-24", "lambda-labs-docs-31", "lambda-labs-probe-rt-1", "lambda-labs-comm-3" ] }, { "productId": "lambda-labs", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Lambda is a GPU cloud/infrastructure platform (instance provisioning, clusters, storage, Slurm/K8s) with no agent or assistant component that could consume external tools via MCP. The evidence even shows the reverse: a third-party built an MCP server *around* Lambda's API so other agents could control Lambda, not Lambda acting as an MCP client itself, so this axis is a category error for this product type.", "evidenceIds": [ "lambda-labs-comm-3" ] }, { "productId": "lambda-labs", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Only an unofficial, community-built CLI/MCP server for Lambda GPU instances is documented; there is no evidence of an official first-party MCP server from Lambda itself.", "evidenceIds": [ "lambda-labs-comm-3" ] }, { "productId": "lambda-labs", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Lambda itself only exposes a REST API (docs-1..41) with no first-party natural-language or agent interface; the only NL-command capability comes from a third-party developer's unofficial CLI/MCP server that lets AI agents launch/terminate instances via commands like 'launch an H100' (lambda-labs-comm-3). This is an extra, unofficial tool rather than a supported product feature. missing for 10: first-party NL/agent interface, official MCP server or chat-based control, documentation of natural-language command support.", "evidenceIds": [ "lambda-labs-comm-3", "lambda-labs-docs-31", "lambda-labs-probe-rt-1" ] }, { "productId": "lambda-labs", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only a REST API (openapi.json) and cloud-init/SSH access, with no first-party CLI tool documented; the only CLI mentioned is an unofficial third-party CLI/MCP server built by a community developer for AI agents, not an official Lambda offering.", "evidenceIds": [ "lambda-labs-docs-31", "lambda-labs-docs-39", "lambda-labs-comm-3", "lambda-labs-probe-rt-1" ] }, { "productId": "lambda-labs", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Lambda publishes a documented public REST API (Lambda Cloud API) with a live OpenAPI 3.1 spec covering instance provisioning, cluster launch, filesystems, and firewall management, confirmed both in docs and via runtime probe returning the full spec and a key-gated endpoint. Community evidence further shows a third-party built a CLI/MCP server on top of this API enabling AI agents to launch/terminate GPU instances programmatically, corroborating real-world agentic usability. Missing for 10: an official first-party SDK/MCP server and an llms.txt (probe found 404), so slight extra integration work is needed.", "evidenceIds": [ "lambda-labs-docs-5", "lambda-labs-docs-23", "lambda-labs-docs-31", "lambda-labs-docs-39", "lambda-labs-probe-rt-1", "lambda-labs-comm-3" ] }, { "productId": "lambda-labs", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Lambda's Cloud API is key-gated (docs-31/39, probe-rt-1 confirms 401 without a key), but there is no evidence of scoped/least-privilege credential issuance (e.g., role-based permissions, read-only vs. write scopes, or per-agent restricted keys) — only that a single API key exists to access all account resources.", "evidenceIds": [ "lambda-labs-docs-31", "lambda-labs-docs-39", "lambda-labs-probe-rt-1" ] }, { "productId": "lambda-labs", "storyId": "agentic-sdks", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Lambda documents a public, key-gated REST/Cloud API with a full OpenAPI 3.1 spec that could underlie SDK development, but no official first-party SDK client libraries (Python/JS/etc.) are evidenced — only an unofficial third-party CLI/MCP server exists. Missing for 10: official SDK packages/libraries, first-party language bindings, docs referencing an 'SDK' rather than raw REST endpoints.", "evidenceIds": [ "lambda-labs-docs-5", "lambda-labs-docs-31", "lambda-labs-probe-rt-1", "lambda-labs-comm-3" ] }, { "productId": "lambda-labs", "storyId": "agentic-webhooks", "verdict": "partial", "quality": 3, "confidence": "medium", "rationale": "Lambda's API docs mention webhook notifications only for support ticket events ('Lambda can send webhook notifications to your URL when support ticket events occur'), not for core compute/instance lifecycle events that an AI-native/agentic user would most want to subscribe to. missing for 10: webhook support for instance state changes or job/cluster events, first-party docs on webhook setup/payload schema, and independent confirmation of use in agentic workflows.", "evidenceIds": [ "lambda-labs-docs-40" ] }, { "productId": "lambda-labs", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Lambda exposes a raw OpenAPI 3.1 spec (cloud.lambda.ai/api/v1/openapi.json) and documents REST endpoints, but there is no evidence of an interactive, browsable API reference (e.g., Swagger UI, 'try it out' console) with runnable examples — probes for docs.lambda.ai/openapi.json, swagger.json, and llms.txt all 404.", "evidenceIds": [ "lambda-labs-docs-31", "lambda-labs-docs-39", "lambda-labs-probe-1", "lambda-labs-probe-2", "lambda-labs-probe-rt-1" ] }, { "productId": "lambda-labs", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Lambda publishes a live, machine-readable OpenAPI 3.1 spec for its Cloud API at https://cloud.lambda.ai/api/v1/openapi.json, confirmed both by docs references and a runtime probe that fetched the full spec keylessly. Minor gap: the probe found the docs.lambda.ai domain itself doesn't serve openapi.json at the expected conventional path (404s), requiring the correct subdomain. missing for 10: independent third-party confirmation of spec completeness/versioning beyond the runtime probe.", "evidenceIds": [ "lambda-labs-docs-31", "lambda-labs-docs-39", "lambda-labs-probe-rt-1", "lambda-labs-probe-2" ] }, { "productId": "lambda-labs", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Lambda's docs describe launching isolated GPU instances, firewalls, and filesystems, but there is no documented sandbox/production separation concept, test-mode, or data-isolation feature aimed at safely testing without touching production data — users would have to build this themselves by manually spinning up separate instances. missing for 10: explicit sandbox/staging environment feature, guidance on isolating test data from production, any mention of a 'sandbox mode' or non-production testing workflow.", "evidenceIds": [ "lambda-labs-docs-16", "lambda-labs-docs-11", "lambda-labs-docs-9" ] }, { "productId": "lambda-labs", "storyId": "api-versioning-policy", "verdict": "partial", "quality": 3, "confidence": "medium", "rationale": "The Lambda Cloud API is clearly versioned (path-based v1, documented via a live OpenAPI 3.1 spec with rate limits), satisfying the 'versioned APIs' half of the story, but no evidence anywhere in the pack mentions a deprecation policy, versioning changelog, or sunset process for older API versions. Missing for 10: documented deprecation/versioning policy, changelog of breaking changes, migration guidance between API versions, independent confirmation of stability guarantees.", "evidenceIds": [ "lambda-labs-docs-31", "lambda-labs-docs-39", "lambda-labs-probe-rt-1" ] }, { "productId": "lambda-labs", "storyId": "auto-shutdown-spend-guards", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No documentation or feature evidence shows auto-shutdown timers, idle detection, or spend-limit controls; on the contrary, community evidence explicitly states Lambda 'does not distinguish between idle and in use instance states' and a user was billed $583 for 391 hours of an idle GH200 instance, confirming the absence of this safeguard.", "evidenceIds": [ "lambda-labs-comm-1" ] }, { "productId": "lambda-labs", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Lambda supports large-scale provisioning (1-Click Clusters of 16–512+ GPUs, Slurm-based multi-node job scheduling) and a REST API for programmatic instance management, which lets an AI-native user manage many GPU resources at once. However, the documented API is rate-limited to 1 req/s (and launches limited per 12s), with no evidence of true bulk/batch endpoints for creating, updating, or deleting many items in a single call. Missing for 10: explicit bulk-create/bulk-delete API operations, batch job submission tooling beyond Slurm's node scheduling, and evidence of high-throughput automation without rate-limit friction.", "evidenceIds": [ "lambda-labs-docs-5", "lambda-labs-docs-8", "lambda-labs-docs-15", "lambda-labs-docs-37", "lambda-labs-probe-rt-1" ] }, { "productId": "lambda-labs", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Lambda is a GPU IaaS provider; the only event-triggered mechanism found is webhook notifications for support-ticket events (docs-40) and cloud-init instructions applied only at launch time (docs-6, docs-24, docs-32) — neither constitutes a general rules/automation engine letting users define arbitrary triggers-to-actions. No autoscaling policies, alert-based actions, or rule-definition UI/API are documented. missing for 10: a general event-rule engine (define trigger conditions + arbitrary actions), autoscaling/alerting automation, and any first-party support beyond narrow support-ticket webhooks.", "evidenceIds": [ "lambda-labs-docs-40", "lambda-labs-docs-6", "lambda-labs-docs-24" ] }, { "productId": "lambda-labs", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Lambda offers Slurm-based job scheduling for HPC workloads and a REST API with cloud-init for launch-time configuration, but none of the evidence describes a mechanism for scheduling recurring/cron-like jobs or automated recurring workflows — Slurm here is described as scheduling submitted workloads, not recurring automation. missing for 10: any cron/recurring job scheduler, workflow orchestration triggers, or documentation of repeat/interval-based job execution.", "evidenceIds": [ "lambda-labs-docs-37", "lambda-labs-docs-14", "lambda-labs-docs-27", "lambda-labs-docs-38", "lambda-labs-docs-31" ] }, { "productId": "lambda-labs", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Lambda is a GPU cloud infrastructure/compute provider, not an automation/workflow-builder product; versioning, reviewing, and rolling back 'automations' is a category mismatch — this is a wrong-axis question for an IaaS/GPU platform.", "evidenceIds": [] }, { "productId": "lambda-labs", "storyId": "billing-usage-api", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Lambda's docs describe billing by hourly/minute increments and a web 'Usage page' with an Instances tab for viewing monthly usage (lambda-labs-docs-10, lambda-labs-docs-20), but this is UI-only, not a programmatic API. The published Lambda Cloud API (lambda-labs-docs-31/39, confirmed live via probe) covers instance provisioning/management, not billing or usage export, and no team/workload cost-attribution tagging or billing API endpoint is documented anywhere in the evidence.", "evidenceIds": [ "lambda-labs-docs-10", "lambda-labs-docs-20", "lambda-labs-docs-31", "lambda-labs-docs-39", "lambda-labs-probe-rt-1" ] }, { "productId": "lambda-labs", "storyId": "custom-docker-images", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Lambda's docs show real support for custom environments: the Cloud API lets you launch instances with 'a different base image' (custom machine template) and cloud-init for launch-time configuration, plus GPU Base images for full control over the Python environment. Containerized workloads (i.e., running an actual Docker image) are supported via Managed Kubernetes (MK8s) rather than as a first-class ODC instance feature. Missing for 10: explicit first-party documentation of directly launching a Docker image as an ODC instance (vs. VM base image), a custom-image/AMI gallery, and independent hands-on confirmation that custom base images work as claimed.", "evidenceIds": [ "lambda-labs-docs-29", "lambda-labs-docs-6", "lambda-labs-docs-24", "lambda-labs-docs-22", "lambda-labs-docs-12", "lambda-labs-docs-33" ] }, { "productId": "lambda-labs", "storyId": "data-transfer-cloud-sync", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Lambda documents an S3-compatible adapter for its filesystems, explicitly supporting rclone and s5cmd for copying data in/out, plus a REST Cloud API and cloud-init for programmatic/automated data and instance management. Missing for 10: no independent hands-on verification of transfer performance/reliability and no native cloud-storage sync service beyond the S3-compatible adapter.", "evidenceIds": [ "lambda-labs-docs-25", "lambda-labs-docs-30", "lambda-labs-docs-9", "lambda-labs-docs-31", "lambda-labs-probe-rt-1" ] }, { "productId": "lambda-labs", "storyId": "exposed-ports-networking", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Lambda's docs show port-level control via Firewall rules restricting incoming traffic (docs-11/18) and SSH/JupyterLab access (docs-7), and high-performance private networking (GPUDirect RDMA) exists within 1-Click Clusters and Managed Kubernetes for GPU node interconnect (docs-36, docs-12/33). However, there's no explicit documentation of a general-purpose private networking/VPC feature connecting arbitrary On-Demand instances, nor clear guidance on opening/exposing custom application ports beyond firewall restriction rules. Missing for 10: explicit port-exposure/ingress configuration for serving apps, dedicated private networking (VPC/VLAN) between standard instances outside of 1CC/K8s clusters, and independent confirmation of these networking features working in practice.", "evidenceIds": [ "lambda-labs-docs-11", "lambda-labs-docs-18", "lambda-labs-docs-7", "lambda-labs-docs-36", "lambda-labs-docs-12", "lambda-labs-docs-33" ] }, { "productId": "lambda-labs", "storyId": "gpu-availability-transparency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes Lambda's instance types, API endpoints for launching/managing instances, and pricing, but nowhere shows a real-time GPU availability/capacity view by type and region that an engineer could check before provisioning. Docs even describe access as 'first-come' (docs-28), implying no visibility into stock levels, and no dashboard or API field for capacity-by-region is mentioned.", "evidenceIds": [ "lambda-labs-docs-21", "lambda-labs-docs-28", "lambda-labs-docs-31", "lambda-labs-docs-39" ] }, { "productId": "lambda-labs", "storyId": "gpu-breadth-latest-hardware", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Lambda's docs clearly list current-gen H100/B200 (and GH200) options alongside older A100 in on-demand instances and clusters, covering both current and previous-generation GPU tiers with flexible on-demand access. Missing for 10: no explicit pricing comparison table or independent benchmark confirming actual availability/pricing tiers side-by-side.", "evidenceIds": [ "lambda-labs-docs-21", "lambda-labs-docs-28", "lambda-labs-docs-8", "lambda-labs-docs-15", "lambda-labs-comm-2" ] }, { "productId": "lambda-labs", "storyId": "instance-lifecycle-management", "verdict": "disputed", "quality": 6, "confidence": "medium", "rationale": "Lambda's API and docs support programmatic launch/terminate of instances with per-minute billing (lambda-labs-docs-5, -10, -23, -31, -39, runtime probe confirms live REST API), but there's no documented stop/start (pause) capability distinct from terminate, and a hands-on community report shows billing continues for idle-but-running instances regardless of usage, contradicting 'pay only for what is running' in the sense of active workload — user was billed $583 for an idle instance because Lambda 'does not distinguish between idle and in use instance states' (lambda-labs-comm-1). missing for 10: documented stop/pause (vs. terminate) lifecycle action, confirmation that idle time is excluded from billing, independent corroboration beyond one HN anecdote.", "evidenceIds": [ "lambda-labs-docs-5", "lambda-labs-docs-10", "lambda-labs-docs-23", "lambda-labs-docs-31", "lambda-labs-probe-rt-1", "lambda-labs-comm-1" ] }, { "productId": "lambda-labs", "storyId": "jupyter-ide-access", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Lambda instances ship with a preinstalled JupyterLab server and SSH access out of the box, letting a developer open Jupyter or SSH in with no extra setup (docs-3, docs-7). However, there's no explicit documentation of a one-step VS Code/Cursor Remote-SSH or dev-container integration — only generic SSH connectivity is mentioned, requiring the developer to manually configure their IDE's remote connection. Missing for 10: explicit VS Code/Cursor connection docs or one-click IDE integration, independent confirmation that IDE remote-attach works smoothly.", "evidenceIds": [ "lambda-labs-docs-3", "lambda-labs-docs-7", "lambda-labs-docs-16" ] }, { "productId": "lambda-labs", "storyId": "keyless-catalog-pricing-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Lambda publishes a full OpenAPI 3.1 spec for its Cloud API keylessly (lambda-labs-probe-rt-1), and docs confirm the API is used to create/manage instances programmatically (lambda-labs-docs-5, lambda-labs-docs-23, lambda-labs-docs-31/39), with a separate public pricing page (lambda-labs-docs-41). An unofficial MCP/CLI already lets agents launch/terminate Lambda GPUs (lambda-labs-comm-3), implying some catalog query capability exists. However, actual provisioning calls (e.g. /instances) require an API key (401 per probe), and no evidence explicitly shows a documented, unauthenticated 'instance-types/catalog' endpoint returning live pricing+availability distinct from the static marketing pricing page. Missing for 10: explicit documentation of a public catalog/instance-types endpoint with live pricing+availability, and independent confirmation an agent can query it pre-spend without a key.", "evidenceIds": [ "lambda-labs-probe-rt-1", "lambda-labs-docs-31", "lambda-labs-docs-39", "lambda-labs-docs-41", "lambda-labs-comm-3", "lambda-labs-docs-5" ] }, { "productId": "lambda-labs", "storyId": "managed-slurm-kubernetes", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Lambda offers Managed Slurm and Managed Kubernetes (MK8s) as first-party managed schedulers on top of 1-Click Clusters, with Slurm handling automatic job scheduling/utilization and daemon health monitoring, and MK8s providing preconfigured GPU/RDMA Kubernetes clusters ready for workload deployment — directly fulfilling the 'managed scheduler instead of building your own' story. Missing for 10: independent/hands-on validation of Slurm or K8s job scheduling in production (community evidence only covers billing and unofficial CLI tooling, not scheduler usage), and no detail on multi-tenancy/queue customization depth.", "evidenceIds": [ "lambda-labs-docs-12", "lambda-labs-docs-13", "lambda-labs-docs-14", "lambda-labs-docs-27", "lambda-labs-docs-33", "lambda-labs-docs-34", "lambda-labs-docs-37", "lambda-labs-docs-38" ] }, { "productId": "lambda-labs", "storyId": "multi-node-clusters", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Lambda offers self-serve 1-Click Clusters (16-512 H100/B200 GPUs) with GPUDirect RDMA up to 3200 Gb/s, launched instantly with no long-term commitment and no sales cycle mentioned, plus Managed Slurm/Kubernetes for orchestration and a live self-serve API confirmed by runtime probe. Community evidence corroborates self-serve on-demand access and even third-party tooling built on the API, though no independent hands-on report specifically confirms multi-node cluster provisioning experience. Missing for 10: independent/hands-on validation of the 1-Click Cluster provisioning flow itself and interconnect performance in practice.", "evidenceIds": [ "lambda-labs-docs-8", "lambda-labs-docs-36", "lambda-labs-docs-14", "lambda-labs-docs-15", "lambda-labs-docs-28", "lambda-labs-docs-31", "lambda-labs-probe-rt-1", "lambda-labs-comm-3" ] }, { "productId": "lambda-labs", "storyId": "on-demand-gpu-provisioning", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Lambda's docs and live API confirm on-demand GPU provisioning via console or REST API, with cloud-init, SSH, and JupyterLab access enabling immediate code execution, and pricing pages claim deployment 'in minutes.' A community-built CLI/MCP tool also independently confirms real users launching and SSHing into instances quickly. Missing for 10: independent benchmark/timing evidence validating the 'minutes' claim and first-hand confirmation of no friction in provisioning flow.", "evidenceIds": [ "lambda-labs-docs-1", "lambda-labs-docs-5", "lambda-labs-docs-6", "lambda-labs-docs-7", "lambda-labs-docs-16", "lambda-labs-docs-28", "lambda-labs-probe-rt-1", "lambda-labs-comm-3" ] }, { "productId": "lambda-labs", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Lambda publishes a live, documented REST API (OpenAPI spec, key-gated) covering core instance lifecycle actions (launch, list, terminate, cloud-init, custom images) that mirror UI capabilities for On-Demand Cloud instances, and third parties have built agent-facing CLIs/MCP wrappers on top of it. However, evidence doesn't show API parity for other UI-managed features like firewall rules, filesystem management, 1-Click Cluster/Slurm cluster provisioning, or usage/billing dashboards. Missing for 10: documented API endpoints for firewalls, filesystems, 1CC/Slurm cluster lifecycle, and usage/billing views; independent confirmation of full UI-API parity.", "evidenceIds": [ "lambda-labs-docs-5", "lambda-labs-docs-23", "lambda-labs-docs-29", "lambda-labs-docs-31", "lambda-labs-docs-39", "lambda-labs-probe-rt-1", "lambda-labs-comm-3" ] }, { "productId": "lambda-labs", "storyId": "openness-full-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Lambda's filesystems support S3-compatible tools like rclone and s5cmd, letting users copy datasets and files out in standard, open formats rather than being locked into a proprietary export mechanism, and the Cloud API allows programmatic access to resource metadata. However, there's no explicit documentation of a full account-data export (billing, usage history, configs) or a one-click 'export everything and leave' workflow. Missing for 10: explicit full-account data export/portability guarantee, documentation of exporting non-file data (billing/usage/API keys), independent verification that egress is unrestricted or free of lock-in fees.", "evidenceIds": [ "lambda-labs-docs-25", "lambda-labs-docs-30", "lambda-labs-docs-9", "lambda-labs-docs-31" ] }, { "productId": "lambda-labs", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Lambda is a GPU cloud/infrastructure product, not an open-source software project; there is no indication its core product source code is published under an open license. This axis (reading product source under an open license) is a category mismatch for a cloud compute service rather than something the evidence contradicts.", "evidenceIds": [] }, { "productId": "lambda-labs", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Lambda is a cloud GPU IaaS/compute platform, not open-source software; self-hosting the 'core product' is a category error since the product itself is the hosted cloud infrastructure being rented out, not a deployable application.", "evidenceIds": [] }, { "productId": "lambda-labs", "storyId": "per-second-billing", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Lambda's docs explicitly state ODC instances are priced hourly but billed in one-minute increments, and billing only accrues while an instance is running/attached — corroborated by a community report where a user was billed for the full duration their instance remained running (idle counted as running, confirming billing follows instance lifecycle rather than usage activity). Missing for 10: true per-second billing granularity (only per-minute is documented), and independent/user confirmation that billing precisely matches the one-minute increment claim.", "evidenceIds": [ "lambda-labs-docs-10", "lambda-labs-comm-1" ] }, { "productId": "lambda-labs", "storyId": "persistent-network-storage", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Lambda's docs explicitly describe attachable persistent filesystems ('high-capacity regional file store you can attach to your instance to store datasets and back up system state') that exist independently of any instance, plus S3-compatible tooling (rclone/s5cmd) for moving data in/out, directly matching the story of datasets/checkpoints surviving GPU teardown. missing for 10: no independent/hands-on confirmation of persistence across teardown or details on durability guarantees/pricing for the filesystem.", "evidenceIds": [ "lambda-labs-docs-9", "lambda-labs-docs-17", "lambda-labs-docs-25", "lambda-labs-docs-30" ] }, { "productId": "lambda-labs", "storyId": "prebuilt-ml-templates", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Lambda's ODC instances preinstall 'Lambda Stack' (AI/ML drivers, tools, frameworks) and offer a 'GPU Base' minimal image, which covers PyTorch/CUDA-style ready environments, plus JupyterLab for quick start. However, there's no evidence of named pre-built templates for vLLM or ComfyUI specifically, only generic 'AI/ML tools and frameworks'. missing for 10: explicit vLLM template/image, explicit ComfyUI template/image, a documented template gallery/catalog beyond Lambda Stack and GPU Base.", "evidenceIds": [ "lambda-labs-docs-2", "lambda-labs-docs-4", "lambda-labs-docs-22", "lambda-labs-docs-3" ] }, { "productId": "lambda-labs", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Lambda's docs mention that filesystems are a 'regional' store (implying infrastructure regions exist) but provide no evidence of user-facing controls to select a specific region or data-residency zone for compliance/privacy purposes. Missing for 10: explicit region-selection UI/API, documented list of available regions, and any data-residency/compliance guarantees.", "evidenceIds": [ "lambda-labs-docs-9", "lambda-labs-docs-17" ] }, { "productId": "lambda-labs", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Lambda is a GPU cloud infrastructure provider (IaaS), not an AI model/chat product that trains models on user data; data-training opt-out is a category error for this kind of product.", "evidenceIds": [] }, { "productId": "lambda-labs", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Lambda is a GPU cloud infrastructure provider (compute instances, clusters, filesystems, API); there is no evidence of any data retention/deletion controls, data lifecycle policies, or privacy dashboard for AI-native users to manage what data is stored or deleted. The evidence covers filesystems for storage and billing/usage tracking but nothing about retention limits or deletion mechanisms. missing for 10: documented data retention policy, deletion/opt-out controls, data lifecycle settings, any privacy-posture documentation.", "evidenceIds": [ "lambda-labs-docs-9", "lambda-labs-docs-17", "lambda-labs-docs-20" ] }, { "productId": "lambda-labs", "storyId": "privacy-telemetry-optout", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Lambda is a GPU cloud/infrastructure provider, not an AI assistant or telemetry-collecting client tool; opting out of telemetry/usage tracking is not a relevant axis for this product category — it's about billing usage of cloud resources, not AI-native telemetry.", "evidenceIds": [] }, { "productId": "lambda-labs", "storyId": "quota-limit-transparency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only generic API rate limits and a vague 'contact us for reserved capacity' pricing note, but no documented per-account instance/GPU quotas nor any defined process (e.g., support ticket workflow, quota dashboard) to request limit increases.", "evidenceIds": [ "lambda-labs-docs-35", "lambda-labs-probe-rt-1" ] }, { "productId": "lambda-labs", "storyId": "reserved-committed-discounts", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Lambda explicitly advertises a path to reserved/committed capacity pricing ('Contact us for reserved capacity at our lowest prices') and also offers Private Cloud for large single-tenant multi-year deployments, confirming the capability exists. However, this is sales-assisted only — there's no self-service reservation mechanism, no published discount tiers/terms, and on-demand pages emphasize 'No long-term commitments,' contrasting with the reserved offering. Missing for 10: self-service reservation/commitment workflow, published discount rates or contract terms, and independent evidence of actual reserved pricing outcomes.", "evidenceIds": [ "lambda-labs-docs-35", "lambda-labs-docs-19", "lambda-labs-docs-1" ] }, { "productId": "lambda-labs", "storyId": "security-compliance-posture", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses SOC 2 certification, compliance attestations, data handling policies, or datacenter tier ratings for Lambda's cloud offerings; the docs focus entirely on GPU provisioning, clusters, filesystems, and API usage. This is a fair, applicable question for any cloud infrastructure provider hosting proprietary models, but no supporting documentation exists in the evidence pack.", "evidenceIds": [] }, { "productId": "lambda-labs", "storyId": "serverless-gpu-endpoints", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Lambda's documented products are on-demand VMs, 1-Click Clusters, Managed Kubernetes, and Managed Slurm — all provisioned/billed as running instances, not autoscaling serverless GPU functions. Community evidence explicitly confirms Lambda 'does not distinguish between idle and in use instance states' and bills continuously even when idle, the opposite of scale-to-zero behavior. No docs mention a serverless endpoint product or autoscale-to-zero deployment model.", "evidenceIds": [ "lambda-labs-docs-16", "lambda-labs-docs-10", "lambda-labs-comm-1" ] }, { "productId": "lambda-labs", "storyId": "spot-interruptible-pricing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Lambda's docs describe only on-demand, reserved, and private-cloud pricing tiers; there is no mention of spot/interruptible/preemptible instances or any preemption semantics. The billing evidence even highlights that Lambda bills continuously for any running instance regardless of use, reinforcing the absence of a spot/interruptible model.", "evidenceIds": [ "lambda-labs-docs-10", "lambda-labs-docs-35", "lambda-labs-docs-41", "lambda-labs-comm-1" ] }, { "productId": "lambda-labs", "storyId": "ssh-root-access", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs confirm direct SSH access to GPU instances with user-controlled base images (GPU Base minimal Ubuntu image for high control), cloud-init customization, and API/CLI provisioning, implying standard root SSH access typical of cloud VM instances. Community evidence (unofficial CLI/MCP tool) corroborates real-world SSH-based workflows for launching and connecting to instances. Missing for 10: explicit first-party documentation confirming root/sudo privileges once SSH'd in, and independent hands-on confirmation of key management specifics.", "evidenceIds": [ "lambda-labs-docs-7", "lambda-labs-docs-22", "lambda-labs-docs-6", "lambda-labs-docs-29", "lambda-labs-comm-3" ] }, { "productId": "lambda-labs", "storyId": "team-access-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents instance provisioning, filesystems, firewalls, and a key-gated REST API, but contains no mention of team/member management, role-based access control, or scoped/restricted API keys for spend control. The billing complaint (lambda-labs-comm-1) actually highlights a lack of granular usage controls rather than confirming governance features.", "evidenceIds": [ "lambda-labs-docs-31", "lambda-labs-probe-rt-1", "lambda-labs-comm-1" ] }, { "productId": "lambda-labs", "storyId": "transparent-gpu-pricing", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Lambda publishes a public pricing page (lambda.ai/pricing) with clear per-GPU-hour pricing and explicitly states 'Clear, straightforward pricing for Instances, 1-Click Clusters, and Superclusters' without requiring sales contact for on-demand instances; community evidence corroborates historical public per-GPU pricing announcements (e.g., $1.99/GPU/Hour H100). Missing for 10: independent third-party confirmation that ALL current GPU types' per-hour rates are listed publicly (reserved/private cloud capacity explicitly requires contacting sales per docs-35), and no direct evidence snippet showing the actual price table contents.", "evidenceIds": [ "lambda-labs-docs-41", "lambda-labs-docs-15", "lambda-labs-docs-35", "lambda-labs-comm-2" ] }, { "productId": "paperspace", "storyId": "agent-provisions-gpu", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Paperspace documents API keys, a Core RESTful API, and an official CLI (pspace) for creating/managing machines, custom templates, and auto-shutdown, and a runtime probe confirms the CLI installs and lists machine/deployment commands — enough to plausibly script provisioning and teardown without a human in the console. However there is no MCP server, no documented end-to-end 'create→monitor→run workload→teardown' workflow example, and community reports describe friction getting SSH/API access working reliably on Core VMs, undercutting a clean automated experience. Missing for 10: an MCP server, explicit workload-run/monitoring API docs, and independent confirmation of a full agent-driven lifecycle without console fallback.", "evidenceIds": [ "paperspace-docs-1", "paperspace-docs-6", "paperspace-docs-23", "paperspace-docs-24", "paperspace-docs-5", "paperspace-probe-4", "paperspace-probe-rt-1", "paperspace-comm-7" ] }, { "productId": "paperspace", "storyId": "agentic-agent-docs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The Paperspace docs live under docs.digitalocean.com, and that domain does serve a working llms.txt (HTTP 200) with a general description of DigitalOcean's documentation corpus, giving agents a discoverable entry point. However, a direct markdown/agent-friendly version of the Paperspace-specific docs page 404s, and there's no evidence the llms.txt specifically indexes or highlights Paperspace content vs. the broader DigitalOcean product catalog. Missing for 10: Paperspace-specific agent-readable docs (e.g. .md endpoint), confirmation llms.txt references Paperspace pages, and any documentation stating this is an intentional agent-onboarding feature.", "evidenceIds": [ "paperspace-probe-1", "paperspace-probe-2" ] }, { "productId": "paperspace", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Paperspace is an infrastructure/GPU-cloud platform for provisioning machines, notebooks, and deployments; it does not itself analyze user data to generate AI-driven insights or suggestions inside the product. This capability is a category error for an infrastructure/compute provider rather than an analytics or AI-assistant product.", "evidenceIds": [] }, { "productId": "paperspace", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Paperspace's 'Workflows' feature is documented as automating ML tasks/pipelines, and 'Deployments' run containerized models continuously, both of which could constitute background automation, plus API/CLI access enables scripting external automations. However there's no documentation of triggers, schedules, or persistent autonomous execution akin to cron/agentic loops, and no community evidence confirming this works well in practice. Missing for 10: scheduling/trigger mechanisms, evidence of long-running unattended execution, and independent confirmation that Workflows/Deployments actually operate autonomously without manual intervention.", "evidenceIds": [ "paperspace-docs-9", "paperspace-docs-8", "paperspace-docs-6", "paperspace-docs-24" ] }, { "productId": "paperspace", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Paperspace is a GPU cloud/infrastructure platform (machines, notebooks, deployments, CLI) with no evidence of any built-in AI assistant feature for delegating tasks; this axis is a category mismatch for an infra provider rather than an agentic assistant product.", "evidenceIds": [] }, { "productId": "paperspace", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Paperspace exposes a REST API, JS SDK, and a working CLI (pspace) that can create/manage machines, deployments, and templates non-interactively, and Workflows offers pipeline-style automation — enough to script headless usage in CI. However there's no first-party CI-integration guide (e.g. GitHub Actions), no documented service-account/non-interactive auth flow for CI runners, and community feedback flags friction around SSH/API access reliability and product quality, so full CI-native support isn't clearly evidenced. missing for 10: documented CI/CD integration examples, non-interactive/service-account auth for automation, independent confirmation of reliable headless workflows in CI pipelines.", "evidenceIds": [ "paperspace-docs-6", "paperspace-docs-23", "paperspace-docs-24", "paperspace-docs-9", "paperspace-probe-4", "paperspace-probe-rt-1", "paperspace-comm-7" ] }, { "productId": "paperspace", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Paperspace is a GPU cloud/compute infrastructure platform (VMs, notebooks, deployments), not an AI agent or agent-hosting product with MCP client/server integration; the story of plugging MCP servers into it for tool use is a category error for this kind of product.", "evidenceIds": [] }, { "productId": "paperspace", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "paperspace", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "paperspace", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Paperspace ships an official CLI (pspace), documented in DigitalOcean docs and independently verified via a runtime probe showing successful install and functional commands for machines, templates, autoscaling, and deployments. missing for 10: independent third-party review specifically praising CLI agentic workflows, and no evidence of AI-native scripting/automation examples beyond basic resource management.", "evidenceIds": [ "paperspace-docs-24", "paperspace-docs-6", "paperspace-probe-4", "paperspace-probe-rt-1" ] }, { "productId": "paperspace", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Paperspace documents a Core RESTful API, JavaScript SDK, and CLI for programmatically managing machines, notebooks, deployments, and workflows, and a runtime probe confirms the official pspace CLI installs and works with commands for machine/deployment management. Missing for 10: no public OpenAPI/swagger spec found, no independent third-party corroboration of API robustness beyond docs.", "evidenceIds": [ "paperspace-docs-6", "paperspace-docs-23", "paperspace-docs-24", "paperspace-probe-4", "paperspace-probe-rt-1", "paperspace-probe-3" ] }, { "productId": "paperspace", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Paperspace offers API keys for programmatic access, but evidence shows no scoped or least-privilege permission model — keys appear to be account-level, not fine-grained or agent-specific credentials.", "evidenceIds": [ "paperspace-docs-6", "paperspace-docs-23" ] }, { "productId": "paperspace", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm an official Core RESTful API and Core JavaScript SDK plus a documented CLI (pspace) verified via runtime probe, giving developers concrete official interfaces to build against. However, evidence only names one language SDK (JavaScript), with no mention of Python or other SDKs, and no independent corroboration of SDK quality or ecosystem breadth. missing for 10: evidence of additional language SDKs (e.g. Python), independent developer corroboration of SDK reliability, and a public SDK repo/changelog.", "evidenceIds": [ "paperspace-docs-6", "paperspace-docs-23", "paperspace-docs-24", "paperspace-probe-4", "paperspace-probe-rt-1" ] }, { "productId": "paperspace", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any webhook or event-subscription mechanism in Paperspace's API, CLI, or docs; only REST API, CLI, and SDK access are documented.", "evidenceIds": [ "paperspace-docs-23", "paperspace-docs-24", "paperspace-probe-4" ] }, { "productId": "paperspace", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of an interactive API reference with runnable examples; probes show no OpenAPI/swagger spec found (404s) and no interactive docs playground mentioned, only static docs and CLI/API mentions.", "evidenceIds": [ "paperspace-probe-3", "paperspace-probe-2", "paperspace-docs-23" ] }, { "productId": "paperspace", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Paperspace/DigitalOcean documents a Core RESTful API and CLI, but explicit probes for an OpenAPI/Swagger spec at all standard locations returned 404, and no machine-readable spec is referenced anywhere in the docs.", "evidenceIds": [ "paperspace-docs-23", "paperspace-probe-3" ] }, { "productId": "paperspace", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Paperspace documentation covers general-purpose VM/notebook/deployment provisioning but never describes a distinct sandbox mode or production-data isolation guarantee; nothing in the evidence ties machine creation to safely testing against non-production data.", "evidenceIds": [ "paperspace-docs-16", "paperspace-docs-25", "paperspace-docs-8" ] }, { "productId": "paperspace", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Paperspace has a REST API, CLI, and SDK, but nothing documents API versioning schemes or a deprecation policy anywhere in the docs or community sources.", "evidenceIds": [ "paperspace-docs-6", "paperspace-docs-23", "paperspace-docs-24" ] }, { "productId": "paperspace", "storyId": "auto-shutdown-spend-guards", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Paperspace documents an auto-shutdown feature that stops a machine after inactivity (from 1 hour to 1 week), directly addressing the 'forgotten instance' risk, and this is corroborated by official docs. However, there is no evidence of configurable spend/budget limits or billing caps, which is the other half of the story. missing for 10: documented spend-limit/budget-cap feature, independent hands-on confirmation that auto-shutdown reliably prevents runaway billing.", "evidenceIds": [ "paperspace-docs-5" ] }, { "productId": "paperspace", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Paperspace exposes an API, CLI (pspace), and Core JS SDK for programmatic resource management, which theoretically could be scripted for bulk actions, but no evidence documents any actual bulk/batch operation feature (e.g., multi-machine create/delete/update in one call, batch endpoints, or CLI loop/apply commands) across many items at once.", "evidenceIds": [ "paperspace-docs-6", "paperspace-docs-23", "paperspace-docs-24", "paperspace-probe-4", "paperspace-probe-rt-1" ] }, { "productId": "paperspace", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "paperspace", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Paperspace docs mention on-demand Workflows for ML pipelines and auto-shutdown after inactivity, but there is no evidence of a scheduler, cron-like trigger, or recurring/periodic job execution capability. Absence of evidence for this applicable capability yields none.", "evidenceIds": [ "paperspace-docs-9", "paperspace-docs-5" ] }, { "productId": "paperspace", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Paperspace is a GPU cloud/VM/notebook infrastructure product, not an automation/agent-workflow builder; there is no concept of 'automations' with version history, review, or rollback in its evidence (Machines, Notebooks, Deployments, Workflows are infra features, not user-authored automations with versioning/review workflows). This is a category mismatch rather than a missing feature.", "evidenceIds": [] }, { "productId": "paperspace", "storyId": "billing-usage-api", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any billing/usage export or cost-attribution API, report, or CLI command — the API/CLI docs cover compute resources (machines, notebooks, deployments) but nothing about programmatic billing or GPU spend breakdown by team/workload.", "evidenceIds": [] }, { "productId": "paperspace", "storyId": "custom-docker-images", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Paperspace lets users create custom machine templates (paperspace-docs-3, paperspace-docs-25) and run container images via Deployments (paperspace-docs-8), plus SSH root access to VMs for custom setups (paperspace-docs-12, paperspace-docs-19). Community reports (paperspace-comm-7) note friction getting reliable SSH/console access on Core VMs, which tempers confidence. Missing for 10: independent hands-on confirmation that custom Docker images run smoothly end-to-end, and clearer detail on template versioning/sharing.", "evidenceIds": [ "paperspace-docs-3", "paperspace-docs-8", "paperspace-docs-25", "paperspace-docs-12", "paperspace-docs-19", "paperspace-comm-7" ] }, { "productId": "paperspace", "storyId": "data-transfer-cloud-sync", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of S3-compatible endpoints, cloud-storage sync, or dedicated data-transfer tooling; only generic 'shared drives' (docs-4) and CLI/API for resource management are documented, and community reports (paperspace-comm-7) describe data transfer to Core VMs as slow and cumbersome, further undermining any efficiency claim. missing for 10: S3-compatible endpoint, documented sync/transfer tool, benchmarked transfer speeds, and any first-party guidance on moving data in/out.", "evidenceIds": [ "paperspace-docs-4", "paperspace-docs-16", "paperspace-comm-7" ] }, { "productId": "paperspace", "storyId": "exposed-ports-networking", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs confirm private networking between machines (shared drives across a private network in paperspace-docs-4) and an option to enable a 'private network' or 'public IP address' when creating a machine (paperspace-docs-25), plus SSH/root access to instances (paperspace-docs-12/19/21). However, there is no explicit documentation of exposing arbitrary application ports or firewall/port-forwarding configuration for serving apps beyond SSH; 'Deployments' (containers-as-a-service, paperspace-docs-8) hints at serving models but isn't tied to port exposure or private networking specifics. missing for 10: explicit port-exposure/firewall configuration docs, hands-on confirmation of connecting instances over private network for app traffic, and any independent corroboration of these networking features working as described.", "evidenceIds": [ "paperspace-docs-4", "paperspace-docs-25", "paperspace-docs-8", "paperspace-docs-12", "paperspace-docs-19" ] }, { "productId": "paperspace", "storyId": "gpu-availability-transparency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any real-time GPU availability dashboard, API endpoint, or CLI command showing stock/quota by type and region; docs only describe choosing machine type/region at creation time and needing approval for high-end GPUs, with no visibility into availability before provisioning.", "evidenceIds": [ "paperspace-docs-17", "paperspace-docs-25" ] }, { "productId": "paperspace", "storyId": "gpu-breadth-latest-hardware", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm access to current-gen H100 (behind an approval gate) and previous-gen A100 GPUs, plus NVLink support and 'largest GPU catalog' claims, supporting a range of GPU tiers and cost/performance tradeoffs (docs-11, docs-17, docs-20, docs-26). However, there's no explicit mention of H200/B200-class GPUs or a clear listing of older, cheaper GPU tiers (e.g., T4/P100/V100) to fully match the story's specificity. missing for 10: explicit H200/B200 listing, explicit older/cheaper GPU tier catalog, independent benchmarking/corroboration of availability.", "evidenceIds": [ "paperspace-docs-11", "paperspace-docs-17", "paperspace-docs-20", "paperspace-docs-26", "paperspace-docs-27" ] }, { "productId": "paperspace", "storyId": "instance-lifecycle-management", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Evidence confirms programmatic creation of machines via API/CLI/console and an auto-shutdown feature that stops machines after inactivity to control costs, and the pspace CLI clearly manages 'machine' resources. However, no direct documentation shows explicit start/stop/restart/terminate commands or billing-only-when-running semantics beyond auto-shutdown. missing for 10: explicit API/CLI start, stop, restart, terminate operations, confirmation that billing pauses immediately on stop, independent verification of lifecycle commands working.", "evidenceIds": [ "paperspace-docs-1", "paperspace-docs-5", "paperspace-docs-6", "paperspace-docs-23", "paperspace-docs-24", "paperspace-probe-4", "paperspace-probe-rt-1" ] }, { "productId": "paperspace", "storyId": "jupyter-ide-access", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Paperspace ships a native web-based Jupyter Notebooks product and documents SSH access with 'bring your own key, full root access' to Machines, which in principle supports VS Code Remote-SSH style connections, but there is no documentation of a genuine one-step IDE handshake (e.g., no VS Code/Cursor-specific integration, devcontainer, or remote-SSH config generator). A first-hand community report directly contradicts the 'easy SSH' claim, saying getting SSH access to Paperspace Core VMs was 'an uphill battle' with a 'confusing GUI' instead. missing for 10: explicit VS Code/Cursor one-click connect docs, evidence of an SSH config/remote extension workflow, and independent confirmation that SSH setup is actually fast/frictionless.", "evidenceIds": [ "paperspace-docs-7", "paperspace-docs-2", "paperspace-docs-12", "paperspace-docs-19", "paperspace-comm-7" ] }, { "productId": "paperspace", "storyId": "keyless-catalog-pricing-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While Paperspace documents machine types, GPU options, and an API/CLI for creating machines, there is no evidence of a documented endpoint or CLI command that returns live pricing or availability data for the GPU catalog before provisioning—only marketing claims like 'save up to 70%' and 'largest GPU catalog' with no queryable pricing API surfaced.", "evidenceIds": [ "paperspace-docs-11", "paperspace-docs-17", "paperspace-docs-23", "paperspace-probe-3", "paperspace-probe-rt-1" ] }, { "productId": "paperspace", "storyId": "managed-slurm-kubernetes", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "paperspace", "storyId": "multi-node-clusters", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Paperspace provisions single machines with up to 8 GPUs and NVLink for intra-node scaling, but nothing describes provisioning a multi-node cluster with fast cross-node interconnect (e.g., InfiniBand/RDMA) for distributed training, nor any self-service multi-node cluster workflow.", "evidenceIds": [ "paperspace-docs-11", "paperspace-docs-20", "paperspace-docs-27", "paperspace-docs-17" ] }, { "productId": "paperspace", "storyId": "on-demand-gpu-provisioning", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Docs and marketing show provisioning via console/API/CLI, SSH root access, and ML-ready templates enabling fast startup (paperspace-docs-1,6,10,12,13,24), and a runtime probe confirms the CLI installs and lists machine-management commands (paperspace-probe-rt-1). However, a hands-on community report directly contradicts the 'connect and run in minutes' claim: 'Getting SSH access is an uphill battle; instead there's a pointless virtual console... very slow internet, takes forever to copy datasets in' (paperspace-comm-7), and another user calls the notebook experience 'really bad' (paperspace-comm-6). Missing for 10: independent verified timing benchmarks of provisioning-to-running-code, resolution of the SSH-access friction reported by users, and more recent hands-on corroboration.", "evidenceIds": [ "paperspace-docs-1", "paperspace-docs-6", "paperspace-docs-10", "paperspace-docs-12", "paperspace-docs-13", "paperspace-docs-24", "paperspace-probe-rt-1", "paperspace-comm-7", "paperspace-comm-6" ] }, { "productId": "paperspace", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Paperspace ships both an API and a first-party pspace CLI that support machine creation, custom templates, autoscaling groups, and deployments (paperspace-docs-1, paperspace-docs-6, paperspace-docs-23/24, paperspace-probe-4, paperspace-probe-rt-1), giving strong API/CLI parity for core resource lifecycle tasks. However, some UI-only features like SSH connection setup are described as done via console/desktop app (paperspace-docs-2), no public OpenAPI/Swagger spec was found (paperspace-probe-3), and there's no explicit evidence that shared drives, auto-shutdown, or notebook management are fully API-controllable. Missing for 10: OpenAPI/spec confirmation of full endpoint coverage, explicit API parity for shared drives/auto-shutdown/notebooks, and independent hands-on confirmation that API-driven workflows match UI capability without gaps.", "evidenceIds": [ "paperspace-docs-1", "paperspace-docs-6", "paperspace-docs-23", "paperspace-docs-24", "paperspace-probe-4", "paperspace-probe-rt-1", "paperspace-docs-2", "paperspace-probe-3" ] }, { "productId": "paperspace", "storyId": "openness-full-export", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Paperspace machines use standard SSH/root access and shared drives, and API/CLI access lets users programmatically pull down VM configs, custom templates, and data via SSH/SCP, which provides some data portability. However there is no documented one-click 'export all data' or bulk data-export feature, no explicit open-format guarantee for notebooks/deployments/workflows metadata, and no evidence of a full account data export or migration tool. missing for 10: dedicated bulk data export/download feature, explicit open-format export guarantees for notebooks/deployments/workflows, documented account-level data portability or migration tooling, independent confirmation of successful full data export.", "evidenceIds": [ "paperspace-docs-2", "paperspace-docs-4", "paperspace-docs-6", "paperspace-docs-23", "paperspace-docs-24", "paperspace-probe-4" ] }, { "productId": "paperspace", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Paperspace is a closed, commercial GPU cloud platform; there is no evidence of its source code being available under an open license, and this is not a fair expectation for this category of product (proprietary SaaS/IaaS offering).", "evidenceIds": [] }, { "productId": "paperspace", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Paperspace is a managed cloud GPU/ML SaaS platform (now part of DigitalOcean); it is not distributed as software that a user could self-host, and no evidence pack item mentions any self-hostable core product or on-prem option — this is a category error for this product type.", "evidenceIds": [] }, { "productId": "paperspace", "storyId": "per-second-billing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack states per-second or per-minute billing granularity; the only granularity mentioned is an 'hourly rate' (paperspace-comm-8) and a flat monthly subscription tier with fixed hours (paperspace-comm-5), neither of which confirms sub-minute billing precision while running.", "evidenceIds": [ "paperspace-comm-8", "paperspace-comm-5" ] }, { "productId": "paperspace", "storyId": "persistent-network-storage", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs describe persistent storage attached to machines (paperspace-docs-16) and separate 'shared drives' accessible from multiple machines in a private network (paperspace-docs-4), plus persistent notebook storage (paperspace-docs-7), which together imply data can outlive a single GPU instance. However, no evidence explicitly confirms shared drives survive full instance deletion/teardown, nor documents attach/detach workflow, pricing, or size limits, and one community comment complains about slow data transfer into Paperspace VMs (paperspace-comm-7). Missing for 10: explicit lifecycle documentation proving storage persists after instance termination, attach/detach mechanics, and independent hands-on confirmation of durability across teardown.", "evidenceIds": [ "paperspace-docs-4", "paperspace-docs-16", "paperspace-docs-7", "paperspace-comm-7" ] }, { "productId": "paperspace", "storyId": "prebuilt-ml-templates", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Paperspace docs confirm a pre-built \"ML in a Box\" template with major ML frameworks and CUDA drivers preinstalled, plus custom-template creation for reuse, supporting the general concept of launching from ready-made ML environments. However, no evidence names specific frameworks like PyTorch, vLLM, or ComfyUI templates, and community comments describe friction (slow setup, SDK upgrade walls, GUI confusion) that undercuts a seamless 'launch instantly' experience. missing for 10: explicit PyTorch/vLLM/ComfyUI-named templates, hands-on confirmation of frictionless template launch, independent corroboration beyond marketing docs.", "evidenceIds": [ "paperspace-docs-13", "paperspace-docs-3", "paperspace-docs-25", "paperspace-comm-2", "paperspace-comm-7" ] }, { "productId": "paperspace", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs confirm you select a 'region' when creating a machine ([paperspace-docs-25]), showing basic region choice for compute placement, but there is no evidence of dedicated data residency controls, storage region selection, compliance certifications (e.g., GDPR, SOC2 data residency), or documentation on where notebooks/deployments data is stored. missing for 10: explicit data residency/compliance documentation, storage-specific region selection, list of available regions, independent confirmation of enforcement.", "evidenceIds": [ "paperspace-docs-25" ] }, { "productId": "paperspace", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Paperspace is a GPU cloud/compute infrastructure product (VMs, notebooks, deployments) rather than an AI model provider or data-processing service that trains models on user data; a data-training opt-out policy is not a relevant capability axis for this kind of product.", "evidenceIds": [] }, { "productId": "paperspace", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack items address data retention policies, deletion controls, or privacy/data lifecycle management for Paperspace resources like notebooks, storage, or machine data; the docs cover creation, connection, and compute features but nothing on retention/deletion control.", "evidenceIds": [] }, { "productId": "paperspace", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items mention telemetry, usage tracking, analytics opt-out, or privacy controls for Paperspace; the docs cover machine creation, CLI, and pricing but never address data collection or opt-out mechanisms.", "evidenceIds": [] }, { "productId": "paperspace", "storyId": "quota-limit-transparency", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs mention that high-end GPU machines (e.g., H100) require an approval request, implying some quota/limit gating exists, but there is no dedicated documentation of quota tiers, numeric limits, or a defined escalation/support process for raising them beyond that single mention. Missing for 10: a documented quotas/limits reference page, explicit instance-count or resource caps, and a clear support ticket/process for requesting increases.", "evidenceIds": [ "paperspace-docs-17" ] }, { "productId": "paperspace", "storyId": "reserved-committed-discounts", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of reserved instances, committed-use contracts, or sustained-use discounts; only pay-as-you-go pricing claims like 'save up to 70%' and 'cancel anytime' are documented, which implies on-demand rather than committed pricing.", "evidenceIds": [ "paperspace-docs-14", "paperspace-docs-26" ] }, { "productId": "paperspace", "storyId": "security-compliance-posture", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items mention SOC 2 compliance, certifications, data handling policies, or datacenter tier information; the evidence is entirely product feature docs and community complaints about performance/support, none of which address compliance posture. missing for 10: SOC 2/ISO certifications, data handling/privacy documentation, datacenter tier specs, any compliance attestations.", "evidenceIds": [] }, { "productId": "paperspace", "storyId": "serverless-gpu-endpoints", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Paperspace's evidence describes VM-based 'Machines' with auto-shutdown after inactivity (docs-5) and container 'Deployments'/autoscaling-groups (docs-8, probe-rt-1), but nothing confirms true serverless GPU workers that scale to zero per-request and back up automatically without idle billing — the model described is always-provisioned instances that stop, not ephemeral serverless invocation.", "evidenceIds": [ "paperspace-docs-5", "paperspace-docs-8", "paperspace-probe-rt-1" ] }, { "productId": "paperspace", "storyId": "spot-interruptible-pricing", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack mentions spot, preemptible, or interruptible instances, discount pricing tiers, or preemption semantics; only standard on-demand GPU rentals and general 'save up to 70%' marketing are documented.", "evidenceIds": [] }, { "productId": "paperspace", "storyId": "ssh-root-access", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Paperspace's marketing explicitly promises 'bring your own SSH key' with full root access to the VM, and DigitalOcean docs describe SSH-based connection to machines, but a hands-on community report states that SSH access on Paperspace's Core VMs was 'an uphill battle' with a 'pointless virtual console and confusing GUI' instead of easy SSH, contradicting the frictionless claim. missing for 10: independent confirmation that SSH access works smoothly as advertised, and no rebuttal to the community complaint about SSH being hard to obtain.", "evidenceIds": [ "paperspace-docs-12", "paperspace-docs-19", "paperspace-docs-21", "paperspace-docs-2", "paperspace-comm-7" ] }, { "productId": "paperspace", "storyId": "team-access-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence confirms API keys exist for programmatic access (paperspace-docs-6, paperspace-docs-23) but there is no mention of team member management, role assignment, scoped/permissioned API keys, or spend-control governance features anywhere in the pack.", "evidenceIds": [ "paperspace-docs-6", "paperspace-docs-23" ] }, { "productId": "paperspace", "storyId": "transparent-gpu-pricing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains marketing claims like 'save up to 70% on compute costs' but no actual published per-GPU-hour pricing table or pricing page content is shown; no citation demonstrates a public price list for each GPU type.", "evidenceIds": [ "paperspace-docs-14", "paperspace-docs-18", "paperspace-docs-22" ] }, { "productId": "runpod", "storyId": "agent-provisions-gpu", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Runpod documents and runtime-probes confirm a REST API (live, API-key-gated) for provisioning/monitoring/terminating Pods, a CLI (runpodctl, verified running keylessly with pod/serverless/template management), and official MCP servers (one OAuth-gated for actual resource management, one no-auth for docs) enabling agent-driven end-to-end lifecycle without console use. Community evidence corroborates real-world CLI/template usage but also notes a locked-down execution environment and occasional GPU availability issues. missing for 10: independent hands-on verification of a full agent-driven create→monitor→teardown cycle via MCP specifically (only docs MCP fully probed keylessly; API MCP only auth-checked), and no third-party report confirming reliability of automated teardown at scale.", "evidenceIds": [ "runpod-docs-8", "runpod-docs-26", "runpod-docs-29", "runpod-docs-46", "runpod-docs-32", "runpod-docs-36", "runpod-probe-3", "runpod-probe-4", "runpod-probe-rt-1", "runpod-probe-rt-2", "runpod-probe-rt-3", "runpod-probe-rt-4", "runpod-comm-2", "runpod-comm-4" ] }, { "productId": "runpod", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Runpod hosts a live llms.txt at docs.runpod.io/llms.txt (HTTP 200, confirmed via probe) plus agent-oriented docs like MCP server guides and an agent-skills plugin, and the docs MCP server was verified to complete a full keyless handshake, confirming agent-reachability of documentation. Missing for 10: no independent/community corroboration of llms.txt usage by real agents outside the vendor probe.", "evidenceIds": [ "runpod-probe-1", "runpod-docs-3", "runpod-probe-rt-3", "runpod-docs-12" ] }, { "productId": "runpod", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Runpod is a GPU cloud compute infrastructure platform (Pods, Serverless, storage, CLI, MCP servers for provisioning); it has no data analytics/BI layer or AI-generated insights/suggestions feature over a user's own data. This story fits an analytics or SaaS product with embedded AI features, not a raw compute infrastructure provider — wrong axis for this product category.", "evidenceIds": [] }, { "productId": "runpod", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Runpod's Serverless endpoints scale workers autonomously (scale-to-zero, pay-per-second) and the REST API/CLI/MCP servers explicitly target integration into 'applications, workflows, and automation systems,' letting a user wire up background GPU jobs. However there's no dedicated scheduler, trigger/cron mechanism, or first-party 'automation' product documented—users would need to build the automation logic themselves on top of the API/Serverless primitives. Missing for 10: a built-in scheduling/trigger system, evidence of persistent background agent workflows (not just on-demand endpoints), and any hands-on report of an autonomous automation actually running unattended.", "evidenceIds": [ "runpod-docs-8", "runpod-docs-20", "runpod-docs-25", "runpod-docs-49", "runpod-docs-12", "runpod-docs-44", "runpod-probe-rt-1" ] }, { "productId": "runpod", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Runpod's evidence describes MCP servers and 'agent skills' that let EXTERNAL coding agents (Claude, Cursor, etc.) control Runpod resources — the reverse of an assistant built into Runpod's own product for users to delegate tasks to. There is no mention of a native chat/assistant surface inside the Runpod console or CLI itself.", "evidenceIds": [ "runpod-docs-3", "runpod-docs-12", "runpod-docs-44", "runpod-docs-36", "runpod-probe-3" ] }, { "productId": "runpod", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Runpod ships both a scriptable open-source CLI (runpodctl) and a REST API explicitly positioned for 'integrating GPU infrastructure into your applications, workflows, and automation systems,' and runtime probes confirm both work keylessly/API-key-gated exactly as documented (CLI version check succeeded, REST API openapi.json served, /v1/pods correctly 401s without a key) — i.e., headless automation is real and verified, not just a claim. Missing for 10: an explicit CI pipeline example/tutorial (e.g., GitHub Actions) and independent third-party confirmation of CI usage beyond the vendor's own runtime probe.", "evidenceIds": [ "runpod-docs-5", "runpod-docs-8", "runpod-docs-16", "runpod-docs-26", "runpod-probe-rt-1", "runpod-probe-rt-4", "runpod-docs-46" ] }, { "productId": "runpod", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Runpod is a GPU cloud/compute platform, not an AI agent or assistant with its own tool-use runtime — the evidence shows Runpod publishes MCP servers (the server role, docs-3, docs-36, probe-3) and integrates with external coding agents via skills, but there's no first-party agent/chat runtime into which a user would 'plug' external MCP servers for Runpod itself to consume their tools. This client-side capability is a category error for an infrastructure platform rather than an applicable-but-unmet axis.", "evidenceIds": [] }, { "productId": "runpod", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Runpod documents two official MCP servers (API MCP and docs MCP) that connect AI tools/coding agents to Runpod, and runtime probes confirm both are live: the docs MCP completes a full keyless handshake, and the API MCP correctly enforces the documented OAuth flow. This is Runpod acting as a service provider exposing an MCP server, not an agent client, so the axis clearly applies and is delivered with first-party docs plus independent runtime verification. Missing for 10: no third-party/community usage reports of agents actually connecting via these MCP servers in practice.", "evidenceIds": [ "runpod-docs-3", "runpod-docs-36", "runpod-probe-3", "runpod-probe-rt-2", "runpod-probe-rt-3" ] }, { "productId": "runpod", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Runpod ships an agent-skills plugin and MCP servers explicitly designed so users can ask their AI agent in natural language to create Pods, deploy Serverless endpoints, transfer files, or deploy code, backed by a live hosted MCP server and API confirmed via runtime probes. This is a first-party, agent-native workflow rather than just CLI/API access repurposed for agents. Missing for 10: independent (non-vendor) hands-on confirmation that the natural-language agent-skills workflow works end-to-end as described.", "evidenceIds": [ "runpod-docs-12", "runpod-docs-44", "runpod-docs-3", "runpod-docs-36", "runpod-probe-3", "runpod-probe-rt-2", "runpod-probe-rt-3" ] }, { "productId": "runpod", "storyId": "agentic-official-cli", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Runpod ships an official open-source CLI (runpodctl) documented for managing pods, serverless endpoints, templates, volumes, and models, and this was independently verified working at runtime (installed via brew, ran keylessly, version and help output confirmed). This is a strong, corroborated case of an official CLI supporting agentic/AI-native workflows (e.g., 'Run Python functions on remote GPUs directly from your local terminal'). Missing for 10: no deeper hands-on exploration of advanced CLI subcommands or agent-specific CLI usage beyond the smoke test.", "evidenceIds": [ "runpod-docs-5", "runpod-docs-16", "runpod-docs-32", "runpod-probe-4", "runpod-probe-rt-1", "runpod-docs-15" ] }, { "productId": "runpod", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Runpod documents and exposes a public REST API (v1) covering pods, endpoints, templates, volumes, and registries, with a live OpenAPI spec confirmed at runtime and proper API-key gating, plus CLI, MCP servers, and S3-compatible storage API as complementary programmatic surfaces. Runtime probes independently corroborate the documented REST API and MCP endpoints are actually live and functioning as described. Missing for 10: no independent third-party review of API completeness/versioning stability beyond docs and probes.", "evidenceIds": [ "runpod-docs-8", "runpod-docs-26", "runpod-docs-36", "runpod-probe-rt-4", "runpod-probe-2", "runpod-docs-19" ] }, { "productId": "runpod", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Runpod documents API-key and OAuth (\"Sign in with Runpod\") authentication for its REST API and MCP servers, but no evidence describes scoped, role-based, or least-privilege API key creation (e.g., read-only or resource-limited keys) that a user could issue specifically for an agent.", "evidenceIds": [ "runpod-docs-8", "runpod-docs-36", "runpod-probe-rt-2", "runpod-probe-rt-4" ] }, { "productId": "runpod", "storyId": "agentic-sdks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Runpod documents a REST API (runpod-docs-8/26, runpod-probe-rt-4), an open-source CLI (runpod-docs-5/16/32), and a Python-function-on-remote-GPU capability (runpod-docs-15) that implies an SDK-like interface, giving AI-native builders programmatic access. However the evidence pack never explicitly names or links a dedicated 'official SDK' page (e.g., a Python/JS client library reference) distinct from the CLI/REST API, so SDK-specific documentation depth is unverified. Missing for 10: dedicated official SDK docs/reference pages, multi-language SDK coverage, and independent developer confirmation of SDK usage.", "evidenceIds": [ "runpod-docs-8", "runpod-docs-26", "runpod-docs-15", "runpod-docs-5", "runpod-docs-16", "runpod-probe-rt-4" ] }, { "productId": "runpod", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of webhook subscription/event notification support anywhere in the docs pack — Runpod offers REST API, CLI, MCP servers, and SSH access, but nothing about outbound event webhooks for job/pod status changes.", "evidenceIds": [] }, { "productId": "runpod", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Runpod documents a full REST API reference (runpod-docs-8, runpod-docs-26) and a probe confirms the OpenAPI spec is live and served at rest.runpod.io/v1/openapi.json (runpod-probe-rt-4), which is the backbone for an interactive reference. However, there is no direct evidence of a 'try it now' interactive console or embedded runnable code examples within the docs UI itself—only the raw spec and static markdown pages are confirmed. Missing for 10: evidence of an interactive Swagger/Redoc-style try-it-out console, runnable code snippets embedded in the docs, and confirmation that the OpenAPI spec is surfaced in the actual docs site rather than only at a separate API host.", "evidenceIds": [ "runpod-docs-8", "runpod-docs-26", "runpod-probe-rt-4", "runpod-probe-2" ] }, { "productId": "runpod", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Runpod documents a REST API v1 and a probe confirms the OpenAPI spec is actually served keylessly at https://rest.runpod.io/v1/openapi.json, matching the documented API. Missing for 10: no first-party download link/documentation explicitly advertising the OpenAPI spec location on the docs site itself (probe found it via the raw REST host, not linked from docs directly).", "evidenceIds": [ "runpod-docs-8", "runpod-docs-26", "runpod-probe-rt-4", "runpod-probe-2" ] }, { "productId": "runpod", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Runpod's docs describe deploying Pods, Serverless endpoints, and Instant Clusters, but nothing in the evidence describes a dedicated sandbox/staging mode that isolates test workloads from 'production' data or endpoints — no environment-separation, test-vs-prod flagging, or data-isolation guarantee is documented. missing for 10: any documented sandbox/staging environment concept, production-data isolation guarantees, or environment-promotion workflow.", "evidenceIds": [] }, { "productId": "runpod", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Runpod documents a versioned REST API (v1) with a live OpenAPI spec, but no evidence in the pack describes any deprecation policy, versioning changelog, or migration/support-window commitments for API versions — only a `/runpod:migrate rest` agent command is mentioned, which is a migration tool, not a policy.", "evidenceIds": [ "runpod-docs-8", "runpod-docs-26", "runpod-probe-rt-4", "runpod-docs-9" ] }, { "productId": "runpod", "storyId": "auto-shutdown-spend-guards", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack shows Runpod billing is pay-per-second and includes an 'auto-pay' feature that reloads balance when low (the opposite of a spend cap), but there is no mention anywhere of auto-shutdown timers, idle-timeout limits, max-runtime settings, or spend/budget caps that would stop a forgotten Pod from accumulating charges. This is a fair and plausible axis for a GPU cloud provider, so absence of evidence means 'none' rather than 'na'.", "evidenceIds": [ "runpod-docs-13", "runpod-docs-50" ] }, { "productId": "runpod", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Runpod's docs describe CLI/API/MCP access for managing individual Pods, endpoints, templates, and volumes, and a general-purpose REST API that could in principle be scripted for bulk actions, but there is no documented bulk/batch operation feature (e.g., batch-create/delete many pods or jobs in one call) or evidence of such usage in practice.", "evidenceIds": [ "runpod-docs-16", "runpod-docs-32", "runpod-docs-8", "runpod-docs-46" ] }, { "productId": "runpod", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only a single fixed automatic behavior (auto-pay reloading balance below a threshold) and autoscaling to zero, neither of which constitutes a user-definable rules/event-trigger system. No documentation shows webhooks, event subscriptions, or a general 'if X then Y' automation engine for Runpod resources.", "evidenceIds": [ "runpod-docs-13", "runpod-docs-25", "runpod-docs-38" ] }, { "productId": "runpod", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents Pods, Serverless endpoints, Instant Clusters, CLI, API, and MCP servers, but nowhere describes a scheduler, cron-like recurring job feature, or workflow orchestration capability for automatically re-running jobs on a schedule. Users could manually trigger jobs via API/CLI, but no evidence shows a built-in recurring/scheduled job mechanism.", "evidenceIds": [] }, { "productId": "runpod", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Runpod is a GPU compute/infrastructure platform (Pods, Serverless, Instant Clusters) — it has no concept of 'automations' with versioning/review/rollback workflows like an automation-builder or agent-orchestration product would. This story targets version control/rollback of automation logic, which is a category mismatch for a compute-provisioning platform.", "evidenceIds": [] }, { "productId": "runpod", "storyId": "billing-usage-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers Pods/Serverless pricing, auto-pay, and a general REST API for provisioning compute resources, but nothing documents a billing/usage endpoint, cost-export, or per-team/per-workload spend attribution mechanism. The CLI's 'view account information' (runpod-docs-32) is the closest hint but is not shown to expose granular usage/billing data programmatically.", "evidenceIds": [ "runpod-docs-8", "runpod-docs-13", "runpod-docs-20", "runpod-docs-32", "runpod-probe-rt-4" ] }, { "productId": "runpod", "storyId": "custom-docker-images", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Runpod Pods explicitly support pulling custom containers from any compatible registry (Docker Hub, GHCR, ECR) or selecting official templates, with full SSH/JupyterLab/VS Code access to the running environment, and CLI/API/console management of templates and pods. missing for 10: independent hands-on verification of deploying a fully custom Docker image end-to-end (only docs/probe evidence, plus community notes that the execution environment is somewhat locked down for advanced networking use cases).", "evidenceIds": [ "runpod-docs-21", "runpod-docs-42", "runpod-docs-14", "runpod-docs-45", "runpod-docs-10", "runpod-docs-29", "runpod-docs-46", "runpod-comm-2", "runpod-comm-3" ] }, { "productId": "runpod", "storyId": "data-transfer-cloud-sync", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Runpod documents an S3-compatible API for direct file management on network volumes, plus CLI (runpodctl) file transfer between local system and Runpod, SSH access, and no ingress/egress fees for Pods — covering multiple documented transfer paths. Missing for 10: independent hands-on verification of the S3 API's throughput/compatibility and broader cloud-storage sync (e.g., rclone/GDrive) integration beyond docs claims.", "evidenceIds": [ "runpod-docs-7", "runpod-docs-19", "runpod-docs-32", "runpod-docs-16", "runpod-docs-50", "runpod-docs-18" ] }, { "productId": "runpod", "storyId": "exposed-ports-networking", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Runpod docs describe exposing ports via 'web proxy for exposed web services' on Pods (runpod-docs-14/45) alongside SSH/JupyterLab/VS Code access, and Instant Clusters provide 'high-performance networking for distributed workloads' enabling multi-node/private networking between instances (runpod-docs-4/17/37/48). Missing for 10: independent hands-on verification of port-exposure and inter-instance private networking, and more detail on how private networking is configured/secured beyond the Instant Clusters feature blurb.", "evidenceIds": [ "runpod-docs-14", "runpod-docs-45", "runpod-docs-4", "runpod-docs-17", "runpod-docs-37", "runpod-docs-48" ] }, { "productId": "runpod", "storyId": "gpu-availability-transparency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item describes a real-time GPU-availability-by-region/type dashboard or API that lets an ML engineer check stock before provisioning. The only related community evidence (runpod-comm-4) describes the opposite: users only learn of 'low availability' via failed/incomplete deploys, i.e. discovering stockouts by failure rather than checking ahead of time.", "evidenceIds": [ "runpod-comm-4", "runpod-docs-33", "runpod-docs-51" ] }, { "productId": "runpod", "storyId": "gpu-breadth-latest-hardware", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence confirms Runpod offers current-gen H100 GPUs and previous-gen (4090) GPUs via community reports, and docs claim 'thousands of GPUs across 30+ regions,' but there is no docs page or listing enumerating the specific GPU classes (H100/H200/B200) or older-gen tiers, and one report notes a period of 4090 unavailability. missing for 10: an explicit GPU catalog/pricing page listing H100/H200/B200 vs older-gen options, docs confirming H200/B200 support, and evidence of consistent availability across tiers.", "evidenceIds": [ "runpod-comm-1", "runpod-comm-4", "runpod-docs-33", "runpod-docs-51" ] }, { "productId": "runpod", "storyId": "instance-lifecycle-management", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Runpod docs and CLI explicitly support create/start/stop/terminate pods via console, CLI (`runpodctl pod create`), and REST API, with per-second billing so users only pay for running compute, plus Serverless auto-scale-to-zero for idle cost avoidance. Runtime probes confirm the REST API and CLI are live and functional as documented. Missing for 10: independent third-party confirmation of full lifecycle (start/stop/restart) beyond docs, and no hands-on community report specifically testing restart/stop behavior.", "evidenceIds": [ "runpod-docs-46", "runpod-docs-29", "runpod-docs-8", "runpod-docs-50", "runpod-docs-20", "runpod-docs-25", "runpod-probe-rt-1", "runpod-probe-rt-4", "runpod-docs-16" ] }, { "productId": "runpod", "storyId": "jupyter-ide-access", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly state Pods can be connected via SSH, JupyterLab, or VS Code/Cursor for local IDE integration, and templates pre-configure PyTorch/JupyterLab so everything is 'ready instantly' without manual setup. Community evidence corroborates ease of use (template-based one-click deploys) though notes the execution environment is 'locked down,' a minor friction point for advanced tooling. Missing for 10: independent hands-on confirmation of the actual one-step VS Code/Cursor connect experience beyond vendor docs.", "evidenceIds": [ "runpod-docs-14", "runpod-docs-45", "runpod-docs-41", "runpod-docs-10", "runpod-docs-22", "runpod-comm-2", "runpod-comm-5" ] }, { "productId": "runpod", "storyId": "keyless-catalog-pricing-api", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Runpod's REST API is documented and its OpenAPI spec is publicly fetchable without auth, and pricing pages describe per-second GPU pricing across regions/plans, so an agent can discover pricing structure before spending. However, the evidence never shows a specific GPU catalog/availability endpoint that returns live pricing/availability, and the actual data-returning REST endpoints (e.g. /v1/pods) require an API key (401 without one), so live catalog querying isn't fully public. Missing for 10: a documented GPU types/availability endpoint, confirmation that pricing/availability data is queryable without an account/API key, and independent verification of live pricing accuracy via the API.", "evidenceIds": [ "runpod-docs-8", "runpod-docs-26", "runpod-docs-33", "runpod-docs-51", "runpod-probe-rt-4" ] }, { "productId": "runpod", "storyId": "managed-slurm-kubernetes", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Runpod's Instant Clusters provide fully managed multi-node compute with high-performance networking for distributed training, but no evidence indicates these clusters are backed by Slurm or Kubernetes scheduling — the docs describe raw multi-node GPU networking, not a managed job scheduler. This is a fair axis for a clusters/scale-focused GPU platform, but the evidence pack never mentions Slurm or K8s support, workload orchestration primitives, or job queueing semantics.", "evidenceIds": [ "runpod-docs-4", "runpod-docs-17", "runpod-docs-37", "runpod-docs-48" ] }, { "productId": "runpod", "storyId": "multi-node-clusters", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Runpod documents Instant Clusters as fully managed multi-node compute with high-performance networking for distributed training, deployable without managing infrastructure/networking/cluster configuration, and explicitly targets training models too large for one GPU or accelerating training across multiple nodes — all self-service via console/API/CLI without a sales process. Missing for 10: no independent/hands-on benchmark of actual interconnect performance (e.g., InfiniBand/NVLink specifics or bandwidth numbers) and no community corroboration of successfully running a multi-node cluster end-to-end.", "evidenceIds": [ "runpod-docs-4", "runpod-docs-17", "runpod-docs-37", "runpod-docs-48", "runpod-docs-33" ] }, { "productId": "runpod", "storyId": "on-demand-gpu-provisioning", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Runpod's docs and live runtime probes confirm the full provisioning path: create a Pod via console, CLI (`runpodctl pod create`), or REST API (verified live and key-gated at rest.runpod.io/v1/pods), select a pre-built template (PyTorch/JupyterLab ready instantly), and immediately access it via SSH, JupyterLab, or VS Code — matching the 'minutes to running code' story. Community posts corroborate ease of use ('few clicks' vs Azure, H100 access easier than Colab) though also note occasional GPU availability/boot issues that can delay provisioning. Missing for 10: independent hands-on timing benchmark of the full provision-to-code-execution flow, and resolution of the community-reported low-availability/boot failures.", "evidenceIds": [ "runpod-docs-1", "runpod-docs-8", "runpod-docs-14", "runpod-docs-22", "runpod-docs-29", "runpod-docs-46", "runpod-probe-rt-1", "runpod-probe-rt-4", "runpod-comm-1", "runpod-comm-5", "runpod-comm-4" ] }, { "productId": "runpod", "storyId": "openness-api-parity", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Runpod's REST API is documented as providing programmatic access to 'all Runpod compute resources' (docs-8, docs-26) and is confirmed live via runtime probe with OpenAPI spec (runpod-probe-rt-4); the CLI/runpodctl and MCP servers also expose pod, serverless, template, network-volume, and registry management matching UI capabilities (docs-16, docs-32, docs-36, runpod-probe-rt-1/2/3). This gives strong evidence of broad UI/API parity for compute-resource management, but there is no explicit confirmation that account/billing settings (e.g., auto-pay, org/team management) or Secure Cloud vetting features are exposed via API/CLI, and community notes describe the execution environment as 'very locked down' for certain use cases (runpod-comm-2, runpod-comm-3). Missing for 10: explicit API coverage of billing/account/org settings, and independent confirmation that literally every UI action (not just compute-resource ones) has an API equivalent.", "evidenceIds": [ "runpod-docs-8", "runpod-docs-26", "runpod-docs-16", "runpod-docs-32", "runpod-docs-36", "runpod-probe-rt-1", "runpod-probe-rt-2", "runpod-probe-rt-3", "runpod-probe-rt-4", "runpod-comm-2", "runpod-comm-3" ] }, { "productId": "runpod", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Runpod provides an S3-protocol-compatible API and CLI for transferring files off network volumes, and a REST API for programmatic access to resources — all standard/open interfaces that let a user pull data out without vendor lock-in. However, there is no explicit documentation of a full account data export, model/metadata export, or an account-closure/data-deletion workflow tying it all together into a genuine 'export everything and leave' capability. Missing for 10: comprehensive account/data export tooling, explicit data portability/GDPR-style export documentation, account closure and full data takeout confirmation.", "evidenceIds": [ "runpod-docs-19", "runpod-docs-7", "runpod-docs-32", "runpod-docs-26", "runpod-docs-18" ] }, { "productId": "runpod", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Runpod is a closed-source GPU cloud platform; only its CLI (runpodctl) is described as open source, but no evidence indicates the core Runpod platform/service source is available under an open license. The story asks about reading the product's source under an open license, which applies to any product but here evidence shows only a peripheral CLI tool is open, not the product itself.", "evidenceIds": [ "runpod-docs-5", "runpod-docs-16" ] }, { "productId": "runpod", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Runpod's core product is a hosted GPU cloud/marketplace; while the CLI (runpodctl) is open source, there is no evidence of a self-hostable version of the actual compute-orchestration platform, control plane, or marketplace that a user could run on their own infrastructure. The evidence pack only shows open-source client tooling (CLI, MCP client integration) and hosted APIs/services, not a self-hostable core product.", "evidenceIds": [ "runpod-docs-5", "runpod-docs-16", "runpod-probe-4" ] }, { "productId": "runpod", "storyId": "per-second-billing", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Runpod docs explicitly state Pods are billed by the second with no ingress/egress fees, and Serverless is pay-per-second with billing running from worker start until fully stopped, rounded to the nearest second — directly matching the story of per-second billing only while running. Multiple docs corroborate scale-to-zero behavior for flex workers, meaning no charges when idle. Missing for 10: independent third-party billing audit or user account confirming exact second-level charges in practice.", "evidenceIds": [ "runpod-docs-50", "runpod-docs-20", "runpod-docs-49", "runpod-docs-25", "runpod-docs-38" ] }, { "productId": "runpod", "storyId": "persistent-network-storage", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Runpod's docs explicitly describe network volumes as persistent storage that exists independently of compute and is retained on termination or scale-to-zero, shareable across machines/products, plus an S3-compatible API to manage files without launching a Pod. This directly matches the story of datasets/checkpoints outliving GPU rentals. Missing for 10: independent hands-on confirmation of durability across teardown and details on volume size/performance limits or region constraints.", "evidenceIds": [ "runpod-docs-18", "runpod-docs-19", "runpod-docs-40", "runpod-docs-7" ] }, { "productId": "runpod", "storyId": "prebuilt-ml-templates", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly describe selecting official PyTorch/CUDA templates to skip manual environment setup, and community evidence corroborates one-click template usage (TheBloke templates in a few clicks) as a differentiator vs cloud giants like Azure. Coverage of specific named templates like vLLM/ComfyUI isn't directly evidenced, though template overview docs generalize the mechanism. missing for 10: explicit named vLLM/ComfyUI template listing, independent hands-on confirmation of template launch speed/completeness.", "evidenceIds": [ "runpod-docs-10", "runpod-docs-22", "runpod-docs-34", "runpod-docs-39", "runpod-docs-47", "runpod-comm-5" ] }, { "productId": "runpod", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Runpod advertises GPUs across '30+ regions' and distinguishes Secure Cloud (T3/T4 data centers) vs Community Cloud, implying some ability to pick a compute location, but there is no explicit documentation of selecting a specific region/data-residency guarantee for storage or network volumes. missing for 10: explicit region-selection UI/API for deployments, documented data-residency/compliance controls for stored data, independent confirmation that chosen region persists for storage.", "evidenceIds": [ "runpod-docs-33", "runpod-docs-51", "runpod-docs-28", "runpod-docs-27" ] }, { "productId": "runpod", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Runpod is a GPU cloud/compute infrastructure platform, not an AI model provider with a data-usage/training policy for user prompts or content; no evidence pack item addresses training-data opt-out or data usage for model training, and this axis is a category error for an IaaS/compute provider rather than a hosted AI model service.", "evidenceIds": [] }, { "productId": "runpod", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers persistent storage (network volumes retaining data across terminations) and general compliance certifications (SOC 2, ISO 27001, PCI DSS) but contains no documentation of user-controlled data retention policies, deletion mechanisms, account/data purge options, or GDPR-style controls that would let an AI-native user manage retention and deletion of their data.", "evidenceIds": [ "runpod-docs-18", "runpod-docs-27" ] }, { "productId": "runpod", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry, usage-tracking, or an opt-out mechanism for Runpod; the docs cover compute, storage, CLI, and API features but never mention privacy settings or telemetry controls.", "evidenceIds": [] }, { "productId": "runpod", "storyId": "quota-limit-transparency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "There is community evidence of GPU availability constraints (low availability messages for 4090s) but no documented quota/limit structure or a defined process to request quota increases anywhere in the docs pack. Absence of evidence for an applicable capability (documented limits and escalation process) means this axis is unmet.", "evidenceIds": [ "runpod-comm-4" ] }, { "productId": "runpod", "storyId": "reserved-committed-discounts", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Runpod docs explicitly mention committing to 3- or 6-month terms upfront for significant discounts on compute costs, which directly addresses reserved/committed-use pricing for sustained GPU capacity. However, the evidence is a single brief doc line with no detail on discount percentages, capacity guarantees, contract terms, or enterprise commitment programs, and no independent/community corroboration of how this works in practice. Missing for 10: detailed terms of committed-use contracts, discount tiers, capacity guarantee mechanics, and hands-on/community confirmation of the reserved pricing program.", "evidenceIds": [ "runpod-docs-11" ] }, { "productId": "runpod", "storyId": "security-compliance-posture", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Runpod docs directly address SOC 2, ISO 27001, PCI DSS certifications and T3/T4 datacenter tiers for its Secure Cloud offering, giving a platform engineer concrete compliance signals to evaluate before deployment. However, this is limited to a single compliance page with no mention of audit reports/trust portal, data handling/residency details, encryption-at-rest specifics, or independent third-party verification, and community comments note the execution environment is 'locked down' without elaborating on security architecture. Missing for 10: downloadable SOC 2 report or trust center, detailed data handling/privacy policy, independent audit corroboration, and clarity on Community Cloud (non-Secure-Cloud) compliance gaps.", "evidenceIds": [ "runpod-docs-27", "runpod-docs-28", "runpod-comm-2", "runpod-comm-3" ] }, { "productId": "runpod", "storyId": "serverless-gpu-endpoints", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Runpod Serverless explicitly documents autoscaling GPU workers with 'Flex workers | Scale to zero when idle' and pay-per-second billing only while workers run, directly matching the story. Deployment is supported via CLI, REST API, or console, with FlashBoot/model caching to reduce cold-start costs. Missing for 10: independent hands-on benchmarks of scale-to-zero latency/cold-start behavior and third-party confirmation of autoscaling reliability under load.", "evidenceIds": [ "runpod-docs-25", "runpod-docs-38", "runpod-docs-20", "runpod-docs-49", "runpod-docs-24", "runpod-docs-8", "runpod-docs-44" ] }, { "productId": "runpod", "storyId": "spot-interruptible-pricing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Runpod's on-demand Pods, Serverless per-second pricing, Community/Secure Cloud tiers, and long-term commitment discounts, but nowhere documents a spot/interruptible/preemptible GPU tier or any preemption semantics (e.g., notice period, reclaim behavior, discount percentage). Community notes about 'low availability' (runpod-comm-4) reflect capacity issues, not a documented spot-pricing product.", "evidenceIds": [ "runpod-docs-11", "runpod-docs-20", "runpod-docs-25", "runpod-docs-28", "runpod-comm-4" ] }, { "productId": "runpod", "storyId": "ssh-root-access", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Runpod's docs explicitly describe SSH access to Pods with 'full shell capabilities' and list SSH among the standard connection methods (console, CLI, or agent-deployed Pods), matching the developer's need for root-level shell control over their GPU instance. Community commentary references SSH as a viable, sometimes primary, access method for Runpod Pods, and doesn't concretely contradict SSH working or granting shell access — it instead notes the execution environment is otherwise locked down, which is orthogonal. Missing for 10: explicit documentation excerpt describing the own-key upload/configuration step and independent hands-on confirmation of root-level privileges once inside.", "evidenceIds": [ "runpod-docs-6", "runpod-docs-14", "runpod-docs-41", "runpod-docs-45", "runpod-comm-3" ] }, { "productId": "runpod", "storyId": "team-access-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers Runpod's compute, storage, CLI, MCP, and API capabilities but contains no mention of team member management, role-based access control, or scoped/restricted API keys for governance purposes. This is a fair axis for a cloud infrastructure platform serving teams, but no documentation or probe evidence demonstrates it.", "evidenceIds": [] }, { "productId": "runpod", "storyId": "transparent-gpu-pricing", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Runpod's public pricing page is referenced (www.runpod.io/pricing) and docs confirm per-second billing for Pods/Serverless without needing sales contact, but the evidence pack never shows an actual itemized per-GPU-hour price table for each GPU type. missing for 10: explicit citation of the per-GPU-hour rate listing (e.g. A100 $x/hr, H100 $y/hr) on the public page, confirmation that all GPU types are listed with prices, independent corroboration that the page requires no sales contact.", "evidenceIds": [ "runpod-docs-33", "runpod-docs-51", "runpod-docs-11", "runpod-docs-50" ] }, { "productId": "vast-ai", "storyId": "agent-provisions-gpu", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Vast.ai documents a full REST API, CLI, and Python SDK covering the entire instance lifecycle (search offers, create/rent, connect, destroy), confirmed by runtime probes showing the CLI and API working keylessly for search and an OpenAPI spec being live. It also ships a dedicated 'vastai agent skill' explicitly for AI coding assistants to create instances, deploy endpoints, and manage keys without human console interaction, plus scoped API keys for automation/CI. Missing for 10: no independent hands-on evidence of a full agent-driven create→monitor→run→teardown cycle end-to-end (only search was probed live), and no first-party MCP server confirmed beyond the 'agent skill' framing.", "evidenceIds": [ "vast-ai-docs-2", "vast-ai-docs-3", "vast-ai-docs-4", "vast-ai-docs-23", "vast-ai-docs-24", "vast-ai-docs-34", "vast-ai-probe-2", "vast-ai-probe-rt-1", "vast-ai-probe-rt-2", "vast-ai-probe-rt-3" ] }, { "productId": "vast-ai", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Vast.ai publishes a live llms.txt at docs.vast.ai/llms.txt (confirmed via probe returning HTTP 200 with structured doc links) and additionally ships an explicit agent-oriented guide/skill ('vastai agent skill') documenting how AI coding assistants can drive the platform directly. This directly satisfies pointing an agent at llms.txt or agent-oriented docs. Missing for 10: no independent/community corroboration of agents actually consuming llms.txt in practice.", "evidenceIds": [ "vast-ai-probe-1", "vast-ai-docs-4", "vast-ai-docs-23", "vast-ai-docs-39" ] }, { "productId": "vast-ai", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Vast.ai is a GPU rental/compute marketplace, not a data-analytics or BI product; there is no user data corpus within the product for it to analyze and generate AI insights from. Its 'agent' features (vastai skill, CLI, SDK) let AI assistants operate the platform (search offers, launch instances) but do not generate insights/suggestions from a user's own data — this is a category mismatch, not a missing feature.", "evidenceIds": [] }, { "productId": "vast-ai", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Vast.ai exposes a CLI/SDK/REST API, scoped API keys for CI/CD, and an 'agent skill' letting AI coding assistants drive the platform (create instances, deploy endpoints, manage keys) — all usable to script automations, and Serverless endpoints let workloads run without manual GPU management. However, there is no evidence of a built-in scheduler, cron/trigger system, or persistent background job orchestration that runs autonomously without an external driver; automation depends on the user's own agent/script staying alive. Missing for 10: native scheduling/trigger mechanism, evidence of long-running autonomous background jobs, and independent confirmation of the agent-skill working unattended in production.", "evidenceIds": [ "vast-ai-docs-4", "vast-ai-docs-23", "vast-ai-docs-39", "vast-ai-docs-5", "vast-ai-docs-19", "vast-ai-docs-24", "vast-ai-docs-11", "vast-ai-docs-29", "vast-ai-probe-rt-1" ] }, { "productId": "vast-ai", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence only shows the reverse relationship: Vast.ai exposes a CLI/API/agent-skill so that external AI coding assistants (e.g., Claude, Copilot) can drive Vast.ai on the user's behalf — not that Vast.ai itself ships a built-in AI assistant a user can delegate tasks to within the product. No docs, UI, or probes mention any native chatbot/copilot embedded in the Vast.ai console.", "evidenceIds": [ "vast-ai-docs-4", "vast-ai-docs-23", "vast-ai-docs-39", "vast-ai-docs-29" ] }, { "productId": "vast-ai", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Vast.ai offers a full CLI and REST API for scriptable, non-interactive control of the entire instance lifecycle (search, create, destroy), confirmed by both docs and hands-on runtime probes running keylessly. Scoped API keys are explicitly recommended for CI/CD and shared tooling, and the CLI was verified to run headlessly via uvx with no login. Missing for 10: no explicit CI pipeline example (e.g., GitHub Actions workflow) or third-party case study of running it in an automated CI pipeline.", "evidenceIds": [ "vast-ai-docs-2", "vast-ai-docs-5", "vast-ai-docs-24", "vast-ai-docs-29", "vast-ai-probe-rt-1", "vast-ai-probe-rt-2", "vast-ai-probe-rt-3" ] }, { "productId": "vast-ai", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Vast.ai is a GPU cloud marketplace/infrastructure platform, not an AI agent or assistant that itself consumes tools. Its 'agent skill' evidence (vast-ai-docs-4/23/39) is the reverse direction — letting external coding assistants drive Vast.ai — not Vast.ai acting as an MCP client plugging in tool servers, so this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "vast-ai", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Vast.ai documents a CLI, REST API, Python SDK, and an 'agent skill' file for coding assistants, but no evidence anywhere describes an official MCP server or MCP integration. Since Vast.ai is a platform/service (not itself an agent), the MCP-server axis applies, and its absence from the evidence pack means the story is unmet.", "evidenceIds": [ "vast-ai-docs-4", "vast-ai-docs-23", "vast-ai-docs-39", "vast-ai-docs-2", "vast-ai-docs-3" ] }, { "productId": "vast-ai", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Vast.ai documents a dedicated 'vastai agent skill' explicitly designed so AI coding assistants can drive the platform on the user's behalf (create instances, deploy endpoints, manage keys, check balances) via natural-language-style interaction, backed by a full CLI/REST API surface. Missing for 10: independent or hands-on verification of the natural-language agent-skill flow itself (probes only confirm raw CLI/REST access, not NL interpretation), and no third-party account of an AI agent successfully using it end-to-end.", "evidenceIds": [ "vast-ai-docs-4", "vast-ai-docs-23", "vast-ai-docs-39", "vast-ai-docs-2", "vast-ai-docs-29", "vast-ai-probe-rt-1" ] }, { "productId": "vast-ai", "storyId": "agentic-official-cli", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Vast.ai ships an official CLI (pypi `vastai`) with full documentation of the entire platform surface (search, instances, templates, volumes, serverless), and runtime probes confirm it installs and works keylessly for real marketplace queries. Missing for 10: independent third-party reviews specifically praising/critiquing the CLI's UX beyond vendor docs and probes.", "evidenceIds": [ "vast-ai-docs-2", "vast-ai-docs-20", "vast-ai-docs-29", "vast-ai-probe-3", "vast-ai-probe-rt-1", "vast-ai-probe-rt-2" ] }, { "productId": "vast-ai", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Vast.ai publishes a documented REST API (with live OpenAPI spec) and CLI/SDK built on top of it, explicitly positioned for programmatic/agentic use, including scoped API keys for automation and a dedicated 'agent skill' for AI coding assistants. Runtime probes confirm the CLI and REST endpoints work keylessly for core operations like search/create/destroy instances, corroborating the documentation with hands-on evidence. Missing for 10: independent third-party developer reports specifically about API robustness/rate limits beyond docs and probes.", "evidenceIds": [ "vast-ai-docs-3", "vast-ai-docs-15", "vast-ai-docs-2", "vast-ai-docs-4", "vast-ai-docs-23", "vast-ai-docs-19", "vast-ai-docs-24", "vast-ai-probe-2", "vast-ai-probe-rt-1", "vast-ai-probe-rt-2", "vast-ai-probe-rt-3" ] }, { "productId": "vast-ai", "storyId": "agentic-scoped-keys", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly describe creating scoped API keys with limited permissions for workloads/CI-CD/shared tooling via `vastai create api-key`, directly matching least-privilege agent credentialing, and this is reinforced by a dedicated api-keys reference page and the agent-skill docs describing agent-driven usage. Missing for 10: independent/hands-on verification of scope enforcement (e.g., testing that a scoped key actually blocks restricted actions) and detail on the granularity of available scopes.", "evidenceIds": [ "vast-ai-docs-5", "vast-ai-docs-19", "vast-ai-docs-24", "vast-ai-docs-4", "vast-ai-docs-23" ] }, { "productId": "vast-ai", "storyId": "agentic-sdks", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Vast.ai documents an official Python SDK (built alongside CLI) and REST API with OpenAPI spec, a documented migration path from the legacy vastai_sdk package, and runtime-probed confirmation that the CLI/SDK surface works live against the marketplace API. Documentation explicitly frames CLI/SDK as 'same operations, different syntax' with Python usage examples for serverless workflows, satisfying an AI-native builder's need for official SDK access. Missing for 10: independent (non-vendor) hands-on reports specifically validating the Python SDK's completeness/stability beyond CLI parity.", "evidenceIds": [ "vast-ai-docs-3", "vast-ai-docs-29", "vast-ai-docs-40", "vast-ai-gh-2", "vast-ai-probe-2", "vast-ai-probe-rt-1", "vast-ai-probe-rt-3" ] }, { "productId": "vast-ai", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of a webhook subscription mechanism anywhere in the docs, CLI, API reference, or probes; Vast.ai exposes REST API, CLI, and SDK for polling/imperative control but nothing describing event-driven push notifications or webhook callbacks. missing for 10: any documentation of webhook registration/endpoints, event types, or delivery/retry semantics.", "evidenceIds": [ "vast-ai-docs-3", "vast-ai-docs-2", "vast-ai-probe-2" ] }, { "productId": "vast-ai", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Vast.ai publishes a machine-readable OpenAPI spec and API-reference docs, and the CLI docs include copy-pasteable, runnable example commands that were verified to work live (search offers, create instance) without auth — showing some 'runnable example' quality. However there is no evidence of an interactive API console (e.g., embedded 'try it' Swagger-UI style playground) on the docs site itself. Missing for 10: an interactive in-browser API explorer/console, explicit 'try it now' runnable request execution within the API reference pages, and independent confirmation of such interactivity.", "evidenceIds": [ "vast-ai-docs-3", "vast-ai-docs-15", "vast-ai-probe-2", "vast-ai-probe-rt-1", "vast-ai-probe-rt-2" ] }, { "productId": "vast-ai", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Vast.ai publishes a live OpenAPI spec at docs.vast.ai/api/openapi.json (verified HTTP 200 with an 'openapi' key), backed by a full REST API reference and CLI/SDK built on it. Missing for 10: no independent third-party confirmation that the spec is comprehensive/kept in sync beyond the probe check.", "evidenceIds": [ "vast-ai-probe-2", "vast-ai-docs-3", "vast-ai-docs-15", "vast-ai-probe-rt-3" ] }, { "productId": "vast-ai", "storyId": "api-sandbox", "verdict": "na", "quality": 0, "confidence": "medium", "rationale": "Vast.ai is a GPU compute marketplace, not a SaaS/API product with a distinct 'production' data environment vs. a sandbox/test mode; the evidence never frames the platform in terms of separating sandbox from production data, since each rented instance is simply an isolated compute unit under the customer's own control. This story's axis (sandbox testing without touching production data) doesn't map onto a bare-metal/GPU rental marketplace's product model.", "evidenceIds": [] }, { "productId": "vast-ai", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Vast.ai has a REST API, OpenAPI spec, and CLI/SDK, but there is no documentation of API versioning scheme or a deprecation policy anywhere in the pack; the only related signal is a backward-compatible import shim for the old SDK (vast-ai-gh-2), which is not a documented policy.", "evidenceIds": [ "vast-ai-docs-3", "vast-ai-probe-2", "vast-ai-gh-2" ] }, { "productId": "vast-ai", "storyId": "auto-shutdown-spend-guards", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack documents autobilling (auto top-up to keep instances running), reserved/interruptible instance pricing, and scoped API keys, but none of these are auto-shutdown timers or spend caps — autobilling in fact works against this story by automatically refilling balance to prevent instance interruption rather than capping spend. No mention of idle-timeout, max-runtime, or budget-limit features exists anywhere in the pack.", "evidenceIds": [ "vast-ai-docs-8", "vast-ai-docs-25", "vast-ai-docs-6", "vast-ai-docs-19" ] }, { "productId": "vast-ai", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "The CLI and REST API expose full programmatic control (search, create, destroy, manage keys) and are explicitly positioned for automation/scripting from shell or Python, which lets an AI-native user loop over many offers/instances (e.g., filtered searches returning many GPU offers at once). However, there is no documented native batch/bulk endpoint (e.g., one call to create or destroy N instances simultaneously) — only single-item CRUD operations that must be scripted in a loop. Missing for 10: explicit bulk/batch API endpoints or CLI flags for multi-item operations, and community/hands-on confirmation of bulk usage at scale.", "evidenceIds": [ "vast-ai-docs-2", "vast-ai-docs-3", "vast-ai-docs-29", "vast-ai-gh-1", "vast-ai-docs-20", "vast-ai-probe-rt-1", "vast-ai-probe-rt-2" ] }, { "productId": "vast-ai", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Vast.ai documents 'autobilling', an explicit event-triggered rule (balance drops low → auto top-up from saved card), which is genuine automation-on-event. Beyond that, evidence only shows generic CLI/SDK/API scripting and agent-skill hooks for manual automation, not a broader rules/trigger engine (no webhooks, alerts, or conditional actions for instance events like preemption, cost thresholds, or health failures). Missing for 10: a general-purpose event/trigger system (webhooks, alerting, conditional rules) beyond the single billing use case, and independent confirmation of its reliability.", "evidenceIds": [ "vast-ai-docs-8", "vast-ai-docs-25", "vast-ai-docs-29", "vast-ai-docs-24" ] }, { "productId": "vast-ai", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Vast.ai's evidence covers CLI/API/SDK for instance and serverless management but contains no mention of a built-in scheduler, cron-like recurring job feature, or workflow orchestration; users would have to bring their own external scheduler to hit the API/CLI repeatedly, which isn't documented as a first-party capability.", "evidenceIds": [] }, { "productId": "vast-ai", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Vast.ai offers templates, CLI/SDK automation, and agent skills for scripting instance management, but there is no evidence of version control, change review, or rollback capability for these templates or automation scripts/workflows.", "evidenceIds": [ "vast-ai-docs-30", "vast-ai-docs-38", "vast-ai-docs-29" ] }, { "productId": "vast-ai", "storyId": "billing-usage-api", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Vast.ai's REST API and CLI provide general programmatic access to the platform, including account balance checks and billing pages (autobilling, credit top-up), and scoped API keys can be created per workload/team for CI/CD use, which could support some cost attribution. However there is no explicit evidence of a dedicated usage/billing breakdown endpoint, cost tagging, or per-team/workload spend reporting API. missing for 10: dedicated billing/usage export API, cost tagging or labels for spend attribution, documented reports/analytics endpoint for team-level breakdowns.", "evidenceIds": [ "vast-ai-docs-3", "vast-ai-docs-8", "vast-ai-docs-19", "vast-ai-docs-24", "vast-ai-docs-25" ] }, { "productId": "vast-ai", "storyId": "custom-docker-images", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Vast.ai's template system explicitly lets developers package their own Docker environment ('Package your environment so any GPU can run it with one click') and launch prebuilt or custom templates with one click, backed by CLI/API/SDK automation for creating and managing instances from these templates. Documentation confirms templates configure the rented machine with whatever software/formatting is needed, directly matching the story.\n\nmissing for 10: no independent/hands-on confirmation of building and running a fully custom Docker image end-to-end (community evidence focuses on pricing/reliability, not template customization).", "evidenceIds": [ "vast-ai-docs-38", "vast-ai-docs-30", "vast-ai-docs-13", "vast-ai-docs-27", "vast-ai-docs-29" ] }, { "productId": "vast-ai", "storyId": "data-transfer-cloud-sync", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Vast.ai documents SSH-based secure file transfer to/from instances and a CLI workflow that includes 'copy data' as a step, plus persistent Volumes for data that survives instance destruction. However there is no evidence of S3-compatible endpoints or native cloud-storage (S3/GCS/Azure Blob) sync tooling, and a community comment notes persistent-disk workflows are still cumbersome over time. Missing for 10: S3-compatible storage endpoint, built-in cloud-storage sync/mirroring tooling, independent hands-on validation of transfer throughput/performance.", "evidenceIds": [ "vast-ai-docs-36", "vast-ai-docs-43", "vast-ai-docs-34", "vast-ai-docs-10", "vast-ai-comm-4" ] }, { "productId": "vast-ai", "storyId": "exposed-ports-networking", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs explicitly describe exposing ports (search filter `direct_port_count>=1`, SSH access) and creating private overlay networks so instances on different machines can share a virtual LAN for multi-node workloads, directly matching both halves of the story. Missing for 10: independent/hands-on confirmation of port-forwarding for arbitrary web apps beyond SSH, and more detail on overlay network setup/limitations.", "evidenceIds": [ "vast-ai-docs-12", "vast-ai-docs-18", "vast-ai-docs-20", "vast-ai-docs-9", "vast-ai-docs-43" ] }, { "productId": "vast-ai", "storyId": "gpu-availability-transparency", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Vast.ai's search/offers API and CLI let engineers query live marketplace inventory (GPU type, count, verified status, price) before renting, and this was independently confirmed via a keyless live probe against both the CLI and the public REST bundles endpoint returning real-time offers with reliability data. This directly satisfies seeing availability before provisioning rather than hitting a stockout on create. missing for 10: explicit region/geolocation filter documentation and evidence of real-time refresh/alerting for capacity changes.", "evidenceIds": [ "vast-ai-docs-20", "vast-ai-docs-41", "vast-ai-gh-1", "vast-ai-probe-rt-2", "vast-ai-probe-rt-3" ] }, { "productId": "vast-ai", "storyId": "gpu-breadth-latest-hardware", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Vast.ai's docs and CLI clearly support searching/filtering the marketplace by GPU model (e.g., `gpu_name=RTX_4090`) and offer tiered pricing (on-demand, reserved, interruptible) that would let an ML engineer pick pricier or cheaper GPU options, and 'Secure Cloud (Datacenter)' verification implies access to datacenter-grade hardware. However, none of the evidence explicitly confirms availability of current-generation H100/H200/B200-class GPUs alongside older cards — all concrete examples in the pack use RTX_4090 (a consumer card), not datacenter-tier flagship GPUs. Missing for 10: explicit documentation or listing showing H100/H200/B200 GPUs are actually rentable on the platform alongside cheaper previous-gen options.", "evidenceIds": [ "vast-ai-gh-1", "vast-ai-docs-20", "vast-ai-docs-14", "vast-ai-docs-28", "vast-ai-docs-6", "vast-ai-docs-7", "vast-ai-probe-rt-2" ] }, { "productId": "vast-ai", "storyId": "instance-lifecycle-management", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Vast.ai's CLI and REST API provide documented commands/endpoints for the full instance lifecycle (search, create, destroy) confirmed by runtime probes, plus billing docs (autobilling, per-hour pricing, reserved/interruptible discounts) demonstrate pay-only-for-what-runs. Community feedback corroborates cost savings but also notes reliability variance, which is a secondary concern rather than a lifecycle-control failure. Missing for 10: explicit documented 'stop'/'restart' CLI subcommands beyond create/destroy in the evidence, and independent hands-on confirmation of stop/restart specifically (only create/destroy verified at runtime).", "evidenceIds": [ "vast-ai-docs-2", "vast-ai-docs-3", "vast-ai-probe-rt-1", "vast-ai-probe-rt-2", "vast-ai-probe-rt-3", "vast-ai-docs-8", "vast-ai-docs-25", "vast-ai-docs-6", "vast-ai-docs-7", "vast-ai-comm-2" ] }, { "productId": "vast-ai", "storyId": "jupyter-ide-access", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Vast.ai documents secure SSH access to instances (key-based auth, run commands, transfer files) and one-click template launches that could include Jupyter environments, which together enable IDE remote-connection workflows (e.g., VS Code Remote-SSH). However, the evidence never explicitly names Jupyter notebook access or a documented one-step VS Code/Cursor connection flow. Missing for 10: explicit Jupyter launch/access docs, explicit VS Code/Cursor remote-connect guide or extension support, and any hands-on confirmation of a true 'one step' connect experience.", "evidenceIds": [ "vast-ai-docs-9", "vast-ai-docs-36", "vast-ai-docs-43", "vast-ai-docs-13", "vast-ai-docs-30", "vast-ai-docs-38" ] }, { "productId": "vast-ai", "storyId": "keyless-catalog-pricing-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Vast.ai provides both a documented public REST endpoint (bundles/offers) and CLI (`vastai search offers`) that return live GPU pricing, specs, and availability without requiring an account or API key, verified via runtime probes. This directly matches the story of an agent querying the catalog before committing spend; missing for 10: independent third-party corroboration beyond vendor docs/probes and no explicit rate-limit/auth-tier documentation for heavy automated querying.", "evidenceIds": [ "vast-ai-probe-rt-2", "vast-ai-probe-rt-3", "vast-ai-docs-20", "vast-ai-gh-1", "vast-ai-probe-2" ] }, { "productId": "vast-ai", "storyId": "managed-slurm-kubernetes", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Vast.ai provides raw GPU instance rental, templates, overlay networking for multi-node training, and CLI/API automation, but nothing about a managed Slurm or Kubernetes scheduling service — users would need to build their own scheduler on top of raw nodes. missing for 10: managed Slurm service, managed Kubernetes service, any orchestration layer beyond raw instance provisioning.", "evidenceIds": [ "vast-ai-docs-12", "vast-ai-docs-18", "vast-ai-docs-30", "vast-ai-docs-2" ] }, { "productId": "vast-ai", "storyId": "multi-node-clusters", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Docs describe self-service, no-sales-cycle provisioning (CLI/API/search offers for multi-GPU nodes) and an overlay-network feature specifically for multi-node NCCL/PyTorch training, which addresses the 'no sales cycle' and 'multi-node' parts of the story. However, independent hands-on reports describe bandwidth as 'lousy' and 'all over the place compared to what's listed in the console,' directly undercutting the 'fast interconnect' claim needed for distributed training performance. Missing for 10: documented interconnect specs (e.g. InfiniBand/NVLink bandwidth guarantees), benchmarked multi-node throughput, and independent confirmation that overlay networking delivers low-latency performance at scale.", "evidenceIds": [ "vast-ai-docs-12", "vast-ai-docs-18", "vast-ai-gh-1", "vast-ai-docs-34", "vast-ai-comm-5", "vast-ai-comm-6" ] }, { "productId": "vast-ai", "storyId": "on-demand-gpu-provisioning", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs and runtime probes show the full quickstart flow — search offers via CLI/API keylessly, one-click templates, instance creation, SSH access with password auth disabled, and REST/CLI parity for automation — matching the 'search, rent, boot, connect' workflow described in vast-ai-docs-34. Community reviews largely corroborate ease and cost savings, though some flag friction/hoops versus AWS/GCP and inconsistent reliability, which tempers but doesn't contradict the core provisioning claim. Missing for 10: an independent hands-on timing test confirming actual boot-to-running-code within minutes, and resolution of community complaints about setup friction/reliability variance.", "evidenceIds": [ "vast-ai-docs-1", "vast-ai-docs-34", "vast-ai-docs-41", "vast-ai-docs-13", "vast-ai-docs-2", "vast-ai-docs-3", "vast-ai-docs-43", "vast-ai-probe-rt-1", "vast-ai-probe-rt-2", "vast-ai-probe-rt-3", "vast-ai-comm-2", "vast-ai-comm-7" ] }, { "productId": "vast-ai", "storyId": "openness-api-parity", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Vast.ai's REST API is explicitly documented as the foundation underlying both the CLI and SDK, giving full programmatic control over the entire platform (search, instance lifecycle, templates, volumes, serverless endpoints), and this is corroborated by a live OpenAPI spec and runtime probes showing the CLI/API perform the same search/rent/manage operations available in the UI, including a dedicated agent skill for AI assistants to drive the platform. Minor gap: no explicit UI-vs-API feature parity audit exists confirming literally every UI-only setting (e.g., some billing/account UI screens) has an API equivalent. Missing for 10: an explicit parity matrix or independent confirmation that 100% of UI actions (not just core lifecycle) are API-exposed.", "evidenceIds": [ "vast-ai-docs-2", "vast-ai-docs-3", "vast-ai-docs-15", "vast-ai-docs-29", "vast-ai-docs-39", "vast-ai-probe-2", "vast-ai-probe-rt-1", "vast-ai-probe-rt-2", "vast-ai-probe-rt-3" ] }, { "productId": "vast-ai", "storyId": "openness-full-export", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Vast.ai instances run standard Linux with SSH access explicitly described as supporting file transfer ('Transfer files without exposing your data') and volumes are persistent, detachable storage, implying no proprietary lock-in on user data — but there is no explicit documentation of a bulk data-export feature, open-format guarantee, or account-closure data portability process. missing for 10: explicit data-export/portability documentation, statement on open formats, and evidence of full account data extraction on leaving.", "evidenceIds": [ "vast-ai-docs-36", "vast-ai-docs-43", "vast-ai-docs-10" ] }, { "productId": "vast-ai", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Vast.ai is a GPU marketplace platform whose core service (the marketplace, matching engine, billing) is closed; while its CLI is on GitHub, there is no evidence of an open-source license covering the platform's source code, and no license is mentioned anywhere in the pack. missing for 10: N/A (verdict is none) — need explicit license file/OSI license reference for the platform or CLI repo, and any statement of open-licensing of the core product.", "evidenceIds": [ "vast-ai-gh-1", "vast-ai-gh-2" ] }, { "productId": "vast-ai", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Vast.ai is a hosted GPU marketplace/cloud platform whose core value is a live network connecting many hosts and renters (bidding, marketplace pricing, datacenter verification) — not standalone software a user could deploy on their own infrastructure. Self-hosting 'the core product' is a category error for this kind of marketplace/cloud service; evidence only shows CLI/API/SDK access to the hosted platform, not an on-prem deployable version.", "evidenceIds": [] }, { "productId": "vast-ai", "storyId": "per-second-billing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Vast.ai quotes prices as hourly rates (dph/dph_total) and discusses reserved/interruptible pricing models and autobilling, but nowhere does the evidence pack confirm sub-hour (per-second or per-minute) billing granularity or explicitly state billing stops precisely when an instance is not running.", "evidenceIds": [] }, { "productId": "vast-ai", "storyId": "persistent-network-storage", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vast.ai docs explicitly describe 'Volumes' as persistent storage that survives instance destruction and can be reattached to new instances, directly matching the story. However, a community user notes wishing it were 'easier to work with persistent disks over time' and now avoids it for anything but one-off jobs, indicating real friction in practice. Missing for 10: independent hands-on verification of volume reattachment reliability, docs detailing size/performance limits, and broader community corroboration beyond a single anecdotal complaint.", "evidenceIds": [ "vast-ai-docs-10", "vast-ai-comm-4" ] }, { "productId": "vast-ai", "storyId": "prebuilt-ml-templates", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Vast.ai's docs explicitly describe templates as a first-class launch mechanism ('Launch prebuilt or custom templates with one click', 'A template is how Vast helps you launch an instance') and directly name PyTorch, vLLM, and ComfyUI as available prebuilt templates ('PyTorch, vLLM, ComfyUI, agents, and more. Real deployments you can copy'). CUDA support is implied through GPU offer filtering and template packaging but not explicitly named as its own template. missing for 10: independent/hands-on confirmation of launching a specific named template (e.g., a community post describing a ComfyUI or vLLM template launch), and explicit CUDA-template documentation rather than inferred CUDA support.", "evidenceIds": [ "vast-ai-docs-13", "vast-ai-docs-27", "vast-ai-docs-30", "vast-ai-docs-38" ] }, { "productId": "vast-ai", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Vast.ai lets users search/filter GPU offers by attributes like GPU model and reliability, and it distinguishes 'Secure Cloud' verified datacenters, but nowhere documents an explicit region/country selection or data-residency guarantee for where data is stored. A community report even flags a case where a host's advertised location (US) was misrepresented (actually China), undermining confidence that location can be reliably chosen or verified.", "evidenceIds": [ "vast-ai-docs-14", "vast-ai-docs-28", "vast-ai-comm-10" ] }, { "productId": "vast-ai", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Vast.ai is a GPU cloud marketplace/compute rental platform, not an AI model provider or SaaS tool that trains on user data/content; there is no product feature (like an AI assistant or model training pipeline) whose data-training-opt-out policy would be a fair axis. The story is a category error for this product type.", "evidenceIds": [] }, { "productId": "vast-ai", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Evidence shows users can destroy instances and volumes (which otherwise persist across instance destruction) via CLI/API, giving basic control over deletion of compute resources and data, but there is no explicit documentation of data retention policies, data-deletion guarantees, or privacy controls beyond simply terminating instances/volumes. missing for 10: explicit data retention/deletion policy docs, guarantees on backend data purge after deletion, account-level data deletion controls, independent verification of deletion behavior.", "evidenceIds": [ "vast-ai-docs-10", "vast-ai-docs-34", "vast-ai-probe-rt-1" ] }, { "productId": "vast-ai", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions telemetry, usage tracking, analytics opt-out, or privacy settings for Vast.ai's own platform/CLI; the docs focus on billing, instances, SSH, and API/CLI usage. missing for 10: any documentation of telemetry collection, an opt-out mechanism, or a privacy policy addressing usage tracking.", "evidenceIds": [] }, { "productId": "vast-ai", "storyId": "quota-limit-transparency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers billing, autobilling, reserved/interruptible instance pricing, API keys, and CLI/SDK usage, but contains no documentation of account quotas, per-user instance limits, or a defined process to request limit increases. This axis is applicable to any capacity-oriented GPU marketplace, so absence of evidence yields 'none'. Missing for 10: documented quota/limit tables, a support/ticket process for raising limits, any mention of default instance caps or how to request more capacity.", "evidenceIds": [] }, { "productId": "vast-ai", "storyId": "reserved-committed-discounts", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Vast.ai documents Reserved Instances that let users pre-pay for GPU time to lock in up to 50% discounts, convertible from any on-demand instance at any time, directly matching the committed-use discount story. Missing for 10: no independent/community corroboration of reserved-instance pricing behavior, no detail on commitment duration/terms flexibility, and no evidence of longer-term (e.g. multi-month/annual) contractual reserved capacity beyond simple pre-payment conversion.", "evidenceIds": [ "vast-ai-docs-6", "vast-ai-docs-16", "vast-ai-docs-21", "vast-ai-docs-31" ] }, { "productId": "vast-ai", "storyId": "security-compliance-posture", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Vast.ai docs claim some infrastructure verification — 'Secure Cloud' machines are checked for TIER 2/3 datacenter rating or ISO 27001 certification (vast-ai-docs-14, vast-ai-docs-28) — but there is no mention of SOC 2 attestation, data-handling/privacy policies, or compliance documentation anywhere in the pack. Community evidence directly undercuts trust in these verifications: a host was found masquerading as US-based while actually located in China (vast-ai-comm-10), and another thread questions whether running consumer GPUs at scale even complies with Nvidia's own terms (vast-ai-comm-9) — concrete contradictions of the 'verified' security posture platform engineers would need to rely on. Missing for 10: SOC 2 or equivalent third-party compliance attestation, explicit data-handling/retention policy, and independent audit corroborating the datacenter tier claims.", "evidenceIds": [ "vast-ai-docs-14", "vast-ai-docs-28", "vast-ai-comm-9", "vast-ai-comm-10" ] }, { "productId": "vast-ai", "storyId": "serverless-gpu-endpoints", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vast.ai explicitly ships 'Vast Serverless', described as letting users run compute-intensive workloads 'without managing GPUs, paying for execution rather than GPU rental time,' with a Python SDK and CLI support for deploying endpoints — matching the pay-per-execution serverless model a developer would want instead of always-on instances. However, the evidence never explicitly confirms autoscaling behavior or scale-to-zero semantics, nor gives hands-on/independent corroboration of how workers scale under load. Missing for 10: explicit autoscaling/scale-to-zero documentation, independent or hands-on validation of serverless endpoint behavior, and details on cold-start/latency tradeoffs.", "evidenceIds": [ "vast-ai-docs-11", "vast-ai-docs-40", "vast-ai-docs-23", "vast-ai-docs-4" ] }, { "productId": "vast-ai", "storyId": "spot-interruptible-pricing", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vast.ai clearly documents an interruptible/bidding instance type offering 50%+ discounts vs on-demand (vast-ai-docs-7/17/22/35/44), which satisfies the 'deep discount' part of the story. However, the evidence pack gives only the pricing angle and doesn't detail concrete preemption mechanics (notice period, how bidding/eviction actually triggers, restart behavior) — missing for 10: explicit preemption trigger/notice documentation, and independent confirmation that discount/preemption behavior matches docs (community notes reliability variance, e.g. vast-ai-comm-6, but this isn't a concrete dispute of interruptible semantics specifically).", "evidenceIds": [ "vast-ai-docs-7", "vast-ai-docs-17", "vast-ai-docs-22", "vast-ai-docs-35", "vast-ai-docs-44", "vast-ai-comm-6" ] }, { "productId": "vast-ai", "storyId": "ssh-root-access", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm SSH access is key-based only (password auth disabled) and lets you log in securely, run commands remotely, and transfer files, which implies developer control of the instance. However, the evidence pack never explicitly confirms root-level privileges or documents the workflow for adding a user's own SSH key. Missing for 10: explicit documentation of root/sudo access inside instances, and details on uploading/managing your own SSH keys.", "evidenceIds": [ "vast-ai-docs-9", "vast-ai-docs-36", "vast-ai-docs-43" ] }, { "productId": "vast-ai", "storyId": "team-access-controls", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Vast.ai documents scoped API keys with limited permissions for CI/CD and shared tooling use cases (create api-key, per-workload scoping), which addresses the credential-control half of the story. However, there is no evidence of team/organization member management, role assignment, or multi-user account governance — the evidence pack only covers individual scoped keys, not team-based RBAC. Missing for 10: team member invitation/management, role definitions (admin/viewer/etc.), org-level spend controls tied to roles, and any audit trail for team activity.", "evidenceIds": [ "vast-ai-docs-5", "vast-ai-docs-19", "vast-ai-docs-24" ] }, { "productId": "vast-ai", "storyId": "transparent-gpu-pricing", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "The marketplace search page and public REST/CLI endpoints let anyone see live per-GPU-hour prices without an account or contacting sales (vast-ai-docs-41, vast-ai-probe-rt-2, vast-ai-probe-rt-3), which covers the core of the story. However, there's no dedicated static 'pricing page' listing every GPU type's rate at a glance—pricing is discovered via a dynamic search/filter interface, and a community user explicitly complained about the number of clicks needed to find even an approximate price (vast-ai-comm-3). Missing for 10: a single canonical pricing table/page enumerating all GPU types' rates, and stronger independent corroboration that pricing is easy to find without friction.", "evidenceIds": [ "vast-ai-docs-41", "vast-ai-probe-rt-2", "vast-ai-probe-rt-3", "vast-ai-comm-3", "vast-ai-docs-26" ] } ]