[ { "productId": "formant", "storyId": "agent-controls-robot", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Formant provides an Agent SDK and Cloud SDK/API that let external programs authenticate, ingest telemetry, and send commands to control real robots (formant-docs-2, formant-docs-4, formant-docs-9, formant-gh-1), which supports end-to-end robot commanding via SDK. However, there is no evidence of scoped/limited-permission credentials or AI-agent-specific auth mechanisms, and the OpenAPI spec probe returned 404s, suggesting the API surface isn't fully documented for programmatic discovery. Missing for 10: explicit scoped-credential/auth-token documentation, AI-agent-specific integration examples, and a public OpenAPI/machine-readable API spec.", "evidenceIds": [ "formant-docs-2", "formant-docs-4", "formant-docs-9", "formant-gh-1", "formant-probe-2" ] }, { "productId": "formant", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A direct probe confirms llms.txt is live and returns a 200 with structured docs content at docs.formant.io/llms.txt, directly satisfying the story. Missing for 10: no independent third-party confirmation of agent usage or broader agent-oriented doc structure beyond the single llms.txt file.", "evidenceIds": [ "formant-probe-1" ] }, { "productId": "formant", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 6, "confidence": "medium", "rationale": "Formant's own product pages describe an 'AI assistant' for building dashboards from device data and a plain-language fleet query feature that returns ML-driven task recommendations, directly matching the AI-generated insights/suggestions story. Missing for 10: deeper technical documentation of how the AI assistant/insights engine works and independent or hands-on corroboration beyond marketing copy.", "evidenceIds": [ "formant-docs-7", "formant-docs-25", "formant-docs-12", "formant-docs-23", "formant-docs-17", "formant-docs-8" ] }, { "productId": "formant", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Formant provides trigger-based automation (webhooks fired from events, automated commands, and fleet orchestration that dispatches robots/help based on operational anomalies), which can run without a human polling the system — a basic form of background automation. However, this is rule/event-based orchestration rather than a documented framework for autonomous, AI-driven background agents, and there's no evidence of scheduling, persistent agent loops, or independent corroboration of unattended operation. Missing for 10: explicit AI-agent-style background automation framework, scheduling/looping capability, and third-party validation of autonomous operation.", "evidenceIds": [ "formant-docs-28", "formant-docs-9", "formant-docs-18", "formant-docs-17", "formant-docs-30", "formant-docs-5" ] }, { "productId": "formant", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Formant documents a built-in AI assistant for building analytics dashboards and a natural-language interface to query fleet status and get ML-driven task recommendations, but delegation is explicitly gated by human approval ('people approving what happens next') rather than autonomous task execution. Missing for 10: detailed docs on assistant capabilities/API, evidence of the assistant autonomously executing delegated tasks, and independent/hands-on confirmation of this feature working.", "evidenceIds": [ "formant-docs-7", "formant-docs-25", "formant-docs-12", "formant-docs-23" ] }, { "productId": "formant", "storyId": "agentic-headless", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Formant provides a Dockerized agent, Python Cloud SDK, Admin API, and webhooks that enable programmatic, non-UI interaction with telemetry, commands, and data ingestion — all of which could be scripted for automated/headless workflows. However, there is no explicit documentation or example of running Formant in a CI pipeline or fully headless automation context; most emphasis is on operator dashboards, teleoperation, and fleet visualization UI.\nmissing for 10: explicit CI/CD integration examples, documented headless mode or automation pipeline guide, evidence of non-interactive/scripted end-to-end workflows.", "evidenceIds": [ "formant-docs-1", "formant-docs-3", "formant-docs-4", "formant-docs-27", "formant-docs-28" ] }, { "productId": "formant", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Formant is a robotics fleet management platform with SDKs/APIs and an AI assistant for analytics, but there is no evidence of MCP server support allowing external AI tools to plug into Formant, nor of Formant consuming MCP servers itself.", "evidenceIds": [] }, { "productId": "formant", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Formant is a robot fleet management platform, not itself an AI agent, so a fair question exists about whether it exposes an official MCP server for agentic access; no evidence pack item mentions MCP at all, only SDKs, REST/webhooks, and Admin API.", "evidenceIds": [] }, { "productId": "formant", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Formant's marketing pages mention asking the fleet 'what's happening in plain language' and getting ML-driven task recommendations, plus an 'AI assistant' for building dashboards, indicating some natural-language interaction capability. However, this is only surfaced in marketing copy, not backed by technical documentation, examples, or independent verification, and the bulk of the 'commands' documentation describes predefined/scripted actions rather than open natural-language control. Missing for 10: technical docs on the NL interface, example commands/queries, independent hands-on verification, and clarity on scope of natural-language coverage versus predefined commands.", "evidenceIds": [ "formant-docs-12", "formant-docs-23", "formant-docs-7", "formant-docs-25", "formant-docs-9" ] }, { "productId": "formant", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Formant offers Python Cloud SDK, Agent SDK, RealtimeSDK/toolkit, and REST/Admin APIs, but no documentation or GitHub repo describes an official command-line interface tool.", "evidenceIds": [ "formant-docs-2", "formant-docs-3", "formant-gh-1", "formant-docs-27" ] }, { "productId": "formant", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Formant documents multiple public APIs/SDKs (Agent SDK, Cloud SDK, Admin API, Realtime SDK/toolkit, ingestion API, webhooks) that let developers ingest telemetry, send commands, teleoperate, and build custom apps, with GitHub toolkit repos for hands-on integration. Missing for 10: a discoverable formal OpenAPI/Swagger spec (probe found only 404s) and independent third-party corroboration of API robustness.", "evidenceIds": [ "formant-docs-2", "formant-docs-3", "formant-docs-4", "formant-gh-1", "formant-docs-27", "formant-docs-28", "formant-probe-2" ] }, { "productId": "formant", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of scoped or least-privilege API credential issuance for agents; Formant docs describe agent SDK, admin API, and commands but nothing about API key scoping, permission levels, or credential minimization. Missing for 10: any documentation of scoped/least-privilege API keys, role-based credential issuance, or agent-specific token permissions.", "evidenceIds": [] }, { "productId": "formant", "storyId": "agentic-sdks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Formant documents multiple official SDKs — the Agent SDK (telemetry, commands, teleop), the Cloud SDK (Python) for data ingestion/query, and a public GitHub 'toolkit' repo exposing RealtimeSDK and APIs for building custom apps — giving AI-native developers concrete, first-party libraries to build against. Missing for 10: independent/hands-on developer corroboration beyond vendor docs, and a working OpenAPI/swagger spec (probe shows all candidate OpenAPI endpoints 404), which weakens confidence in full API-contract completeness.", "evidenceIds": [ "formant-docs-2", "formant-docs-3", "formant-docs-20", "formant-docs-27", "formant-gh-1", "formant-gh-5", "formant-probe-2" ] }, { "productId": "formant", "storyId": "agentic-webhooks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Formant docs explicitly document webhooks triggered from platform events, allowing an HTTP call to any endpoint based on a Formant event trigger, which directly matches the story's subscribe-to-events-via-webhooks capability. Missing for 10: independent/hands-on corroboration beyond first-party docs, and details on payload schema, retries, or auth for webhook subscriptions.", "evidenceIds": [ "formant-docs-28" ] }, { "productId": "formant", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows documentation and SDK references but no interactive API reference with runnable examples; a probe explicitly found no OpenAPI/swagger spec (404s across candidate paths), and no mention of a try-it-now console or code sandbox.", "evidenceIds": [ "formant-probe-2" ] }, { "productId": "formant", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Formant references an Admin API, Cloud SDK, and query API, but a direct probe for OpenAPI/Swagger specs at common paths returned 404 for all candidates, and no evidence pack item links to a downloadable machine-readable spec.", "evidenceIds": [ "formant-probe-2", "formant-docs-4", "formant-docs-3" ] }, { "productId": "formant", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a sandbox/staging environment, test devices, or any mechanism to isolate testing from production robot fleets and data; Formant's docs focus on production device management, telemetry, teleoperation, and analytics with no mention of sandboxing.", "evidenceIds": [] }, { "productId": "formant", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of API versioning scheme or a documented deprecation policy; probes for an OpenAPI spec all returned 404, and none of the docs mention version numbers or deprecation timelines.", "evidenceIds": [ "formant-probe-2" ] }, { "productId": "formant", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Formant offers fleet-wide monitoring, orchestration that assigns jobs across many robots, and a single view to manage multiple fleets simultaneously, plus a programmatic Admin API/Agent SDK that could be scripted to issue commands to many devices at once. However, there is no explicit documentation of a dedicated bulk-operation API or UI (e.g., select-many-and-apply-action) for arbitrary bulk actions across devices. Missing for 10: explicit bulk-command/bulk-update API or UI documentation, evidence of batch processing limits/throughput, and independent confirmation of bulk workflows in practice.", "evidenceIds": [ "formant-docs-8", "formant-docs-17", "formant-docs-19", "formant-docs-22", "formant-docs-18", "formant-docs-4", "formant-docs-9" ] }, { "productId": "formant", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Formant docs describe event-triggered webhooks (\"make an API call to any endpoint over HTTP based on a trigger from a Formant event\") and automated incident-response/alerting for anomalies, which directly support rule-based automatic actions on events. However, there's no detailed documentation of a full rules/condition-builder UI, complex logic composition, or independent corroboration of this feature working at scale. Missing for 10: detailed rule-definition interface/logic docs, independent/hands-on validation, examples of multi-condition or chained automations.", "evidenceIds": [ "formant-docs-28", "formant-docs-18", "formant-docs-5", "formant-docs-9" ] }, { "productId": "formant", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Formant offers event-triggered webhooks, commands, and fleet orchestration automation, but no evidence describes a scheduler, cron-like recurring job system, or workflow automation that runs on a time-based schedule. Missing for 10: any mention of scheduled/recurring task configuration, cron syntax, or workflow scheduling UI/API.", "evidenceIds": [ "formant-docs-28", "formant-docs-9", "formant-docs-18" ] }, { "productId": "formant", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Formant is a robot fleet operations/teleoperation platform; there is no evidence of automation versioning, review workflows, or rollback capability for automations/workflows anywhere in the docs or GitHub toolkit. No mention of version history, change review, or revert functionality for commands, triggers, or workflows.", "evidenceIds": [] }, { "productId": "formant", "storyId": "ci-sim-testing", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Formant is a fleet operations/observability/teleoperation platform for deployed robots, not a CI/testing or simulation framework; nothing in the evidence relates to running automated tests or simulation-based regression testing in CI pipelines. This is a category mismatch rather than a missing feature.", "evidenceIds": [] }, { "productId": "formant", "storyId": "command-access-control", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers telemetry, commands, teleoperation, analytics, and fleet orchestration but contains no mention of roles, permissions, access control, or user/team management features that would let an ops lead restrict who can command, configure, or view robots.", "evidenceIds": [] }, { "productId": "formant", "storyId": "community-modules", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Formant provides SDKs, agent installation, and custom module/view building, but there is no evidence of a registry or marketplace of reusable packages for common hardware/behaviors that developers can pull from — modules are custom-built per app rather than shared/discoverable community assets.", "evidenceIds": [] }, { "productId": "formant", "storyId": "data-capture-sync", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Formant's on-robot agent (Docker install, Agent SDK) captures telemetry and sensor data and syncs it to the cloud via the ingestion/query APIs (formant-docs-1, formant-docs-2, formant-docs-3, formant-docs-27), and each data stream can be rate-limited to control bandwidth usage (formant-docs-31), directly supporting bandwidth-aware policies. missing for 10: explicit documentation of offline buffering/store-and-forward behavior during connectivity loss, and independent/hands-on validation of bandwidth policy effectiveness beyond vendor docs.", "evidenceIds": [ "formant-docs-1", "formant-docs-2", "formant-docs-3", "formant-docs-27", "formant-docs-31" ] }, { "productId": "formant", "storyId": "data-query-export", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Formant provides a Cloud SDK/query API for pulling telemetry data for offline analysis (formant-docs-3, formant-docs-27) and Analytics with custom SQL queries for aggregating/analyzing device data over time (formant-docs-7, formant-docs-25, formant-docs-16). Missing for 10: a documented OpenAPI/swagger spec (probe found 404s) and independent third-party corroboration of the export workflow beyond first-party docs.", "evidenceIds": [ "formant-docs-3", "formant-docs-27", "formant-docs-7", "formant-docs-25", "formant-docs-16", "formant-probe-2" ] }, { "productId": "formant", "storyId": "estop-safety-patterns", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers teleoperation, commands, telemetry, fleet orchestration, and analytics, but contains no mention of emergency-stop or safety-interlock patterns, safety certifications, or documented safety-critical control flows. Since fleet-management/robotics-ops platforms are a fair category for this axis, absence of any e-stop/safety-interlock documentation yields 'none'.", "evidenceIds": [] }, { "productId": "formant", "storyId": "fleet-alerts", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Formant supports alerting-related capabilities: 'Set up alerts for anomalies... respond to issues before they become downtime' and webhook triggers from events, plus fleet-wide visibility and analytics dashboards that could surface battery/connectivity/fault data. However, there's no explicit documentation naming battery level, connectivity loss, or fault-code alerting specifically, nor detail on notification channels (email/SMS/Slack) or configurable thresholds. missing for 10: explicit mention of battery/connectivity/fault-specific alert types, notification channel/escalation details, independent/hands-on confirmation of alerting reliability.", "evidenceIds": [ "formant-docs-5", "formant-docs-28", "formant-docs-18", "formant-docs-25" ] }, { "productId": "formant", "storyId": "fleet-config-rollout", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Formant supports device configuration (formant-docs-14/26), custom commands via Admin API (formant-docs-4/9), and multi-device 'coherence group views' (formant-gh-2) plus managing multiple fleets from one view (formant-docs-22), suggesting some grouped/bulk operational capability. However there is no explicit documentation of pushing configuration changes in bulk with per-group targeting as a named feature. Missing for 10: explicit bulk config-push workflow, documented per-group targeting UI/API, and independent confirmation of at-scale config rollout.", "evidenceIds": [ "formant-docs-4", "formant-docs-9", "formant-docs-14", "formant-docs-26", "formant-gh-2", "formant-docs-22" ] }, { "productId": "formant", "storyId": "fleet-dashboard", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Formant's documentation and marketing explicitly describe fleet-wide visibility ('instant visibility into every robot in your fleet', 'current location and status', unified dashboard across vendors, anomaly alerts), directly matching the ops-lead story of monitoring health, location, and status from one dashboard. Missing for 10: independent/hands-on third-party validation of the dashboard experience beyond vendor docs.", "evidenceIds": [ "formant-docs-5", "formant-docs-19", "formant-docs-24", "formant-docs-25", "formant-docs-22" ] }, { "productId": "formant", "storyId": "hardware-abstraction", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Formant advertises connecting to 'multiple robot types with minimal lift' with uniform, searchable data ingestion, and unifying different manufacturers' hardware (AMRs, quadrupeds, arms) into one dashboard/data model, which is a coarse form of hardware abstraction at the fleet-telemetry/command level. However, this is about fleet monitoring and teleop integration, not a documented hardware-abstraction API layer that lets a robotics engineer's application code swap out individual sensors/actuators without code changes — there is no driver-interface or HAL SDK evidence for this. Missing for 10: a documented sensor/actuator driver abstraction layer, evidence of swapping specific hardware components without app-code changes, and independent/hands-on confirmation of this working across heterogeneous hardware.", "evidenceIds": [ "formant-docs-10", "formant-docs-19", "formant-docs-20", "formant-gh-1" ] }, { "productId": "formant", "storyId": "intervention-workflows", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Formant supports alerting on anomalies, dispatching help when 'something looks wrong,' and time-synced 'intervention records,' plus teleoperation for operators to take direct control, suggesting an intervention workflow exists. However there is no explicit documentation of a dedicated human-intervention queue, assignment/resolution workflow, or an automated hand-back-to-autonomy mechanism after operator resolution. Missing for 10: explicit intervention-queue UI/workflow docs, operator resolution/hand-back-to-autonomy mechanics, and independent confirmation of this specific failure-routing flow.", "evidenceIds": [ "formant-docs-5", "formant-docs-13", "formant-docs-17", "formant-docs-18", "formant-docs-21", "formant-docs-30" ] }, { "productId": "formant", "storyId": "ml-training-loop", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Formant provides data export via the Cloud SDK for querying/collecting telemetry (formant-docs-3, formant-docs-27) and a commands mechanism that can trigger scripts or actions on robots (formant-docs-4, formant-docs-9, formant-docs-29), which could theoretically be used to push a retrained model or trigger redeployment. However, there is no explicit documentation of a model-training integration or a dedicated 'deploy model to fleet' feature — the closest is generic 'AI assistant' and 'ML-driven task recommendations' language which is not about custom model training/deployment. Missing for 10: explicit model training pipeline integration, a documented model artifact deployment/update mechanism, and evidence of round-tripping trained models back onto devices.", "evidenceIds": [ "formant-docs-3", "formant-docs-27", "formant-docs-4", "formant-docs-9", "formant-docs-29", "formant-docs-12" ] }, { "productId": "formant", "storyId": "multi-language-sdks", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Formant offers a Python Cloud SDK, an Agent SDK, and a JS/TS-based RealtimeSDK/toolkit for building custom apps, but there is no evidence of official Go or C++ SDKs, nor any claim that these are strongly typed or cover the platform's 'full surface' (fleet orchestration, analytics, teleop APIs are referenced mostly via REST/Admin API, not dedicated SDKs). missing for 10: Go SDK, C++ SDK, explicit typing guarantees, and full API-surface coverage beyond telemetry/commands/data export.", "evidenceIds": [ "formant-docs-2", "formant-docs-3", "formant-docs-27", "formant-gh-1", "formant-docs-4" ] }, { "productId": "formant", "storyId": "nl-to-robot-task", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Formant advertises a natural-language interface where users can 'ask your fleet what's happening in plain language and get governed, ML-driven task recommendations,' but execution explicitly requires human approval ('with people approving what happens next'), not autonomous platform planning/execution as the story describes. This is a marketing-page mention with no technical documentation of the underlying NL-to-plan pipeline or execution semantics. Missing for 10: technical docs on the NL planning engine, evidence of autonomous execution without human gating, and independent/hands-on verification of the feature.", "evidenceIds": [ "formant-docs-12", "formant-docs-23" ] }, { "productId": "formant", "storyId": "offline-resilience", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes the Formant agent, telemetry ingestion, and data rate-limiting, but nothing states that robots continue operating autonomously or buffer data locally during a cloud connectivity outage and then resync on reconnection. This is a reasonable expectation for an edge agent supporting fleet operations, but no documentation or GitHub evidence confirms offline buffering/recovery behavior.", "evidenceIds": [ "formant-docs-1", "formant-docs-2", "formant-docs-31" ] }, { "productId": "formant", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Formant exposes broad API/SDK surfaces (Agent SDK, Cloud SDK, Admin API, RealtimeSDK, webhooks, custom commands) covering telemetry, teleop, commands, and data export, and the toolkit lets developers embed/build custom apps mirroring UI functionality. However, some UI-centric features (fleet orchestration dashboards, AI assistant chat, analytics dashboard builder, alerting UI) are not clearly exposed as documented API endpoints, and no public OpenAPI spec was found (probe returned 404s), making full UI-parity via API unverifiable. missing for 10: a published OpenAPI/API reference proving full endpoint parity, API access to fleet-orchestration/AI-assistant/analytics-dashboard features, and independent confirmation of complete UI-to-API parity.", "evidenceIds": [ "formant-docs-2", "formant-docs-3", "formant-docs-4", "formant-gh-1", "formant-gh-4", "formant-docs-27", "formant-docs-28", "formant-probe-2" ] }, { "productId": "formant", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Formant provides a documented Python Cloud SDK and query API specifically for exporting telemetry data, and an ingestion API for getting data in, suggesting some data portability. However, there is no explicit documentation of exporting *all* account data, no stated open-format guarantees (e.g., CSV/JSON/Parquet), and no discussion of account closure/data portability policy. Missing for 10: explicit 'export all data' workflow, named open file formats, and confirmation of no vendor lock-in upon leaving.", "evidenceIds": [ "formant-docs-3", "formant-docs-27", "formant-docs-4" ] }, { "productId": "formant", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only a client-side toolkit/SDK repo (FormantIO/toolkit) for building integrations against Formant's APIs, not the platform's own source code, and no license is mentioned anywhere. Formant is a closed commercial SaaS robotics platform with no indication its core source is open or licensed for reading.", "evidenceIds": [ "formant-gh-1", "formant-gh-3", "formant-gh-4" ] }, { "productId": "formant", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Formant is a cloud-hosted SaaS platform; the evidence shows only an on-device agent (Docker-installed) and SDKs that connect to Formant's cloud, with no documentation of a self-hostable core platform/backend. No mention of on-prem or self-hosted deployment options anywhere in the docs or GitHub toolkit.", "evidenceIds": [ "formant-docs-1", "formant-docs-2", "formant-docs-14", "formant-probe-1" ] }, { "productId": "formant", "storyId": "ota-deployment", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers agent installation, telemetry ingestion, commands, teleoperation, analytics, and fleet orchestration, but nothing describes OTA software update deployment, staged rollouts, or rollback mechanisms for robot fleets. This is a plausible capability for a robot fleet management platform, so absence of evidence yields 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "formant", "storyId": "physics-simulation", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Formant is a fleet operations/observability/teleoperation platform for real robots, not a physics simulation tool; nothing in the evidence pack relates to simulating robots or environments with physics/sensor models. This axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "formant", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, region selection, or geographic storage controls for Formant's cloud platform; all evidence covers robot fleet management, telemetry SDKs, and orchestration features. missing for 10: any mention of data region/residency options, compliance certifications tied to geography, or configurable storage location.", "evidenceIds": [] }, { "productId": "formant", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items mention AI training data policies, opt-outs, or data usage terms regarding model training; this is a robotics fleet management platform whose evidence never addresses this privacy-posture question at all.", "evidenceIds": [] }, { "productId": "formant", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items address data retention policies, data deletion controls, or privacy/compliance configuration options for AI-native users; the docs focus on telemetry ingestion, teleoperation, fleet orchestration, and SDKs. missing for 10: retention policy documentation, data deletion/export controls, privacy/compliance settings, user-facing controls for data lifecycle.", "evidenceIds": [] }, { "productId": "formant", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Formant is a robotics fleet management platform whose core function is telemetry ingestion; no evidence in the pack addresses any opt-out mechanism for telemetry/usage tracking related to the product's own analytics or data collection about users/AI-native usage.", "evidenceIds": [] }, { "productId": "formant", "storyId": "remote-debugging", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Formant provides remote observability via the Agent SDK ingesting telemetry, an Observe view for live data, Analytics for historical/aggregated queries with custom SQL, and ROS/Docker agent installation for deployed robots, all matching remote debugging of topics/logs/state. missing for 10: no explicit mention of raw log tailing or independent third-party hands-on validation of the debugging workflow.", "evidenceIds": [ "formant-docs-1", "formant-docs-2", "formant-docs-5", "formant-docs-16", "formant-docs-25", "formant-docs-31" ] }, { "productId": "formant", "storyId": "robot-bring-up", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Formant provides quick agent installation (Docker/ROS2), an SDK for telemetry/commands, and low-latency teleoperation that can get a robot reporting data and driving within a short setup window, which speaks to fast bring-up of the software/fleet layer. However, it doesn't provide actual robot driver development or low-level hardware integration — it assumes drivers already exist and layers fleet ops on top — and there's no explicit 'hours not weeks' benchmark or case study demonstrating time-to-first-motion.\nmissing for 10: evidence of driver creation/hardware integration tooling, a concrete time-to-bring-up case study or benchmark, first-motion/actuation examples beyond teleop.", "evidenceIds": [ "formant-docs-1", "formant-docs-2", "formant-docs-4", "formant-docs-10", "formant-docs-15", "formant-docs-21" ] }, { "productId": "formant", "storyId": "sensor-integration", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Formant's docs show a ROS 2 agent install path and a generic Agent/Cloud SDK for ingesting telemetry, plus multi-robot-type connectivity claims, which implies some level of standard-message ingestion (e.g., via ROS topics) but never explicitly names ready-made drivers or standard message types for cameras, lidars, IMUs, or GPS. Missing for 10: explicit driver/adapter documentation per sensor type (camera, lidar, IMU, GPS), confirmation of standard message-type support (e.g., sensor_msgs), and any hands-on evidence of plug-and-play sensor bring-up.", "evidenceIds": [ "formant-docs-1", "formant-docs-10", "formant-docs-2", "formant-docs-27" ] }, { "productId": "formant", "storyId": "sim-to-real", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Formant is a fleet operations/observability/teleop platform for real robots, not a simulation environment or robot-code framework; the story concerns simulation-to-real code portability, which is outside Formant's product category entirely.", "evidenceIds": [] }, { "productId": "formant", "storyId": "synthetic-data-rl", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Formant is a robot fleet operations, telemetry, and teleoperation platform for real-world deployed robots—not a simulation or RL training platform. Synthetic data generation and simulation-based RL are entirely outside its product category (wrong axis).", "evidenceIds": [] }, { "productId": "formant", "storyId": "teleoperation", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Formant explicitly documents low-latency, peer-to-peer teleoperation with joystick/button control, configurable teleop interfaces, and secure remote control over real-world networks, plus live video/sensor visualization via Analytics and Observe views. Missing for 10: independent/hands-on network-performance validation and detail on video streaming latency/quality under adverse network conditions beyond vendor claims.", "evidenceIds": [ "formant-docs-6", "formant-docs-15", "formant-docs-21", "formant-docs-11", "formant-docs-29", "formant-docs-25" ] }, { "productId": "formant", "storyId": "vla-foundation-models", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Formant's docs describe an Agent SDK/Cloud SDK for telemetry ingestion, commands, and teleoperation, plus AI-assisted dashboards/orchestration, but there is no evidence of tooling specifically for integrating vision-language-action or robotics foundation models into a robot's autonomy stack.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "agent-controls-robot", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence describes Gazebo as a physics simulator with C++/plugin APIs, transport messaging, and CLI tools for simulated robots, but there is no mention of authentication, scoped credentials, or any mechanism for an AI agent to command a real (non-simulated) robot end-to-end via an API/SDK. Probes for API/auth surfaces (llms.txt, openapi.json) also returned 404s.", "evidenceIds": [ "gazebo-gh-6", "gazebo-gh-9", "gazebo-gh-5", "gazebo-gh-8", "gazebo-probe-3" ] }, { "productId": "gazebo", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Direct probes show no llms.txt (404), no markdown-alt docs, and no OpenAPI spec, indicating no agent-oriented documentation format is provided; no evidence of any agent-discoverable docs endpoint.", "evidenceIds": [ "gazebo-probe-1", "gazebo-probe-2", "gazebo-probe-3" ] }, { "productId": "gazebo", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gazebo is a robotics simulation engine, not a data analytics or AI-insights product; nothing in the evidence pack relates to AI-generated insights/suggestions from user data, and this axis is a category error for a simulator.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "agentic-autonomous-automation", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gazebo is a robotics simulator, not an automation/agent-orchestration platform; the concept of setting up autonomous background 'automations' (e.g., scheduled agent tasks) is a category error for this product type. Its plugins and headless server mode run simulations, not user-defined autonomous automations.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gazebo is a robotics simulation engine, not an AI assistant product; there is no built-in AI assistant concept applicable to this category, and no evidence suggests one exists.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Gazebo explicitly supports headless server-only operation via the -s flag and even offers a dedicated lightweight server-only package (gz-sim-server) without GUI/Qt dependencies, plus CLI tools and TCP/IP transport for remote/automated control, all of which are well suited to CI pipelines. Missing for 10: explicit first-party CI/CD example (e.g., GitHub Actions workflow) or independent report confirming headless CI usage in practice.", "evidenceIds": [ "gazebo-docs-2", "gazebo-docs-4", "gazebo-docs-12", "gazebo-gh-3", "gazebo-gh-6" ] }, { "productId": "gazebo", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gazebo is a robotics simulator, not an AI agent or MCP-capable platform; there is no evidence of MCP server integration and this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Gazebo is a robotics simulator, not an AI agent, so it is a plausible candidate to expose an official MCP server for agent integration, but no evidence in the pack mentions MCP, agent connectivity, or any AI-agent integration protocol at all.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Gazebo is a robotics simulator controlled via CLI tools, SDF files, and C++ plugins; no evidence of any natural-language command interface, LLM integration, or agentic control layer is present in the evidence pack.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "agentic-official-cli", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Gazebo ships an official `gz` CLI with documented commands (e.g. `gz sim shapes.sdf`), headless/server-only mode (`-s` flag) and server-only packages that are ideal for scripted/automated (agentic) invocation without a GUI, plus GitHub docs calling out 'extensive command line tools for increased simulation introspection and control'. Missing for 10: explicit framing of the CLI for AI-agent orchestration use-cases and independent (non-vendor) corroboration of CLI usage in automated pipelines.", "evidenceIds": [ "gazebo-docs-10", "gazebo-docs-2", "gazebo-docs-11", "gazebo-docs-4", "gazebo-gh-3", "gazebo-gh-10" ] }, { "productId": "gazebo", "storyId": "agentic-public-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Gazebo exposes documented, non-GUI ways to drive the simulator programmatically: a headless server mode (-s flag), gz-transport for socket-based TCP/IP message passing to remote-control simulations, extensive CLI tools for introspection/control, and C++ library APIs (gz-math, sdformat, gz-transport) for building custom integrations/plugins. However, probes for a modern machine-readable API spec (OpenAPI/swagger) and an llms.txt all returned 404, and there is no evidence of a REST/HTTP API or AI-specific SDK. missing for 10: OpenAPI/swagger spec or any public HTTP API, explicit AI-agent-oriented API docs or llms.txt, independent confirmation that an AI agent can drive Gazebo via gz-transport.", "evidenceIds": [ "gazebo-docs-11", "gazebo-gh-6", "gazebo-gh-10", "gazebo-docs-14", "gazebo-probe-1", "gazebo-probe-3" ] }, { "productId": "gazebo", "storyId": "agentic-scoped-keys", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gazebo is a robotics simulator with no API-credential/authentication system for agents; scoped API credential issuance is not a concept that applies to this product category.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Gazebo documents genuine official SDKs/libraries (gz-math, gz-transport, sdformat) and a plugin API for building custom C++ applications/plugins, with a dedicated 'Developers' docs section (gazebo-docs-1/8/14/16). However, evidence shows only C++-centric libraries with no Python or other language SDK bindings, no AI-agent-oriented SDK, and probes show no llms.txt or OpenAPI spec, suggesting no AI-native tooling layer. Missing for 10: language bindings beyond C++/plugins, an AI-agent-specific SDK or API spec, independent developer corroboration of SDK usability.", "evidenceIds": [ "gazebo-docs-1", "gazebo-docs-14", "gazebo-docs-16", "gazebo-gh-1", "gazebo-docs-6", "gazebo-probe-1", "gazebo-probe-3" ] }, { "productId": "gazebo", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of webhook subscription support; Gazebo uses TCP/IP transport, plugins, and CLI tools for integration, but nothing in the evidence pack mentions webhooks or event push notifications.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an interactive API reference or runnable examples; probes for llms.txt, markdown docs, and OpenAPI specs all returned 404, and the docs are static C++ library documentation without interactive tooling.", "evidenceIds": [ "gazebo-probe-1", "gazebo-probe-2", "gazebo-probe-3" ] }, { "productId": "gazebo", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Gazebo is a robotics simulator, not an HTTP API service, and probes explicitly show no OpenAPI/swagger endpoints or llms.txt exist. No evidence of any machine-readable API spec being offered.", "evidenceIds": [ "gazebo-probe-3", "gazebo-probe-1", "gazebo-probe-2" ] }, { "productId": "gazebo", "storyId": "api-sandbox", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Gazebo is fundamentally a robot/environment simulator that lets users test robots, sensors, and physics in a virtual sandbox instead of on real (production) hardware — it can run headless or on remote servers via TCP/IP transport, and supports numerous pre-built simulation models via Gazebo Fuel, letting an AI-native user iterate safely without touching production systems. Missing for 10: explicit framing around isolating from 'production data' (e.g., data provenance, reset/snapshot guarantees) and no first-party mention of AI/agent-specific sandbox workflows.", "evidenceIds": [ "gazebo-docs-2", "gazebo-docs-4", "gazebo-gh-5", "gazebo-gh-6", "gazebo-gh-2" ] }, { "productId": "gazebo", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of a versioned API policy or documented deprecation schedule; ROS/Gazebo version compatibility docs mention supporting multiple versions but don't describe an API deprecation policy, and API probes (openapi, llms.txt) all 404. Missing for 10: any explicit versioning/deprecation policy documentation, API changelogs, or stability guarantees for AI/agentic consumers.", "evidenceIds": [ "gazebo-docs-9", "gazebo-docs-13", "gazebo-probe-3", "gazebo-probe-1" ] }, { "productId": "gazebo", "storyId": "automation-bulk-operations", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gazebo is a robotics simulator; 'bulk operations across many items' (e.g. batch editing of records/tickets/documents) is not a fair axis for this product category — it has no notion of item collections to bulk-operate on, only simulation entities/plugins/CLI tools for simulation control.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Gazebo's evidence shows a plugin architecture for developers to extend robot/sensor/environment behavior in C++, but there is no documented rule-based trigger-action automation system for AI-native/non-developer users to define event-driven rules declaratively.", "evidenceIds": [ "gazebo-gh-1", "gazebo-gh-7", "gazebo-docs-17" ] }, { "productId": "gazebo", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gazebo is a robotics simulator, not a workflow/task automation platform; scheduling recurring jobs is outside its product category (wrong axis).", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gazebo is a robotics simulator, not an automation/workflow tool with versioned automations to review or roll back; this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "ci-sim-testing", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Gazebo supports headless server-only execution and CLI-driven simulation control, which are prerequisites for CI-based regression testing, but there is no evidence of a documented CI integration, test harness, or automated regression-test framework/examples. missing for 10: CI pipeline examples (e.g. GitHub Actions), automated test/assertion framework for simulation regression, documented headless test workflows, third-party corroboration of CI usage.", "evidenceIds": [ "gazebo-docs-2", "gazebo-docs-4", "gazebo-gh-3", "gazebo-gh-6" ] }, { "productId": "gazebo", "storyId": "command-access-control", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any role-based access control, user permissions, or authentication system in Gazebo; it is a robotics simulator with no mention of RBAC or multi-user access governance.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "community-modules", "verdict": "full", "quality": 6, "confidence": "medium", "rationale": "Gazebo Fuel is documented as a registry/ecosystem from which developers can pull ready-made robot models (PR2, TurtleBot, etc.) and physically accurate environment models, and the docs show copying SDF snippets directly from a model's Fuel page into a project; plugins extend hardware/behavior reuse further. missing for 10: no evidence of versioning/dependency management, search/discovery tooling details, or independent hands-on corroboration of the Fuel registry experience.", "evidenceIds": [ "gazebo-gh-5", "gazebo-gh-8", "gazebo-docs-5", "gazebo-gh-1", "gazebo-gh-7" ] }, { "productId": "gazebo", "storyId": "data-capture-sync", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gazebo is a robotics simulator, not an on-robot data-capture/cloud-sync product; there is no on-robot deployment or bandwidth-aware cloud sync concept in its evidence, and this capability is outside the simulator's category.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "data-query-export", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any SQL database, data export tool, or query API for offline analysis of collected robot/simulation data; evidence only covers transport messaging, plugins, and CLI simulation control, and API probes for OpenAPI/docs endpoints returned 404.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "estop-safety-patterns", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "While Gazebo supports custom plugins for robot/sensor/environment control and command-line introspection tools that could theoretically be used to build safety logic, none of the evidence mentions emergency-stop, safety-interlock, or any documented safety pattern/support specifically. missing for 10: any documentation or example referencing e-stop or safety-interlock patterns, guidance on implementing safety-critical control logic, or built-in safety plugin APIs.", "evidenceIds": [ "gazebo-gh-1", "gazebo-gh-7", "gazebo-gh-10", "gazebo-docs-1" ] }, { "productId": "gazebo", "storyId": "fleet-alerts", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gazebo is a robotics simulator, not a fleet-management/monitoring platform for physical robots; alerting on real-world battery, connectivity, or fault status of deployed robots is outside its product category.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "fleet-config-rollout", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gazebo is a robotics simulator, not a fleet-management or device-management platform; it has no concept of deploying configuration to physical robot fleets grouped by tags/groups. This is a category mismatch, not a missing feature.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "fleet-dashboard", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gazebo is a robotics simulator, not a fleet-management/monitoring platform; the evidence pack is entirely about simulation, plugins, physics engines, and CLI tooling with no fleet dashboard concept. This story is a category error for a simulation tool.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "hardware-abstraction", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Gazebo's plugin architecture lets robot, sensor, environment, and physics-engine components be swapped as plugins (gazebo-gh-1/7, gazebo-gh-2, gazebo-docs-15), and simulated hardware is accessed via SDF/gz-transport rather than hard-coded app logic, which supports a hardware-abstraction-like workflow. However, the evidence never explicitly discusses interface stability guarantees or confirms application code is untouched when swapping sensors/actuators. Missing for 10: explicit documentation of stable/versioned sensor-actuator interface contracts, and case evidence that application code needed zero changes across a hardware swap.", "evidenceIds": [ "gazebo-gh-1", "gazebo-gh-7", "gazebo-gh-2", "gazebo-docs-15", "gazebo-gh-5" ] }, { "productId": "gazebo", "storyId": "intervention-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gazebo is a robotics simulator, not a teleoperation/monitoring platform with human-in-the-loop intervention workflows; there is no concept of routing autonomy failures to a human operator queue. This story is a category error for a simulation engine.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "ml-training-loop", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gazebo is a robotics simulator, not an ML training/deployment pipeline tool; there is no concept of feeding data into model training and deploying models to a fleet within its scope. This story is a category error for a simulation platform.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "multi-language-sdks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows Gazebo is primarily a C++ library/simulator with plugin APIs, ROS integration, and CLI tools, but there is no mention of official typed SDKs in Python, TypeScript, or Go, nor any unified multi-language SDK covering the platform surface. Missing for 10: Python/TypeScript/Go SDK docs, typed client libraries, and evidence of full API-surface coverage across languages.", "evidenceIds": [ "gazebo-docs-1", "gazebo-docs-6", "gazebo-gh-1" ] }, { "productId": "gazebo", "storyId": "nl-to-robot-task", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gazebo is a robotics simulation engine (physics, SDF models, plugins, GUI/CLI tools) with no natural-language task planning or LLM-driven execution layer; this story concerns AI-native NL-to-action planning which is outside its category.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "offline-resilience", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gazebo is a robotics simulator/development tool, not a fleet operations or cloud-connectivity product; offline buffering and cloud-reconnect recovery for deployed robots is outside its scope as evidenced by docs (physics engine, GUI, plugins, transport tools).", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Gazebo can run fully headless via the `-s` server flag and exposes CLI tools plus gz-transport for programmatic introspection/control, suggesting most simulation functionality is scriptable outside the GUI. However, there is no explicit documentation claiming full feature parity between the GUI (which is plugin-based and includes visualization-only components) and the API/CLI surface. Missing for 10: an explicit statement or evidence that every GUI capability (especially visualization/plugin-specific GUI features) is exposed via API/CLI, and any third-party confirmation of parity.", "evidenceIds": [ "gazebo-docs-2", "gazebo-docs-3", "gazebo-docs-11", "gazebo-gh-3", "gazebo-gh-9", "gazebo-docs-7" ] }, { "productId": "gazebo", "storyId": "openness-full-export", "verdict": "na", "quality": 0, "confidence": "medium", "rationale": "Gazebo is an open-source, locally-installed robotics simulator rather than a hosted data platform that accumulates user data subject to lock-in; there is no 'account data' or walled garden to export from before leaving. The evidence pack discusses plugins, CLI tools, physics engines, and ROS integration, none of which relate to a data-export/exit story, making this axis a category mismatch for this product type.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "openness-open-license", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Evidence confirms the source code is publicly hosted on GitHub (github.com/gazebosim/gz-sim) with docs describing internals, implying open availability, but no citation explicitly states an open-source license (e.g., Apache-2.0) or links a LICENSE file. Missing for 10: explicit license text/citation, confirmation of license type, and any independent verification of licensing terms.", "evidenceIds": [ "gazebo-gh-1", "gazebo-gh-4", "gazebo-docs-1" ] }, { "productId": "gazebo", "storyId": "openness-self-host", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Gazebo is open-source, self-installable software with documented server-only installs (gz-sim10-server package, headless -s flag) and remote TCP/IP transport for running simulation on remote servers, enabling full self-hosting of the core simulation engine. missing for 10: no explicit containerized/docker self-hosting guide or independent third-party confirmation of a full self-hosted deployment at scale.", "evidenceIds": [ "gazebo-docs-2", "gazebo-docs-4", "gazebo-docs-11", "gazebo-docs-12", "gazebo-gh-6", "gazebo-gh-9" ] }, { "productId": "gazebo", "storyId": "ota-deployment", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gazebo is a robotics simulator, not a fleet management/OTA deployment tool; deploying software updates to physical robots with staged rollouts and rollback is entirely outside its product category.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "physics-simulation", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Gazebo's core purpose is physics-and-sensor simulation: docs/GitHub evidence confirms multiple physics engines (ODE, Bullet, DART), plugin-based sensor and robot control, access to physically accurate robot/environment models via Fuel, and SDF-based world/robot description before hardware deployment. Missing for 10: independent/hands-on validation of sensor model realism (e.g. lidar/camera noise fidelity) and no third-party benchmark or case study corroborating physics accuracy claims beyond vendor docs.", "evidenceIds": [ "gazebo-gh-2", "gazebo-docs-15", "gazebo-gh-1", "gazebo-gh-5", "gazebo-gh-4", "gazebo-docs-5" ] }, { "productId": "gazebo", "storyId": "privacy-data-residency", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gazebo is an open-source robotics simulator run locally/self-hosted, not a cloud data-storage or SaaS service with region/residency options; this axis is a category error for this type of product.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gazebo is an open-source robotics simulator run locally/self-hosted; it is not an AI model provider or SaaS collecting user data for training, so an AI-training data opt-out policy is a category error for this product.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "privacy-retention-controls", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gazebo is a robotics simulator, not a data-processing/AI service that stores user data requiring retention/deletion controls; this privacy-posture axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "privacy-telemetry-optout", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gazebo is an open-source, locally-run robotics simulator with no telemetry or cloud tracking service described anywhere in the evidence; the privacy-opt-out axis is a category error for a self-hosted open-source simulation tool.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "remote-debugging", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Gazebo provides remote TCP/IP transport (gz-transport) for interfacing with simulations over the network, a headless server mode, and extensive CLI tools for 'simulation introspection and control', which could plausibly extend to inspecting topics/state on a running (simulated or transport-connected) robot. However, none of the evidence explicitly addresses inspecting logs or state on a physically deployed robot, or dedicated remote debugging/monitoring dashboards. Missing for 10: log inspection tooling, explicit remote/robot (non-simulated) debugging workflow, and independent hands-on confirmation of remote monitoring in production robot deployments.", "evidenceIds": [ "gazebo-gh-6", "gazebo-gh-9", "gazebo-gh-10", "gazebo-gh-3", "gazebo-docs-2", "gazebo-docs-11", "gazebo-gh-4" ] }, { "productId": "gazebo", "storyId": "robot-bring-up", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Gazebo provides reusable robot/sensor plugins, Fuel-hosted models (PR2, TurtleBot, etc.), SDF snippet copy-paste, and CLI tools that could speed up assembling a simulated robot and getting first motion quickly, but there is no first-party or independent evidence of an actual hours-vs-weeks bring-up workflow, driver integration, or hardware configuration case study. missing for 10: quantified bring-up time claims, driver/hardware configuration workflow evidence, independent case study or hands-on report of a new robot going from zero to first motion", "evidenceIds": [ "gazebo-gh-5", "gazebo-gh-7", "gazebo-docs-5", "gazebo-gh-10", "gazebo-docs-17" ] }, { "productId": "gazebo", "storyId": "sensor-integration", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence confirms Gazebo supports sensor/robot plugins generally (gazebo-gh-7) and ROS integration via ros_gz (gazebo-docs-9, gazebo-docs-13, gazebo-docs-18), implying standard message compatibility, but no evidence specifically names ready-made camera, lidar, IMU, or GPS driver plugins or standard message types used. missing for 10: explicit documentation of built-in sensor plugin types (camera/lidar/IMU/GPS), confirmation of standard ROS/Gazebo message schemas for each, and hands-on verification of sensor bring-up.", "evidenceIds": [ "gazebo-gh-7", "gazebo-docs-9", "gazebo-docs-13", "gazebo-docs-18" ] }, { "productId": "gazebo", "storyId": "sim-to-real", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Gazebo's plugin/transport architecture and ROS integration (ros_gz, TCP/IP transport, command-line/introspection tools) suggest that robot control code communicating via standard ROS topics/services could target both simulated and real robots, but no evidence explicitly describes a sim-to-real code-reuse workflow or hardware-in-the-loop testing. missing for 10: explicit documentation or case study of running identical control code against simulated and physical robots, no independent corroboration of a no-rewrite workflow.", "evidenceIds": [ "gazebo-gh-6", "gazebo-gh-9", "gazebo-docs-6", "gazebo-docs-9", "gazebo-gh-7" ] }, { "productId": "gazebo", "storyId": "synthetic-data-rl", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Gazebo provides infrastructure that supports large-scale simulation runs: headless server-only mode, a lightweight server-only package, TCP/IP transport for remote servers, and CLI tools for introspection/control, which together enable running many simulation instances programmatically. However, there is no direct evidence of built-in synthetic data generation pipelines, RL training integrations, or scaling/orchestration tooling specifically for RL workflows. Missing for 10: explicit RL framework integration (e.g. gym-style APIs), documented synthetic dataset export tooling, and evidence of distributed/parallel scaling for training at scale.", "evidenceIds": [ "gazebo-docs-2", "gazebo-docs-4", "gazebo-gh-6", "gazebo-gh-9", "gazebo-gh-10", "gazebo-docs-12" ] }, { "productId": "gazebo", "storyId": "teleoperation", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gazebo is a robotics simulator, not a real-world teleoperation platform; its evidence is about simulating robots and remote servers running simulations, not operating a physical robot over real-world networks with live video feed. This is a wrong-axis story for a simulation tool.", "evidenceIds": [] }, { "productId": "gazebo", "storyId": "vla-foundation-models", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers Gazebo's general plugin architecture, transport, physics engines, and ROS integration, but there is no mention of VLA models, robotics foundation models, or any supported tooling/SDK for integrating such models into an autonomy stack. Nothing in the pack ties Gazebo to AI/foundation-model integration workflows.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "agent-controls-robot", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Isaac docs describe simulation, synthetic data, ROS 2 integration, and Python packages, but nothing addresses AI agent authentication with scoped credentials or an API/SDK contract for an agent to command a real robot end-to-end. Probe results show no OpenAPI/API discovery artifacts either. missing for 10: scoped-credential authentication mechanism, agent-facing command API/SDK for real (non-simulated) robots, any security/authorization model for agent access.", "evidenceIds": [ "nvidia-isaac-docs-4", "nvidia-isaac-docs-7", "nvidia-isaac-docs-12", "nvidia-isaac-probe-3" ] }, { "productId": "nvidia-isaac", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Direct probes show no llms.txt (404) and no markdown-friendly docs endpoint (404), and no evidence of agent-oriented documentation formats elsewhere in the pack.", "evidenceIds": [ "nvidia-isaac-probe-1", "nvidia-isaac-probe-2", "nvidia-isaac-probe-3" ] }, { "productId": "nvidia-isaac", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NVIDIA Isaac is a robotics simulation/training platform (simulation, synthetic data, robot learning frameworks), not a data analytics product that surfaces AI-generated insights/suggestions from a user's own data corpus; this story targets an analytics-copilot axis that doesn't fit this product category.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "agentic-autonomous-automation", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NVIDIA Isaac is a robotics simulation/training platform (simulation, synthetic data, robot learning, ROS integration), not a background-automation/workflow-agent tool; the notion of setting up autonomous background automations doesn't map to this product category.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NVIDIA Isaac is a robotics simulation/training platform, not a chat-style product with a built-in AI assistant for delegating general tasks; the axis of an in-app AI assistant for task delegation is a category error for this kind of product.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Isaac Lab explicitly advertises 'standalone headless operation, from workstation to data center' and Isaac Sim exposes Python namespace packages installable via pip and ROS 2 services/actions to script simulation steps, supporting automation-friendly usage. However, there's no explicit CI pipeline example, Docker/CI integration guide, or independent confirmation of headless CI runs for Isaac Sim itself. Missing for 10: concrete CI/CD examples, Docker headless run docs, independent verification of headless Isaac Sim usage in automated pipelines.", "evidenceIds": [ "nvidia-isaac-docs-10", "nvidia-isaac-docs-12", "nvidia-isaac-docs-4", "nvidia-isaac-docs-20" ] }, { "productId": "nvidia-isaac", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NVIDIA Isaac is a robotics simulation/training platform (simulation, ROS 2, synthetic data, robot learning), not an AI agent or assistant that consumes tools via MCP; no evidence pack content relates to MCP integration at all. This axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "agentic-mcp-server", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NVIDIA Isaac is a robotics simulation/training platform, not an agentic coding assistant; there is no evidence it acts as an MCP client or server, and the story's framing (connecting an agent via an official MCP server) is a category mismatch for this product type rather than a missing feature.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "NVIDIA Isaac is a robotics simulation/training platform (Isaac Sim, Lab, ROS, GR00T) with Python APIs, ROS 2 interfaces, and CAD/USD import, but no evidence shows a natural-language command interface for operating the product itself. GR00T models involve perception/action policies for robots, not for controlling the Isaac tooling via NL prompts.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence describes Python namespace packages installable via pip and various APIs, but there is no mention of a dedicated official CLI tool for driving Isaac Sim/Lab/ROS workflows in an AI-native/agentic manner. missing for 10: any documented CLI command reference, CLI installation instructions, or CLI-based automation examples.", "evidenceIds": [ "nvidia-isaac-docs-12", "nvidia-isaac-docs-20" ] }, { "productId": "nvidia-isaac", "storyId": "agentic-public-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Isaac Sim exposes a documented Python API (namespace packages installable via pip) and ROS 2 services/actions for loading worlds, spawning entities, and stepping simulations, which lets AI-native users drive the simulator programmatically. However, probes found no REST/OpenAPI-style public API (404s on openapi.json, swagger.json, llms.txt), so the 'documented public API' is limited to Python/ROS2 SDK-style access rather than a formal web API. Missing for 10: a REST/OpenAPI specification, independent/hands-on confirmation of API usage beyond vendor docs.", "evidenceIds": [ "nvidia-isaac-docs-4", "nvidia-isaac-docs-12", "nvidia-isaac-docs-20", "nvidia-isaac-probe-1", "nvidia-isaac-probe-3" ] }, { "productId": "nvidia-isaac", "storyId": "agentic-scoped-keys", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NVIDIA Isaac is a robotics simulation/training platform (simulation, synthetic data, robot learning), not an API/identity service that issues scoped credentials to agents; this credential-management axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "NVIDIA Isaac provides multiple official, documented SDKs (Isaac Sim Python packages installable via pip, Isaac Lab open-source framework, Isaac ROS packages, Isaac GR00T reference platform) with extensibility and integration points for custom pipelines, giving AI-native developers concrete official SDKs to build against. missing for 10: independent third-party hands-on corroboration of SDK usability/API stability and no official OpenAPI/llms.txt machine-readable spec (probes returned 404s).", "evidenceIds": [ "nvidia-isaac-docs-12", "nvidia-isaac-docs-9", "nvidia-isaac-docs-17", "nvidia-isaac-docs-7", "nvidia-isaac-docs-19", "nvidia-isaac-docs-16", "nvidia-isaac-gh-3" ] }, { "productId": "nvidia-isaac", "storyId": "agentic-webhooks", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NVIDIA Isaac is a robotics simulation/training platform, not an event-driven SaaS or automation product where webhook subscriptions would be a natural integration point; no evidence pack item relates to webhooks at all.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of an interactive API reference with runnable examples; probes for llms.txt, markdown docs, and OpenAPI specs all returned 404s, and no documentation item mentions runnable/interactive examples.", "evidenceIds": [ "nvidia-isaac-probe-1", "nvidia-isaac-probe-2", "nvidia-isaac-probe-3" ] }, { "productId": "nvidia-isaac", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "NVIDIA Isaac is a robotics simulation/training platform, not a web API service, but the probes explicitly checked for an OpenAPI spec and found only 404s at all candidate paths, and no docs mention a machine-readable API spec being available for download. Since probes were run confirming absence, this is 'none' rather than 'na'.", "evidenceIds": [ "nvidia-isaac-probe-3", "nvidia-isaac-probe-1", "nvidia-isaac-probe-2" ] }, { "productId": "nvidia-isaac", "storyId": "api-sandbox", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Isaac Sim itself is inherently a simulation sandbox that lets users test robot stacks, perception, and policies in synthetic/simulated environments (software-in-the-loop/hardware-in-the-loop testing) without touching real production robots or data, and it supports synthetic data generation as an alternative to production data. However, evidence doesn't explicitly frame this in 'sandbox vs production data' terms typical of SaaS/agentic tooling — it's inferred from simulation-first design. Missing for 10: explicit documentation framing simulation as a non-production/sandbox testing environment distinct from real deployment data, and independent hands-on confirmation that simulated data safely substitutes for production data in real workflows.", "evidenceIds": [ "nvidia-isaac-docs-26", "nvidia-isaac-docs-21", "nvidia-isaac-docs-2", "nvidia-isaac-docs-16" ] }, { "productId": "nvidia-isaac", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of versioned APIs with a documented deprecation policy; probes for API/OpenAPI specs returned 404s, and none of the docs mention API versioning or deprecation guarantees.", "evidenceIds": [ "nvidia-isaac-probe-3", "nvidia-isaac-probe-1", "nvidia-isaac-probe-2" ] }, { "productId": "nvidia-isaac", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Isaac Lab's GPU-optimized, CUDA-graphable simulation supports large-scale parallel training across many simulated robots/environments, and Isaac Sim's Replicator can generate synthetic data across many scenes at once, which functions as a form of bulk operation. However, there is no explicit documentation of a general 'bulk operate on many items' interface (e.g., batch API for arbitrary assets or tasks) beyond the training/data-generation context.\nmissing for 10: explicit bulk-operation API/CLI documentation, examples of batch processing arbitrary asset sets, independent verification of scale claims", "evidenceIds": [ "nvidia-isaac-docs-10", "nvidia-isaac-docs-3", "nvidia-isaac-docs-9" ] }, { "productId": "nvidia-isaac", "storyId": "automation-rules-engine", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NVIDIA Isaac is a robotics simulation/training platform (simulation, synthetic data, robot learning, ROS integration); defining event-triggered automation rules is a workflow-automation axis unrelated to its product category, with no evidence of such a rules/trigger engine.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Isaac is a robotics simulation/training platform (simulation, synthetic data, robot learning, ROS integration), not a workflow orchestration or job-scheduling product; recurring job/workflow scheduling is outside its category scope.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NVIDIA Isaac is a robotics simulation/training platform, not an automation/workflow tool with versionable 'automations' to review or roll back; this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "ci-sim-testing", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Isaac Sim/Lab supports headless operation, Python scripting/namespace packages, and explicit software-in-the-loop/hardware-in-the-loop validation of robot stacks with ROS 2, which are foundational building blocks for CI-based regression testing, but there is no documented CI pipeline integration (e.g., GitHub Actions, Jenkins) or automated test-runner/regression-suite tooling. missing for 10: explicit CI/CD integration examples, documented automated regression-test framework, evidence of headless simulation being invoked from a test harness.", "evidenceIds": [ "nvidia-isaac-docs-26", "nvidia-isaac-docs-10", "nvidia-isaac-docs-12", "nvidia-isaac-docs-25" ] }, { "productId": "nvidia-isaac", "storyId": "command-access-control", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Isaac is a robotics simulation/training/development platform (simulation, synthetic data, policy training, ROS integration); role-based access control for commanding/viewing robots is an operations/fleet-management concern outside this product's category, not something evidenced or expected here.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "community-modules", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "NVIDIA Isaac provides ready-to-use ROS 2 packages (Isaac ROS) for navigation, perception, computer vision, object detection and trajectory optimization, plus Isaac Sim namespace packages installable via pip, and Isaac Lab/GR00T for reusable learning frameworks and pretrained robot policies. These constitute a genuine ecosystem of reusable modules for common hardware/behaviors, backed by first-party docs and GitHub feature descriptions. Missing for 10: no evidence of a centralized package registry/marketplace with versioning or independent third-party corroboration of ecosystem breadth.", "evidenceIds": [ "nvidia-isaac-docs-7", "nvidia-isaac-docs-8", "nvidia-isaac-docs-27", "nvidia-isaac-docs-12", "nvidia-isaac-docs-9", "nvidia-isaac-docs-11", "nvidia-isaac-gh-3" ] }, { "productId": "nvidia-isaac", "storyId": "data-capture-sync", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NVIDIA Isaac is a simulation, ROS, and robot-learning platform; the story describes on-robot telemetry capture with cloud-sync bandwidth-aware policies, which is a fleet/cloud-ops data-pipeline concern outside Isaac's simulation/training/perception scope. No evidence pack items address cloud syncing or bandwidth-aware upload policies, confirming this is a different product category (fleet management/IoT), not a fair axis for a sim/training toolkit.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "data-query-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of any API or SQL interface for querying/exporting collected robot data for offline analysis; evidence covers simulation, synthetic data generation, and ROS integration, but no data query/export/analytics API or SQL access is documented. Probes also confirm no discoverable OpenAPI/API docs.", "evidenceIds": [ "nvidia-isaac-probe-3", "nvidia-isaac-probe-1", "nvidia-isaac-probe-2" ] }, { "productId": "nvidia-isaac", "storyId": "estop-safety-patterns", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers simulation, synthetic data, ROS 2 integration, robot learning, and hardware acceleration, but contains no mention of emergency-stop, safety-interlock, fault-detection, or safety-certification patterns documented or supported by the platform. Missing for 10: any documentation of e-stop mechanisms, safety-rated I/O, interlock design patterns, or safety validation workflows.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "fleet-alerts", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NVIDIA Isaac is a robotics simulation, training, and development platform (Isaac Sim, Isaac Lab, Isaac ROS, GR00T) — not a fleet-management/ops monitoring product. Fault/battery/connectivity alerting for deployed robot fleets is outside its category; the evidence is entirely about simulation, synthetic data, training pipelines, and ROS integration.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "fleet-config-rollout", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NVIDIA Isaac is a simulation/robot-learning platform (Isaac Sim, Isaac Lab, Isaac ROS, GR00T) for developing, training, and testing robot software, not a fleet-management/ops tool for deploying and managing config across a live robot fleet. Pushing configuration changes with per-group targeting across deployed robots is outside its category.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "fleet-dashboard", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NVIDIA Isaac is a robotics simulation, training, and development platform (Isaac Sim, Isaac Lab, Isaac ROS, GR00T) focused on building and training robot software, not a fleet operations/monitoring dashboard product. Fleet health/location/status monitoring is a different product category (fleet management software) and is outside Isaac's scope as evidenced.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "hardware-abstraction", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Isaac ROS provides GPU-accelerated perception/navigation packages deployable across workstations and Jetson, and Isaac Sim supports multiple sensor types and physics engines (PhysX, Newton, MuJoCo) plus adding custom robots, suggesting some hardware abstraction. However, evidence is about simulation/training flexibility rather than a documented stable HAL for swapping real sensors/actuators without app-code changes. missing for 10: explicit documentation of a hardware abstraction layer/driver interface contract for physical sensors/actuators, evidence of swapping real hardware without rewriting application code, and independent hands-on confirmation of this specific capability.", "evidenceIds": [ "nvidia-isaac-docs-7", "nvidia-isaac-docs-8", "nvidia-isaac-docs-13", "nvidia-isaac-docs-18", "nvidia-isaac-docs-28" ] }, { "productId": "nvidia-isaac", "storyId": "intervention-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Isaac is a simulation/training/robot-learning platform (Isaac Sim, Lab, ROS, GR00T) focused on building and training robot policies, not an operations/teleoperation-monitoring product with human-in-the-loop intervention queues; routing autonomy failures to human operators is outside its product category.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "ml-training-loop", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Isaac's stack covers most pipeline stages: Isaac Sim ingests real-world captures (NuRec/TeleOp) and CAD/URDF, generates synthetic data (Replicator), Isaac Lab trains policies at scale, and Isaac ROS packages deploy trained models to workstations and Jetson-based robots. However, there is no explicit documentation of a closed-loop fleet-update mechanism (e.g., pushing new trained models back to many deployed robots, fleet management/OTA), so the 'deploy improved model back to the fleet' half of the story is only weakly evidenced. Missing for 10: explicit fleet-wide deployment/update workflow, evidence of real collected field data being fed back into retraining loops, independent case studies confirming this end-to-end loop in production.", "evidenceIds": [ "nvidia-isaac-docs-3", "nvidia-isaac-docs-6", "nvidia-isaac-docs-7", "nvidia-isaac-docs-9", "nvidia-isaac-docs-10", "nvidia-isaac-docs-15", "nvidia-isaac-docs-21" ] }, { "productId": "nvidia-isaac", "storyId": "multi-language-sdks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence only shows Python namespace packages/pip-based composition for Isaac Sim; there is no mention of official typed SDKs in TypeScript, Go, or C++, nor evidence of multi-language coverage of the platform's full surface. This story is a fair question for a developer platform but is unsupported by the evidence provided.", "evidenceIds": [ "nvidia-isaac-docs-12", "nvidia-isaac-docs-20" ] }, { "productId": "nvidia-isaac", "storyId": "nl-to-robot-task", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Isaac GR00T is marketed as a generalist robot foundation model that can generalize across tasks and perform multi-step tasks requiring long context, hinting at high-level task capability, but there is no explicit evidence of a natural-language interface where a user types a task and the platform plans/executes it end-to-end. Missing for 10: explicit NL command parsing/planning pipeline, documented examples of text-to-task execution, and independent hands-on verification of this workflow.", "evidenceIds": [ "nvidia-isaac-docs-11", "nvidia-isaac-docs-19", "nvidia-isaac-docs-22" ] }, { "productId": "nvidia-isaac", "storyId": "offline-resilience", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NVIDIA Isaac is a robotics simulation/training/development platform (Isaac Sim, Isaac Lab, Isaac ROS, GR00T) — offline connectivity resilience and cloud-buffering during runtime operation is a fleet-management/edge-ops concern, not something this simulation/SDK toolset addresses; no evidence pack content relates to runtime connectivity loss or data buffering.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Isaac Sim exposes Python packages for scripting/composing apps and supports headless/standalone operation and ROS 2 control, implying broad API-driven access, but no evidence explicitly confirms full parity between UI and API functionality, and probes show no public REST/OpenAPI surface. missing for 10: explicit documentation of UI-API parity, a public REST/GraphQL API spec, and independent confirmation that all UI-exposed features (e.g., asset import GUI tools, robot tuning UI) are scriptable via API.", "evidenceIds": [ "nvidia-isaac-docs-12", "nvidia-isaac-docs-20", "nvidia-isaac-docs-10", "nvidia-isaac-docs-4", "nvidia-isaac-docs-16", "nvidia-isaac-probe-3" ] }, { "productId": "nvidia-isaac", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Isaac Sim supports import/export of robots and scenes in open formats (URDF, MJCF, CAD, USD) and Isaac Lab/GR00T are released open-source (Apache 2.0), showing openness in model/format terms, but this is about interoperable asset formats, not a genuine 'export all your data and leave' data-portability guarantee for an AI-native user's own project/runtime state. missing for 10: explicit data-export/account-portability documentation, evidence of full project or telemetry data export, independent confirmation of lock-in-free migration.", "evidenceIds": [ "nvidia-isaac-docs-1", "nvidia-isaac-gh-1", "nvidia-isaac-docs-15", "nvidia-isaac-comm-2" ] }, { "productId": "nvidia-isaac", "storyId": "openness-open-license", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Isaac Lab is explicitly described as 'open-source' (nvidia-isaac-docs-17/6), GR00T N1 was released under an Apache-2.0 license per community coverage (nvidia-isaac-comm-2), and Isaac Sim's source is hosted publicly on GitHub with feature docs (nvidia-isaac-gh-1/2/3). However, the pack lacks an explicit license file citation for Isaac Sim or Isaac ROS, and Isaac ROS's status (open vs. proprietary NITROS components) is unclear. Missing for 10: explicit license text/citation for Isaac Sim and Isaac ROS repos, clarity on which sub-components are closed vs open.", "evidenceIds": [ "nvidia-isaac-docs-17", "nvidia-isaac-docs-6", "nvidia-isaac-comm-2", "nvidia-isaac-gh-1", "nvidia-isaac-gh-2", "nvidia-isaac-gh-3" ] }, { "productId": "nvidia-isaac", "storyId": "openness-self-host", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Isaac Sim, Isaac Lab, and Isaac GR00T are distributed as open-source/downloadable software (GitHub repos, pip-installable packages, Apache 2 license for GR00T N1) that can run on local workstations or data centers, satisfying self-hosting for an AI-native user. missing for 10: explicit self-hosting/on-prem deployment guide or Docker/Helm instructions, independent hands-on confirmation of a fully self-hosted install outside NVIDIA cloud services, and clarity on which components (e.g., Omniverse services) may still require cloud dependencies.", "evidenceIds": [ "nvidia-isaac-gh-1", "nvidia-isaac-docs-12", "nvidia-isaac-docs-10", "nvidia-isaac-comm-2" ] }, { "productId": "nvidia-isaac", "storyId": "ota-deployment", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NVIDIA Isaac is a robotics simulation, training, and perception SDK/platform (Isaac Sim, Isaac Lab, Isaac ROS, GR00T) — it has no fleet management or OTA deployment/rollback tooling in evidence, and this axis pertains to fleet ops infrastructure, not a simulation/AI training toolkit.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "physics-simulation", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Isaac Sim provides realistic physics (PhysX/Newton), RTX and physics-based sensor simulation, robot/scene import from URDF/MJCF/CAD/USD, synthetic data generation, and ROS 2 validation loops — directly matching the pre-hardware simulation story, corroborated by first-party docs and GitHub feature list. Missing for 10: independent/hands-on third-party validation of sensor/physics fidelity beyond vendor docs.", "evidenceIds": [ "nvidia-isaac-docs-1", "nvidia-isaac-docs-2", "nvidia-isaac-docs-13", "nvidia-isaac-docs-23", "nvidia-isaac-docs-25", "nvidia-isaac-docs-26", "nvidia-isaac-gh-1", "nvidia-isaac-gh-2" ] }, { "productId": "nvidia-isaac", "storyId": "privacy-data-residency", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NVIDIA Isaac is a robotics simulation/development SDK (Isaac Sim, Lab, ROS) run locally on workstations or embedded devices like Jetson, not a hosted SaaS service with data storage regions; the evidence pack shows no cloud-hosted data storage or residency controls, and this axis is a category mismatch for a locally-deployed simulation toolkit.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NVIDIA Isaac is a robotics simulation/training SDK, not a consumer or enterprise data-processing service with an AI-training opt-out policy; there is no user data collection/training relationship applicable here, making this a category error for this product type.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "privacy-retention-controls", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NVIDIA Isaac is a robotics simulation/training SDK, not a data-processing SaaS with user data retention controls; the evidence pack contains no mention of data retention/deletion policies, and this axis is a category mismatch for a local/on-prem simulation and robot-learning toolkit.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry, usage tracking, or opt-out controls for NVIDIA Isaac; the axis is plausible for a developer platform/SDK but is entirely unaddressed by the provided docs, GitHub, or community items.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "remote-debugging", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NVIDIA Isaac is a simulation/training/robot-learning stack (Isaac Sim, Isaac Lab, Isaac ROS, GR00T); remote debugging of live topics/logs/state on a deployed robot is a fleet-management/observability tool concern, not part of this simulation and training platform's category.", "evidenceIds": [] }, { "productId": "nvidia-isaac", "storyId": "robot-bring-up", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Isaac ROS provides ready-to-use packages for navigation/perception deployable on workstations and Jetson, and Isaac Sim supports importing robot URDF/CAD models and adding custom robots, which supports faster bring-up. However, there is no concrete evidence of an hours-not-weeks bring-up timeline, no case study of driver/config automation for new hardware, and community commentary focuses on GR00T model performance skepticism rather than hardware bring-up speed. missing for 10: quantified bring-up time evidence, first-party or independent case study of new robot onboarding, driver auto-configuration tooling specifics.", "evidenceIds": [ "nvidia-isaac-docs-7", "nvidia-isaac-docs-15", "nvidia-isaac-docs-28", "nvidia-isaac-gh-1", "nvidia-isaac-gh-3" ] }, { "productId": "nvidia-isaac", "storyId": "sensor-integration", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Isaac Sim/Isaac ROS docs show RTX/physics-based sensor support and ROS 2 bridge/message integration for camera-type sensors and general robot stacks (e.g., nvidia-isaac-docs-2, -13, -23, gh-3), and Isaac ROS advertises perception/navigation packages using standard ROS 2 messages (nvidia-isaac-docs-7, -27). However, there is no explicit evidence of ready-made lidar, IMU, or GPS driver packages or their specific standard message-type support. Missing for 10: explicit lidar/IMU/GPS driver documentation, concrete standard message-type mappings for each sensor class, and independent hands-on confirmation of hardware bring-up beyond cameras.", "evidenceIds": [ "nvidia-isaac-docs-2", "nvidia-isaac-docs-13", "nvidia-isaac-docs-23", "nvidia-isaac-docs-7", "nvidia-isaac-docs-27", "nvidia-isaac-gh-3" ] }, { "productId": "nvidia-isaac", "storyId": "sim-to-real", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Isaac Sim's ROS 2 bridge lets robot stacks validated in simulation be evaluated via software-in-the-loop or hardware-in-the-loop testing, and Isaac ROS packages are explicitly designed to run unchanged on both workstations and embedded Jetson hardware, which is the core of write-once run-in-sim-and-on-robot. missing for 10: independent/hands-on case study confirming zero code changes between sim and real deployment, and no mention of common friction points (sensor timing, physics mismatch) that engineers typically must handle.", "evidenceIds": [ "nvidia-isaac-docs-26", "nvidia-isaac-docs-7", "nvidia-isaac-docs-4", "nvidia-isaac-gh-3", "nvidia-isaac-docs-8" ] }, { "productId": "nvidia-isaac", "storyId": "synthetic-data-rl", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Isaac Sim provides Replicator for generating labeled synthetic training data with randomized scenes and physics/RTX sensors, while Isaac Lab is explicitly built for large-scale, GPU-accelerated reinforcement learning with pluggable RL libraries (skrl, RLLib, rl_games) and CUDA-graphable, headless scaling from workstation to data center. This directly covers both synthetic data generation and RL-at-scale in simulation. Missing for 10: independent/hands-on benchmarks validating RL throughput claims beyond vendor docs.", "evidenceIds": [ "nvidia-isaac-docs-3", "nvidia-isaac-docs-5", "nvidia-isaac-docs-9", "nvidia-isaac-docs-10", "nvidia-isaac-docs-17", "nvidia-isaac-docs-21" ] }, { "productId": "nvidia-isaac", "storyId": "teleoperation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only mentions 'Isaac TeleOp' in passing as a data-ingestion source for converting real-world captures into USD (nvidia-isaac-docs-15), with no documentation of live video streaming, network-resilient control, or remote operator interfaces. The rest of the pack focuses on simulation, synthetic data, ROS 2 validation, and policy training, not real-time teleoperation over real-world networks. Missing for 10: any documented live video/control-loop architecture, latency/network-resilience handling, or hands-on teleoperation demo evidence.", "evidenceIds": [ "nvidia-isaac-docs-15" ] }, { "productId": "nvidia-isaac", "storyId": "vla-foundation-models", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "NVIDIA Isaac explicitly ships Isaac GR00T, an open reference platform/foundation model for humanoid robots with generalizable VLA-style skills (grasping, manipulation, multi-step tasks), plus Isaac Lab for training robot policies that integrate with custom libraries, and Isaac Sim/ROS 2 tooling to deploy and validate these stacks end-to-end. Community commentary corroborates GR00T's open-source release (Apache 2.0) though also raises skepticism about real-world performance limits, which tempers but doesn't contradict the core integration capability. Missing for 10: independent hands-on validation of VLA model integration into a full autonomy stack beyond vendor docs, and more detail on API-level integration points for third-party foundation models.", "evidenceIds": [ "nvidia-isaac-docs-11", "nvidia-isaac-docs-19", "nvidia-isaac-docs-22", "nvidia-isaac-docs-9", "nvidia-isaac-docs-6", "nvidia-isaac-docs-17", "nvidia-isaac-docs-7", "nvidia-isaac-gh-3", "nvidia-isaac-comm-2", "nvidia-isaac-comm-1" ] }, { "productId": "ros2", "storyId": "agent-controls-robot", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "ROS 2 provides DDS-Security with per-node 'security enclaves' encapsulating a single policy (scoped permissions) plus keystore/authentication tooling (ros2-docs-12,13,15,40,42,49), and its client libraries (rclcpp/rclpy) plus topics/services/actions let a program (including an AI agent) command a robot end-to-end via a real API/SDK (ros2-docs-7,10,24,35,36). However, there is no documentation of an AI-agent-oriented credential model (e.g., token/OAuth-style scoped API keys), no explicit workflow for an agent authenticating and issuing end-to-end commands, and no independent evidence of this being used by AI agents in practice. missing for 10: agent-specific scoped credential/token issuance workflow, explicit AI-agent authentication example, independent/hands-on validation of an agent driving a real robot end-to-end via this security model.", "evidenceIds": [ "ros2-docs-12", "ros2-docs-13", "ros2-docs-15", "ros2-docs-40", "ros2-docs-42", "ros2-docs-49", "ros2-docs-7", "ros2-docs-10", "ros2-docs-24", "ros2-docs-35", "ros2-docs-36" ] }, { "productId": "ros2", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an llms.txt file, agent-oriented documentation format, or any mention of AI agent-targeted docs; the evidence pack only covers standard human-oriented ROS 2 documentation and community discussion.", "evidenceIds": [] }, { "productId": "ros2", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "ROS 2 is a robotics middleware/communication framework, not an analytics or AI-insights product; the evidence pack covers messaging, QoS, security, tooling, and client libraries with no mention of AI-generated insights or suggestions on data, making this axis a category error for this product type.", "evidenceIds": [] }, { "productId": "ros2", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "ROS 2 provides the core building blocks for autonomous background execution — a launch system to coordinate multiple nodes, managed node lifecycles, a background daemon process maintaining graph state, and actions for long-running tasks with feedback/cancellation — which together let a robotics system run continuously and autonomously once configured. However, this is generic robotics middleware automation, not anything framed around AI-agent-triggered or AI-native workflow automation specifically. Missing for 10: explicit support for AI-driven/agentic triggering or scheduling of automations, independent hands-on evidence of long-running unattended deployments succeeding, and any AI-specific orchestration tooling.", "evidenceIds": [ "ros2-docs-6", "ros2-docs-5", "ros2-docs-52", "ros2-docs-24", "ros2-docs-28" ] }, { "productId": "ros2", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "ROS 2 is a robotics middleware/framework, not an AI assistant product; nothing in the evidence describes a built-in AI assistant to delegate tasks to. The story asks about a wrong axis for this kind of product.", "evidenceIds": [] }, { "productId": "ros2", "storyId": "agentic-headless", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "ROS 2 documents a headless-capable install (ROS-Base 'Bare Bones' with no GUI tools) plus a full CLI toolset (ros2 command, topic pub, bag record/play) that can be scripted for automation, supporting non-interactive/CI-style operation. However there's no explicit documentation of CI pipelines, Docker images, or automated test harnesses for ROS 2 itself. Missing for 10: explicit CI/automation docs or examples, containerized/headless deployment guidance, and independent evidence of running ROS 2 in a CI pipeline.", "evidenceIds": [ "ros2-docs-44", "ros2-docs-8", "ros2-docs-37", "ros2-docs-9", "ros2-docs-39", "ros2-docs-52" ] }, { "productId": "ros2", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "ROS 2 is a robotics middleware/framework, not an AI agent or agent-facing platform; the evidence pack contains no mention of MCP servers or tool-plugin integration at all, so this axis is a category error for this product type rather than a missing capability.", "evidenceIds": [] }, { "productId": "ros2", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "ROS 2 is a robotics middleware/framework, not an AI agent — publishing an official MCP server is a plausible ecosystem capability, but no evidence in the pack mentions MCP, agent connectivity, or any AI-agent integration server. Missing for 10: any mention of an official MCP server, MCP protocol support, or agent-facing API for ROS 2.", "evidenceIds": [] }, { "productId": "ros2", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "ROS 2's interfaces are CLI-based (ros2 command with sub-commands) and programmatic client libraries, not natural-language control; no evidence of NL command parsing, LLM integration, or agentic natural-language interface is present in the evidence pack.", "evidenceIds": [] }, { "productId": "ros2", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "ROS 2 ships an official, extensively documented `ros2` CLI with sub-commands for nodes, topics, services, bags, security, etc., making it fully scriptable and usable by any automated/agentic caller. Missing for 10: explicit documentation or examples targeting AI-agent/automated usage patterns, and independent hands-on confirmation of CLI reliability for such workflows.", "evidenceIds": [ "ros2-docs-8", "ros2-docs-30", "ros2-docs-37", "ros2-docs-38", "ros2-docs-39", "ros2-docs-52", "ros2-docs-9" ] }, { "productId": "ros2", "storyId": "agentic-public-api", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "ROS 2 exposes a well-documented public API surface — client libraries in multiple languages (rclpy/rclcpp) for nodes, topics, services, actions, parameters, plus an extensible CLI (`ros2 topic`, `ros2 service`, `ros2 bag`, etc.) and Python APIs for bag recording — all of which let an external program or agent drive the system programmatically. Missing for 10: no explicit mention of AI-agent-specific SDKs, and community commentary flags friction/complexity in actually using these APIs in practice (dependency hell, steep learning curve), so real-world ease for an AI-native driver is unproven.", "evidenceIds": [ "ros2-docs-7", "ros2-docs-8", "ros2-docs-9", "ros2-docs-10", "ros2-docs-17", "ros2-docs-25", "ros2-docs-37", "ros2-comm-9", "ros2-comm-15" ] }, { "productId": "ros2", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "ROS 2's security framework (sros2/DDS-Security) lets you create a keystore and define per-participant 'security enclaves' that encapsulate a distinct authentication/authorization policy, which maps loosely onto the idea of scoped, least-privilege credentials issued to individual nodes/agents. However this is a general robotics middleware security feature, not an API-credential system designed for AI agents, and there's no documented workflow for dynamically issuing/revoking scoped tokens to an 'agent' persona. Missing for 10: agent-oriented credential issuance workflow, dynamic/temporary scoped tokens, any AI-agent-specific access control examples, and independent evidence of least-privilege enforcement in practice.", "evidenceIds": [ "ros2-docs-12", "ros2-docs-13", "ros2-docs-15", "ros2-docs-29", "ros2-docs-40", "ros2-docs-42" ] }, { "productId": "ros2", "storyId": "agentic-sdks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "ROS 2 ships official client libraries (rclcpp, rclpy, etc.) that expose the core ROS 2 functionality, letting developers build in the language best suited to their application, and this is the backbone of any ROS 2 application development (ros2-docs-7, ros2-docs-35, ros2-docs-36, ros2-docs-21). This constitutes an official SDK story that any AI-native builder could target when writing perception/planning nodes. Missing for 10: explicit AI-agent-oriented SDK features or examples (e.g., LLM/agent integration guides), broader language coverage beyond C++/Python emphasis, and independent hands-on corroboration of SDK ergonomics.", "evidenceIds": [ "ros2-docs-7", "ros2-docs-35", "ros2-docs-36", "ros2-docs-21" ] }, { "productId": "ros2", "storyId": "agentic-webhooks", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "ROS 2 is a robotics middleware using pub/sub topics, services, and actions over DDS for inter-node communication; webhooks are an HTTP-based external event-notification pattern that is not part of ROS 2's architecture or documented feature set. This story applies to web/SaaS-style integration products, not a robotics middleware framework.", "evidenceIds": [] }, { "productId": "ros2", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack shows standard reStructuredText documentation, tutorials, and CLI examples, but nothing describing an interactive API reference (e.g., a searchable/browsable API explorer with runnable/executable code examples in-browser). No mention of interactive docs, sandboxed code execution, or API playgrounds.", "evidenceIds": [] }, { "productId": "ros2", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "ROS 2 defines interfaces via .msg/.srv/.action IDL files and provides CLI introspection tools, but no evidence in the pack shows a downloadable machine-readable API spec in OpenAPI or equivalent format for AI-native consumption. missing for 10: any OpenAPI/JSON-schema export, machine-readable spec endpoint, or documented spec-generation tool.", "evidenceIds": [] }, { "productId": "ros2", "storyId": "api-sandbox", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "ROS 2 supports Gazebo simulation (ros2-docs-16) and rosbag recording/playback (ros2-docs-17, ros2-docs-23, ros2-docs-39) which together enable testing against simulated/recorded data instead of live production systems, and separate ROS domains/DDS partitioning could isolate test traffic. However there is no explicit first-party guidance framing a dedicated 'sandbox vs production' workflow or environment isolation for AI-native testing. missing for 10: explicit sandbox/staging environment documentation, guidance on isolating test data from production ROS graphs, AI-agent-specific test harness integration, independent corroboration of safe sandbox testing practice.", "evidenceIds": [ "ros2-docs-16", "ros2-docs-17", "ros2-docs-23", "ros2-docs-39" ] }, { "productId": "ros2", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack references API versioning policy, deprecation guidelines, or backward-compatibility guarantees for ROS 2 APIs/interfaces; the docs cover features, tools, and QoS but not a documented deprecation/versioning policy. missing for 10: documented API versioning scheme, formal deprecation policy/timeline, compatibility guarantees across releases.", "evidenceIds": [] }, { "productId": "ros2", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "ROS 2's launch system lets you coordinate and start many nodes at once, and its CLI/introspection framework can operate across the whole graph (nodes, topics, services), which gives a loose analog to bulk operations, but there's no dedicated batch/bulk API (e.g., batch topic remap, bulk parameter set across nodes, mass import/export) described in the docs. Missing for 10: explicit bulk/batch operation tooling (e.g., scripted mass parameter updates, bulk topic/service operations), any evidence of an AI-native workflow performing such operations, and independent confirmation of this use case.", "evidenceIds": [ "ros2-docs-6", "ros2-docs-30", "ros2-docs-37", "ros2-docs-25" ] }, { "productId": "ros2", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "ROS 2's pub/sub, services, and actions model is inherently event-driven — subscribing to a topic or lifecycle event triggers a callback/action, which is a form of automatic reaction to events (ros2-docs-10, ros2-docs-24, ros2-docs-5). However, there is no evidence of a declarative 'rules engine' or AI-native trigger-condition-action system; this capability exists only as a byproduct of general-purpose programming against the pub/sub API, not as a built-in automation-rule feature. Missing for 10: a documented rules/automation-definition interface (e.g. declarative if-event-then-action config), AI-native tooling to define such rules without writing node code, and independent evidence of this pattern being used for automation purposes.", "evidenceIds": [ "ros2-docs-10", "ros2-docs-24", "ros2-docs-5", "ros2-docs-6" ] }, { "productId": "ros2", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "ROS 2 is a robotics middleware/framework for real-time communication between nodes, not a job/workflow scheduling system; the evidence pack covers pub/sub, services, actions, QoS, security, and CLI tools but nothing about scheduling recurring jobs or workflows, which is outside its category.", "evidenceIds": [] }, { "productId": "ros2", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "ROS 2 is a robotics middleware/framework for building distributed node-based systems, not an automation/workflow platform with versioned, reviewable, rollback-able 'automations'; version control of code/config would use external tools like git, not a ROS 2 feature. This axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "ros2", "storyId": "ci-sim-testing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows ROS 2 can launch simulations (Gazebo) and record/playback data via rosbag, but contains no documentation or evidence of automated testing frameworks (e.g., colcon test, launch_testing), CI pipeline integration, or simulation-based regression testing workflows. Without concrete evidence of a CI-oriented test harness or regression-testing tooling, the story is unevidenced even though the axis clearly applies to a robotics middleware/deployment story.", "evidenceIds": [ "ros2-docs-16", "ros2-docs-17", "ros2-docs-23" ] }, { "productId": "ros2", "storyId": "command-access-control", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "ROS 2's SROS2 security layer provides per-node 'security enclaves' with DDS-Security authentication and permissions policies that can restrict which participants may publish, subscribe, or call services — a mechanism that could be used to gate who can command/configure/view a robot's topics and services (ros2-docs-12,13,15,29,40,42,49). However, this is a low-level, certificate-based node/participant authorization system, not a role-based access model for human operators (no user/role management UI, no built-in 'ops vs viewer' role concept), and no community evidence corroborates real-world use for this purpose. Missing for 10: explicit human-role/permission management (RBAC for ops users), UI/dashboard for granting per-robot view/command rights, and independent hands-on validation of the security enclave workflow.", "evidenceIds": [ "ros2-docs-12", "ros2-docs-13", "ros2-docs-15", "ros2-docs-29", "ros2-docs-40", "ros2-docs-42", "ros2-docs-49" ] }, { "productId": "ros2", "storyId": "community-modules", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Community comments confirm a broad ecosystem of pre-built ROS drivers/nodes exists that developers rely on ('a lot of nodes and drivers out there', 'data bagging, visualization, teleop, and sim... for free'), and docs show packages are distributed via apt-installable debs (Desktop/ROS-Base installs), implying a package ecosystem. However there is no first-party documentation in the pack describing an actual registry/index (e.g. ROS Index, rosdep, ros--* apt packages) or concrete examples of pulling hardware-specific packages. Missing for 10: explicit registry/package-manager documentation, examples of installing a specific hardware driver or behavior package, and independent corroboration beyond passing community remarks.", "evidenceIds": [ "ros2-comm-7", "ros2-comm-9", "ros2-comm-10", "ros2-docs-51", "ros2-docs-44" ] }, { "productId": "ros2", "storyId": "data-capture-sync", "verdict": "partial", "quality": 3, "confidence": "medium", "rationale": "ROS 2 provides strong on-robot data capture via rosbag (record/play, Python API for custom recording) and configurable QoS policies for tuning network reliability, but there is no evidence of any built-in cloud-sync mechanism or bandwidth-aware upload policy — bags are local artifacts and QoS applies to live DDS pub/sub between nodes, not to offloading recorded data to a cloud backend. Missing for 10: cloud upload/sync capability, bandwidth-aware transfer policies, any first-party cloud integration or partner tooling for telemetry sync.", "evidenceIds": [ "ros2-docs-17", "ros2-docs-23", "ros2-docs-39", "ros2-docs-22", "ros2-docs-47" ] }, { "productId": "ros2", "storyId": "data-query-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "ROS 2 provides rosbag2 with a Python API for reading and writing recorded data (ros2-docs-17, ros2-docs-23) and CLI tools for recording/playing bags (ros2-docs-39), enabling offline analysis of collected data programmatically. However, no evidence documents a SQL query interface or export mechanism, despite rosbag2's known SQLite backend not being mentioned in this pack. Missing for 10: explicit SQL querying/export documentation, examples of offline analysis workflows, and independent corroboration of API usability for this purpose.", "evidenceIds": [ "ros2-docs-17", "ros2-docs-23", "ros2-docs-39", "ros2-docs-11" ] }, { "productId": "ros2", "storyId": "estop-safety-patterns", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "ROS 2 documents managed lifecycle nodes (ros2-docs-5), which are the standard building block for implementing safety-interlock/state-transition patterns like emergency stop, plus QoS tuning (ros2-docs-14/22/41) and preliminary real-time support (ros2-docs-32) that engineers rely on for safety-critical control loops. However, there is no explicit documented emergency-stop tutorial, reference pattern, or safety-certification guidance in the evidence. Missing for 10: dedicated emergency-stop/safety-interlock tutorial or design guide, explicit safety-certification or functional-safety documentation, and independent confirmation that lifecycle nodes are used successfully for e-stop in production.", "evidenceIds": [ "ros2-docs-5", "ros2-docs-14", "ros2-docs-22", "ros2-docs-32", "ros2-docs-41" ] }, { "productId": "ros2", "storyId": "fleet-alerts", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "ROS 2 docs describe core middleware capabilities (topics, services, QoS, security, CLI introspection, bag recording) but there is no evidence of any built-in alerting, notification, or ops-dashboard system that would notify an ops lead about robot faults, battery levels, or connectivity issues proactively. Diagnostics-like data could theoretically be built on top of topics, but no such alerting feature or fleet-ops tooling is documented.", "evidenceIds": [] }, { "productId": "ros2", "storyId": "fleet-config-rollout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "ROS 2's documented features cover per-node parameters, launch systems for coordinating multiple nodes on one robot, and CLI introspection tools, but there is no evidence of a fleet-wide configuration push mechanism with per-group targeting across many robots — that is a higher-level fleet-management capability (e.g., separate tools like open-rmf) not shown here.", "evidenceIds": [ "ros2-docs-6", "ros2-docs-25", "ros2-docs-37" ] }, { "productId": "ros2", "storyId": "fleet-dashboard", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "ROS 2 evidence covers single-robot/graph introspection (CLI tools, RViz via desktop install, topic/service introspection) but nothing about a unified dashboard aggregating health, location, and status across a whole fleet of robots — fleet-level monitoring is typically built by third-party layers on top of ROS 2, not documented here.", "evidenceIds": [ "ros2-docs-8", "ros2-docs-37", "ros2-docs-51", "ros2-docs-52" ] }, { "productId": "ros2", "storyId": "hardware-abstraction", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "ROS 2's core architecture—standard message packages, topics/services/actions interfaces, and multi-language client libraries—is exactly the mechanism that lets drivers for different sensors/actuators publish/subscribe to the same standardized interface, enabling swap-in-place hardware. However, the evidence pack never explicitly documents a 'hardware abstraction layer' concept, driver ecosystem practices, or a concrete example of swapping a sensor/actuator without app changes. Missing for 10: explicit hardware-abstraction/driver-interface documentation, named sensor/actuator driver packages demonstrating swap-ability, and independent hands-on confirmation that application code needs no changes when hardware is swapped.", "evidenceIds": [ "ros2-docs-10", "ros2-docs-45", "ros2-docs-46", "ros2-docs-7", "ros2-docs-35", "ros2-docs-44", "ros2-docs-36" ] }, { "productId": "ros2", "storyId": "intervention-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "ROS 2 provides low-level building blocks (actions with feedback/cancel, lifecycle-managed nodes, topics) that could underpin a human-intervention workflow, but the evidence pack shows no actual feature for detecting autonomy failures, routing them to an operator queue, or handing control back after resolution — that is an application-layer capability ROS 2 does not ship out of the box. Missing for 10: any documentation of a fault-detection/escalation mechanism, an intervention-queue or ticketing construct, or operator handback protocol.", "evidenceIds": [ "ros2-docs-24", "ros2-docs-5", "ros2-docs-10" ] }, { "productId": "ros2", "storyId": "ml-training-loop", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "ROS 2 provides rosbag recording/playback (ros2-docs-17, ros2-docs-23) which supports collecting robot data for later use, but there is no evidence of any built-in or documented pathway for feeding that data into model training or deploying updated models back to a robot fleet. missing for 10: model-training integration, fleet-wide model deployment/update mechanism, MLOps tooling, and any end-to-end example connecting bag data to a training loop and redeployment.", "evidenceIds": [ "ros2-docs-17", "ros2-docs-23" ] }, { "productId": "ros2", "storyId": "multi-language-sdks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "ROS 2 docs confirm official client libraries in multiple languages, but only explicitly evidence C++ and Python (rclcpp/rclpy) as officially verified APIs, with no mention of official TypeScript or Go SDKs covering the full ROS 2 surface (nodes, topics, services, actions, QoS, security, etc.). Community remarks confirm long-standing use of C++/Python but do not corroborate TypeScript/Go as first-party. Missing for 10: official TypeScript SDK, official Go SDK, evidence that all four languages have full-surface parity and 'typed' guarantees.", "evidenceIds": [ "ros2-docs-7", "ros2-docs-19", "ros2-docs-21", "ros2-docs-35", "ros2-docs-36" ] }, { "productId": "ros2", "storyId": "nl-to-robot-task", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "ROS 2 is a middleware/robotics framework providing communication, tooling, and libraries; natural-language task planning and execution is an application-layer AI capability built on top of ROS 2, not a feature of the platform itself — no evidence pack item describes any NLP/LLM-based task planning or execution component.", "evidenceIds": [] }, { "productId": "ros2", "storyId": "offline-resilience", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "ROS 2's QoS policies (reliable/best-effort spanning TCP-to-UDP behavior) are explicitly designed to handle non-ideal networks, and rosbag tooling lets nodes record/buffer data locally, which are adjacent capabilities to 'operate offline and buffer data.' However, there is no documented feature specifically addressing cloud-connectivity loss detection, local buffering during an outage, or a clean reconnection/sync-recovery workflow — the evidence is about node-to-node DDS network resilience, not cloud uplink resilience. missing for 10: explicit cloud-connectivity-loss handling, local buffering-for-cloud-sync mechanism, and documented clean-recovery-on-reconnect behavior.", "evidenceIds": [ "ros2-docs-3", "ros2-docs-14", "ros2-docs-41", "ros2-docs-47", "ros2-docs-17", "ros2-docs-23" ] }, { "productId": "ros2", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "ROS 2's architecture is fundamentally API/CLI-first: docs show that features like bag recording (ros2 bag CLI and equivalent Python API [ros2-docs-17],[ros2-docs-23]), security keystore creation (CLI [ros2-docs-42]), and topic pub/sub ([ros2-docs-9],[ros2-docs-38]) are all backed by client-library APIs, and the CLI tools themselves are described as an 'extensible framework' built on the same graph introspection layer as the daemon ([ros2-docs-30],[ros2-docs-52]). However there is no explicit documentation claiming full parity between GUI tools (e.g., RViz, which is only mentioned as part of the Desktop install [ros2-docs-51]) and the client-library APIs, nor any statement that everything visualizable/configurable in RViz can be replicated purely via API. Missing for 10: explicit API/GUI parity claim, documentation of RViz-equivalent functionality via API, and independent confirmation that AI-native/headless workflows can fully replace GUI-based configuration.", "evidenceIds": [ "ros2-docs-8", "ros2-docs-9", "ros2-docs-17", "ros2-docs-23", "ros2-docs-30", "ros2-docs-37", "ros2-docs-42", "ros2-docs-51", "ros2-docs-52" ] }, { "productId": "ros2", "storyId": "openness-full-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "ROS 2 is fully open-source with rosbag recording/playback and a Python API for reading/writing bag data, meaning all system data lives in inspectable, non-proprietary formats accessible via CLI or code, and there's no vendor SaaS lock-in to 'leave' from since it runs locally. However, the evidence never explicitly addresses bulk data export, format standardization/versioning guarantees, or migration tooling framed as 'export and leave'. Missing for 10: explicit documentation on bag format portability/interoperability guarantees, an explicit statement on no lock-in or data export completeness, and independent confirmation that recorded data is fully portable across tools/versions.", "evidenceIds": [ "ros2-docs-17", "ros2-docs-23", "ros2-docs-39", "ros2-gh-1" ] }, { "productId": "ros2", "storyId": "openness-open-license", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "ROS 2 is hosted publicly on GitHub (github.com/ros2/ros2) as an open-source project, consistent with its well-known Apache 2.0 licensing, and the docs/source are openly accessible for inspection. Missing for 10: explicit license file citation and independent confirmation of license terms beyond the GitHub repo listing.", "evidenceIds": [ "ros2-gh-1" ] }, { "productId": "ros2", "storyId": "openness-self-host", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "ROS 2 is a fully open-source framework distributed via apt/source with no cloud dependency, installable and run entirely on user-owned infrastructure (ROS-Base and Desktop install options, CLI tools, client libraries). Community evidence confirms it is widely self-hosted in production and research despite operational complaints. missing for 10: no explicit first-party 'self-hosting' guide or containerized deployment reference architecture, and no independent audit confirming zero external calls.", "evidenceIds": [ "ros2-docs-44", "ros2-docs-51", "ros2-docs-18", "ros2-gh-1", "ros2-comm-9", "ros2-comm-13" ] }, { "productId": "ros2", "storyId": "ota-deployment", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "ROS 2 is a robotics middleware/framework for communication, computation graph management, and tooling (pub/sub, services, actions, CLI tools, security, QoS) — it is not a fleet management or OTA deployment platform. Staged rollouts and rollback of software updates across a robot fleet is a distinct product category (e.g., fleet management/OTA tools built on top of ROS), so this axis does not apply to ROS 2 itself.", "evidenceIds": [] }, { "productId": "ros2", "storyId": "physics-simulation", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "ROS 2 docs reference a tutorial for launching a Gazebo simulation with ROS 2, indicating an integration path exists, but there is no detail on physics fidelity, sensor model realism, or independent corroboration of simulation quality. missing for 10: documentation of physics engine accuracy, sensor plugin catalog (cameras, lidar, IMU), community/hands-on validation of sim-to-real fidelity.", "evidenceIds": [ "ros2-docs-16" ] }, { "productId": "ros2", "storyId": "privacy-data-residency", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "ROS 2 is a robotics middleware/framework running on self-hosted infrastructure chosen entirely by the deploying organization; it has no cloud data-storage service or hosted offering where 'region/residency' selection would be a relevant configuration. This axis is a category error for a local/on-prem middleware framework.", "evidenceIds": [] }, { "productId": "ros2", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "ROS 2 is a robotics middleware framework, not an AI model training/data platform — there is no concept of 'AI training data opt-out' in this product category; the axis is a category error for this kind of product.", "evidenceIds": [] }, { "productId": "ros2", "storyId": "privacy-retention-controls", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "ROS 2 is a robotics middleware/communication framework, not a data platform with retention/deletion policies for AI-native usage; the evidence covers security, QoS, bagging, and CLI tools but nothing about data retention or deletion controls, and this is a category mismatch rather than a gap in an applicable feature.", "evidenceIds": [] }, { "productId": "ros2", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no mention of telemetry, usage tracking, or any opt-out mechanism for ROS 2; documentation covers middleware, QoS, CLI tools, and security but nothing about data collection practices. Since telemetry controls are a plausible axis for any software toolchain and no evidence confirms or denies its presence, this is 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "ros2", "storyId": "remote-debugging", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "ROS 2 provides a comprehensive suite of CLI introspection tools (ros2 topic, ros2 node, ros2 service, ros2 bag) plus a daemon for graph state, topic statistics, service introspection, and rosbag recording/playback for logs, all of which support remote debugging of live topics and state on deployed robots. This is corroborated by direct docs on command-line tools, topic statistics, and bag recording, and community comments confirm CLI tools/bags/viewers are valued 'for free' benefits of using ROS's ecosystem conventions. Missing for 10: explicit first-party or independent evidence of remote/networked debugging over real deployments (e.g., cross-machine domain setup docs) and no hands-on account specifically describing remote robot debugging via these tools.", "evidenceIds": [ "ros2-docs-8", "ros2-docs-30", "ros2-docs-37", "ros2-docs-52", "ros2-docs-11", "ros2-docs-17", "ros2-docs-23", "ros2-docs-39", "ros2-docs-34", "ros2-comm-9", "ros2-comm-10" ] }, { "productId": "ros2", "storyId": "robot-bring-up", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "ROS 2 docs show strong building blocks for hardware bring-up (launch system for coordinating nodes, CLI tools for introspection/pub-sub, multi-language client libraries, parameters, lifecycle nodes, drivers ecosystem implied by ros2-gh-1) that in principle let an engineer wire up a robot quickly. However, independent hands-on community reports concretely contradict the 'hours not weeks' claim: reports of dependency hell, a convoluted build system, disorganized tutorials causing 'weird errors,' and even advice to budget a full 'Epic: Migrate off of ROS' once shipping — while one commenter admits it's fast only for hobbyist/prototype use, not production bring-up. Missing for 10: a first-party quantified bring-up example (e.g., time-to-first-motion tutorial) and independent corroboration that real hardware bring-up is fast, not just prototyping.", "evidenceIds": [ "ros2-docs-6", "ros2-docs-7", "ros2-docs-8", "ros2-gh-1", "ros2-comm-4", "ros2-comm-6", "ros2-comm-7", "ros2-comm-10", "ros2-comm-15" ] }, { "productId": "ros2", "storyId": "sensor-integration", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Evidence confirms ROS 2 ships standard message packages and a large driver/node ecosystem in general terms (ros2-docs-44 'message packages'; ros2-comm-7 'a lot of nodes and drivers out there for ROS already'; ros2-comm-16 on standardized robotics datatypes), but there is no explicit documentation of specific camera, lidar, IMU, or GPS driver packages or sensor_msgs/standard sensor message types. missing for 10: explicit docs listing camera/lidar/IMU/GPS driver packages, sensor_msgs or NavSat message specifics, first-party hardware integration guides.", "evidenceIds": [ "ros2-docs-44", "ros2-comm-7", "ros2-comm-16" ] }, { "productId": "ros2", "storyId": "sim-to-real", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "ROS 2's architecture (topics/services/actions abstraction, pub/sub) lets nodes interact with either real hardware drivers or simulated ones (e.g., Gazebo) using the identical ROS API, and the docs explicitly walk through launching a Gazebo simulation alongside ROS 2 nodes, plus a community user notes ROS gives you sim 'for free' as part of its ecosystem. Missing for 10: an explicit vendor statement that the same node code requires zero changes to move from sim to real robot, and independent hands-on corroboration/case study confirming no-rewrite in practice.", "evidenceIds": [ "ros2-docs-16", "ros2-docs-20", "ros2-docs-10", "ros2-docs-27", "ros2-comm-9" ] }, { "productId": "ros2", "storyId": "synthetic-data-rl", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only a basic Gazebo simulation launch tutorial (ros2-docs-16); there is no mention of synthetic training data generation, reinforcement learning frameworks, or scaled RL training pipelines integrated with ROS 2. The axis is plausible for a robotics middleware (simulation is core to ROS use cases), but no evidence supports the specific RL/synthetic-data-at-scale capability. Missing for 10: RL framework integration (e.g., Isaac Gym/Gazebo RL bridges), synthetic data generation tooling, evidence of scaled/parallel simulation training.", "evidenceIds": [ "ros2-docs-16" ] }, { "productId": "ros2", "storyId": "teleoperation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "ROS 2's QoS policies (reliable/best-effort tuning for non-ideal networks), pub/sub topics for continuous sensor/video streams, and actions for long-running feedback-driven tasks provide the underlying building blocks for teleoperation, and a community user explicitly notes ROS gives teleop 'for free' via its ecosystem conventions [ros2-comm-9]. However, there is no first-party documentation of a video-streaming pipeline, latency benchmarks over real-world/lossy networks, or a dedicated teleop stack/tooling in this evidence pack. missing for 10: dedicated video transport/compression tooling (e.g., image_transport/web video), real-network latency/responsiveness benchmarks, explicit teleoperation reference architecture or case study.", "evidenceIds": [ "ros2-docs-3", "ros2-docs-14", "ros2-docs-22", "ros2-docs-24", "ros2-docs-45", "ros2-comm-9" ] }, { "productId": "ros2", "storyId": "vla-foundation-models", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers ROS 2's core middleware, client libraries, CLI tools, QoS, security, and simulation features, but contains no mention of VLA models, robotics foundation models, or any supported tooling/integration path (e.g., official packages, tutorials, or SDKs) for incorporating such AI models into the autonomy stack. Missing for 10: any documentation or examples of VLA/foundation-model integration, official model-serving nodes, or partner tooling (e.g., NVIDIA Isaac, Hugging Face robotics integrations) referenced by ROS 2 itself.", "evidenceIds": [] }, { "productId": "viam", "storyId": "agent-controls-robot", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Viam has a dedicated doc page explicitly framing this exact story—'An AI agent with a shell or an SDK can operate a Viam machine the same way a person or a program can'—and details how an agent lists resources/status, drives motion via the motion service, and calls any API method via CLI or SDK with JSON in/out. Fleet/auth docs show permission and authorization management at org/location/machine granularity, supporting scoped credentials, and the platform provides multi-language SDKs (Python, Go, C++, TypeScript, Flutter) plus a CLI for end-to-end control. Missing for 10: a concrete walkthrough of creating/scoping an API key specifically for an agent, and independent/hands-on corroboration beyond first-party docs.", "evidenceIds": [ "viam-docs-12", "viam-docs-31", "viam-docs-32", "viam-docs-30", "viam-docs-13", "viam-docs-20", "viam-docs-34", "viam-docs-3", "viam-gh-1" ] }, { "productId": "viam", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Probe confirms llms.txt is live at https://docs.viam.com/llms.txt (HTTP 200) with a structured summary, and Viam also documents an explicit agent-oriented doc page (use-viam-from-an-agent) describing how an AI agent with a shell or SDK can operate a machine. missing for 10: no independent/community corroboration that agents actually consume llms.txt successfully, and no evidence of broader agent-oriented doc index beyond the single page.", "evidenceIds": [ "viam-probe-1", "viam-docs-12", "viam-docs-31", "viam-docs-32" ] }, { "productId": "viam", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Viam documents data capture, SQL/MQL querying, and ML model training/deployment for machines, plus agent/CLI access to data, but nothing shows the product itself generating AI insights or suggestions from that data for the user (e.g., automated analytics, anomaly summaries, or recommendations surfaced in the app).", "evidenceIds": [ "viam-docs-8", "viam-docs-9", "viam-docs-11", "viam-docs-23", "viam-docs-33" ] }, { "productId": "viam", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Viam supports logic modules that react to sensor data, coordinate components, run continuous processes, and schedule actions autonomously (viam-docs-17, viam-docs-38, viam-docs-46), plus scheduled data sync and cloud automation triggers (viam-docs-7, viam-docs-9). This is genuine background automation for physical/robotic systems, but it's not framed as an AI-agent-orchestrated automation builder — it's code-based module logic requiring developer setup rather than a no-code 'set up automation' AI-native workflow. missing for 10: a dedicated no-code automation/trigger builder UI, evidence of AI-native natural-language automation setup, and independent verification that these automations run reliably unattended in production long-term.", "evidenceIds": [ "viam-docs-17", "viam-docs-38", "viam-docs-46", "viam-docs-7", "viam-docs-9", "viam-docs-12" ] }, { "productId": "viam", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows that external AI agents can control Viam machines via the SDK/CLI/shell (viam-docs-12, viam-docs-31, viam-docs-32) — this is the reverse direction (an agent driving Viam), not a built-in assistant inside the Viam product that a user can delegate tasks to. No evidence of an in-app AI assistant/copilot for configuring machines, writing modules, or querying data.", "evidenceIds": [ "viam-docs-12", "viam-docs-31", "viam-docs-32" ] }, { "productId": "viam", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Viam has a documented CLI that can call any machine API method with JSON in/out, supports SQL queries for data, and explicitly documents agent/headless usage patterns (shell or SDK operating a machine like a person or program), which supports CI/automation workflows. However, evidence lacks explicit CI/CD pipeline integration examples, headless installation/authentication flows for automation contexts, or exit-code/scripting conventions typical of CI tooling. missing for 10: CI pipeline examples/integration guides, non-interactive auth/service-account setup for headless CLI use, independent confirmation of CI usage in the wild.", "evidenceIds": [ "viam-docs-12", "viam-docs-13", "viam-docs-30", "viam-docs-11", "viam-docs-31", "viam-probe-3" ] }, { "productId": "viam", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers Viam being controlled BY an AI agent (agents operating a Viam machine via SDK/CLI) but there is no mention of Viam itself connecting to or consuming external MCP servers/tools as a client.", "evidenceIds": [] }, { "productId": "viam", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Viam is a robotics platform (not itself an agent), so publishing an official MCP server is a fair question, but the evidence only shows agent access via CLI/SDK/shell (viam-docs-12, viam-docs-13, viam-docs-31, viam-docs-32) with no mention of the Model Context Protocol or an MCP server offering anywhere in docs or probes.", "evidenceIds": [ "viam-docs-12", "viam-docs-13", "viam-probe-1", "viam-probe-2" ] }, { "productId": "viam", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Viam has a dedicated 'use Viam from an agent' doc explicitly describing AI agents operating machines via shell/SDK/CLI with JSON in/out (viam-docs-12, viam-docs-13, viam-docs-30, viam-docs-31), which is strong evidence of agentic operability, but this is API/CLI-driven control rather than a built-in natural-language command interface—an agent must translate NL to API calls itself. Missing for 10: explicit natural-language parsing/command interface within Viam itself, and independent/hands-on confirmation of an AI agent successfully operating a machine via plain-language instructions.", "evidenceIds": [ "viam-docs-12", "viam-docs-13", "viam-docs-30", "viam-docs-31", "viam-docs-32" ] }, { "productId": "viam", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Viam ships an official documented CLI (docs.viam.com/cli/reference) supporting data queries, resource control, fleet/org management, and JSON-in/JSON-out API calls that an AI agent can drive directly, with a dedicated guide on using Viam from an agent via shell/SDK. This is agent-friendly and well-documented but lacks independent hands-on corroboration of the CLI itself. Missing for 10: independent/community verification of CLI usage, more detail on agent-specific CLI workflows beyond the single doc page.", "evidenceIds": [ "viam-docs-11", "viam-docs-12", "viam-docs-13", "viam-docs-30", "viam-docs-23", "viam-probe-3" ] }, { "productId": "viam", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Viam publishes a dedicated doc page explicitly stating 'An AI agent with a shell or an SDK can operate a Viam machine the same way a person or a program can,' backed by CLI/API references (JSON in/out for any API method), fleet API docs, and multi-language SDKs, directly satisfying the AI-native documented public API story. Missing for 10: standard machine-readable OpenAPI spec discoverable (probe found 404s at all candidate openapi.json paths), independent third-party hands-on verification of agent API usage.", "evidenceIds": [ "viam-docs-12", "viam-docs-13", "viam-docs-30", "viam-docs-31", "viam-docs-32", "viam-docs-20", "viam-docs-11", "viam-gh-1", "viam-probe-2", "viam-probe-3" ] }, { "productId": "viam", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Viam's Fleet API/CLI documentation mentions the ability to 'manage permissions and authorization' at the org/location/machine level (viam-docs-20/34/43), and machine credentials are auto-injected for hosted apps (viam-docs-52), implying some access-control primitives exist. However, there is no explicit documentation of issuing scoped or least-privilege API keys/tokens specifically for AI agents, nor any agent-specific credential-scoping workflow beyond generic 'permissions and authorization' management. Missing for 10: explicit API key/token scoping mechanism, agent-specific credential issuance docs, and any least-privilege role examples tied to agent use.", "evidenceIds": [ "viam-docs-20", "viam-docs-34", "viam-docs-43", "viam-docs-52", "viam-docs-12" ] }, { "productId": "viam", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Viam documents official SDKs in Python, Go, C++, TypeScript, and Flutter for building client applications and control logic, with consistent APIs across components (e.g., GetImage) and explicit support for AI agents using an SDK or shell to operate machines. This is corroborated by the GitHub RDK repo listing multiple SDKs. Missing for 10: independent hands-on developer reviews of SDK ergonomics/quality and more detail on SDK completeness/parity across languages.", "evidenceIds": [ "viam-docs-3", "viam-docs-1", "viam-docs-12", "viam-gh-1", "viam-docs-16" ] }, { "productId": "viam", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack mentions fleet monitoring, alerting, and data sync, but nowhere describes a webhook subscription mechanism for events. Webhooks are a plausible feature for an IoT/robotics platform, so absence of evidence means 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "viam", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of an interactive, runnable API reference (like a Swagger/OpenAPI explorer or executable code sandbox); a direct probe for OpenAPI/Swagger endpoints returned 404s, and docs only show static CLI/SDK reference pages and a llms.txt file, not a runnable API explorer.", "evidenceIds": [ "viam-probe-2", "viam-docs-11", "viam-probe-3" ] }, { "productId": "viam", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Viam's APIs are built on gRPC/protobuf (viam-comm-3) and documented via CLI/SDK references, but a direct probe for OpenAPI/swagger endpoints returned 404 on all candidate paths, and no evidence pack item mentions a downloadable OpenAPI spec or equivalent machine-readable API description.", "evidenceIds": [ "viam-probe-2", "viam-comm-3", "viam-docs-13" ] }, { "productId": "viam", "storyId": "api-sandbox", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Viam offers 'test panels' and browser-based controls to 'debug without deploying code' and mentions 'canary testing and rollbacks for safe versioning', which suggest some isolated testing capability, but there is no explicit sandbox environment or data isolation from production described. missing for 10: dedicated sandbox/staging environment concept, explicit guarantee that test data doesn't touch production datasets, and any first-party or community confirmation of production/sandbox separation.", "evidenceIds": [ "viam-docs-6", "viam-docs-16", "viam-docs-40" ] }, { "productId": "viam", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows SDK/CLI/API surface documentation but no mention of API versioning scheme (e.g., v1/v2 endpoints) or any documented deprecation policy for APIs; references to 'versioning' only concern ML model deployment, not API contracts.", "evidenceIds": [] }, { "productId": "viam", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Viam's CLI supports bulk data operations — exporting, querying, deleting, and tagging 'all data that matches a given filter' — and Fleet Management APIs let users manage many machines/organizations at once (OTA updates, permissions, fragments) rather than one-by-one. This covers bulk operations on data records and fleet-wide updates, but the evidence doesn't show a general-purpose batch API for bulk manipulating arbitrary resources (e.g., configuring many components/machines in one call) or independent confirmation of these bulk features working at scale. Missing for 10: independent/hands-on verification of bulk operations, explicit batch API for machine/component configuration beyond data tagging, and scale/performance evidence for large fleets.", "evidenceIds": [ "viam-docs-23", "viam-docs-33", "viam-docs-47", "viam-docs-11", "viam-docs-20", "viam-docs-34", "viam-docs-43", "viam-docs-10", "viam-docs-26" ] }, { "productId": "viam", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Viam supports event-driven automation via logic modules that react to sensor thresholds and coordinate components (e.g., 'A temperature sensor crosses a threshold, and a notification fires' and 'React to sensor data: trigger an alert or an actuator when a reading crosses a threshold'), which requires writing a custom module rather than a built-in no-code rules engine. missing for 10: a declarative/no-code rules-builder UI, scheduling/cron-style trigger config in the app, and independent/hands-on confirmation that this event-action pattern works reliably in production.", "evidenceIds": [ "viam-docs-17", "viam-docs-38", "viam-docs-46" ] }, { "productId": "viam", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Viam's docs mention that a 'logic module' can be used to 'Schedule actions' alongside reacting to sensor data, implying developers can build custom scheduled/recurring workflows into a module, but there is no dedicated scheduler, cron-like job system, or workflow orchestration UI documented. missing for 10: a native recurring-job/cron scheduling feature, documentation of periodic execution intervals, and evidence of built-in workflow automation beyond a one-line mention.", "evidenceIds": [ "viam-docs-17", "viam-docs-38", "viam-docs-46" ] }, { "productId": "viam", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Viam documents versioning and rollback for ML models and OTA software/modules (inline modules have Viam-managed 'source code, builds, versioning, and deployment', and canary testing/rollbacks are offered for 'safe versioning'), which extends to automation logic deployed via modules. However there is no explicit evidence of a review/approval workflow for automations, and rollback evidence is framed mainly around ML models and fleet software updates rather than automation logic specifically. Missing for 10: explicit review/approval workflow for automations, direct hands-on evidence of rolling back a logic/automation module (not just ML models or generic OTA software).", "evidenceIds": [ "viam-docs-9", "viam-docs-18", "viam-docs-25", "viam-docs-26", "viam-docs-40", "viam-docs-48" ] }, { "productId": "viam", "storyId": "ci-sim-testing", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence describes CI integration, automated test pipelines, or simulation-based regression testing for robot software; docs only mention manual web-based test panels and debugging tools, which is a different capability.", "evidenceIds": [ "viam-docs-6", "viam-docs-16" ] }, { "productId": "viam", "storyId": "command-access-control", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Viam's Fleet Management docs explicitly mention 'access control' and the Fleet API reference states you can 'manage permissions and authorization' for organizations, locations, and machines, supporting role-based access control at a high level. However, the evidence never details specific role types (e.g., viewer vs operator vs admin) or how granularly permissions map to command/configure/view actions per robot. missing for 10: granular role definitions, documentation of specific permission scopes (command vs configure vs view), independent/hands-on confirmation of RBAC enforcement.", "evidenceIds": [ "viam-docs-10", "viam-docs-20", "viam-docs-34", "viam-docs-43" ] }, { "productId": "viam", "storyId": "community-modules", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Viam has a Registry providing pre-built drivers/modules for common hardware (cameras, motors, arms, sensors) that viam-server pulls automatically based on config, plus 'pre-built modules or build your own' for reusable behaviors, and searchable models by manufacturer/type. This is well-documented first-party but lacks independent hands-on corroboration of registry breadth/quality. Missing for 10: independent/community verification of registry module quality and coverage, and any registry browsing UI/API evidence beyond docs claims.", "evidenceIds": [ "viam-docs-2", "viam-docs-21", "viam-docs-27", "viam-docs-29", "viam-docs-36", "viam-docs-37", "viam-docs-45", "viam-gh-1" ] }, { "productId": "viam", "storyId": "data-capture-sync", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Viam's docs explicitly describe on-device data capture with a built-in data manager that queues locally during outages/low-bandwidth conditions and syncs automatically when connectivity returns, handling storage constraints and restarts (viam-docs-7, viam-docs-28, viam-docs-51). Independent community evidence corroborates offline-first behavior (cached configs, working reboots without internet) (viam-comm-1), and CLI/query tooling for the synced tabular/binary data is documented (viam-docs-11, viam-docs-23, viam-docs-33). missing for 10: granular bandwidth-throttling/scheduling policy configuration details (e.g., rate limits, sync frequency tuning) beyond generic 'low-bandwidth' handling, and independent hands-on benchmarking of sync behavior under constrained bandwidth", "evidenceIds": [ "viam-docs-7", "viam-docs-28", "viam-docs-51", "viam-comm-1", "viam-docs-23" ] }, { "productId": "viam", "storyId": "data-query-export", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Viam's CLI explicitly supports SQL and MQL querying of tabular/binary data with export to a destination path, plus data management APIs and SDKs for programmatic access, directly satisfying the story. missing for 10: independent/hands-on confirmation of SQL export workflow and details on API-based (non-CLI) query mechanics beyond docs.", "evidenceIds": [ "viam-docs-11", "viam-docs-23", "viam-docs-33", "viam-docs-47", "viam-docs-28" ] }, { "productId": "viam", "storyId": "estop-safety-patterns", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows general capabilities like motion planning with obstacle avoidance (viam-docs-14/22/39) and logic modules that can react to sensor thresholds (viam-docs-17/38/46), which could theoretically be used to build custom safety logic, but there is no documentation of an emergency-stop primitive, safety-interlock pattern, safety-rated I/O, or any dedicated safety/E-stop API or guide anywhere in the pack.", "evidenceIds": [] }, { "productId": "viam", "storyId": "fleet-alerts", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Viam explicitly markets 'Cloud-managed fleet monitoring and alerting' and machine health status APIs (GetMachineStatus reporting unhealthy/configuring resources), plus a generic pattern where 'a temperature sensor crosses a threshold and a notification fires,' which supports fault/threshold-based alerting. However, there's no explicit documentation of dedicated battery-level or connectivity-loss alert configuration, nor any hands-on/community confirmation that these alerts actually fire reliably in practice. Missing for 10: explicit battery/connectivity-specific alert setup docs, and independent evidence that alerting reliably catches issues before customers notice.", "evidenceIds": [ "viam-docs-49", "viam-docs-26", "viam-docs-46", "viam-docs-31", "viam-docs-38" ] }, { "productId": "viam", "storyId": "fleet-config-rollout", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Viam's Fleet Management API lets you manage organizations, locations, and machines and 'create and manage fragments' — fragments are Viam's mechanism for defining shared config that can be applied across multiple machines, and locations/organizations provide a grouping structure for targeting. However, no evidence explicitly describes a workflow for pushing a config change to a defined subset/group of robots at once or shows a UI/CLI command for group-scoped rollout beyond generic OTA update mentions. missing for 10: explicit documentation of per-group/segment targeting UI or CLI command, evidence of bulk config push confirmation/rollback across a named group, independent/hands-on confirmation of fragment-based fleet-wide updates.", "evidenceIds": [ "viam-docs-20", "viam-docs-34", "viam-docs-43", "viam-docs-10", "viam-docs-26", "viam-docs-48" ] }, { "productId": "viam", "storyId": "fleet-dashboard", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Viam explicitly offers cloud-managed fleet monitoring/alerting, remote monitoring, access control, and OTA updates, plus a fleet management API to manage organizations, locations, and machines, and configuration to connect and monitor machines 'from anywhere.' This covers health (GetMachineStatus, alerting), status, and centralized dashboard-style fleet management across locations. Missing for 10: explicit mention of a unified 'location' map/view or independent hands-on confirmation of the dashboard UI beyond vendor docs.", "evidenceIds": [ "viam-docs-10", "viam-docs-26", "viam-docs-49", "viam-docs-34", "viam-docs-31", "viam-docs-15" ] }, { "productId": "viam", "storyId": "hardware-abstraction", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Viam's docs explicitly describe standardized component APIs (e.g., GetImage() for all cameras) so swapping hardware doesn't require code changes, backed by a Registry of drivers for common hardware and a module system for unsupported hardware that then 'behaves like any built-in component.' This is corroborated by community evidence confirming the architecture is real (gRPC-based, open-source RDK) and by explicit doc statements ('If you change a camera or motor in your config, your code keeps working'). Missing for 10: independent hands-on verification from a robotics engineer swapping real hardware in production (only vendor docs and general community architecture confirmation, not a specific swap case study).", "evidenceIds": [ "viam-docs-1", "viam-docs-19", "viam-docs-41", "viam-docs-21", "viam-docs-36", "viam-docs-37", "viam-docs-16", "viam-docs-45", "viam-comm-3" ] }, { "productId": "viam", "storyId": "intervention-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Viam documents fleet monitoring, alerting, remote control/teleoperation via web interface, and OTA updates, but nothing describes an automated routing of autonomy failures to a human intervention queue with resolve-and-handback workflow. Missing for 10: any documented intervention-queue mechanism, failure-triggered escalation to a human operator, or handback-to-autonomy control flow.", "evidenceIds": [ "viam-docs-6", "viam-docs-10", "viam-docs-26", "viam-docs-49" ] }, { "productId": "viam", "storyId": "ml-training-loop", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly describe the full loop: capturing data to the cloud, creating/annotating datasets and training ML models (or bringing your own), then pushing trained models to the cloud where machines pull and run inference automatically with versioning/staging/rollback and canary testing, backed by OTA updates for ML models on the fleet. This directly matches the story of feeding data into training and deploying improved models fleet-wide. Missing for 10: independent/hands-on verification of the training-to-deployment loop working in practice and more detail on training architecture options beyond marketing copy.", "evidenceIds": [ "viam-docs-7", "viam-docs-8", "viam-docs-9", "viam-docs-26", "viam-docs-40", "viam-docs-48", "viam-docs-28" ] }, { "productId": "viam", "storyId": "multi-language-sdks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Viam's docs explicitly state that developers can write control logic in Python, Go, C++, and TypeScript (plus Flutter), and GitHub confirms multiple official SDKs exist for the RDK, directly matching the story's language list. However, there's no independent verification of full API-surface parity across all SDKs or detailed typed-API documentation per language beyond marketing copy. Missing for 10: independent hands-on corroboration of SDK completeness/parity, and detailed reference docs showing full API coverage per language.", "evidenceIds": [ "viam-docs-3", "viam-gh-1", "viam-docs-41", "viam-probe-3" ] }, { "productId": "viam", "storyId": "nl-to-robot-task", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Viam explicitly documents that an AI agent with a shell or SDK can operate a machine via the same APIs as a human, including querying resource status and invoking the motion service to plan collision-free paths from a declared goal, obstacles, and constraints (viam-docs-12, viam-docs-31, viam-docs-32). This supports an agent translating intent into planned, executed motion, but there's no evidence of a natural-language interface or LLM-driven task planner built into the platform itself — the agent (e.g., an LLM) must already exist externally and call Viam's APIs/CLI to do the NL-to-plan translation. missing for 10: no first-party natural-language front-end or built-in LLM planner that maps free-text task descriptions to motion/API calls, no end-to-end demo/case study of NL command to execution, no independent corroboration of this agent workflow succeeding in practice.", "evidenceIds": [ "viam-docs-12", "viam-docs-31", "viam-docs-32", "viam-docs-14", "viam-docs-30" ] }, { "productId": "viam", "storyId": "offline-resilience", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly state data queues locally during outages/low-bandwidth and syncs when connectivity returns, with automatic handling of restarts and storage constraints (viam-docs-7), and independent community testimony confirms robots keep operating offline (cached configs, works even through full reboot without internet, TLS cert grace period of 30+ days) (viam-comm-1). This directly matches the ops story of continued operation plus buffered local data and clean recovery. missing for 10: no first-party documentation detailing exact buffer size/retention limits or a hands-on failure/recovery test log beyond the anecdotal HN report.", "evidenceIds": [ "viam-docs-7", "viam-comm-1" ] }, { "productId": "viam", "storyId": "openness-api-parity", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Viam explicitly documents that AI agents can operate machines identically to UI/human users: 'An AI agent with a shell or an SDK can operate a Viam machine the same way a person or a program can' and 'The CLI can call any API method a machine serves, with JSON in and JSON out; no code to write.' Fleet management, data query/export, motion planning, and component control are all exposed via CLI/SDK/API (viam-docs-11, 13, 20, 23, 30-34), mirroring UI capabilities like test panels and driving components (viam-docs-6). Missing for 10: a discoverable OpenAPI spec (probe found 404s across candidate paths) and independent/hands-on confirmation that every UI action (e.g., dataset annotation workflows) has a full API equivalent.", "evidenceIds": [ "viam-docs-12", "viam-docs-13", "viam-docs-30", "viam-docs-20", "viam-docs-34", "viam-docs-11", "viam-docs-23", "viam-docs-33", "viam-docs-6", "viam-docs-32", "viam-probe-2" ] }, { "productId": "viam", "storyId": "openness-full-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Viam's CLI documents exporting data in 'a variety of formats' and querying tabular/binary data via SQL or MQL, and the core RDK is confirmed open source (gRPC-based, self-hostable, robots can run offline/on their own server), supporting a genuine 'leave' path. However, the docs don't spell out full-fidelity export of configs/models/permissions or a formal migration guide, and a community comment flags uncertainty about how much of the platform (beyond RDK) is truly open, which tempers confidence without constituting a concrete contradiction. Missing for 10: comprehensive documentation of exporting all data types/configs (not just sensor data) in open formats, and independent verification of successful full platform migration/exit.", "evidenceIds": [ "viam-docs-23", "viam-docs-33", "viam-docs-47", "viam-comm-3", "viam-comm-4", "viam-comm-1" ] }, { "productId": "viam", "storyId": "openness-open-license", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "A Hacker News community comment states the core RDK is open source on GitHub (github.com/viamrobotics/rdk) while the cloud app itself is closed, which is corroborated by a GitHub reference. However, no first-party documentation confirms an explicit open-source license, and another commenter expresses uncertainty about how much of the codebase is genuinely open, indicating the openness story is only partially substantiated. missing for 10: explicit license file/name in first-party docs, clarity on which components (SDKs, modules, app) are covered vs proprietary, and confirmation the readable source is complete rather than partial.", "evidenceIds": [ "viam-comm-3", "viam-comm-4", "viam-gh-1" ] }, { "productId": "viam", "storyId": "openness-self-host", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "viam-server (the core robot software / RDK) is open source and runs locally on the user's own compute device, and community evidence confirms it can operate fully offline/self-hosted with robots talking to each other without outside communication, but the management 'cloud app' itself remains a proprietary hosted service not shown to be self-hostable, and a community commenter voices uncertainty about how much is genuinely open source. Missing for 10: first-party documentation for self-hosting the full fleet-management/app layer, and resolution of community doubts about open-source scope and production robustness.", "evidenceIds": [ "viam-comm-3", "viam-comm-1", "viam-comm-4", "viam-docs-15", "viam-docs-24" ] }, { "productId": "viam", "storyId": "ota-deployment", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Viam's marketing/docs explicitly list 'OTA updates for software and ML models' alongside 'Canary testing and rollbacks for safe versioning' as part of Fleet Management, directly matching staged rollout + rollback + OTA requirements, and model versioning/rollback is echoed elsewhere ('Version, stage, and roll back models just like code'). Missing for 10: detailed technical documentation of how staged/canary rollout is configured, and independent/hands-on confirmation that OTA rollback works reliably in practice.", "evidenceIds": [ "viam-docs-10", "viam-docs-26", "viam-docs-40", "viam-docs-48", "viam-docs-9" ] }, { "productId": "viam", "storyId": "physics-simulation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack mentions simulation, physics engines, digital twins, or sensor-model simulation; the documented capability set (real hardware drivers, motion planning against physical machines, fleet management, data capture) is entirely about operating and managing real hardware, not simulating it before deployment. missing for 10: any mention of a simulator/physics engine, virtual sensor models, or a documented sim-to-real workflow.", "evidenceIds": [] }, { "productId": "viam", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions data region selection, residency controls, or geographic storage location choices for Viam's cloud data management; only self-hosting option (viam-comm-3) is mentioned, which is not region/residency selection.", "evidenceIds": [] }, { "productId": "viam", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Viam is a robotics/machine platform; the evidence pack contains no AI-model-training-data policy or opt-out controls, and this axis (preventing data from being used to train AI models) is a category mismatch for a robotics fleet management platform rather than a consumer AI assistant/chatbot product.", "evidenceIds": [] }, { "productId": "viam", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The CLI reference documents a data query/delete capability — `delete data, add or remove tags from all data that matches a given filter` — giving users explicit control to remove captured cloud data (viam-docs-23/33/47). However, there's no documentation of configurable retention periods, automated expiration policies, or org-wide deletion/export controls beyond ad-hoc CLI filtering. Missing for 10: documented retention-period settings, data lifecycle/expiration policies, and any independent confirmation that deletion is durable/complete across storage tiers.", "evidenceIds": [ "viam-docs-23", "viam-docs-33", "viam-docs-47", "viam-docs-28" ] }, { "productId": "viam", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry/usage-tracking opt-out settings or privacy controls for Viam's platform or CLI; docs focus on device data capture/cloud sync, not vendor telemetry about usage of the product itself.", "evidenceIds": [] }, { "productId": "viam", "storyId": "remote-debugging", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Viam docs show real-time remote visibility into machine state, sensor data, and camera feeds from the browser (viam-docs-5, viam-docs-6), machine status/resource health via GetMachineStatus (viam-docs-31), and cloud-managed fleet monitoring/alerting (viam-docs-10, viam-docs-26). NAT traversal (viam-docs-4) supports remote access to deployed robots without networking hassle.\n\nmissing for 10: explicit documentation of a dedicated 'logs' viewer/log streaming feature (only alerting/monitoring and state inspection are directly evidenced, not log tailing specifically), and independent hands-on confirmation of the debugging workflow in production.", "evidenceIds": [ "viam-docs-5", "viam-docs-6", "viam-docs-10", "viam-docs-26", "viam-docs-31", "viam-docs-4", "viam-docs-15" ] }, { "productId": "viam", "storyId": "robot-bring-up", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Viam's docs describe a first-machine wizard, a hardware Registry that auto-pulls drivers, no-code web test panels to drive motors/arms immediately, and a motion-planning service for first motion — directly matching the 'drivers, config, first motion in hours' story. Community evidence (HN thread) corroborates the architecture (gRPC, open-source RDK, local caching) but raises general skepticism about production fit for industrial use rather than concretely refuting bring-up speed. Missing for 10: independent hands-on timing benchmarks proving 'hours not weeks' and a case where an engineer contradicts the ease-of-bring-up claim.", "evidenceIds": [ "viam-docs-15", "viam-docs-24", "viam-docs-36", "viam-docs-6", "viam-docs-27", "viam-docs-14", "viam-comm-3", "viam-comm-2" ] }, { "productId": "viam", "storyId": "sensor-integration", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Viam's Registry provides a searchable catalog of pre-built driver modules for cameras, arms, sensors, and other hardware (explicitly matching by manufacturer/type e.g. webcam, rpi5, xArm6), with standard component APIs like GetImage() ensuring uniform message types across brands, and modules for unsupported hardware can be added easily. Docs and SDK repo corroborate multi-language SDK support and automatic driver initialization from config. Missing for 10: explicit named lidar/IMU/GPS driver examples in the Registry and independent hands-on validation of driver quality/breadth beyond vendor docs.", "evidenceIds": [ "viam-docs-1", "viam-docs-2", "viam-docs-27", "viam-docs-36", "viam-docs-37", "viam-docs-41", "viam-docs-45", "viam-gh-1" ] }, { "productId": "viam", "storyId": "sim-to-real", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Viam's evidence shows strong hardware abstraction (same API across camera/motor brands, swap hardware without rewriting code) and driver registry, which is architecturally adjacent to sim-to-real, but there is no mention anywhere of a simulation environment, simulated components, digital twin, or explicit sim-to-real workflow. The axis is fair for a robotics platform, but no evidence demonstrates Viam actually supports running the same code against a simulated robot.", "evidenceIds": [] }, { "productId": "viam", "storyId": "synthetic-data-rl", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Viam is a physical robotics/IoT fleet management platform focused on real hardware, ML model deployment, and data pipelines; there is no evidence of a simulation environment, synthetic data generation, or RL training infrastructure — this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "viam", "storyId": "teleoperation", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Viam documents live camera streaming, keyboard/teleop controls from a web interface, and NAT traversal for connecting to machines behind firewalls without VPN/port-forwarding, which directly supports remote teleoperation with live video (viam-docs-4, viam-docs-6, viam-docs-42, viam-docs-44). However, there is no evidence quantifying latency/responsiveness over real-world (e.g., cellular, high-latency) networks, and community commentary focuses on offline resilience rather than teleop performance. Missing for 10: independent benchmarks or hands-on reports of control responsiveness/latency over real networks, and explicit teleoperation-specific case studies beyond generic 'drive with keyboard' docs.", "evidenceIds": [ "viam-docs-4", "viam-docs-6", "viam-docs-5", "viam-docs-42", "viam-docs-44", "viam-comm-1" ] }, { "productId": "viam", "storyId": "vla-foundation-models", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Viam documents generic ML model training/deployment ('bring your own model', push trained models to cloud for local inference) and a motion-planning service that could host custom control logic, which together could support integrating a VLA or robotics foundation model — but no evidence specifically mentions VLA, foundation models, or dedicated tooling/integrations for such models. Missing for 10: explicit VLA/foundation-model support, documented integration examples, prebuilt model-hub connectors, or partner/third-party corroboration of foundation-model deployment on Viam.", "evidenceIds": [ "viam-docs-8", "viam-docs-9", "viam-docs-14", "viam-docs-22", "viam-docs-50" ] } ]