[ { "productId": "crossplane", "storyId": "agent-authors-change", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Crossplane's CLI supports rendering/testing Compositions standalone (crossplane-docs-7) which could let an agent preview a change, but the probe explicitly found no headless local plan/preview loop — every real plan/apply requires a live Kubernetes cluster with Crossplane installed (crossplane-probe-2), unlike Terraform-style plan/diff tools. There's no documented diff-and-approve workflow tailored for agent-authored changes.\nmissing for 10: a genuine headless 'plan' command producing a diff against live state, documented agent/approval integration, evidence of an AI agent actually driving this workflow end-to-end.", "evidenceIds": [ "crossplane-docs-7", "crossplane-probe-2", "crossplane-docs-16" ] }, { "productId": "crossplane", "storyId": "agent-least-privilege-gates", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Crossplane's RBAC-based control plane and 'developer only given permission to perform operations at the interface' model (comm-8, docs-17) support a least-privilege foundation, and its declarative API design is explicitly pitched as safe for AI systems to act on (docs-16), but there is no documented mechanism for agent-specific credential scoping, plan/approval review gates, or blocking unapproved applies before they take effect. missing for 10: explicit approval/review-gate workflow for AI-driven changes, agent-scoped least-privilege credential issuance, evidence of a 'plan preview' step analogous to Terraform plan, and independent validation that unauthorized changes are actually blocked.", "evidenceIds": [ "crossplane-docs-16", "crossplane-docs-17", "crossplane-comm-8" ] }, { "productId": "crossplane", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an llms.txt file, agent-oriented documentation format, or any mention of AI-agent-consumable docs; docs-16 mentions AI systems acting on APIs but does not describe agent-oriented documentation discovery. missing for 10: llms.txt file or equivalent, agent-facing documentation format, any explicit reference to AI agents consuming docs directly.", "evidenceIds": [] }, { "productId": "crossplane", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence only shows marketing language about exposing APIs that 'AI systems can safely act on' (crossplane-docs-16), which is about external agents calling Crossplane's API, not the product itself generating AI-driven insights or suggestions from data. No feature, dashboard, or docs describe AI-generated analysis, recommendations, or insights surfaced to users.", "evidenceIds": [ "crossplane-docs-16" ] }, { "productId": "crossplane", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Crossplane's control plane continuously reconciles desired state and self-corrects drift autonomously (crossplane-docs-18), and its declarative APIs are explicitly framed as actionable by automation/AI systems (crossplane-docs-16), which supports background autonomous operation of infrastructure. However, there is no evidence of AI-native 'automations' or agentic workflows being set up specifically for AI systems, scheduled/triggered agent tasks, or any AI-specific orchestration features beyond generic drift-correction and declarative reconciliation. missing for 10: explicit AI-agent-triggered automation examples, scheduling/triggers for autonomous AI tasks, evidence of AI systems actually driving Crossplane resources end-to-end, independent hands-on validation of autonomous background operation.", "evidenceIds": [ "crossplane-docs-18", "crossplane-docs-16", "crossplane-docs-17" ] }, { "productId": "crossplane", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Crossplane is a Kubernetes control-plane infrastructure orchestration tool, not an AI assistant product; there is no built-in AI assistant persona for users to delegate tasks to. One doc line mentions APIs that 'AI systems can safely act on' but this refers to external systems calling Crossplane's APIs, not a built-in assistant feature — this is a category error for this axis.", "evidenceIds": [ "crossplane-docs-16" ] }, { "productId": "crossplane", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Crossplane is fundamentally a Kubernetes control-plane operator: everything is driven by declarative YAML/CRDs reconciled by controllers, which is inherently automatable via kubectl/GitOps pipelines (ArgoCD/Flux) as community evidence confirms, and the CLI itself is keyless and scriptable. However the probe shows every real operation requires a live Kubernetes cluster with Crossplane installed — there's no standalone local plan/apply loop like other CLI-first tools, meaning 'headless CI' usage requires provisioning/maintaining a cluster as a CI dependency rather than a lightweight headless binary. missing for 10: evidence of a documented CI-specific workflow/example (e.g. official GitHub Action or CI recipe), confirmation of non-interactive apply without cluster bootstrap overhead, and independent hands-on reports of running Crossplane in CI pipelines.", "evidenceIds": [ "crossplane-probe-2", "crossplane-probe-1", "crossplane-docs-7", "crossplane-comm-9", "crossplane-comm-10", "crossplane-docs-18" ] }, { "productId": "crossplane", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Crossplane is a Kubernetes-based infrastructure control plane, not an AI agent or assistant capable of consuming external tools; the evidence pack shows no agentic/LLM tool-use component that could plug in MCP servers to gain tool capabilities. The only AI-related mention (crossplane-docs-16) describes AI systems acting on Crossplane's APIs, the opposite direction of an MCP-client story, so this axis is a category error for this product type.", "evidenceIds": [ "crossplane-docs-16" ] }, { "productId": "crossplane", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Crossplane is a Kubernetes control-plane framework, not an AI agent, so publishing an official MCP server is a plausible ecosystem capability, but no evidence in the pack mentions an MCP server, MCP integration, or agent connectivity of any kind—only marketing language about 'AI systems' acting on APIs, which is not an MCP server claim.", "evidenceIds": [ "crossplane-docs-16" ] }, { "productId": "crossplane", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Crossplane's interface is entirely declarative YAML/Compositions plus a CLI for build/test/push workflows; the closest hit is marketing copy saying APIs can be 'acted on' by AI systems, but that describes external consumers, not natural-language command input into Crossplane itself. No documentation, CLI feature, or probe shows any natural-language command interface. Missing for 10: any NL command parser, chat/agent interface, or evidence of accepting free-text instructions instead of declarative manifests.", "evidenceIds": [ "crossplane-docs-16", "crossplane-docs-1", "crossplane-probe-1", "crossplane-probe-2" ] }, { "productId": "crossplane", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Crossplane ships an official CLI (docs and hands-on probe confirm install via brew and working `crossplane version --client`) supporting building/testing/rendering Compositions and managing packages, but it is not framed or evidenced as an AI-native/agentic interface (no MCP integration, no agent-friendly output modes, no natural-language or automation-specific tooling beyond generic CLI usage). missing for 10: explicit AI-native design/marketing for the CLI itself, evidence of agent-friendly structured output or automation hooks, and independent corroboration beyond one hands-on probe.", "evidenceIds": [ "crossplane-docs-7", "crossplane-docs-8", "crossplane-docs-9", "crossplane-probe-1", "crossplane-probe-2" ] }, { "productId": "crossplane", "storyId": "agentic-public-api", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Crossplane explicitly documents that it exposes 'declarative APIs that humans, automation, and AI systems can safely act on' (docs-16), and its architecture is built on the Kubernetes API server extended via CRDs/Managed Resources (docs-3, docs-13, docs-20, docs-21), which forms a standard, machine-readable, documented API surface. This is reinforced by an official CLI (probe-1, probe-2) for interacting with the control plane programmatically. missing for 10: independent/hands-on evidence of an AI agent actually driving Crossplane via the API, and more detail on API schemas/authentication for agentic use beyond marketing copy.", "evidenceIds": [ "crossplane-docs-16", "crossplane-docs-21", "crossplane-docs-3", "crossplane-docs-13", "crossplane-probe-1", "crossplane-probe-2" ] }, { "productId": "crossplane", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Crossplane's own marketing claims it exposes 'declarative APIs that humans, automation, and AI systems can safely act on' and leverages Kubernetes RBAC for security, which implies a path to scoped, least-privilege access control for agentic consumers. However, there is no concrete documentation of an agent-specific credential-issuance mechanism (e.g., token minting, scoped API keys, or short-lived credentials tailored for AI agents) — it relies on generic Kubernetes RBAC rather than a purpose-built agent credentialing feature. Missing for 10: explicit agent-credential issuance workflow, scoping granularity examples, and independent verification that RBAC is actually used this way in practice.", "evidenceIds": [ "crossplane-docs-16", "crossplane-docs-17" ] }, { "productId": "crossplane", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence describes Crossplane's function pipeline supporting multiple languages (YAML, KCL, Python, Go) and declarative APIs that 'AI systems can safely act on,' but there is no mention of an official SDK (e.g., function-sdk-go/python) or any AI-native developer tooling for building against Crossplane programmatically.", "evidenceIds": [] }, { "productId": "crossplane", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack of an event/webhook subscription mechanism for AI agents or other consumers; Crossplane's model is Kubernetes-native reconciliation/drift-correction, not webhook-based event push. This is a fair capability to ask of an infrastructure control plane, but nothing in the evidence documents it.", "evidenceIds": [] }, { "productId": "crossplane", "storyId": "ai-generate-iac", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows Crossplane exposes declarative APIs that AI systems could act on, but there is no documented feature for generating infrastructure code from natural language within the Crossplane toolchain itself (CLI, docs, or otherwise).", "evidenceIds": [] }, { "productId": "crossplane", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an interactive API reference with runnable examples exists in the pack; documentation is described only as static pages and CLI docs, with no mention of a live/interactive reference or embedded runnable code snippets.", "evidenceIds": [] }, { "productId": "crossplane", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Crossplane's CRD-based schemas (crossplane-docs-13) imply an OpenAPI-compatible structure inherent to Kubernetes, but the evidence pack contains no explicit documentation or feature describing a downloadable OpenAPI or machine-readable API spec for Crossplane's Composite Resources or Providers. Missing for 10: explicit docs on exporting/downloading OpenAPI schema, CLI or API endpoint for spec retrieval, and any community/hands-on confirmation of this capability.", "evidenceIds": [ "crossplane-docs-13" ] }, { "productId": "crossplane", "storyId": "api-sandbox", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Crossplane's CLI supports rendering/testing Composition Functions locally without needing a live Kubernetes cluster (crossplane-docs-7), which enables some local, production-safe testing. However, a probe shows that real plan/apply operations still require a running Kubernetes cluster with Crossplane installed, with no built-in local sandbox/preview loop for full resource provisioning — so isolating test workloads from production relies on the user provisioning a separate cluster/cloud account rather than a first-class sandbox feature. missing for 10: dedicated sandbox-mode or dry-run against real cloud providers, evidence of safe multi-tenant test isolation, and confirmation that render-only testing covers full MR/Composition behavior rather than just function outputs.", "evidenceIds": [ "crossplane-docs-7", "crossplane-probe-2" ] }, { "productId": "crossplane", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "While Crossplane exposes CRD-based APIs and has a 'Composition Revisions' feature for managing changes, the evidence pack contains no documentation of a formal API versioning scheme or deprecation policy for its own APIs (e.g., CRD version graduation, deprecation timelines, migration guides). Missing for 10: explicit deprecation policy docs, CRD version stability guarantees, migration/upgrade guidance for deprecated APIs.", "evidenceIds": [] }, { "productId": "crossplane", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence describes Crossplane's declarative APIs, Compositions, and controller-based reconciliation, but nothing in the pack documents or demonstrates bulk/batch operations across many resources at once (e.g., batch apply, multi-resource templating at scale, or AI-driven bulk actions).", "evidenceIds": [] }, { "productId": "crossplane", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Crossplane's control-plane model is inherently reactive: it continuously watches Kubernetes resource state and automatically corrects drift, and its composition function pipelines process changes to composite resources as they occur, which functions as an implicit event-driven automation model. However, there is no evidence of an explicit rule-authoring interface (e.g., 'on event X do Y' triggers, webhooks, custom event sources) — the automation is confined to declarative reconciliation loops built into the resource model. Missing for 10: explicit event-trigger/rule definition mechanism beyond drift reconciliation, documentation of custom event sources or conditional trigger logic, and independent confirmation of this automation depth.", "evidenceIds": [ "crossplane-docs-18", "crossplane-docs-1", "crossplane-docs-23", "crossplane-docs-26" ] }, { "productId": "crossplane", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Crossplane is a Kubernetes control-plane/infrastructure-orchestration tool, not a scheduler or workflow-automation engine for recurring jobs; its reconciliation loop continuously corrects drift rather than running scheduled/recurring tasks. This story is a category error for this product type.", "evidenceIds": [] }, { "productId": "crossplane", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Crossplane's Composition Revisions explicitly let you manage and track changes to Compositions, and its YAML-based Kubernetes resources are naturally committed to source control and managed via GitOps, as community members note ('clear yaml syntax... easy enough to commit to source control'). Drift correction also provides a form of automatic rollback to desired state. missing for 10: explicit documentation of a review/approval workflow (e.g., PR-gated promotion), and a clear step-by-step rollback procedure using Composition Revisions rather than just their existence.", "evidenceIds": [ "crossplane-docs-15", "crossplane-docs-18", "crossplane-comm-10", "crossplane-comm-9" ] }, { "productId": "crossplane", "storyId": "destroy-teardown", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack never documents a destroy/teardown operation or workflow for Crossplane; it covers install, packages, functions, and drift-correction, but the only deletion-related item (crossplane-docs-6) is about *protecting* providers from deletion, not enabling clean teardown. No CLI or docs evidence shows a 'crossplane destroy' or cascading delete guarantee for an environment. Missing for 10: any documentation of a destroy/delete command, cascading resource cleanup behavior, or a described teardown workflow.", "evidenceIds": [ "crossplane-docs-6", "crossplane-docs-9", "crossplane-probe-2" ] }, { "productId": "crossplane", "storyId": "drift-detection", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Crossplane's control-plane model continuously reconciles declared desired state against actual cloud resources and self-corrects drift automatically, as documented directly ('If your software ever drifts from your desired state, the control plane automatically corrects the drift'). This is reinforced by its reconciliation-based architecture (managed resources, compositions) rather than one-off plan/apply. Missing for 10: independent hands-on verification/benchmarks of drift-detection behavior in production, and no detail on drift visibility/reporting tooling (e.g., diffing UI) beyond auto-correction.", "evidenceIds": [ "crossplane-docs-18", "crossplane-docs-23", "crossplane-docs-3", "crossplane-comm-3" ] }, { "productId": "crossplane", "storyId": "environment-config-stacks", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Crossplane has 'Environment Configs' (in-memory data stores for Compositions) and 'Configurations' as portable packages, which could be used to parameterize per-environment settings, but there is no documented workspace/stack concept analogous to Terraform workspaces for isolating dev/staging/prod as separate stacks. missing for 10: explicit dev/staging/prod workspace pattern, first-party guidance on structuring multiple environments, independent corroboration of this use case.", "evidenceIds": [ "crossplane-docs-14", "crossplane-docs-11", "crossplane-docs-15" ] }, { "productId": "crossplane", "storyId": "gp-language-support", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Crossplane's core resource model (Compositions, XRDs) is YAML/CRD-based, but its Functions pipeline explicitly supports writing composition logic in general-purpose languages like Python and Go (as well as KCL), which brings type systems and IDE tooling into play (crossplane-docs-2, crossplane-docs-26). However, this applies only to the Function extension mechanism, not to defining the full infrastructure spec itself, and there's no evidence of dedicated SDKs, typed client libraries, or IDE integration for this workflow. missing for 10: dedicated general-purpose-language SDK for defining infra resources (not just functions), documented IDE/autocomplete support, type-safety examples, loop/programmatic composition examples, independent hands-on validation of this workflow.", "evidenceIds": [ "crossplane-docs-2", "crossplane-docs-26", "crossplane-docs-1" ] }, { "productId": "crossplane", "storyId": "headless-ci-apply", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Crossplane operates as a continuously-reconciling Kubernetes control plane rather than a plan/apply tool with saved plan artifacts and approval gates; the evidence pack contains no documentation of a CI-oriented plan/apply/approve workflow. The hands-on probe explicitly notes that unlike Terraform-style tools, 'every real plan/apply operation requires a running Kubernetes cluster... no local plan/preview loop exists,' confirming the absence of this capability.", "evidenceIds": [ "crossplane-probe-2", "crossplane-docs-7" ] }, { "productId": "crossplane", "storyId": "import-existing-infra", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack shows a Crossplane feature for importing existing/unmanaged cloud resources or auto-generating matching Composition/Claim YAML; the docs cover installation, packages, and compositions but nothing about resource discovery or import tooling. missing for 10: an import/adopt command or workflow, generated MR/Composition code from existing infra, any docs or community reports of migrating pre-existing resources into Crossplane management.", "evidenceIds": [] }, { "productId": "crossplane", "storyId": "migrate-from-terraform", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any Terraform-state import, HCL conversion, or migration tooling in Crossplane's docs, CLI, or community discussion; Crossplane uses an entirely different declarative Kubernetes-native model with no documented path to ingest existing Terraform codebases or state files.", "evidenceIds": [] }, { "productId": "crossplane", "storyId": "open-license-governance", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of Crossplane's license (e.g., Apache 2.0) or its governance model (e.g., CNCF graduation, foundation stewardship) — nothing here addresses licensing-governance at all. missing for 10: license identification, governance/foundation structure, any statement on relicensing protection.", "evidenceIds": [] }, { "productId": "crossplane", "storyId": "openness-api-parity", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Crossplane is fundamentally a Kubernetes control-plane extension: all functionality (Providers, Compositions, Functions, Managed Resources) is exposed as declarative Kubernetes API objects (CRDs) manipulated via kubectl/API, with docs explicitly framing it as exposing 'declarative APIs that humans, automation, and AI systems can safely act on' and letting users 'design the APIs and abstractions' for their control planes — there is no separate UI gating capability. Missing for 10: no evidence pack mention of any Crossplane-native UI/console to explicitly confirm parity claims, and no independent/hands-on corroboration of full UI-API equivalence.", "evidenceIds": [ "crossplane-docs-16", "crossplane-docs-21", "crossplane-docs-20", "crossplane-docs-13", "crossplane-docs-10" ] }, { "productId": "crossplane", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Crossplane's resources (Compositions, Configurations, Managed Resources) are defined as plain YAML/KCL/etc. stored in Kubernetes, which is inherently portable and inspectable, and packages are designed to be portable across clusters (crossplane-docs-11, crossplane-docs-2). However there is no explicit documented 'export all data' feature or bulk-export/migration tool, and no evidence addressing exporting live state or leaving the platform cleanly. missing for 10: explicit data-export/backup tooling, documentation of exporting full state (not just config) in open formats, and any evidence of migration/exit workflow.", "evidenceIds": [ "crossplane-docs-2", "crossplane-docs-11", "crossplane-docs-13", "crossplane-docs-15" ] }, { "productId": "crossplane", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions Crossplane's source code license, GitHub repository, or open-source licensing terms (e.g., Apache 2.0) — the pack covers only product docs, community discussion, and CLI probes. Missing for 10: license file/declaration, repository link, any statement about open-source status or license type.", "evidenceIds": [] }, { "productId": "crossplane", "storyId": "openness-self-host", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Crossplane is an open-source Kubernetes control-plane extension that users install and run themselves via Helm on their own cluster, with full control over feature flags, package installation, and versions (docs-4, docs-5, docs-6, docs-19, docs-22, docs-25). The probe confirms hands-on that the CLI and control plane operate entirely against a self-managed Kubernetes cluster with no forced SaaS dependency (probe-2). Missing for 10: no explicit vendor statement contrasting self-hosting vs. any managed offering, and no independent write-up specifically about self-hosting experience at scale.", "evidenceIds": [ "crossplane-docs-4", "crossplane-docs-5", "crossplane-docs-6", "crossplane-docs-19", "crossplane-docs-22", "crossplane-docs-25", "crossplane-probe-2" ] }, { "productId": "crossplane", "storyId": "policy-enforcement", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes Crossplane's control-plane architecture, RBAC, packages, and CLI tooling, but contains no mention of a policy-as-code engine (e.g., OPA/Gatekeeper/Kyverno integration, admission webhooks, or built-in policy checks) that blocks non-compliant infrastructure changes before apply. RBAC is referenced only as an access-control feature, not as a compliance gate.", "evidenceIds": [] }, { "productId": "crossplane", "storyId": "preview-before-apply", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Crossplane operates on a continuous-reconciliation model rather than an explicit plan/apply diff workflow; the CLI's `render` command only tests Composition Function output offline, and probe evidence explicitly states no local plan/preview loop exists — every real change requires a live cluster with the control plane reconciling live, not previewing creates/updates/destroys beforehand. missing for 10: any documented `crossplane plan`/diff command showing pending creates/updates/deletes, first-party guidance on previewing resource changes before applying, and independent confirmation of such a preview workflow.", "evidenceIds": [ "crossplane-probe-2", "crossplane-docs-7", "crossplane-docs-18" ] }, { "productId": "crossplane", "storyId": "privacy-data-residency", "verdict": "na", "quality": 0, "confidence": "low", "rationale": "Crossplane is a Kubernetes-based control plane / infrastructure-provisioning tool for managing cloud resources declaratively; it is not itself a data-storing service that an AI-native end user interacts with to have 'my data' stored somewhere. The residency/region-choice story presumes a SaaS product holding user data, which is a category mismatch for an IaC/control-plane tool, so this axis does not apply.", "evidenceIds": [] }, { "productId": "crossplane", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Crossplane is an infrastructure-orchestration control plane; it has no role in AI-model-training data usage or user-data privacy policies, so the axis of 'preventing data from being used to train AI models' does not apply to this category of product.", "evidenceIds": [] }, { "productId": "crossplane", "storyId": "privacy-retention-controls", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Crossplane is a Kubernetes control-plane/infrastructure-orchestration tool, not a data-handling AI product that stores or retains user/conversational data. Data retention and deletion policy for AI-native privacy posture is a category error for this kind of infrastructure provisioning tool.", "evidenceIds": [] }, { "productId": "crossplane", "storyId": "privacy-telemetry-optout", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Crossplane is a Kubernetes control-plane infrastructure orchestration tool, not an AI assistant/agent product that collects usage telemetry from an end user's interactions; opting out of telemetry/usage tracking is not a relevant axis for this category of product based on the evidence provided.", "evidenceIds": [] }, { "productId": "crossplane", "storyId": "programmatic-deployments", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Crossplane exposes Kubernetes-native declarative APIs and explicitly markets itself as something 'humans, automation, and AI systems can safely act on', implying resources can be created/managed programmatically (e.g., via the Kubernetes API) rather than only via CLI. However, the evidence pack never documents an SDK, client library, or concrete example of driving Crossplane from application code — it only shows CLI tooling and YAML-based composition, and the probe confirms all real plan/apply operations require a running cluster with no programmatic preview path shown. missing for 10: explicit SDK/client-library documentation, a hands-on example of application code creating/managing Crossplane resources, and confirmation that this workflow is idiomatic (not just theoretically possible via raw Kubernetes API).", "evidenceIds": [ "crossplane-docs-16", "crossplane-docs-21", "crossplane-probe-2", "crossplane-docs-22" ] }, { "productId": "crossplane", "storyId": "provider-breadth", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm a package-based Provider model that connects Crossplane to external APIs (e.g. AWS RDS) and describes a 'library of components' to assemble a platform, and community discussion references AWS/GCP/Azure-style resources being managed this way, supporting a broad multi-cloud provider ecosystem claim. However, the evidence pack never enumerates specific major-cloud or SaaS provider packages (Azure, GCP, Datadog, etc.) beyond AWS RDS, nor cites a provider marketplace/registry size or independent confirmation of breadth.\nmissing for 10: explicit list/marketplace evidence of major cloud providers (Azure, GCP) and SaaS providers, independent verification of ecosystem breadth beyond AWS example.", "evidenceIds": [ "crossplane-docs-3", "crossplane-docs-10", "crossplane-docs-27", "crossplane-comm-1", "crossplane-comm-4" ] }, { "productId": "crossplane", "storyId": "remote-state-locking", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Crossplane is a Kubernetes control plane that stores desired/actual state as custom resources in the cluster's own etcd, reconciled continuously — it has no separate 'state file' or pluggable remote backend/locking concept like Terraform-style IaC tools. The evidence pack (installation, packages, compositions) never discusses backend state storage or locking because the architecture makes this axis inapplicable.", "evidenceIds": [] }, { "productId": "crossplane", "storyId": "reusable-modules", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Crossplane docs describe Providers and Configurations as portable, installable packages and the CLI supports building, installing, updating and pushing Crossplane Packages, plus a 'library of components to assemble your platform faster' — all indicating a publish/consume workflow for reusable modules. However, there is no explicit evidence of a named public registry (e.g., Upbound Marketplace) or details on registry discovery/versioning workflows. Missing for 10: explicit public registry reference/marketplace evidence, developer-facing publish workflow walkthrough, independent confirmation of module reuse via registry.", "evidenceIds": [ "crossplane-docs-9", "crossplane-docs-10", "crossplane-docs-11", "crossplane-docs-27", "crossplane-docs-22" ] }, { "productId": "crossplane", "storyId": "secret-config-values", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Crossplane orchestrates infrastructure via Kubernetes and secret/credential handling (e.g., Provider credentials, Kubernetes Secrets) is a plausible axis for such a control plane, but the evidence pack contains no mention of secrets management, credential injection, or log redaction mechanisms. missing for 10: any documentation or community evidence of secret injection, secure credential storage, or protection from log exposure.", "evidenceIds": [] }, { "productId": "crossplane", "storyId": "state-encryption", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of encryption at rest, state files, secrets handling, or etcd encryption for Crossplane's managed resource state. Crossplane stores its state as Kubernetes custom resources rather than flat state files, but no documentation here addresses how sensitive values are protected from plaintext exposure. missing for 10: any docs on etcd encryption, secret handling for provider credentials, or state-at-rest encryption guarantees.", "evidenceIds": [] }, { "productId": "crossplane", "storyId": "state-surgery", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Crossplane CLI features for troubleshooting and rendering Compositions/Managed Resources, but there is no documentation or community evidence of a safe state-inspection/modification workflow (e.g., moving, renaming, or removing resources from state) analogous to Terraform's state mv/rm/import commands. Refactoring-safety for live infrastructure state is a fair axis for any IaC/control-plane tool, so absence of evidence here yields 'none' rather than 'na'.", "evidenceIds": [ "crossplane-docs-8", "crossplane-probe-1", "crossplane-probe-2" ] }, { "productId": "crossplane", "storyId": "structured-plan-output", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Crossplane's model is continuous reconciliation (drift correction) rather than a discrete plan/apply workflow, and the probe evidence explicitly states no local plan/preview loop exists — every operation requires a live cluster. There is no documented JSON plan output or diff artifact an agent could parse.", "evidenceIds": [ "crossplane-probe-2", "crossplane-docs-18" ] }, { "productId": "crossplane", "storyId": "targeted-partial-apply", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Crossplane's docs describe compositions, functions, and CLI rendering/troubleshooting, but nothing in the evidence shows a mechanism to target or exclude specific resources during a plan/apply-like operation; the probe explicitly notes there is no local plan/preview loop at all, only continuous reconciliation via a running cluster. missing for 10: any documented `--target`/`--exclude` style flag or resource-selection mechanism for plan or apply operations.", "evidenceIds": [ "crossplane-probe-2", "crossplane-docs-7", "crossplane-docs-8" ] }, { "productId": "crossplane", "storyId": "unit-integration-tests", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Docs claim the Crossplane CLI supports 'testing and rendering standalone Composition Functions without the need to access a Kubernetes cluster running Crossplane' (crossplane-docs-7), which would let developers test infra code without touching production. However, a hands-on probe explicitly found that 'unlike other tools probed here every real plan/apply operation requires a running Kubernetes cluster with the Crossplane control plane installed — no local plan/preview loop exists' (crossplane-probe-2), directly contradicting the claim of a self-contained local testing workflow. Missing for 10: independent confirmation that `crossplane render` works fully offline for realistic compositions, CI/CD test-suite examples, and reconciliation of the docs/probe discrepancy.", "evidenceIds": [ "crossplane-docs-7", "crossplane-probe-2" ] }, { "productId": "crossplane", "storyId": "validate-fmt", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The Crossplane CLI supports offline testing/rendering of Compositions without a cluster (crossplane-docs-7, crossplane-docs-8), which partially covers 'validate,' but there is no evidence of an auto-format command, and crossplane-probe-2 explicitly notes there is no local plan/preview loop — every real operation requires a live cluster, unlike a typical validate→format→plan workflow. missing for 10: dedicated auto-format tooling, an actual 'plan' equivalent step, and independent confirmation of the validate/render workflow's completeness.", "evidenceIds": [ "crossplane-docs-7", "crossplane-docs-8", "crossplane-probe-2" ] }, { "productId": "crossplane", "storyId": "version-stability", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows install/config docs, feature flags, and package management, but contains no explicit compatibility promises (e.g., API versioning policy) or dedicated upgrade/migration guides between Crossplane releases. This axis clearly applies to a platform like Crossplane, but no supporting evidence is present.", "evidenceIds": [] }, { "productId": "opentofu", "storyId": "agent-authors-change", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Hands-on probes confirm a full headless plan→apply loop: `tofu plan -detailed-exitcode -out=plan.out` runs non-interactively and signals changes, `tofu show -json` emits machine-readable diff output parseable by agent tooling, and `tofu apply plan.out` executes the saved plan for approval-gated apply. This directly matches the story of an agent authoring changes, planning headlessly, and presenting a diff for approval. missing for 10: no first-party agent/CI orchestration example or official docs showing an approval-gate workflow built around the JSON plan output, and no independent (non-vendor) corroboration beyond the probe run.", "evidenceIds": [ "opentofu-probe-5", "opentofu-probe-6", "opentofu-probe-7", "opentofu-docs-3", "opentofu-docs-21" ] }, { "productId": "opentofu", "storyId": "agent-least-privilege-gates", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "OpenTofu's plan/apply separation (saved plan files, machine-readable JSON output via `tofu show -json`, and exit codes) gives an agent a natural review gate where a human or policy layer can inspect a plan before `apply` executes it, as shown in probes -5/-6/-7 and docs-3/-4/-21. However, there is no evidence of built-in least-privilege credential scoping, RBAC, approval workflows, or a policy-as-code engine (Sentinel/OPA equivalent) enforcing that gate — the review step is only possible via external tooling, not something OpenTofu itself provides. Missing for 10: native policy-as-code/policy enforcement, credential scoping/RBAC features, and any documented approval-gate mechanism tied to identity or role.", "evidenceIds": [ "opentofu-docs-3", "opentofu-docs-4", "opentofu-docs-21", "opentofu-probe-5", "opentofu-probe-6", "opentofu-probe-7" ] }, { "productId": "opentofu", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe of https://opentofu.org/llms.txt returned HTTP 404, and no other evidence pack items reference an llms.txt file or agent-oriented documentation format; the docs are standard human-facing pages only.", "evidenceIds": [ "opentofu-probe-1", "opentofu-probe-2" ] }, { "productId": "opentofu", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "OpenTofu is an infrastructure-as-code provisioning tool; the evidence pack shows only plan/apply/test/import/state-encryption features with no AI-generated insights, analytics, or suggestion capability, and this kind of product (an IaC engine) is not the type expected to surface AI-driven data insights about a user's infrastructure — it's a wrong-axis question rather than a missing feature.", "evidenceIds": [] }, { "productId": "opentofu", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "OpenTofu's CLI is fully non-interactive/keyless and scriptable (headless plan→apply→destroy verified in probes), which is a prerequisite for wiring it into cron jobs or CI/CD pipelines for autonomous runs, but there is no evidence of any built-in scheduler, trigger, webhook, or agent runtime that autonomously initiates or manages background runs on its own. missing for 10: native scheduling/trigger mechanism, evidence of autonomous (not just headless) background operation, first-party automation/orchestration features beyond CLI scriptability.", "evidenceIds": [ "opentofu-probe-5", "opentofu-probe-6", "opentofu-probe-7" ] }, { "productId": "opentofu", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "OpenTofu is an infrastructure-as-code CLI/engine, not an agent or assistant product; it has no built-in AI assistant feature and this axis is a category error for its product type.", "evidenceIds": [] }, { "productId": "opentofu", "storyId": "agentic-headless", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Hands-on probes confirm OpenTofu supports non-interactive, scriptable CI workflows: `-input=false`, `-auto-approve`, `-detailed-exitcode` for change detection, machine-readable JSON plan output via `tofu show -json`, and a full plan→apply→destroy loop with correct exit codes, all installable without a login/keys via brew — exactly what CI/agent automation needs. Missing for 10: no explicit vendor CI/CD integration guide (e.g., GitHub Actions/GitLab CI docs) or third-party CI pipeline examples in the evidence pack.", "evidenceIds": [ "opentofu-probe-5", "opentofu-probe-6", "opentofu-probe-7", "opentofu-docs-3", "opentofu-docs-21" ] }, { "productId": "opentofu", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "OpenTofu is an infrastructure-as-code CLI/engine, not an agent or MCP client host; plugging MCP servers into it to give it tool access is a category mismatch — no evidence exists of MCP integration, and the product's role doesn't call for consuming MCP tools.", "evidenceIds": [] }, { "productId": "opentofu", "storyId": "agentic-mcp-server", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "OpenTofu is an infrastructure-as-code CLI tool, not an agent product; the evidence shows only CLI-based automation (plan/apply/JSON output) and no MCP server offering, but this axis is about connecting an agent via an official MCP server which is a wrong-fit ask for this kind of tool rather than an agentic feature gap—no evidence even hints at an MCP endpoint.", "evidenceIds": [] }, { "productId": "opentofu", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "OpenTofu's interface is HCL configuration plus a fixed CLI command set (plan, apply, test, import, etc.); nothing in the evidence pack shows any natural-language command interface, NL parsing, or AI-native control surface — llms.txt and other AI-discovery probes even returned 404s.", "evidenceIds": [ "opentofu-probe-1", "opentofu-probe-2", "opentofu-probe-4", "opentofu-docs-3" ] }, { "productId": "opentofu", "storyId": "agentic-official-cli", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "OpenTofu ships an official, documented CLI (opentofu.org/docs/cli/) and hands-on probes confirm a full headless workflow — init, plan with machine-readable JSON output, auto-approved apply, and destroy — all scriptable without interactive input, which is exactly what an AI-native/agentic caller needs. It is also a drop-in Terraform replacement, so existing agent tooling built for Terraform's CLI works unchanged.\n\nmissing for 10: no first-party documentation specifically addressing AI-agent usage patterns or guardrails (e.g., rate limits, agent-specific auth flows) beyond generic CLI/JSON support.", "evidenceIds": [ "opentofu-probe-4", "opentofu-probe-5", "opentofu-probe-6", "opentofu-probe-7", "opentofu-docs-3", "opentofu-docs-21" ] }, { "productId": "opentofu", "storyId": "agentic-public-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "OpenTofu is drivable programmatically via its CLI (headless plan/apply/destroy loop, machine-readable JSON plan output) as confirmed by hands-on probes, which supports AI-native agentic automation, but there is no documented public HTTP/REST/OpenAPI API — probes for llms.txt, docs-md, and openapi/swagger endpoints all 404. missing for 10: a documented public API (REST/OpenAPI/SDK) beyond the CLI, official llms.txt or machine-readable docs endpoint, and any first-party API reference for programmatic/agentic access.", "evidenceIds": [ "opentofu-probe-5", "opentofu-probe-6", "opentofu-probe-7", "opentofu-probe-1", "opentofu-probe-2", "opentofu-probe-3", "opentofu-probe-4" ] }, { "productId": "opentofu", "storyId": "agentic-scoped-keys", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "OpenTofu is an IaC CLI/engine that consumes provider credentials but has no product surface for issuing or scoping its own API credentials to agents; this axis belongs to identity/credential-issuing platforms, not an infrastructure-as-code tool.", "evidenceIds": [] }, { "productId": "opentofu", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows only CLI usage, JSON plan output, and website docs; there is no mention of any official SDK (Go, Python, REST, or agent-oriented) that AI-native developers could build against, and probes explicitly show llms.txt, docs-md, and OpenAPI endpoints returning 404. Missing for 10: any documented official SDK, API/OpenAPI spec, or programmatic library for AI agents to integrate with OpenTofu beyond the CLI.", "evidenceIds": [ "opentofu-probe-1", "opentofu-probe-2", "opentofu-probe-3", "opentofu-probe-4" ] }, { "productId": "opentofu", "storyId": "agentic-webhooks", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "OpenTofu is an IaC CLI tool that operates via local/CI plan-apply workflows, not an event-driven or SaaS platform; webhook subscriptions are not a fit axis for this product category, and no evidence suggests such a feature exists or is planned.", "evidenceIds": [] }, { "productId": "opentofu", "storyId": "ai-generate-iac", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "OpenTofu is an IaC engine/CLI (HCL execution, state management, plan/apply) with no built-in AI/NLG feature for generating infrastructure code from natural language; this is a category error for this product type, not a missing feature of it.", "evidenceIds": [] }, { "productId": "opentofu", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "OpenTofu's docs are static CLI command references (plan/apply/test/import) with no interactive API explorer or runnable-example sandbox; explicit probes confirm no OpenAPI/swagger spec and no llms.txt discoverability aid.", "evidenceIds": [ "opentofu-probe-3", "opentofu-probe-1", "opentofu-docs-3", "opentofu-docs-7" ] }, { "productId": "opentofu", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Explicit probes confirm no OpenAPI/Swagger spec or llms.txt is available (404s at all candidate paths), and no evidence of any machine-readable API spec being offered.", "evidenceIds": [ "opentofu-probe-3", "opentofu-probe-1", "opentofu-probe-2" ] }, { "productId": "opentofu", "storyId": "api-sandbox", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "OpenTofu's `tofu test` command explicitly creates real infrastructure to test configuration and then destroys it, and workspaces/variables allow separate dev/staging/prod environments (multi-environment support mentioned in docs), which supports sandbox-style testing without touching production if configured with separate state/backends. However, there is no dedicated built-in sandboxing feature, no first-party guidance on isolating a sandbox from production data specifically, and no mocking/dry-run-only testing mode beyond plan preview. missing for 10: dedicated sandbox/mock-infrastructure feature, explicit doc guidance on isolating prod vs sandbox state, independent verification of test-command safety in practice.", "evidenceIds": [ "opentofu-docs-7", "opentofu-docs-25", "opentofu-docs-18", "opentofu-docs-3", "opentofu-probe-5" ] }, { "productId": "opentofu", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack shows OpenTofu has versioned CLI output formats (e.g., plan JSON format_version) but contains no documentation of a formal API/CLI deprecation policy or version-support guarantees; the axis is applicable to an IaC tool with a CLI/JSON interface, but no evidence supports a documented deprecation policy. Missing for 10: explicit deprecation policy docs, versioning/SLA guarantees, changelog of breaking changes.", "evidenceIds": [ "opentofu-probe-6", "opentofu-probe-4" ] }, { "productId": "opentofu", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "OpenTofu supports bulk-style operations via resource graph parallelization, for_each-driven dynamic provider/module generation, -exclude flag for scoping bulk plan/apply, and programmatic module version updates across many modules, which together enable acting on many items at once. However this is infrastructure-as-code bulk management, not an AI-native batch/agent API, and there's no evidence of a dedicated bulk API, batch job orchestration, or AI-specific tooling for mass operations. missing for 10: dedicated bulk/batch API or agent-facing interface for large-scale multi-item operations, independent hands-on evidence of bulk operations at scale, AI-native tooling specifically for orchestrating many items programmatically.", "evidenceIds": [ "opentofu-gh-1", "opentofu-docs-10", "opentofu-docs-11", "opentofu-docs-12", "opentofu-docs-18" ] }, { "productId": "opentofu", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "OpenTofu's evidence pack covers CLI plan/apply/test/import workflows but shows no rule-based or event-triggered automation engine (e.g., webhooks, event listeners, policy-triggered actions) built into the tool itself.", "evidenceIds": [] }, { "productId": "opentofu", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "OpenTofu is an infrastructure-as-code CLI/engine for provisioning resources via plan/apply, not a workflow orchestrator or job scheduler; scheduling recurring jobs is outside its product category and not addressed anywhere in the evidence.", "evidenceIds": [] }, { "productId": "opentofu", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "OpenTofu's state is plain text-based (HCL) which enables version control via git, and plan files (tofu plan -out) provide a preview/review mechanism, with tofu destroy/apply providing rollback-like operations, but there's no native built-in versioning, review workflow, or rollback/history feature beyond what git or external tools (e.g. Terraform Cloud-like state history) provide. missing for 10: no first-party state history/rollback mechanism, no built-in review/approval workflow beyond plan output, no independent evidence of an audit trail or diff-based rollback UI.", "evidenceIds": [ "opentofu-docs-16", "opentofu-docs-3", "opentofu-docs-21", "opentofu-probe-5", "opentofu-probe-7" ] }, { "productId": "opentofu", "storyId": "destroy-teardown", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "OpenTofu docs explicitly describe destroy-mode planning that removes all remote objects leaving an empty state, and a hands-on probe confirms a full plan→apply→destroy loop exiting cleanly (exit 0). This directly matches the story of tearing down an environment cleanly. Missing for 10: independent (non-probe) third-party confirmation of destroy behavior at scale or with complex multi-resource environments.", "evidenceIds": [ "opentofu-docs-5", "opentofu-probe-7" ] }, { "productId": "opentofu", "storyId": "drift-detection", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "OpenTofu's `tofu plan` explicitly supports a refresh-only mode that updates state to reflect out-of-band changes without applying corrections, which is the canonical drift-detection workflow, and a hands-on probe confirms plan surfaces detected changes via -detailed-exitcode in a real run. Missing for 10: an explicit end-to-end hands-on test showing detection of an externally-modified resource (drift) specifically, and independent/community validation of drift detection accuracy.", "evidenceIds": [ "opentofu-docs-6", "opentofu-docs-3", "opentofu-probe-5", "opentofu-probe-6" ] }, { "productId": "opentofu", "storyId": "environment-config-stacks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "OpenTofu's docs explicitly mention using for_each to build 'multi-environment setups (dev/staging/prod)' via provider configuration generation, showing the product can model separate environments, and its drop-in Terraform compatibility implies inherited workspace/state-separation mechanisms. However, there is no direct documentation or probe evidence of a dedicated `tofu workspace` command, backend workspace isolation, or a 'stacks' feature being exercised, so the core mechanism developers would use for per-environment config isn't concretely demonstrated. Missing for 10: explicit `tofu workspace` command docs, evidence of stacks/state isolation per environment, and hands-on proof of switching environments.", "evidenceIds": [ "opentofu-docs-18", "opentofu-docs-1", "opentofu-docs-14" ] }, { "productId": "opentofu", "storyId": "gp-language-support", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "OpenTofu's evidence pack only shows HCL-based declarative configuration (for_each loops, dynamic provider blocks) rather than a general-purpose programming language with types and IDE support; there is no mention of a CDK-like SDK (e.g., TypeScript/Python bindings) that would satisfy this story.", "evidenceIds": [] }, { "productId": "opentofu", "storyId": "headless-ci-apply", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs describe saving plans with `-out=FILE` and applying them via `tofu apply plan.out`, and hands-on probes confirm a full non-interactive plan→apply→destroy loop using `-input=false -auto-approve -detailed-exitcode`, matching the CI approval-flag workflow described in the story. missing for 10: explicit CI/CD pipeline examples (e.g., GitHub Actions/GitLab CI snippets) showing artifact upload/download between plan and apply stages.", "evidenceIds": [ "opentofu-docs-21", "opentofu-probe-5", "opentofu-probe-6", "opentofu-probe-7" ] }, { "productId": "opentofu", "storyId": "import-existing-infra", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "OpenTofu documents `tofu import` for importing existing resources into state, and community evidence confirms `for_each` support for import blocks (bulk import). However, evidence does not show automatic configuration-code generation from imported resources (e.g., a `-generate-config-out` style feature) — missing for 10: documentation/proof of automatic config generation during import, hands-on demonstration of the full import-to-config workflow, and independent verification of accuracy for generated code.", "evidenceIds": [ "opentofu-docs-8", "opentofu-comm-8" ] }, { "productId": "opentofu", "storyId": "migrate-from-terraform", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "OpenTofu explicitly positions itself as a drop-in, backwards-compatible replacement for Terraform, and community reports (including real client migrations and \"s/terraform/opentofu/g\") corroborate that existing Terraform configs and state can be migrated with minimal effort; hands-on probes confirm the CLI runs standard Terraform-style plan/apply/destroy workflows without modification. Missing for 10: an explicit first-party migration guide or documented state-file conversion/validation step (beyond general 'drop-in' claims), and independent verification of state-file compatibility across complex/large codebases.", "evidenceIds": [ "opentofu-docs-1", "opentofu-docs-14", "opentofu-docs-23", "opentofu-comm-5", "opentofu-comm-7", "opentofu-probe-5", "opentofu-probe-7" ] }, { "productId": "opentofu", "storyId": "open-license-governance", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "OpenTofu explicitly ties its open-source guarantee to being hosted under the Linux Foundation, a neutral governance body designed to prevent unilateral relicensing — directly addressing the devops-lead's concern. Community commentary raises cultural/governance grumbles but no concrete evidence of relicensing risk or license violation, so this doesn't rise to a dispute.\nmissing for 10: explicit license text/name (e.g., MPL 2.0) in evidence, details on governance structure (TSC, contribution/veto rules), and independent third-party corroboration of the Linux Foundation guarantee.", "evidenceIds": [ "opentofu-docs-26", "opentofu-comm-3", "opentofu-comm-12" ] }, { "productId": "opentofu", "storyId": "openness-api-parity", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "OpenTofu is a CLI-based IaC tool with no product UI to compare against an API — its docs, probes, and community feedback describe CLI commands (plan/apply/test/import) and installation, not a UI/API parity story. This axis is a category error for a CLI-first tool rather than an applicable-but-failed capability.", "evidenceIds": [ "opentofu-probe-4", "opentofu-probe-5", "opentofu-probe-7", "opentofu-probe-3" ] }, { "productId": "opentofu", "storyId": "openness-full-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "OpenTofu's state and plan files are plain JSON/HCL (confirmed by the probed `tofu show -json` machine-readable output) and the project is governed as 'truly open source' under the Linux Foundation, meaning there is no proprietary lock-in and configs/state can be freely moved or forked. However, there is no explicit documentation of a dedicated 'export and leave' workflow, no discussion of migrating state to other tools, and no llms.txt/data-export interface (llms.txt 404, no OpenAPI) that would make this openness explicit for AI-native tooling. missing for 10: explicit data-export/portability docs, llms.txt or machine-readable manifest, third-party confirmation of migrating state out of OpenTofu.", "evidenceIds": [ "opentofu-docs-26", "opentofu-docs-23", "opentofu-probe-6", "opentofu-probe-1", "opentofu-comm-5" ] }, { "productId": "opentofu", "storyId": "openness-open-license", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "OpenTofu's source is publicly hosted on GitHub (opentofu-gh-1) and the manifesto explicitly commits to remaining 'Truly open source' now under the Linux Foundation (opentofu-docs-26); community discussion confirms the license is readily found in the GitHub repo even if not prominent on the marketing site (opentofu-comm-3). Missing for 10: explicit citation of the license text/name (e.g., MPL 2.0) and independent legal/audit confirmation beyond community mention.", "evidenceIds": [ "opentofu-gh-1", "opentofu-docs-26", "opentofu-comm-3" ] }, { "productId": "opentofu", "storyId": "openness-self-host", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "OpenTofu is fully open-source (Linux Foundation governed), can be installed and run entirely on one's own infrastructure via multiple install methods, and probes confirm a fully local, keyless CLI workflow (init/plan/apply/destroy) with no SaaS dependency — this is inherently self-hostable since it's just a binary operating on local/backend state. missing for 10: no explicit vendor documentation section titled 'self-hosting' or dedicated guide addressing enterprise/air-gapped self-hosted deployment scenarios, and no independent hands-on report specifically validating self-hosted production use at scale.", "evidenceIds": [ "opentofu-docs-2", "opentofu-docs-26", "opentofu-probe-5", "opentofu-probe-6", "opentofu-probe-7", "opentofu-docs-23" ] }, { "productId": "opentofu", "storyId": "policy-enforcement", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows OpenTofu's plan workflow and machine-readable plan JSON output (e.g., opentofu-probe-5, opentofu-probe-6) which could theoretically feed external policy engines, but there is no mention of any built-in or integrated policy-as-code framework (e.g., Sentinel, OPA/Conftest, Checkov) or documented workflow for blocking non-compliant applies. Missing for 10: any policy-as-code engine or gating mechanism, documentation of policy enforcement in CI/CD, and evidence that non-compliant plans are actually blocked before apply.", "evidenceIds": [ "opentofu-docs-3", "opentofu-probe-5", "opentofu-probe-6" ] }, { "productId": "opentofu", "storyId": "preview-before-apply", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "OpenTofu's tofu plan command is explicitly documented to preview creates, updates, and destroys before applying, with dedicated destroy and refresh-only modes, and this was independently verified via a hands-on probe showing a real plan proposing a create, saved to a file, and later applied and destroyed successfully. Machine-readable plan JSON output further supports precise change previewing. missing for 10: independent third-party review of plan accuracy on complex multi-resource updates beyond the single-resource probe", "evidenceIds": [ "opentofu-docs-3", "opentofu-docs-5", "opentofu-docs-6", "opentofu-docs-21", "opentofu-probe-5", "opentofu-probe-6", "opentofu-probe-7" ] }, { "productId": "opentofu", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "OpenTofu's evidence covers state encryption and backend usage but contains no documentation or claims about choosing a storage region/data-residency for state or configuration data. While backend configuration (e.g., S3) inherently involves region settings, none of the evidence addresses residency controls as a feature.", "evidenceIds": [] }, { "productId": "opentofu", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "OpenTofu is an infrastructure-as-code tool, not an AI model provider or data-processing service that trains models on user data; the axis of preventing training-data usage does not apply to this product category.", "evidenceIds": [] }, { "productId": "opentofu", "storyId": "privacy-retention-controls", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "OpenTofu is a local/self-hosted infrastructure-as-code CLI tool, not an AI service or SaaS platform that collects, stores, or retains user conversational/personal data. Data retention and deletion controls are not a meaningful axis for this product category — the evidence is entirely about state file encryption and Terraform-compatible plan/apply workflows, unrelated to retention/deletion policies for user data.", "evidenceIds": [] }, { "productId": "opentofu", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry, usage tracking, or an opt-out mechanism for OpenTofu; the axis is plausible for a CLI tool but nothing documents it.", "evidenceIds": [] }, { "productId": "opentofu", "storyId": "programmatic-deployments", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All evidence describes CLI commands (`tofu plan`, `apply`, `show -json`) and JSON output that can be parsed by external scripts, but there is no official SDK, Go library, or API for embedding OpenTofu operations directly in application code, and probes explicitly show no OpenAPI/API endpoints exist. This means programmatic control still requires shelling out to the CLI rather than a first-class application-integration path.", "evidenceIds": [ "opentofu-probe-5", "opentofu-probe-6", "opentofu-probe-7", "opentofu-probe-3", "opentofu-docs-3" ] }, { "productId": "opentofu", "storyId": "provider-breadth", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "OpenTofu explicitly claims a large provider/module ecosystem (3,900+ providers, 23,600+ modules) covering every major cloud, and being Terraform-compatible means it inherits the entire Terraform provider registry ecosystem; community evidence corroborates drop-in compatibility and easy migration. Missing for 10: independent third-party audit of provider parity/coverage beyond vendor-stated numbers.", "evidenceIds": [ "opentofu-docs-13", "opentofu-docs-24", "opentofu-docs-1", "opentofu-comm-5", "opentofu-comm-7" ] }, { "productId": "opentofu", "storyId": "remote-state-locking", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers plan/apply/destroy workflows and state encryption, but contains no mention of remote backends, state locking, or concurrent-run protection. Missing for 10: any documentation or probe evidence of remote backend configuration (e.g., S3/consul/etc.), locking mechanism, or verification that concurrent runs are prevented from corrupting state.", "evidenceIds": [ "opentofu-docs-9", "opentofu-docs-20" ] }, { "productId": "opentofu", "storyId": "reusable-modules", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "OpenTofu's docs highlight a large public registry (23,600+ modules) and version-controlled module usage (opentofu-docs-13, opentofu-docs-24, opentofu-docs-12), confirming module consumption is core to the product. However, there is no direct evidence describing a publishing workflow for modules to the registry (e.g., versioning/publish CLI or registry submission process). missing for 10: explicit publish/versioning workflow documentation, independent confirmation of registry usability, and any hands-on evidence of publishing a module.", "evidenceIds": [ "opentofu-docs-13", "opentofu-docs-24", "opentofu-docs-12" ] }, { "productId": "opentofu", "storyId": "secret-config-values", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "OpenTofu supports encrypting state and plan files at rest (including AES-GCM configuration) so secrets don't leak in stored state/plan artifacts, and it is Terraform-compatible with variable-based configuration that supports sensitive input passing patterns. However, there is no direct documentation on marking variables as sensitive to suppress console/log output, no evidence of secret-manager integrations, and community commentary raises unresolved concerns about safety of checked-in encrypted state. missing for 10: explicit sensitive-variable/log-redaction documentation, secrets-manager/vault integration evidence, independent verification that logs never expose secret values.", "evidenceIds": [ "opentofu-docs-9", "opentofu-docs-19", "opentofu-docs-20", "opentofu-comm-10" ] }, { "productId": "opentofu", "storyId": "state-encryption", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "OpenTofu docs explicitly state built-in state and plan file encryption at rest, for both local and backend storage, with configurable methods like AES-GCM, directly addressing the story. Community sentiment even highlights excitement specifically about this state encryption feature, though the same commenter raises open questions about best practices for encrypted state handling (not a contradiction of the capability itself). missing for 10: independent hands-on verification/probe of the encryption feature actually working (only docs cited, no probe test like the plan/apply probes).", "evidenceIds": [ "opentofu-docs-9", "opentofu-docs-19", "opentofu-docs-20", "opentofu-comm-10" ] }, { "productId": "opentofu", "storyId": "state-surgery", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The evidence pack shows related state-adjacent capabilities—`tofu import` for bringing existing resources into state, refresh-only mode for reconciling state with real infrastructure, and state encryption—but never documents the actual state-surgery commands (e.g., `tofu state mv`, `tofu state rm`, `moved` blocks) that platform engineers use for refactoring resource addresses. Missing for 10: explicit docs/examples of `state mv`/`state rm`/`moved` block workflows, safety guardrails (backups, dry-run) during state edits, and any hands-on/independent confirmation these operations work reliably.", "evidenceIds": [ "opentofu-docs-6", "opentofu-docs-8", "opentofu-docs-9" ] }, { "productId": "opentofu", "storyId": "structured-plan-output", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Hands-on probe confirms `tofu show -json plan.out` emits machine-readable plan JSON (format_version 1.2) with planned_values, parseable by agent tooling, alongside documented plan/apply CLI workflow supporting `-out=FILE` for later JSON conversion. missing for 10: no independent third-party corroboration beyond the single probe run.", "evidenceIds": [ "opentofu-probe-6", "opentofu-probe-5", "opentofu-docs-21", "opentofu-docs-3" ] }, { "productId": "opentofu", "storyId": "targeted-partial-apply", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "OpenTofu explicitly documents the -exclude flag for excluding resources from operations, and -target flag support is implied by drop-in Terraform compatibility plus verified hands-on plan/apply workflows. missing for 10: explicit docs/example for the -target flag itself and independent community confirmation of targeted apply usage.", "evidenceIds": [ "opentofu-docs-10", "opentofu-docs-3", "opentofu-probe-5", "opentofu-probe-7" ] }, { "productId": "opentofu", "storyId": "unit-integration-tests", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "OpenTofu ships a native `tofu test` command that creates real infrastructure, checks assertions, then destroys it — directly enabling automated infra tests without touching production, and `tofu plan` speculative plans support pre-review verification. Evidence is first-party docs only with no independent/hands-on corroboration of the test framework's mocking or unit-test capabilities.\nmissing for 10: independent/hands-on validation of `tofu test`, details on mock providers or isolated test environments, community confirmation of test workflow reliability.", "evidenceIds": [ "opentofu-docs-7", "opentofu-docs-25", "opentofu-docs-4", "opentofu-docs-22" ] }, { "productId": "opentofu", "storyId": "validate-fmt", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers plan, apply, test, import, and encryption commands but contains no mention of `tofu validate` or `tofu fmt` (or any auto-formatting/validation capability), so there is no evidence supporting this specific story despite the axis clearly applying to a Terraform-compatible IaC CLI. Missing for 10: any doc or probe evidence of a validate command, a fmt/auto-format command, or their integration into a pre-plan workflow.", "evidenceIds": [] }, { "productId": "opentofu", "storyId": "version-stability", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "OpenTofu publicly commits to backwards compatibility and positions itself as a drop-in replacement for Terraform, preserving existing configs/workflows, which implies a compatibility promise, but the evidence pack contains no explicit version-to-version upgrade guide documentation for OpenTofu releases themselves, and one community comment notes OpenTofu is 'already diverging from upstream,' raising uncertainty about long-term compatibility guarantees. missing for 10: dedicated upgrade-guide docs between OpenTofu minor/major releases, changelog/breaking-change policy evidence, independent confirmation of smooth upgrades across OpenTofu versions.", "evidenceIds": [ "opentofu-docs-23", "opentofu-docs-14", "opentofu-docs-1", "opentofu-comm-11", "opentofu-comm-5" ] }, { "productId": "pulumi", "storyId": "agent-authors-change", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Pulumi's Automation API and CLI allow headless plan generation (`pulumi preview --non-interactive --json`) with structured diff output that an agent can parse, and hands-on probes confirm this and the full apply loop work headlessly without Pulumi Cloud; the official MCP server (verified via handshake) lets AI assistants generate infrastructure code and run CLI commands, supporting an agent-authored-and-approved workflow. missing for 10: a single end-to-end demonstrated workflow tying MCP-based code authoring directly into the headless preview/approval loop, and independent (non-vendor) verification of that combined flow.", "evidenceIds": [ "pulumi-docs-2", "pulumi-docs-8", "pulumi-docs-19", "pulumi-docs-26", "pulumi-probe-6", "pulumi-probe-7", "pulumi-probe-8", "pulumi-docs-7", "pulumi-docs-18" ] }, { "productId": "pulumi", "storyId": "agent-least-privilege-gates", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Pulumi provides the building blocks — Policy as Code guardrails (pulumi-docs-5/16), a preview/dry-run step that shows changes before they're applied (pulumi-docs-8/19, pulumi-probe-6), and an MCP server letting AI assistants query/generate/delegate infra changes (pulumi-docs-7/18/26, pulumi-probe-8) — but there is no documented mechanism tying an agent's MCP/CLI actions specifically to least-privilege scoped credentials or an explicit human-approval gate before 'pulumi up' executes for agent-initiated changes. Missing for 10: explicit least-privilege credential scoping for agent identities, a documented approval/review workflow gating agent-triggered applies, and evidence the MCP server itself is constrained from calling 'pulumi up' without human sign-off.", "evidenceIds": [ "pulumi-docs-5", "pulumi-docs-8", "pulumi-docs-19", "pulumi-docs-7", "pulumi-docs-18", "pulumi-docs-26", "pulumi-probe-6", "pulumi-probe-8" ] }, { "productId": "pulumi", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A direct probe confirms Pulumi serves a working llms.txt at https://www.pulumi.com/llms.txt (HTTP 200) with a concise agent-oriented summary, and Pulumi also maintains extensive agent-oriented docs (MCP server docs, CLI docs) that AI assistants can be pointed at. missing for 10: independent/community confirmation that agents successfully consume llms.txt in practice, and a docs.md/markdown-flavored docs endpoint (which returned 404).", "evidenceIds": [ "pulumi-probe-1", "pulumi-docs-7", "pulumi-docs-26", "pulumi-probe-4" ] }, { "productId": "pulumi", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Pulumi's 'Insights' policy features and Neo agent provide some AI-driven analysis of infrastructure (policy guardrails, plain-English task delegation), and the MCP server lets AI assistants query/search cloud resources, but there is no dedicated in-product 'insights/suggestions' dashboard shown generating proactive AI analysis of a user's own data beyond policy compliance and Neo's provisioning actions; one community report calls Pulumi's genAI documentation poor. missing for 10: concrete evidence of an in-product AI insights/dashboard analyzing existing resource data and generating proactive suggestions (not just policy enforcement or Neo's task execution), and independent corroboration of quality.", "evidenceIds": [ "pulumi-docs-5", "pulumi-docs-9", "pulumi-docs-20", "pulumi-docs-26", "pulumi-comm-16" ] }, { "productId": "pulumi", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Pulumi documents both an Automation API for embedding Pulumi programs into custom background services (pulumi-docs-2) and 'Neo', an agent described as autonomously provisioning, governing, and optimizing infrastructure from plain-English instructions (pulumi-docs-9, pulumi-docs-20), plus headless CLI runs confirmed via probe (pulumi-probe-6, pulumi-probe-7). However, Neo's autonomous background operation is only vendor marketing copy with no hands-on or independent verification, and Automation API requires the user to build their own scheduling/triggering layer rather than shipping a ready background-automation feature. Missing for 10: hands-on/independent evidence Neo (or another built-in scheduler) actually runs unattended in production, native trigger/schedule primitives beyond Automation API, and community corroboration of autonomous operation.", "evidenceIds": [ "pulumi-docs-2", "pulumi-docs-9", "pulumi-docs-20", "pulumi-probe-6", "pulumi-probe-7" ] }, { "productId": "pulumi", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 6, "confidence": "medium", "rationale": "Pulumi ships 'Neo', a built-in AI agent described as taking plain-English infrastructure requests and autonomously handling dependencies, executing changes, and monitoring outcomes, and the MCP docs explicitly note tasks can be delegated to Neo. This directly matches the story of delegating tasks to an in-product AI assistant. Missing for 10: independent or hands-on verification of Neo's actual task execution/autonomy (only vendor docs), and detail on scope/guardrails of what Neo can do unsupervised.", "evidenceIds": [ "pulumi-docs-9", "pulumi-docs-20", "pulumi-docs-26" ] }, { "productId": "pulumi", "storyId": "agentic-headless", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Pulumi's CLI supports non-interactive flags and JSON output for scripted use, plus an Automation API explicitly designed for running Pulumi programs without the CLI in orchestration/CI contexts; hands-on probes confirm `pulumi preview/up/destroy --non-interactive` work headlessly on a local backend without Pulumi Cloud, and VCS integrations (GitHub/GitLab/Azure DevOps) support CI pipelines. Missing for 10: no first-party CI-specific documentation (e.g., GitHub Actions workflow examples) or independent CI-pipeline case study in the evidence pack.", "evidenceIds": [ "pulumi-docs-2", "pulumi-probe-6", "pulumi-probe-7", "pulumi-docs-12", "pulumi-docs-19" ] }, { "productId": "pulumi", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All Pulumi MCP evidence describes Pulumi acting as an MCP *server* (exposing its own tools to external AI assistants like Cursor, Claude Code, Windsurf) — the opposite direction from the story, which asks whether Pulumi itself can consume/plug in external MCP servers as a client. Even Pulumi Neo, the AI agent component, is only documented as executing infrastructure changes from plain-English prompts, with no evidence it can attach to third-party MCP servers to use their tools.", "evidenceIds": [ "pulumi-docs-7", "pulumi-docs-18", "pulumi-docs-26", "pulumi-probe-4", "pulumi-probe-8", "pulumi-docs-9", "pulumi-docs-20" ] }, { "productId": "pulumi", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Pulumi ships an official, documented MCP server (@pulumi/mcp-server) that exposes Pulumi Cloud stacks/resources and CLI commands to AI assistants like Cursor, Claude Code/Desktop, and Windsurf, and a hands-on probe confirms it actually performs a JSON-RPC MCP handshake and responds correctly. Missing for 10: no independent third-party (non-Pulumi) review of the MCP server's real-world reliability/behavior beyond the single probe.", "evidenceIds": [ "pulumi-docs-7", "pulumi-docs-18", "pulumi-docs-26", "pulumi-probe-4", "pulumi-probe-8" ] }, { "productId": "pulumi", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Pulumi Neo is explicitly marketed to accept plain-English infrastructure requests ('Tell Neo what you need in plain English...') and the official MCP server, verified hands-on via JSON-RPC handshake, lets AI assistants translate natural-language prompts into Pulumi CLI/registry operations. However, Neo's NL capability is only vendor-claimed (no hands-on or community verification) and is a separate enterprise product, while the core Pulumi CLI itself remains code/command-driven, not NL-native. Missing for 10: independent or hands-on verification of Neo's natural-language handling, and evidence that natural-language control works without relying on Neo or an external AI coding assistant via MCP.", "evidenceIds": [ "pulumi-docs-9", "pulumi-docs-20", "pulumi-docs-26", "pulumi-docs-18", "pulumi-probe-8" ] }, { "productId": "pulumi", "storyId": "agentic-official-cli", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Pulumi ships an official CLI with documented commands (preview, up, import, destroy) and hands-on probes confirm it runs headlessly with structured JSON output an agent can parse, plus non-interactive/automated flags well suited to AI-native workflows. Missing for 10: no independent third-party review specifically praising the CLI's AI-native ergonomics beyond the vendor probes.", "evidenceIds": [ "pulumi-probe-5", "pulumi-probe-6", "pulumi-probe-7", "pulumi-docs-19", "pulumi-docs-25" ] }, { "productId": "pulumi", "storyId": "agentic-public-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Pulumi offers a documented Automation API (programmatic, language-based interface) and a verified MCP server (probed handshake works) plus a CLI with JSON output, giving AI-native users multiple programmatic ways to drive it. However, there is no true public REST/OpenAPI API (openapi.json/swagger endpoints all 404) and no llms.txt-driven docs-md endpoint, so 'documented public API' in the traditional sense is unconfirmed. missing for 10: a documented REST/OpenAPI-style public API, independent third-party corroboration of Automation API usage, and confirmation that MCP/Automation API cover full CRUD parity with the CLI.", "evidenceIds": [ "pulumi-docs-2", "pulumi-docs-7", "pulumi-docs-26", "pulumi-probe-3", "pulumi-probe-4", "pulumi-probe-8", "pulumi-probe-6", "pulumi-probe-7" ] }, { "productId": "pulumi", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence covers Pulumi's MCP server for AI agent integration and CLI/API access broadly, but nothing describes issuing scoped or least-privilege API credentials/tokens specifically for an agent's use (e.g., token scoping, RBAC-limited service accounts, or credential minting for AI agents). This is a fair question for a cloud IaC platform with an API/token system, but no evidence supports it.", "evidenceIds": [] }, { "productId": "pulumi", "storyId": "agentic-sdks", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Pulumi provides official SDKs across multiple mainstream languages (TypeScript, Python, Go, .NET, Java, YAML) with full provider surface area, plus an Automation API for programmatic use and a documented, hands-on-verified MCP server enabling AI assistants to interact with Pulumi programs. This is strongly corroborated by official docs and a live probe confirming the MCP server handshake works. Missing for 10: independent third-party (non-vendor) benchmarking of SDK completeness/parity across all languages, and no evidence of official SDKs in additional languages like Rust or Java-adjacent ecosystems beyond what's listed.", "evidenceIds": [ "pulumi-docs-1", "pulumi-docs-2", "pulumi-docs-22", "pulumi-docs-26", "pulumi-docs-18", "pulumi-probe-4", "pulumi-probe-8" ] }, { "productId": "pulumi", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions webhooks or event subscriptions of any kind; Pulumi's AI-facing integrations documented are MCP server access and Automation API, not webhook-based event subscription. missing for 10: any documentation or mention of webhook support, event subscription APIs, or notification triggers.", "evidenceIds": [] }, { "productId": "pulumi", "storyId": "ai-generate-iac", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Pulumi ships a first-party MCP server explicitly documented to let AI assistants 'generate infrastructure code' from Pulumi Cloud context, and Pulumi Neo lets users 'tell Neo what you need in plain English' to provision infrastructure — both built into the toolchain. A hands-on probe confirms the MCP server actually runs and responds to JSON-RPC handshakes, corroborating the vendor claim beyond just documentation. Missing for 10: independent/hands-on evidence of actual natural-language-to-IaC code generation quality (only the handshake, not code-gen output, was tested) and no third-party corroboration of Neo's real-world behavior.", "evidenceIds": [ "pulumi-docs-26", "pulumi-docs-20", "pulumi-docs-9", "pulumi-docs-18", "pulumi-probe-8", "pulumi-probe-4" ] }, { "productId": "pulumi", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of an interactive API reference with runnable examples; probes explicitly show no OpenAPI/Swagger spec (404s across candidate paths) and no mention of a live/sandboxed API explorer. Pulumi's docs describe language SDKs and CLI commands but not an interactive, runnable API reference.", "evidenceIds": [ "pulumi-probe-3", "pulumi-docs-22" ] }, { "productId": "pulumi", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe for an OpenAPI/equivalent spec at all standard candidate paths returned 404 across the board, and no evidence pack item shows Pulumi publishing a downloadable machine-readable API spec; the llms.txt found is a marketing/context file, not an API spec.", "evidenceIds": [ "pulumi-probe-3", "pulumi-probe-1" ] }, { "productId": "pulumi", "storyId": "api-sandbox", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Pulumi's testing docs describe integration tests that 'deploy ephemeral infrastructure and run external tests against it' and preview command that makes 'no changes to the stack,' and probes confirm a full headless up/destroy loop against a local file backend without touching Pulumi Cloud or production state. This supports safe sandbox testing, but the evidence never explicitly describes stack-based prod/sandbox isolation or guardrails preventing accidental production impact from an AI agent. Missing for 10: explicit multi-stack environment isolation guidance, agent-specific guardrails against touching prod stacks, and independent hands-on confirmation of safe sandboxing beyond the vendor-run probe.", "evidenceIds": [ "pulumi-docs-3", "pulumi-docs-8", "pulumi-probe-6", "pulumi-probe-7" ] }, { "productId": "pulumi", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item discusses API versioning or a documented deprecation policy for Pulumi's APIs (CLI, SDK, Automation API, or MCP server); the OpenAPI probe returned 404s and no changelog/versioning policy is cited. Community evidence even mentions Pulumi lagging on deprecated Azure resources with no fix, but this is unrelated to a documented deprecation policy for Pulumi's own APIs.", "evidenceIds": [ "pulumi-probe-3", "pulumi-comm-1" ] }, { "productId": "pulumi", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Pulumi supports bulk-oriented workflows: pulumi preview --import-file can generate an import covering every resource a program would create, the Automation API allows programmatic orchestration across many stacks or resources, and real-world usage reports deployments of 500+ resources at once. These give an AI agent, via CLI or MCP server, levers for bulk operations, but the docs never explicitly frame a bulk operation across many items workflow for AI agents, and the MCP server evidence only demonstrates single CLI command execution, not batch or bulk semantics. missing for 10: explicit bulk-operation API/CLI docs such as bulk import across many resources in one call or bulk stack updates framed for AI-agent consumption, and hands-on evidence of an agent performing such bulk ops via MCP", "evidenceIds": [ "pulumi-docs-21", "pulumi-docs-2", "pulumi-comm-7", "pulumi-docs-26", "pulumi-probe-8" ] }, { "productId": "pulumi", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence covers Pulumi's languages, testing, state backends, policy-as-code guardrails, import, and MCP/Automation API integrations, but none of it describes a mechanism for defining rules that automatically trigger actions in response to events (e.g., webhooks, event-driven pipelines, or reactive automation). Policy as code (pulumi-docs-5/16) enforces guardrails during deployments rather than triggering actions on arbitrary events, and the Automation API (pulumi-docs-2) is for programmatic invocation, not event-driven rule execution.", "evidenceIds": [ "pulumi-docs-2", "pulumi-docs-5", "pulumi-docs-16" ] }, { "productId": "pulumi", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Pulumi's evidence shows Automation API for programmatic runs, CLI commands, and Neo for AI-driven provisioning, but nothing describes built-in support for scheduling recurring jobs or workflows (e.g., cron-like triggers, scheduled drift detection, or workflow orchestration). Such a capability is plausible for an IaC/automation platform, so the axis applies, but no evidence confirms it.", "evidenceIds": [ "pulumi-docs-2", "pulumi-docs-9", "pulumi-docs-20" ] }, { "productId": "pulumi", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Pulumi's code-based IaC is inherently versionable via standard VCS (git integrations documented), and `pulumi preview`/`pulumi destroy`/state backends give rollback-like control validated hands-on via CLI probes; policy-as-code adds review guardrails. However there's no dedicated 'automation review' workflow, no built-in diff/approval UI beyond preview output, and community feedback flags documentation/complexity issues around dependency management that complicate confident rollback for AI-driven changes. missing for 10: a first-party automated review/approval gate for AI-initiated changes, explicit rollback command/semantics (vs. re-apply prior state), and independent verification of policy-as-code enforcement in practice.", "evidenceIds": [ "pulumi-docs-4", "pulumi-docs-5", "pulumi-docs-8", "pulumi-docs-12", "pulumi-probe-6", "pulumi-probe-7", "pulumi-comm-20", "pulumi-comm-12" ] }, { "productId": "pulumi", "storyId": "destroy-teardown", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Pulumi's `pulumi destroy` command is directly evidenced by a hands-on probe showing it exits 0 and cleanly tears down resources after an apply, running headlessly without cloud dependency, confirming the standard destroy workflow works as documented. missing for 10: no independent/community corroboration specifically discussing destroy behavior (only a first-party probe and docs).", "evidenceIds": [ "pulumi-probe-7", "pulumi-docs-4", "pulumi-probe-5" ] }, { "productId": "pulumi", "storyId": "drift-detection", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Pulumi's `pulumi preview` command computes a diff between declared program state and actual live resources, and this was verified hands-on (JSON steps output) without requiring cloud backend. State-and-backends docs also confirm Pulumi tracks resource state to compare against real infrastructure. However, the evidence never explicitly documents a dedicated `pulumi refresh` command or scheduled/automated drift-detection workflows (e.g. CI drift alerts), which are the more complete drift-detection story. missing for 10: explicit refresh command docs, automated/scheduled drift detection, independent community confirmation of drift-detection accuracy.", "evidenceIds": [ "pulumi-docs-8", "pulumi-docs-19", "pulumi-probe-6", "pulumi-docs-4" ] }, { "productId": "pulumi", "storyId": "environment-config-stacks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Pulumi's core primitive of \"stacks\" is confirmed hands-on (pulumi stack init dev, pulumi preview/up/destroy per stack) and state/config is backend-scoped per stack (pulumi-docs-4/14/23, pulumi-probe-6/7), which is exactly the mechanism developers use to separate dev/staging/prod environments. Missing for 10: explicit documentation citation showing pulumi config set per-stack values and a documented dev→staging→prod promotion workflow, and no independent community confirmation specifically about multi-stack environment management.", "evidenceIds": [ "pulumi-probe-6", "pulumi-probe-7", "pulumi-docs-4", "pulumi-docs-14", "pulumi-docs-23" ] }, { "productId": "pulumi", "storyId": "gp-language-support", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Pulumi's core value proposition is defining infrastructure in general-purpose languages (TypeScript, Python, Go, .NET, Java) with full access to language features like loops, functions, classes, and types, backed by standard IDE tooling via SDKs; this is corroborated by community testimonials praising the type system and language ergonomics. Missing for 10: no direct evidence of specific IDE plugin/autocomplete features beyond inferred SDK typing benefits, and some community complaints about complexity/documentation temper a perfect score.", "evidenceIds": [ "pulumi-docs-1", "pulumi-docs-13", "pulumi-gh-1", "pulumi-comm-6", "pulumi-comm-8", "pulumi-comm-7" ] }, { "productId": "pulumi", "storyId": "headless-ci-apply", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Evidence confirms Pulumi CLI supports fully non-interactive plan/apply in CI: `pulumi preview --non-interactive --json` and `pulumi up --non-interactive --yes --skip-preview` both ran headlessly with approval flags (--yes) and produced machine-parseable output, per hands-on probes [pulumi-probe-6][pulumi-probe-7], with `pulumi preview` docs confirming plan-only dry-run behavior [pulumi-docs-8][pulumi-docs-19]. However, the pack contains no evidence of the specific 'saved plan artifact' workflow (e.g., generating and re-applying a persisted plan file) that the story calls out, only generic non-interactive preview/apply. Missing for 10: documentation or hands-on proof of saved-plan-artifact generation/application (e.g., --save-plan/--plan flags), and independent (non-vendor) confirmation of this exact CI pattern.", "evidenceIds": [ "pulumi-docs-8", "pulumi-docs-19", "pulumi-probe-6", "pulumi-probe-7" ] }, { "productId": "pulumi", "storyId": "import-existing-infra", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Pulumi provides a documented `pulumi import` command that adopts existing resources into stack state and generates matching code in the target language, plus `pulumi preview --import-file` to bulk-generate import files for existing resources, directly matching the story. Docs explicitly describe this as core migration functionality for resources created manually or by other IaC tools. Missing for 10: independent/hands-on verification of the import workflow itself (only preview/apply loop was probed, not import).", "evidenceIds": [ "pulumi-docs-6", "pulumi-docs-17", "pulumi-docs-25", "pulumi-docs-21", "pulumi-docs-10" ] }, { "productId": "pulumi", "storyId": "migrate-from-terraform", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Pulumi documents importing existing (e.g., Terraform-created) resources via `pulumi import` and `pulumi preview --import-file`, generating Pulumi code and adding resources to Pulumi state, and explicitly markets migration from Terraform/CloudFormation. However, this is resource-by-resource adoption of live infrastructure, not a direct conversion of Terraform HCL codebase or import of Terraform state files themselves — no evidence of a bulk Terraform-state-to-Pulumi-state converter or HCL-to-native-language transpiler tool. missing for 10: evidence of direct Terraform state file import/conversion, bulk/automated HCL-to-Pulumi code conversion tooling, and independent/hands-on confirmation of a full codebase migration success story.", "evidenceIds": [ "pulumi-docs-6", "pulumi-docs-10", "pulumi-docs-17", "pulumi-docs-21", "pulumi-docs-25" ] }, { "productId": "pulumi", "storyId": "open-license-governance", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of Pulumi's license type (e.g., Apache 2.0), its governance model, or any foundation/neutral steward that would prevent unilateral relicensing — only GitHub repo links and generic self-hosting/backend docs. Given the well-known industry concern (post-Terraform BSL) that devops leads would want addressed, this axis clearly applies but no evidence in the pack speaks to license or governance guarantees.", "evidenceIds": [ "pulumi-gh-1", "pulumi-gh-2", "pulumi-docs-27" ] }, { "productId": "pulumi", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Pulumi offers a scriptable CLI and Automation API for running programs without the console, and hands-on probes confirm headless preview/up/destroy and MCP server operation work fully outside the UI, but there is no evidence of a full REST/OpenAPI surface (probe found no openapi.json) or explicit documentation that Pulumi Cloud UI features like Insights, Policy dashboards, or Neo's console-only controls are all reachable programmatically. missing for 10: explicit API/CLI parity claim for Pulumi Cloud console features (Insights, Policy management, Neo dashboard), a discoverable OpenAPI/REST spec, and independent confirmation that no UI-only functionality exists.", "evidenceIds": [ "pulumi-docs-2", "pulumi-probe-6", "pulumi-probe-7", "pulumi-probe-8", "pulumi-probe-3", "pulumi-docs-9", "pulumi-docs-5" ] }, { "productId": "pulumi", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Pulumi code is written in standard, open programming languages (TS, Python, Go, etc.) rather than a proprietary DSL, and state can be stored in a self-managed 'DIY' backend (S3, Azure Blob, GCS) instead of Pulumi Cloud, giving a path to keep data in open formats and avoid full platform lock-in. However, community evidence notes Pulumi still 'calls into the mothership' for some functionality and there is no documented explicit bulk-export/migration tool for leaving the platform. Missing for 10: dedicated data-export/migration tooling to leave, clear documentation on state file/format portability guarantees, and independent verification that self-hosted deployments have no residual cloud dependency.", "evidenceIds": [ "pulumi-docs-4", "pulumi-docs-14", "pulumi-docs-23", "pulumi-docs-27", "pulumi-comm-9", "pulumi-gh-1" ] }, { "productId": "pulumi", "storyId": "openness-open-license", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence confirms Pulumi's core CLI/SDK is published on GitHub (pulumi-gh-1, pulumi-gh-2), so an AI-native user could read source there, but the evidence pack never states an explicit open-source license, and Pulumi's cloud/state backend and Neo agent components are proprietary SaaS (pulumi-docs-4, pulumi-docs-9) rather than open source. missing for 10: explicit license text/statement, confirmation that all product components (not just core CLI/SDK) are open, independent confirmation of license terms.", "evidenceIds": [ "pulumi-gh-1", "pulumi-gh-2", "pulumi-docs-4", "pulumi-docs-9" ] }, { "productId": "pulumi", "storyId": "openness-self-host", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Pulumi's core engine/CLI is open-source and can run fully locally without any cloud dependency — confirmed hands-on via probes showing `pulumi preview`/`up`/`destroy` running headlessly against a local file:// backend with PULUMI_CONFIG_PASSPHRASE for state encryption (pulumi-probe-6, pulumi-probe-7). Docs further describe a 'DIY backend' storing state in self-managed object storage (S3/Azure Blob/GCS) as an alternative to the hosted Pulumi Cloud (pulumi-docs-4, pulumi-docs-14, pulumi-docs-23), and the pricing page explicitly states 'Self-hosting available' for the broader service (pulumi-docs-27), corroborated by a community note that Pulumi's state service has a self-management option (pulumi-comm-19). missing for 10: detailed first-party documentation of deploying the full self-hosted Pulumi Cloud service (UI/API), and independent hands-on verification of that enterprise self-hosted deployment rather than just the DIY state-backend/local-CLI path.", "evidenceIds": [ "pulumi-docs-4", "pulumi-docs-14", "pulumi-docs-23", "pulumi-docs-27", "pulumi-comm-19", "pulumi-probe-6", "pulumi-probe-7" ] }, { "productId": "pulumi", "storyId": "policy-enforcement", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Pulumi's Policy-as-Code product explicitly lets teams define guardrails in code and apply them consistently across resources managed by Pulumi, Terraform, CloudFormation, or created manually — directly addressing pre-apply compliance enforcement, and this integrates with the CLI's preview/apply workflow to block non-compliant changes before they take effect. Missing for 10: no independent/hands-on evidence of a policy pack actually blocking a non-compliant `pulumi up`, and no detail on CI/CD gating specifics beyond docs claims.", "evidenceIds": [ "pulumi-docs-5", "pulumi-docs-16", "pulumi-docs-8", "pulumi-docs-19" ] }, { "productId": "pulumi", "storyId": "preview-before-apply", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Pulumi's `pulumi preview` command explicitly shows creates, updates, and destroys without applying any changes, and this was independently verified via hands-on probe producing structured JSON steps[] output that an agent/tool can consume. Docs and probe evidence corroborate this core plan-apply workflow works headlessly. Missing for 10: independent third-party review specifically praising/critiquing preview accuracy in production use (community evidence is about Pulumi generally, not preview diffing specifically).", "evidenceIds": [ "pulumi-docs-8", "pulumi-docs-19", "pulumi-probe-6", "pulumi-docs-21" ] }, { "productId": "pulumi", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Pulumi lets users choose a self-managed 'DIY' state backend (AWS S3, Azure Blob Storage, Google Cloud Storage) or self-host, which effectively lets a user pick the storage region/provider for state data, and self-hosting is explicitly offered on the pricing page. However, there is no documented region-selection control for the default Pulumi Cloud managed backend itself, and no explicit data-residency/compliance documentation (e.g., EU-only region option) for AI-native workflows. Missing for 10: explicit region-selection UI/API for Pulumi Cloud's managed backend, documented data-residency guarantees or compliance certifications tied to region choice.", "evidenceIds": [ "pulumi-docs-4", "pulumi-docs-14", "pulumi-docs-23", "pulumi-docs-27" ] }, { "productId": "pulumi", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Pulumi is an infrastructure-as-code/cloud provisioning tool, not an AI model provider or data-training relationship; there's no axis for 'training data opt-out' in this product category — none of the evidence pertains to AI training data usage policies.", "evidenceIds": [] }, { "productId": "pulumi", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Pulumi offers a self-managed 'DIY' state backend (own S3/Azure Blob/GCS bucket) and self-hosting option, which indirectly lets users control where their infrastructure state data lives and delete it at will, rather than relying on Pulumi Cloud retention. However, there is no explicit documentation of data retention policies, deletion guarantees, or controls specifically for AI-related data (e.g., MCP server or Neo interaction logs). Missing for 10: explicit retention/deletion policy documentation, AI-specific data handling disclosures, and any deletion API/workflow for Pulumi Cloud-stored data.", "evidenceIds": [ "pulumi-docs-4", "pulumi-docs-14", "pulumi-docs-23", "pulumi-docs-27" ] }, { "productId": "pulumi", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no mention of a telemetry/usage-tracking opt-out setting (e.g., an env var, CLI flag, or config option) for Pulumi CLI or MCP server; no documentation or community evidence addresses telemetry controls at all.", "evidenceIds": [] }, { "productId": "pulumi", "storyId": "programmatic-deployments", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Pulumi's Automation API is explicitly documented as a programmatic interface for running Pulumi programs without the CLI, letting developers drive previews/deploys/destroys directly from application code, and this is corroborated by community mention of coding infra migrations via Automation. missing for 10: independent hands-on verification of the Automation API itself (the probes validate CLI/MCP headless flows but not an actual Automation API SDK call).", "evidenceIds": [ "pulumi-docs-2", "pulumi-comm-20" ] }, { "productId": "pulumi", "storyId": "provider-breadth", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Pulumi's provider ecosystem is well documented with 300+ providers spanning AWS, Azure, GCP, Kubernetes, and SaaS offerings via the Pulumi Registry, corroborated by GitHub docs and confirmed hands-on via CLI probes deploying resources. Community evidence corroborates real-world multi-cloud usage (AWS, GCP, Azure, DigitalOcean) though some note documentation/provider lag issues for niche services. Missing for 10: independent quantification of SaaS-provider breadth beyond major clouds and no dispute-level evidence of provider gaps preventing broad usage.", "evidenceIds": [ "pulumi-gh-2", "pulumi-docs-22", "pulumi-comm-7", "pulumi-comm-15", "pulumi-comm-3", "pulumi-probe-6", "pulumi-probe-7" ] }, { "productId": "pulumi", "storyId": "remote-state-locking", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Pulumi docs explicitly describe remote backends (Pulumi Cloud managed service, and DIY backends like S3/Azure Blob/GCS) for storing state, and Pulumi Cloud/state services are designed to provide locking to prevent concurrent-run corruption; community evidence corroborates Pulumi's own state service option. missing for 10: explicit documentation/citation of locking mechanics for DIY object-store backends and independent third-party verification of lock behavior under concurrent runs.", "evidenceIds": [ "pulumi-docs-4", "pulumi-docs-14", "pulumi-docs-23", "pulumi-comm-19" ] }, { "productId": "pulumi", "storyId": "reusable-modules", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Pulumi has a well-documented public Registry of providers/packages (referenced implicitly via 'Pulumi Registry providers' in pulumi-docs-1 and the MCP server description 'querying Pulumi Registry information' in pulumi-probe-8), and 300+ providers are consumable per pulumi-gh-2. However, the evidence pack lacks explicit documentation of a publishing workflow for developers to submit/publish their own reusable components/packages to the public Registry, versioning/discovery mechanics, or community corroboration of publishing experience. missing for 10: first-party docs on the publish/submission process for community packages, evidence of component/package authoring guides, independent/hands-on confirmation of publishing a package to the Registry.", "evidenceIds": [ "pulumi-docs-1", "pulumi-gh-2", "pulumi-probe-8" ] }, { "productId": "pulumi", "storyId": "secret-config-values", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "A hands-on probe confirms Pulumi encrypts stack secrets via PULUMI_CONFIG_PASSPHRASE and can run headlessly with encrypted state (pulumi-probe-7), which supports the ability to store sensitive config outside code. However, the evidence pack lacks first-party docs on the `pulumi config set --secret` workflow, secret providers (KMS/Vault), or log redaction behavior, and there is no independent confirmation of masking-in-CLI-output claims. Missing for 10: docs on secret config commands and secret providers, evidence of automatic log/output redaction, and independent (non-vendor) verification of secrets never leaking in logs.", "evidenceIds": [ "pulumi-probe-7", "pulumi-docs-4", "pulumi-docs-14", "pulumi-docs-23" ] }, { "productId": "pulumi", "storyId": "state-encryption", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Pulumi documents encrypted state backends and hands-on probe evidence confirms secrets are encrypted using PULUMI_CONFIG_PASSPHRASE even on a local file backend, verifying secret values are not stored in plaintext. Missing for 10: explicit documentation of KMS/cloud secrets-provider options for encrypting the whole state, independent third-party verification of encryption strength, and clarity on whether non-secret resource properties are also encrypted at rest.", "evidenceIds": [ "pulumi-probe-7", "pulumi-docs-4", "pulumi-docs-14", "pulumi-docs-23" ] }, { "productId": "pulumi", "storyId": "state-surgery", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Pulumi's docs show related state-safety capabilities — `pulumi import`/adoption for bringing existing resources under management (pulumi-docs-6,17,25), and `pulumi preview` for safely previewing changes without applying them (pulumi-docs-8,19,21) — plus general state backend concepts (pulumi-docs-4,14,23). However, the evidence pack never cites the actual `pulumi state mv`, `pulumi state rename`, or `pulumi state delete` commands that platform engineers use specifically for refactoring (moving/renaming/removing resources in state), so the core of the story is only indirectly supported. missing for 10: explicit documentation/evidence of `pulumi state move`, `pulumi state rename`, and `pulumi state delete` commands, plus hands-on confirmation these work safely during refactors.", "evidenceIds": [ "pulumi-docs-6", "pulumi-docs-17", "pulumi-docs-25", "pulumi-docs-8", "pulumi-docs-19", "pulumi-docs-21", "pulumi-docs-4", "pulumi-docs-14" ] }, { "productId": "pulumi", "storyId": "structured-plan-output", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Pulumi's `pulumi preview --json` produces a structured JSON steps[] array (op, urn, etc.) that an agent can parse to reason about a proposed change, and this was independently verified hands-on in a local, non-interactive run without Pulumi Cloud. Documentation and CLI reference corroborate this as an officially supported flag. missing for 10: no independent third-party (non-vendor) documentation of agents actually consuming this JSON for reasoning, only a single hands-on probe.", "evidenceIds": [ "pulumi-docs-8", "pulumi-docs-19", "pulumi-probe-6", "pulumi-probe-7" ] }, { "productId": "pulumi", "storyId": "targeted-partial-apply", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions Pulumi's targeting/exclusion flags (e.g., `--target`, `--exclude`) for `pulumi preview`/`pulumi up`/`pulumi destroy`; the pack only covers preview mechanics, import, and state/backends without addressing resource-scoped plan/apply. missing for 10: documentation or hands-on proof of --target/--exclude (or equivalent) flags limiting a plan/apply to specific resources.", "evidenceIds": [] }, { "productId": "pulumi", "storyId": "unit-integration-tests", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Pulumi documents a dedicated testing framework with Unit Tests (mocked, no real cloud calls), Property Tests, and Integration Tests using standard language test frameworks, plus 'pulumi preview' to check changes without touching production and Automation API for programmatic test orchestration. This directly satisfies writing/running infra tests without touching production. Missing for 10: independent hands-on validation of the testing framework specifically (probes verify preview/up/destroy but not the unit-test mocking flow) and community commentary on test-writing experience.", "evidenceIds": [ "pulumi-docs-3", "pulumi-docs-15", "pulumi-docs-24", "pulumi-docs-8", "pulumi-docs-19", "pulumi-docs-2", "pulumi-probe-6" ] }, { "productId": "pulumi", "storyId": "validate-fmt", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of a dedicated 'pulumi validate' or 'pulumi fmt' style command that checks/auto-formats configuration prior to planning; Pulumi's docs only show 'pulumi preview' as a plan-preview step, and formatting is left to standard language tooling since Pulumi programs are written in general-purpose languages. missing for 10: no documented config validation command, no auto-formatting feature, no evidence of pre-plan syntax/config linting distinct from preview.", "evidenceIds": [] }, { "productId": "pulumi", "storyId": "version-stability", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no documentation of versioning/compatibility promises, deprecation policy, or upgrade guides between Pulumi CLI/SDK releases; the closest items (resource import/adoption docs) address migrating existing cloud resources into Pulumi, not release-to-release compatibility. Missing for 10: any semver/compatibility policy docs, breaking-change/deprecation notices, or dedicated upgrade guides between major versions.", "evidenceIds": [] }, { "productId": "terraform", "storyId": "agent-authors-change", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Terraform's CLI supports the core mechanics: headless plan generation with -out, machine-readable JSON output via show -json and -json streaming, and apply requiring approval unless -auto-approve is passed, plus an official MCP server for AI integration with provider docs/modules/policies. However, there is no first-party documented workflow or tooling that specifically wires an agent to author config, run plan headlessly, and present a diff for human approval as an integrated feature — this is assembled by the user from separate primitives rather than a turnkey agent-approval pipeline. missing for 10: an end-to-end documented agent-authoring-to-approval workflow, evidence of the MCP server itself invoking plan/apply or diff presentation, and independent/hands-on confirmation of this exact human-in-the-loop pattern.", "evidenceIds": [ "terraform-docs-15", "terraform-docs-30", "terraform-docs-25", "terraform-docs-39", "terraform-docs-22", "terraform-docs-31", "terraform-docs-23", "terraform-docs-40" ] }, { "productId": "terraform", "storyId": "agent-least-privilege-gates", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Terraform's plan/apply workflow provides a clear review gate (plan preview before apply, manual approval unless -auto-approve is used) and policy-as-code enforcement (Sentinel/OPA-style policies) that can block noncompliant changes, plus a dedicated MCP server giving AI agents structured access to docs, modules, and policies — together these support an agent proposing plans that are policy-checked and gated before apply. However, there's no explicit documentation of least-privilege credential scoping specifically for agent-driven runs (e.g., ephemeral/scoped run credentials tied to an agent identity) or of an agent-specific approval workflow beyond the standard CLI/CLOUD run gates. missing for 10: explicit least-privilege credential/service-account guidance for AI agents, documented agent-specific approval/RBAC workflow, independent hands-on evidence of an agent being blocked by policy gates.", "evidenceIds": [ "terraform-docs-15", "terraform-docs-16", "terraform-docs-17", "terraform-docs-9", "terraform-docs-35", "terraform-docs-10", "terraform-docs-23", "terraform-docs-40" ] }, { "productId": "terraform", "storyId": "agentic-agent-docs", "verdict": "partial", "quality": 3, "confidence": "medium", "rationale": "Terraform has no llms.txt or docs.md agent-readable file (confirmed 404 by probes), so the specific artifact requested by the story is absent. However, HashiCorp does provide an official MCP server that gives AI models real-time access to Terraform provider documentation, modules, and policies, which is an agent-oriented documentation access mechanism, giving partial credit. Missing for 10: an actual llms.txt/docs.md file, and any evidence the MCP server is a drop-in substitute for llms.txt-style doc consumption.", "evidenceIds": [ "terraform-probe-1", "terraform-probe-2", "terraform-docs-10", "terraform-docs-40" ] }, { "productId": "terraform", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows an MCP server that lets external AI models query Terraform docs/modules/policies for building configurations — this is agentic tool access, not Terraform itself generating insights or suggestions from a user's own infrastructure data. No documentation shows in-product AI analysis (e.g., cost/security recommendations, drift insights) surfaced to users.", "evidenceIds": [ "terraform-docs-10", "terraform-docs-23", "terraform-docs-36", "terraform-docs-40" ] }, { "productId": "terraform", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Terraform's CLI supports scripted, non-interactive runs (-auto-approve, machine-readable -json output for plan/apply/show) that can be wired into external schedulers or CI/CD pipelines, and the MCP server exposes provider/module/policy context to AI agents for generating configs (terraform-docs-3, terraform-docs-17, terraform-docs-18, terraform-docs-22, terraform-docs-23/40). However, there is no evidence of a native scheduler, trigger, or always-on autonomous run loop within Terraform itself — automation depends on external orchestration (CI, HCP Terraform runs) not documented here. Missing for 10: native scheduling/trigger mechanism, evidence of unattended background execution loop, and independent confirmation of AI agents autonomously running plan/apply cycles.", "evidenceIds": [ "terraform-docs-3", "terraform-docs-17", "terraform-docs-18", "terraform-docs-22", "terraform-docs-23", "terraform-docs-40", "terraform-docs-9" ] }, { "productId": "terraform", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows an MCP server that lets external AI models query Terraform docs/modules/policies, but this is Terraform acting as a server for outside agents, not a built-in assistant inside the product that a user can delegate tasks to. No documentation or probe shows an in-product AI assistant/chat feature.", "evidenceIds": [] }, { "productId": "terraform", "storyId": "agentic-headless", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Terraform CLI is fully scriptable/headless: non-interactive apply via -auto-approve, machine-readable JSON output (-json flag on show, plan, apply), plan files for two-step CI pipelines, and Terraform tests for CI validation — all well documented and widely used in CI/CD pipelines. missing for 10: independent hands-on report of a specific CI pipeline configuration (e.g., GitHub Actions) rather than just docs.", "evidenceIds": [ "terraform-docs-3", "terraform-docs-17", "terraform-docs-8", "terraform-docs-22", "terraform-docs-30", "terraform-docs-7", "terraform-docs-28", "terraform-probe-5" ] }, { "productId": "terraform", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Terraform is an infrastructure-as-code CLI/engine, not an AI agent with a tool-use loop, so 'plugging MCP servers into it so it can use their tools' is a category error — the evidence instead shows Terraform ships an official MCP *server* (terraform-docs-10/23/36/40) that exposes its own docs/modules/policies to external AI models, which is the reverse (server) role, not client consumption of other servers' tools.", "evidenceIds": [ "terraform-docs-10", "terraform-docs-23", "terraform-docs-36", "terraform-docs-40", "terraform-probe-4" ] }, { "productId": "terraform", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Terraform is not itself an agent but an IaC platform, and HashiCorp documents an official Terraform MCP server that gives AI models real-time access to provider docs, modules, and policies, confirmed by both docs and a probe hit. missing for 10: independent/hands-on third-party verification of the MCP server in production use and details on its transport/auth setup.", "evidenceIds": [ "terraform-docs-10", "terraform-docs-23", "terraform-docs-36", "terraform-docs-40", "terraform-probe-4" ] }, { "productId": "terraform", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Terraform's core interface remains HCL config files and explicit CLI commands (plan/apply/import), not natural language. The official MCP server gives AI models real-time access to provider docs, modules, and policies to help generate more accurate configurations, which is a step toward AI-native operation, but there is no evidence of a natural-language command interface for running Terraform actions directly. Missing for 10: a documented NL-to-command translation layer, evidence of AI agents actually driving terraform plan/apply via natural language, and independent corroboration of the MCP server enabling full operational control rather than just documentation lookup.", "evidenceIds": [ "terraform-docs-10", "terraform-docs-23", "terraform-docs-40", "terraform-probe-4" ] }, { "productId": "terraform", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Terraform ships an official CLI with extensive documented commands (plan, apply, show, import, state, test) plus machine-readable JSON output modes (-json flags) that make it scriptable/consumable by AI agents, and HashiCorp also documents an official MCP server for AI integration. missing for 10: independent hands-on confirmation of AI-native agentic CLI usage patterns beyond docs, and no evidence of AI-specific CLI enhancements (e.g., natural-language CLI mode) beyond the JSON output and MCP server.", "evidenceIds": [ "terraform-probe-5", "terraform-docs-18", "terraform-docs-22", "terraform-docs-34", "terraform-docs-10", "terraform-probe-4" ] }, { "productId": "terraform", "storyId": "agentic-public-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Terraform exposes machine-readable interfaces AI agents could drive: JSON output from plan/apply/show, a documented CLI, and an official MCP server built specifically to give AI models access to Terraform docs/modules/policies (terraform-docs-10/23/36/40, terraform-probe-4). However, probes confirm there is no discoverable OpenAPI/REST API spec or llms.txt (terraform-probe-1/2/3), so the 'documented public API' is really CLI+JSON+MCP rather than a conventional API surface. Missing for 10: a formal OpenAPI/REST API specification, an llms.txt or machine-readable docs index, and independent confirmation that AI agents successfully drive Terraform end-to-end via these interfaces.", "evidenceIds": [ "terraform-docs-8", "terraform-docs-18", "terraform-docs-22", "terraform-docs-34", "terraform-docs-10", "terraform-docs-23", "terraform-docs-36", "terraform-docs-40", "terraform-probe-1", "terraform-probe-2", "terraform-probe-3", "terraform-probe-4", "terraform-probe-5" ] }, { "productId": "terraform", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence that Terraform (or its MCP server) supports issuing scoped/least-privilege API credentials for an agent; credential/auth scoping is typically delegated to cloud provider IAM, and nothing in the docs describes Terraform generating or managing agent-specific least-privilege credentials.", "evidenceIds": [ "terraform-docs-10", "terraform-docs-23", "terraform-docs-36" ] }, { "productId": "terraform", "storyId": "agentic-sdks", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence shows Terraform exposes a plugin SDK for writing custom providers (terraform-docs-29) and machine-readable JSON output for programmatic consumption (terraform-docs-4/8/18/22/34), plus an official MCP server for AI models (terraform-docs-10/23/36/40), but there is no documented official SDK (e.g., CDK for Terraform or a language-native SDK) that AI-native builders could target directly. missing for 10: explicit official SDK documentation (e.g., CDKTF), examples of programmatic/AI-driven SDK usage, independent hands-on confirmation of SDK reliability.", "evidenceIds": [ "terraform-docs-29", "terraform-docs-4", "terraform-docs-8", "terraform-docs-10", "terraform-docs-23", "terraform-docs-40" ] }, { "productId": "terraform", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of webhook subscription capability for events in Terraform's core product; while Terraform Cloud has run notifications/webhooks in some contexts, nothing in this evidence pack documents a webhook subscription mechanism for AI-native event consumption. Terraform is fundamentally an IaC provisioning tool, so this axis could apply but is unevidenced.", "evidenceIds": [] }, { "productId": "terraform", "storyId": "ai-generate-iac", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Terraform ships an official MCP server that gives AI models real-time access to provider docs, modules, and policies specifically to enable 'more accurate and actionable Terraform configuration generation,' which supports natural-language-to-IaC workflows — but this is an integration layer for external AI assistants/agents rather than a natural-language generation feature built directly into the Terraform CLI/toolchain itself. missing for 10: no evidence of a native in-CLI NL-to-HCL generation feature, no independent/hands-on validation of the MCP server's generation quality, and no confirmation that generation is embedded rather than delegated to third-party AI tools.", "evidenceIds": [ "terraform-docs-10", "terraform-docs-23", "terraform-docs-36", "terraform-docs-40", "terraform-probe-4" ] }, { "productId": "terraform", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack shows standard static documentation pages and CLI references, but no interactive API reference or runnable code examples; probes explicitly confirm absence of machine-readable docs formats (llms.txt, docs.md, OpenAPI spec all 404).", "evidenceIds": [ "terraform-probe-1", "terraform-probe-2", "terraform-probe-3" ] }, { "productId": "terraform", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Explicit probes for llms.txt, docs.md, and multiple OpenAPI/swagger endpoints all returned 404, and no evidence pack item references a downloadable OpenAPI or equivalent machine-readable API spec for Terraform.", "evidenceIds": [ "terraform-probe-1", "terraform-probe-2", "terraform-probe-3" ] }, { "productId": "terraform", "storyId": "api-sandbox", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Terraform's built-in `terraform test` framework explicitly runs against test-specific, short-lived resources to avoid touching real infrastructure/state, and v1.7.0 added provider-response mocking so tests can run without any real cloud calls at all — a genuine sandbox mechanism. However, this is a general IaC testing feature, not something built or documented specifically for 'AI-native' agentic use, and there's no independent/hands-on evidence validating it in an AI-agent context. Missing for 10: explicit AI-agent-oriented sandbox tooling or workflow guidance, and independent corroboration that the test/mock framework reliably isolates AI-driven runs from production.", "evidenceIds": [ "terraform-docs-27", "terraform-docs-28", "terraform-docs-7", "terraform-docs-21" ] }, { "productId": "terraform", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While Terraform configurations support versioning (terraform-docs-1, terraform-docs-14) and providers/modules use semantic versioning via the registry (terraform-docs-11, terraform-docs-41), the evidence pack shows no documented API versioning scheme or deprecation policy for Terraform's APIs (e.g., Terraform Cloud API, provider protocol) — probes for openapi specs and machine-readable docs all 404 (terraform-probe-1, terraform-probe-2, terraform-probe-3). This axis applies to an infrastructure automation tool with APIs, but no evidence demonstrates a documented deprecation policy.", "evidenceIds": [ "terraform-probe-1", "terraform-probe-2", "terraform-probe-3", "terraform-docs-11", "terraform-docs-41" ] }, { "productId": "terraform", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Terraform's graph-based execution parallelizes non-dependent resource changes, and -auto-approve plus -json machine-readable output enable scripted, bulk plan/apply workflows across many resources, with an official MCP server for AI agent integration. However, evidence doesn't show a native 'bulk operation' primitive (e.g., bulk import/move across many items) beyond per-resource state/import commands. missing for 10: documented bulk import/bulk state-move across many resources in one command, independent hands-on evidence of AI agents driving large-scale bulk operations, and any explicit bulk-editing API beyond graph parallelization.", "evidenceIds": [ "terraform-gh-1", "terraform-docs-17", "terraform-docs-22", "terraform-docs-30", "terraform-docs-23", "terraform-probe-4" ] }, { "productId": "terraform", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows Terraform's policy enforcement (Sentinel/OPA) validates plans against rules and its plan/apply workflow can be scripted, but there is no evidence of a built-in mechanism to define rules that automatically trigger actions in response to arbitrary events (e.g., event-driven automation, webhooks-to-action rules). Terraform relies on external CI/CD or HCP Terraform triggers, which aren't documented here.", "evidenceIds": [ "terraform-docs-9", "terraform-docs-35" ] }, { "productId": "terraform", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Terraform's plan/apply/test/state workflow and MCP server for AI documentation lookup, but contains no mention of a scheduler, cron-like trigger, or recurring workflow execution feature (native or via HCP Terraform). Since infrastructure-automation platforms could plausibly offer scheduled/recurring runs, the axis applies, but no evidence supports it.", "evidenceIds": [] }, { "productId": "terraform", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Terraform configs are text files that can be versioned in git (terraform-docs-1/14/37), and `terraform plan`/`show -json` give strong review capabilities before apply (terraform-docs-2/15/26/39). However, there is no first-class 'rollback' command — reverting infrastructure requires reapplying an older config version or manual state manipulation via `terraform state` (terraform-docs-6/20), and HCP remote state storage is only recommended, not a rollback mechanism. Missing for 10: a documented native rollback/undo feature, and hands-on evidence of rollback workflows succeeding in practice.", "evidenceIds": [ "terraform-docs-1", "terraform-docs-2", "terraform-docs-6", "terraform-docs-20", "terraform-docs-26", "terraform-docs-33", "terraform-comm-15" ] }, { "productId": "terraform", "storyId": "destroy-teardown", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents plan, apply, state, import, and test workflows in detail but never mentions a `terraform destroy` command or any teardown/cleanup operation. Without direct evidence of a destroy capability, this story cannot be credited even though it is a core plan-apply workflow axis for an IaC tool.", "evidenceIds": [] }, { "productId": "terraform", "storyId": "drift-detection", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Terraform's plan command (and underlying refresh) explicitly compares declared configuration against real infrastructure state to preview differences, which is the core drift-detection mechanism, and terraform show -json / state commands expose that comparison programmatically; community commentary corroborates execution plans as a trusted way to see what will change before applying. Missing for 10: explicit vendor documentation using the term 'drift detection' or a dedicated drift-detection guide, and independent hands-on verification of drift-detection accuracy at scale.", "evidenceIds": [ "terraform-docs-2", "terraform-docs-15", "terraform-docs-20", "terraform-docs-4", "terraform-docs-39", "terraform-comm-15" ] }, { "productId": "terraform", "storyId": "environment-config-stacks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Terraform's state and module system (terraform-docs-20, terraform-docs-33, terraform-docs-13) supports separating configuration into reusable modules and remote state per environment, which is the underlying mechanism for dev/staging/prod separation, but the evidence pack never explicitly documents the `terraform workspace` command or a documented pattern for separate environment stacks. Missing for 10: explicit docs on `terraform workspace new/select`, guidance on directory-per-environment or workspace-per-environment patterns, and independent confirmation of how teams structure environments.", "evidenceIds": [ "terraform-docs-20", "terraform-docs-33", "terraform-docs-13", "terraform-docs-6" ] }, { "productId": "terraform", "storyId": "gp-language-support", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "All evidence describes Terraform's native configuration language (HCL) as a 'human-readable configuration file' language, not a general-purpose programming language with types/loops in the sense of Python/TypeScript; there is no CDKTF or similar general-purpose-language authoring mentioned anywhere in the pack. Community feedback even criticizes it as 'the awful half-baked language,' reinforcing that Terraform's core authoring experience is a declarative DSL rather than a GPL with IDE-native type/loop support.", "evidenceIds": [ "terraform-docs-1", "terraform-docs-14", "terraform-comm-1", "terraform-comm-3" ] }, { "productId": "terraform", "storyId": "headless-ci-apply", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Terraform's docs directly support the CI workflow: `terraform plan -out=FILE` saves a plan artifact, `terraform apply FILE` executes that exact saved plan, and `-auto-approve` skips interactive approval for non-interactive automation. Machine-readable JSON output (`-json` on show/plan/apply) further supports CI pipelines that need to parse results. missing for 10: no independent/hands-on CI pipeline example or third-party corroboration of this exact workflow.", "evidenceIds": [ "terraform-docs-30", "terraform-docs-3", "terraform-docs-31", "terraform-docs-38", "terraform-docs-16", "terraform-docs-18", "terraform-docs-34" ] }, { "productId": "terraform", "storyId": "import-existing-infra", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Docs confirm `terraform import` can bring existing resources into state (terraform-docs-5/19/32), but the evidence pack has no mention of the newer -generate-config-out feature that produces matching HCL, and a first-hand community report explicitly says there's no good built-in way to import resources created outside Terraform, having to resort to a third-party tool of uncertain maintenance status (terraform-comm-10). This directly contradicts the 'generate matching configuration code' part of the story. Missing for 10: first-party documentation of automatic config-code generation during import, and corroboration that it works reliably in practice.", "evidenceIds": [ "terraform-docs-5", "terraform-docs-19", "terraform-docs-32", "terraform-comm-10" ] }, { "productId": "terraform", "storyId": "migrate-from-terraform", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "This story presumes migrating a Terraform-format codebase/state into a different tool; since the product being evaluated is Terraform itself, there is no cross-tool migration to perform — the axis is a category error for this product.", "evidenceIds": [] }, { "productId": "terraform", "storyId": "open-license-governance", "verdict": "disputed", "quality": 3, "confidence": "medium", "rationale": "Terraform is marketed as open (public registry, GitHub repo) but the evidence shows concrete counter-signals to the 'open governance that can't be relicensed' promise: HashiCorp requires a CLA that community members explicitly object to, and users report HashiCorp 'yanked the okta/mfa feature from OSS' calling it 'pretty hostile' — both indicating the vendor retains unilateral control over licensing/features rather than a neutral, foundation-governed open project. missing for 10: any first-party statement on license terms (e.g. MPL/BSL) or a governance body/foundation guaranteeing the license can't change, and independent confirmation that community contributions are protected from relicensing.", "evidenceIds": [ "terraform-comm-4", "terraform-comm-13", "terraform-comm-11", "terraform-docs-41" ] }, { "productId": "terraform", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Terraform's CLI/JSON tooling (terraform plan/apply/show -json, machine-readable UI output, state and import commands) lets automation and AI agents drive nearly every core workflow without the UI, and Terraform Cloud exposes policy results in its workspace UI, implying an underlying API layer. However, there is no direct evidence of a documented, comprehensive REST/OpenAPI spec proving true UI/API parity, and probes for an OpenAPI schema and machine-readable docs all 404, suggesting the API surface for Terraform Cloud's UI-specific features (e.g., policy override, workspace management) isn't clearly documented for AI consumption. missing for 10: explicit Terraform Cloud/Enterprise API docs proving 1:1 parity with UI actions, a discoverable OpenAPI/schema for AI tooling, and confirmation that UI-only actions (like policy overrides) are also API-accessible.", "evidenceIds": [ "terraform-docs-4", "terraform-docs-8", "terraform-docs-22", "terraform-docs-35", "terraform-probe-3", "terraform-probe-1" ] }, { "productId": "terraform", "storyId": "openness-full-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Terraform's core artifacts are open by design: configurations are plain human-readable HCL files that users own, version, and share (terraform-docs-1/14/37), and state/plan data can be exported to machine-readable JSON via `terraform show -json` or manipulated via `terraform state` commands (terraform-docs-4/6/18/26/39), enabling a user to walk away with all their data in open formats. However there's no explicit 'export everything and migrate away' workflow or tooling documented, and community evidence notes friction around state portability (e.g., resources outside TF state being ignored, terraform-comm-10) and remote/Cloud-hosted state recommendations (terraform-docs-33) that could complicate a clean exit if using HCP Terraform. Missing for 10: an explicit data-export/migration guide, independent confirmation of a smooth 'leave the platform' path, and clarity on exporting HCP Terraform-hosted state without vendor dependency.", "evidenceIds": [ "terraform-docs-1", "terraform-docs-14", "terraform-docs-4", "terraform-docs-6", "terraform-docs-18", "terraform-docs-26", "terraform-docs-33", "terraform-comm-10" ] }, { "productId": "terraform", "storyId": "openness-open-license", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Terraform's source is publicly hosted on GitHub (terraform-gh-1), which lets an AI-native user read the code, and community discussion references a CLA-based open-source contribution model (terraform-comm-4). However, the evidence never states or links an explicit open-source license (e.g. MPL/BUSL identifier) for the repo, and community commentary notes HashiCorp \"yanked\" a feature from OSS (terraform-comm-13), suggesting the openness of the codebase is not unconditional. Missing for 10: explicit license documentation/SPDX reference, confirmation the full CLI source (vs. only some components) remains under an OSI-approved license, and resolution of the OSS-feature-removal concern.", "evidenceIds": [ "terraform-gh-1", "terraform-comm-4", "terraform-comm-13" ] }, { "productId": "terraform", "storyId": "openness-self-host", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Terraform CLI itself is open-source, self-hostable software that runs locally without requiring HashiCorp's cloud, and evidence confirms open registry, self-authored providers, and CLI-only workflows (state, plan, apply) that don't require any hosted backend; community comments corroborate long-term self-hosted/open-source usage. missing for 10: no explicit evidence pack statement enumerating self-hosting deployment instructions or license details, and no independent audit confirming full feature parity without HCP/Cloud services.", "evidenceIds": [ "terraform-docs-12", "terraform-docs-14", "terraform-docs-29", "terraform-docs-41", "terraform-comm-7", "terraform-gh-1" ] }, { "productId": "terraform", "storyId": "policy-enforcement", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Terraform Cloud/Enterprise documents a policy-enforcement feature (Sentinel/OPA-style) that validates plans against security rules and best practices before allowing an apply, which is the core policy-as-code capability a devops-lead would want. However, evidence is a single thin doc reference with no detail on enforcement levels (advisory vs hard-mandatory), no Sentinel/OPA specifics, and no independent or hands-on confirmation that non-compliant changes are actually blocked. missing for 10: detailed docs on enforcement levels and policy language (Sentinel/OPA), hands-on proof of a blocked apply, independent community corroboration of policy gating in practice.", "evidenceIds": [ "terraform-docs-9" ] }, { "productId": "terraform", "storyId": "preview-before-apply", "verdict": "full", "quality": 10, "confidence": "high", "rationale": "terraform plan generates a detailed execution plan previewing creates/updates/destroys before any changes are made, with human-readable and JSON output options, and apply prompts for confirmation unless auto-approved; this core workflow is corroborated by both official docs and independent community testimony praising execution plans as a key safety feature.", "evidenceIds": [ "terraform-docs-2", "terraform-docs-15", "terraform-docs-16", "terraform-docs-25", "terraform-docs-30", "terraform-docs-39", "terraform-comm-15" ] }, { "productId": "terraform", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While Terraform lets users choose remote backends (e.g., HCP Terraform or self-hosted state stores) per terraform-docs-33, the evidence pack contains no documentation of region/residency selection controls for where Terraform Cloud/HCP or its data is stored. This axis is fair for a product with a hosted SaaS offering (HCP Terraform) that stores state data, so absence of evidence yields 'none' rather than 'na'.", "evidenceIds": [ "terraform-docs-33" ] }, { "productId": "terraform", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Terraform is an infrastructure-as-code provisioning tool; it does not collect or process user data for AI model training, so an AI-training opt-out privacy control is a category error for this product type.", "evidenceIds": [] }, { "productId": "terraform", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of data-retention policies, deletion controls, GDPR/CCPA compliance, or privacy settings for Terraform or HCP Terraform's stored state/logs. State-management docs (terraform state rm, import) address infrastructure resource tracking, not user data privacy controls, so they cannot substitute. missing for 10: any documentation of retention windows, deletion APIs/commands for user or telemetry data, or privacy/compliance policy pages.", "evidenceIds": [] }, { "productId": "terraform", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items mention telemetry, usage tracking, or an opt-out mechanism for Terraform CLI; this is a plausible axis for a CLI tool but no documentation addresses it.", "evidenceIds": [] }, { "productId": "terraform", "storyId": "programmatic-deployments", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows CLI-based workflows (plan/apply/show -json, machine-readable UI output) that can be scripted or wrapped, but nothing indicates an official SDK, REST/HTTP API, or library for driving terraform plan/apply from application code without shelling out to the CLI. No mention of Terraform Cloud API, CDK for Terraform, or any programmatic library in the pack.", "evidenceIds": [ "terraform-docs-15", "terraform-docs-16", "terraform-docs-18", "terraform-docs-22", "terraform-docs-30", "terraform-probe-3" ] }, { "productId": "terraform", "storyId": "provider-breadth", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Terraform's core value proposition is a broad provider ecosystem via the public Terraform Registry, letting anyone publish/consume providers and modules across clouds, on-prem, and SaaS, with community evidence corroborating real-world multi-cloud/cross-account use cases. missing for 10: independent quantitative data on registry breadth (e.g., number of providers) and no direct evidence contradicting ecosystem breadth.", "evidenceIds": [ "terraform-docs-12", "terraform-docs-11", "terraform-docs-41", "terraform-docs-29", "terraform-comm-2", "terraform-docs-14" ] }, { "productId": "terraform", "storyId": "remote-state-locking", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack only covers generic `terraform state` CLI commands and local state mapping (terraform-docs-6, terraform-docs-20) plus a community complaint about setting up backend state (terraform-comm-5) — none of it documents remote backends, state locking, or concurrent-run protection. Missing for 10: any docs/tests referencing S3/Consul/Terraform Cloud remote backends, state locking mechanism, or a hands-on probe demonstrating locking prevents concurrent corruption.", "evidenceIds": [ "terraform-docs-6", "terraform-docs-20", "terraform-comm-5", "terraform-comm-3" ] }, { "productId": "terraform", "storyId": "reusable-modules", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Terraform's public Registry explicitly supports publishing and consuming providers, modules, and policies, and modules are a core reusable/versioned configuration primitive documented extensively. missing for 10: independent hands-on evidence of publishing workflow specifics or community friction reports about registry publishing itself.", "evidenceIds": [ "terraform-docs-11", "terraform-docs-41", "terraform-docs-13", "terraform-docs-14" ] }, { "productId": "terraform", "storyId": "secret-config-values", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation of Terraform's sensitive-variable marking, environment-variable injection, or secret-manager integrations that would let a developer pass secrets without exposing them in code or logs. The only related evidence is a community comment noting secrets are stored unencrypted in state (terraform-comm-3), which points to a gap rather than a documented capability.", "evidenceIds": [ "terraform-comm-3" ] }, { "productId": "terraform", "storyId": "state-encryption", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation of a Terraform state encryption-at-rest feature, and community evidence explicitly states 'Secrets are still stored in the state without encryption when retrieving them from the CLI' (terraform-comm-3), indicating this capability is not delivered per the provided evidence.", "evidenceIds": [ "terraform-comm-3", "terraform-docs-20", "terraform-docs-6" ] }, { "productId": "terraform", "storyId": "state-surgery", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Terraform explicitly documents `terraform state` subcommands for basic state modifications (mv/rm/import equivalents) plus `terraform import` for bringing existing resources under management, directly supporting safe state refactoring. missing for 10: no explicit mention of `moved`/`removed` config blocks or `terraform state mv/rm` command details, and no independent/hands-on evidence confirming safe refactoring workflows in practice.", "evidenceIds": [ "terraform-docs-6", "terraform-docs-20", "terraform-docs-5", "terraform-docs-19", "terraform-docs-33" ] }, { "productId": "terraform", "storyId": "structured-plan-output", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Terraform provides `terraform show -json` to output plan/state/config as JSON, and `-json` on plan/apply/refresh/test streams machine-readable JSON UI messages line-by-line for programmatic processing — both well-documented first-party capabilities directly matching the story. missing for 10: no independent/hands-on third-party corroboration of an agent actually parsing this JSON in practice.", "evidenceIds": [ "terraform-docs-4", "terraform-docs-18", "terraform-docs-8", "terraform-docs-22", "terraform-docs-34", "terraform-docs-39" ] }, { "productId": "terraform", "storyId": "targeted-partial-apply", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents `terraform plan`/`apply`, `-auto-approve`, `-out`, and JSON output flags, but nowhere mentions the `-target` or `-exclude` flags that let a platform engineer scope plan/apply to specific resources. Without citable evidence for this specific capability, it cannot be credited even though it is plausible for this type of tool.", "evidenceIds": [] }, { "productId": "terraform", "storyId": "unit-integration-tests", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Terraform has a dedicated `terraform test` framework (terraform-docs-7, -21, -27, -28) that runs tests against test-specific, short-lived resources, explicitly preventing risk to existing infrastructure/state, and even supports mocking provider data so tests can run without touching real infrastructure. This directly satisfies writing automated infra tests without touching production. missing for 10: independent/hands-on evidence of `terraform test` execution (the probes only cover plan/apply/destroy, not the test framework itself), and more detail on CI integration patterns for the test command.", "evidenceIds": [ "terraform-docs-7", "terraform-docs-21", "terraform-docs-27", "terraform-docs-28" ] }, { "productId": "terraform", "storyId": "validate-fmt", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack never mentions `terraform validate` or `terraform fmt`, the actual commands for config validation and auto-formatting. Closest evidence (terraform tests, policy enforcement) address post-write module testing and plan compliance, not pre-plan syntax validation/formatting, so the specific story is unevidenced.", "evidenceIds": [] }, { "productId": "terraform", "storyId": "version-stability", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation about version compatibility guarantees, semantic versioning policy, or upgrade guides between Terraform releases; it only covers CLI commands, state management, testing, and the MCP server. This axis clearly applies to a widely-used infra-as-code tool, but no supporting citations exist here.", "evidenceIds": [] } ]