[ { "productId": "make", "storyId": "agent-creates-workflow-via-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Make's public REST API is documented as resource-oriented and requires token auth, and the MCP server explicitly allows AI systems to view and modify scenarios, run them, and manage account contents, implying create/update/activate capabilities. However, no evidence explicitly documents an API endpoint or example for creating a new scenario or toggling activation state, and the docs snippets are largely generic descriptions rather than concrete workflow-lifecycle examples. missing for 10: explicit API endpoint documentation for scenario creation, explicit 'activate' endpoint/example, and independent confirmation of programmatic activation.", "evidenceIds": [ "make-docs-11", "make-docs-14", "make-docs-20", "make-docs-1", "make-probe-4" ] }, { "productId": "make", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Make hosts a live llms.txt at developers.make.com/llms.txt (HTTP 200, confirmed by probe) plus Markdown-formatted 'Make Skills' docs designed for AI assistants, directly supporting agent-oriented documentation consumption. missing for 10: confirmation that all doc subpages (not just the root) resolve as .md, and independent/third-party evidence of agents successfully using llms.txt in practice.", "evidenceIds": [ "make-probe-1", "make-docs-4", "make-docs-24", "make-probe-2" ] }, { "productId": "make", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence shows Make's AI Agents app and MCP server let external AI systems build/run automations and manage scenarios, but there is no evidence of a feature that surfaces AI-generated insights or suggestions from a user's own data inside the product UI.", "evidenceIds": [ "make-docs-9", "make-docs-20" ] }, { "productId": "make", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Make scenarios are core automations that run autonomously on triggers (webhooks, schedules) in the background without human intervention, with error handling and history/monitoring built in, and can even be exposed as callable tools via MCP for AI orchestration. missing for 10: independent/hands-on evidence of long-running autonomous scenarios at scale, and no third-party corroboration beyond vendor docs.", "evidenceIds": [ "make-docs-5", "make-docs-6", "make-docs-15", "make-docs-16", "make-docs-21", "make-docs-7", "make-docs-18", "make-docs-2", "make-docs-20" ] }, { "productId": "make", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Make documents a 'Make AI Agents' app for 'agentic automation' inside the platform, suggesting a built-in AI agent/assistant capability, but the evidence is a single glancing mention with no detail on how tasks are delegated or what the assistant can do. Missing for 10: detailed docs on the AI Agents app's task-delegation UX, concrete examples of use, and independent/hands-on corroboration of its capabilities.", "evidenceIds": [ "make-docs-9" ] }, { "productId": "make", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Make exposes a REST API (with auth/scopes) and webhooks that let external systems or CI pipelines trigger, schedule, and manage scenarios without using the UI, which supports headless/automated invocation. However, there is no evidence of an official CLI, containerized runner, or CI-specific integration guide (e.g., GitHub Actions), and the openapi probe returned 404s, suggesting weaker machine-readable API tooling. Missing for 10: dedicated CLI or CI/CD integration docs, confirmed OpenAPI spec, and hands-on evidence of running scenarios in a pipeline.", "evidenceIds": [ "make-docs-1", "make-docs-11", "make-docs-5", "make-docs-15", "make-docs-6", "make-docs-16", "make-probe-3" ] }, { "productId": "make", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All evidence describes Make exposing its own scenarios as an MCP server to external AI clients (Claude, ChatGPT), the reverse of this story's requirement that Make itself act as an MCP client consuming external MCP servers' tools. No documentation shows Make importing or calling third-party MCP servers as tool providers within its scenarios/agents.", "evidenceIds": [ "make-docs-2", "make-docs-3", "make-docs-12", "make-docs-20", "make-docs-9" ] }, { "productId": "make", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Make documents an official MCP server that turns scenarios into callable tools for AI systems like Claude/ChatGPT, exposing scenario run and management capabilities, plus Make Skills to help agents connect. Missing for 10: independent/hands-on third-party corroboration and details on auth scopes specific to MCP server usage.", "evidenceIds": [ "make-docs-2", "make-docs-3", "make-docs-12", "make-docs-20", "make-docs-24", "make-probe-4" ] }, { "productId": "make", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Make ships an official MCP server and 'Make AI Agents' app that let LLMs (Claude, ChatGPT) run and manage scenarios via natural-language tool calls, plus 'Make Skills' markdown files to guide assistants — this is real agentic/NL operation support. However, this capability is mediated entirely through external AI clients rather than a built-in natural-language command interface in Make itself, and there is no independent/hands-on corroboration of reliability. Missing for 10: evidence of a native in-product NL command bar/assistant, and third-party/hands-on validation of the MCP-driven workflow.", "evidenceIds": [ "make-docs-2", "make-docs-3", "make-docs-9", "make-docs-20", "make-docs-24", "make-probe-4" ] }, { "productId": "make", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack documents Make's REST API, MCP server, webhooks, and Skills, but contains no mention of an official command-line interface (CLI) tool for Make. Since automation platforms could plausibly ship a CLI, this axis applies, but no evidence supports it being delivered.", "evidenceIds": [] }, { "productId": "make", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Make documents a public REST API (resource-oriented URLs, token auth, troubleshooting section) plus a first-party MCP server exposing scenario run/management as callable tools, enabling AI-native driving of the product beyond just the raw API. Missing for 10: a working OpenAPI/swagger spec (probe found 404s) and independent hands-on corroboration of the API's completeness/reliability.", "evidenceIds": [ "make-docs-1", "make-docs-11", "make-docs-23", "make-docs-2", "make-docs-3", "make-docs-20", "make-probe-3", "make-probe-4" ] }, { "productId": "make", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Make API tokens support 'relevant scopes enabled' for authentication, indicating some scoped credential capability, and MCP server access can be scoped to specific scenarios/tools rather than full account access. However there's no documentation of granular least-privilege scope definitions, no scope list, no per-agent credential issuance workflow, and no independent verification of enforcement. Missing for 10: a documented list of available API scopes, evidence of fine-grained least-privilege controls (e.g., read-only vs write, resource-level restrictions), and independent/hands-on confirmation that scoped tokens actually restrict agent access as claimed.", "evidenceIds": [ "make-docs-1", "make-docs-11", "make-docs-20", "make-docs-14" ] }, { "productId": "make", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows a REST API and an MCP server, but no official client SDKs (Python/JS/etc.) or OpenAPI spec are documented—probes for openapi/swagger endpoints returned 404, indicating no formal SDK-generation artifact. The API docs describe raw REST endpoints and auth tokens, not packaged SDK libraries.", "evidenceIds": [ "make-docs-1", "make-docs-11", "make-probe-3" ] }, { "productId": "make", "storyId": "agentic-webhooks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Make's custom webhooks let a scenario expose a URL that receives events from external systems and trigger scenario runs, effectively letting a user 'subscribe' to external events via webhook (make-docs-5, make-docs-15, make-docs-25), with queuing/scheduling options (make-docs-6, make-docs-21). However, this is a general automation feature, not something exposed or documented specifically for AI-native/agentic consumption (e.g., no MCP tool or agent-specific API for creating/managing webhook subscriptions is shown). Missing for 10: evidence of webhook subscription management being agent-callable via MCP or API, and any outbound event-notification (pub/sub) webhook mechanism for external AI systems.", "evidenceIds": [ "make-docs-5", "make-docs-15", "make-docs-25", "make-docs-6", "make-docs-21" ] }, { "productId": "make", "storyId": "ai-agent-workflow-steps", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Make has a dedicated 'AI Agents' app (make-docs-9) for agentic automation within workflows, implying model choice and tool use, but the evidence pack lacks first-party documentation detailing how to configure model selection or tool/function definitions within an agent step — most evidence instead focuses on the MCP server (which exposes Make scenarios as tools to external AI clients like Claude/ChatGPT, the reverse direction). missing for 10: detailed docs on adding an AI/LLM step inside a scenario, configuring model provider/choice, and defining tool use within that step, plus independent/hands-on confirmation of this workflow.", "evidenceIds": [ "make-docs-9", "make-docs-2", "make-docs-20" ] }, { "productId": "make", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Make provides REST API documentation describing resource-oriented endpoints and authentication (make-docs-1, make-docs-11, make-docs-23), but no evidence shows an interactive reference with runnable/try-it examples; probes for OpenAPI/Swagger specs and even the docs page in machine-readable form returned 404s (make-probe-2, make-probe-3), suggesting no such interactive tooling exists.", "evidenceIds": [ "make-docs-1", "make-docs-11", "make-docs-23", "make-probe-2", "make-probe-3" ] }, { "productId": "make", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Make documents its REST API but explicit probes for OpenAPI/swagger specs at standard paths all returned 404, and no downloadable machine-readable spec is referenced anywhere in the docs pack.", "evidenceIds": [ "make-docs-11", "make-probe-2", "make-probe-3" ] }, { "productId": "make", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a sandbox environment, staging account, or test/production data separation for Make; the docs cover webhooks, scenarios, error handling, and MCP server but nothing about isolating test runs from production data.", "evidenceIds": [] }, { "productId": "make", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows Make has a REST API and MCP server documentation, but nowhere is there mention of API versioning scheme, version numbers, or a documented deprecation policy for breaking changes. Probes even show broken/missing OpenAPI spec links, further suggesting no formal versioning artifact is exposed.", "evidenceIds": [ "make-docs-11", "make-docs-23", "make-probe-2", "make-probe-3" ] }, { "productId": "make", "storyId": "app-event-triggers", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Make's docs describe webhooks that let external apps or events trigger scenario execution (make-docs-5,15,25) plus scheduling to batch-process trigger events (make-docs-6,16,21), which is the mechanism ops users use to fire workflows off new records, messages, forms, etc. from connected apps/modules (make-docs-10 lists Modules/Webhooks as core app components). Missing for 10: explicit named examples of app-specific instant triggers (e.g., new Gmail email, new Typeform submission) and independent/hands-on confirmation beyond vendor docs.", "evidenceIds": [ "make-docs-5", "make-docs-15", "make-docs-25", "make-docs-6", "make-docs-16", "make-docs-21", "make-docs-10" ] }, { "productId": "make", "storyId": "automatic-step-retries", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Make's docs repeatedly reference a dedicated 'error handling' system with 'error handlers' for diagnosing and resolving scenario failures, implying some built-in mechanism for handling failed steps, but none of the evidence explicitly mentions configurable retry counts or backoff intervals. missing for 10: explicit documentation of a retry directive with configurable attempts/backoff, and any hands-on confirmation of this behavior.", "evidenceIds": [ "make-docs-7", "make-docs-19", "make-docs-22", "make-docs-27" ] }, { "productId": "make", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Make's webhook queue processing and API/data-store functions imply some capacity for handling batches of items via scenarios, and the REST API with scoped tokens could be scripted for bulk actions, but no evidence documents a native bulk-operation feature (e.g., batch endpoints, bulk item processing UI, or iterator-based bulk automation) for AI-native use. missing for 10: explicit documentation of batch/bulk API endpoints or iterator modules for processing many items in one call, evidence of AI agents invoking bulk operations via MCP server, and any hands-on confirmation of bulk throughput/performance.", "evidenceIds": [ "make-docs-5", "make-docs-6", "make-docs-21", "make-docs-8", "make-docs-11", "make-docs-20" ] }, { "productId": "make", "storyId": "automation-rules-engine", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Make's core scenario model is built on trigger-action automation: webhooks let users create URLs that trigger scenario execution on external events, schedules can batch-process queued events, and error handlers manage automated fault-response actions. This directly matches the story of defining rules that fire actions on events, though evidence is entirely first-party docs. Missing for 10: independent/hands-on verification and concrete examples of complex conditional rule logic beyond webhooks/schedules.", "evidenceIds": [ "make-docs-5", "make-docs-6", "make-docs-15", "make-docs-16", "make-docs-21", "make-docs-25", "make-docs-7", "make-docs-19" ] }, { "productId": "make", "storyId": "automation-scheduled-jobs", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Make's docs confirm native scenario scheduling ('schedule a scenario') and webhook queues that can be processed on a periodic schedule, directly supporting recurring automated workflows. Missing for 10: independent/hands-on confirmation of scheduling reliability, details on schedule granularity/frequency options, and any AI-specific scheduling interface beyond generic scenario scheduling.", "evidenceIds": [ "make-docs-26", "make-docs-6", "make-docs-16", "make-docs-21" ] }, { "productId": "make", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Make documents 'scenario history' and the ability to 'clone a scenario', which suggest some versioning/backup capability, but there is no explicit documentation of a review/diff UI or a true rollback mechanism restoring a prior version. missing for 10: explicit rollback/revert feature docs, version diff/review UI, and independent confirmation that history can restore a scenario to an earlier state.", "evidenceIds": [ "make-docs-17", "make-docs-18", "make-docs-26" ] }, { "productId": "make", "storyId": "central-credentials-management", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack only lists 'Connections' as an app component category in custom-app developer docs (make-docs-10); there is no evidence describing centralized credential storage, team sharing, or per-credential access control for ops users.", "evidenceIds": [ "make-docs-10" ] }, { "productId": "make", "storyId": "code-steps-in-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Make's API, webhooks, MCP server, error handling, and functions, but contains no mention of a custom code step, JavaScript/Python module, or code-execution capability inside a Make scenario. No evidence supports the ability to write and run custom code as a workflow step.", "evidenceIds": [] }, { "productId": "make", "storyId": "community-built-components", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Make provides a Custom Apps SDK (Base, Connections, Webhooks, Modules, RPCs) that lets developers build custom integrations, implying a framework where community-built apps could exist, but there is no evidence of a marketplace, directory, or install flow for discovering and installing third-party/community-contributed apps or modules into a Make account. missing for 10: evidence of a public app marketplace/store, discovery of community apps, one-click install of third-party nodes, and any review/vetting process for community contributions.", "evidenceIds": [ "make-docs-10" ] }, { "productId": "make", "storyId": "concurrency-rate-controls", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Make's webhook documentation shows a queuing mechanism where incoming webhook data can be scheduled to be processed periodically in batches rather than immediately, which functions as a basic queue (make-docs-6, make-docs-16, make-docs-21). However, there is no documented general-purpose throttling/rate-limit control for scenario executions against downstream APIs, and no explicit concurrency or rate-limit configuration feature is evidenced. Missing for 10: explicit rate-limit/throttle settings for scenario modules calling downstream APIs, documentation of concurrency controls, and any independent/hands-on confirmation that queuing reliably respects downstream limits.", "evidenceIds": [ "make-docs-6", "make-docs-16", "make-docs-21", "make-docs-5", "make-docs-15" ] }, { "productId": "make", "storyId": "conditional-branching-logic", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no mention of routers, filters, conditional branching, or parallel path merging in Make scenarios — it only covers API auth, MCP server, webhooks, error handling, and scenario management pages. Missing for 10: any documentation of router/filter modules, branch conditions, or parallel-path merge behavior.", "evidenceIds": [] }, { "productId": "make", "storyId": "cron-schedules", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Evidence confirms Make has a scenario scheduling feature (\"schedule a scenario\") and periodic webhook queue processing, but no documentation snippet describes cron-style expressions or explicit timezone configuration options. missing for 10: cron/interval expression syntax details, explicit timezone selection UI/API evidence, independent confirmation of scheduling behavior.", "evidenceIds": [ "make-docs-26", "make-docs-6", "make-docs-16", "make-docs-21" ] }, { "productId": "make", "storyId": "custom-connector-sdk", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Make provides a documented Custom Apps platform (App components: Base, Connections, Webhooks, Modules, Remote Procedure Calls) plus a REST API for building custom connectors/integrations, directly matching the story. Missing for 10: no confirmed official CLI or OpenAPI spec (probe found 404s for openapi.json/swagger.json and a broken docs-md page), and no independent/hands-on corroboration of connector-building success.", "evidenceIds": [ "make-docs-10", "make-docs-11", "make-docs-1", "make-probe-3", "make-probe-2" ] }, { "productId": "make", "storyId": "durable-long-running-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Make is a scenario-based automation platform; while it supports scheduling and webhook queuing, there is no evidence of durable long-running workflow execution primitives (e.g., multi-day waits with guaranteed state persistence across worker/platform restarts) comparable to durable-execution frameworks. Evidence only covers webhooks, scheduling, error handling, and MCP server — none address long-lived stateful workflow durability.", "evidenceIds": [ "make-docs-5", "make-docs-6", "make-docs-21", "make-docs-7" ] }, { "productId": "make", "storyId": "embed-white-label", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers Make's API, webhooks, MCP server, and app-building docs, but contains no mention of white-labeling, embeddable widgets, custom branding, or an embed SDK for reselling Make inside a third-party product. Absence of evidence for this applicable capability yields 'none'.", "evidenceIds": [] }, { "productId": "make", "storyId": "environments-source-control", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows scenario cloning and scenario history features, with no documentation of source-control integration, environment separation, or dev-to-production promotion workflows for Make scenarios.", "evidenceIds": [ "make-docs-17", "make-docs-18", "make-docs-26" ] }, { "productId": "make", "storyId": "error-workflows-alerts", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Make's help docs confirm a dedicated error-handling system with 'error handlers' for diagnosing and resolving failed runs, which is the core mechanism ops users would use to define error-handling paths. However, the evidence never explicitly describes configurable failure notifications (e.g., email/Slack alerts on scenario failure) or how error routes are wired into scenarios beyond generic mentions. Missing for 10: explicit notification-on-failure documentation, concrete error-handler route configuration details, and independent/hands-on confirmation.", "evidenceIds": [ "make-docs-7", "make-docs-19", "make-docs-22", "make-docs-27" ] }, { "productId": "make", "storyId": "expressions-data-mapping", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Make docs confirm a dedicated functions system for transforming/formatting data between modules (make-docs-8), which is the core mapping/expression mechanism ops users use between steps. However, evidence lacks depth on formula syntax, custom function scripting (e.g., IML/JS custom functions), or hands-on examples of complex transformations. missing for 10: detailed docs/examples of expression syntax, custom JS functions, and independent validation of transformation capabilities.", "evidenceIds": [ "make-docs-8" ] }, { "productId": "make", "storyId": "human-approval-steps", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers webhooks, error handling, MCP server, and API docs, but contains no mention of a human-in-the-loop approval step, pause/resume mechanism, or manual confirmation module within Make scenarios. missing for 10: any documentation of a 'wait for input/approval' or manual confirmation module, pause/resume scenario capability, or timeout-based human approval feature.", "evidenceIds": [] }, { "productId": "make", "storyId": "inbound-webhook-triggers", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Make's custom webhooks feature lets developers create a URL that receives external HTTP requests and triggers scenario (workflow) execution with the received payload, with options for immediate or scheduled/batched processing of the queue. missing for 10: independent/hands-on corroboration beyond first-party docs, and no detail on payload parsing/validation specifics.", "evidenceIds": [ "make-docs-5", "make-docs-6", "make-docs-15", "make-docs-21", "make-docs-25" ] }, { "productId": "make", "storyId": "local-dev-instance", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Make is a cloud-hosted automation platform; the evidence pack shows only cloud scenario execution, webhooks, REST API, and MCP server management — there is no mention of a local runtime, CLI, self-hosted/dev instance, or CI testing workflow for scenarios.", "evidenceIds": [] }, { "productId": "make", "storyId": "nl-workflow-generation", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Make's docs show 'Make Skills' (Markdown files) enabling an AI assistant to build and configure scenarios via the MCP server, and the MCP server lets AI systems like Claude/ChatGPT manage scenario contents — implying prompt-driven workflow creation/editing through external AI clients. However, this is an indirect, third-party-assistant-mediated path rather than a documented native in-product 'type a prompt, get a workflow' feature, and there's no hands-on or independent verification of it working. Missing for 10: a native first-party natural-language-to-workflow builder in the Make UI, concrete examples/screenshots of prompt-generated scenarios, and independent hands-on corroboration.", "evidenceIds": [ "make-docs-4", "make-docs-24", "make-docs-20", "make-docs-9" ] }, { "productId": "make", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Make ships a documented, resource-oriented REST API covering scenarios, connections, webhooks, and data stores (make-docs-11, make-docs-14), plus an MCP server that lets AI systems run and manage scenarios (make-docs-3, make-docs-20). However, there's no evidence of a formal, discoverable OpenAPI spec (probe found 404s at standard locations, make-probe-3) and no explicit documentation confirming full parity for scenario-building/editing logic via API as opposed to the UI or Make Skills workflow. missing for 10: explicit API endpoints for full scenario creation/editing parity, public OpenAPI schema, independent confirmation of 1:1 UI/API feature parity.", "evidenceIds": [ "make-docs-11", "make-docs-14", "make-docs-3", "make-docs-20", "make-probe-3" ] }, { "productId": "make", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers Make's REST API, webhooks, MCP server, and scenario management, but nothing addresses a bulk/full account data export feature or open-format portability guarantee that would let a user leave with all their data. Missing for 10: documented data export/download feature, open format (e.g., JSON/CSV) export of scenarios and data stores, any GDPR-style account export or migration tooling.", "evidenceIds": [] }, { "productId": "make", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "make", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "make", "storyId": "prebuilt-connector-breadth", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Make's API, webhooks, MCP server, and custom-app-building framework (Base/Connections/Webhooks/Modules/RPC), but contains no citation confirming a marketplace of thousands of prebuilt, vendor-maintained app connectors. Missing for 10: any documentation or listing of the app/connector marketplace, connector count, or vendor-maintenance claims.", "evidenceIds": [ "make-docs-10", "make-docs-11" ] }, { "productId": "make", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of data residency, regional data storage, or EU/US hosting options for Make; all citations concern MCP server, webhooks, API auth, and error handling. No evidence for data residency capability.", "evidenceIds": [] }, { "productId": "make", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of AI training data opt-out, data usage policies for AI model training, or any privacy controls addressing this specific concern; all evidence covers API auth, MCP server, webhooks, and error handling instead.", "evidenceIds": [] }, { "productId": "make", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no documentation of data retention policies, data deletion requests, or privacy/data lifecycle controls for AI-native users; only tangential scenario-management references (e.g., 'delete a scenario') exist, which do not address retention or deletion of underlying data/logs.", "evidenceIds": [ "make-docs-26" ] }, { "productId": "make", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items mention telemetry, usage tracking, analytics opt-out, or privacy settings; the pack covers API, MCP server, webhooks, and scenario docs only.", "evidenceIds": [] }, { "productId": "make", "storyId": "run-history-replay", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Make's docs confirm scenario execution history for inspecting past runs and dedicated error-handling documentation, which supports viewing logs and diagnosing failures, but none of the evidence describes a feature to re-run a failed execution and resume specifically from the failing step. Missing for 10: explicit documentation of a 'rerun/resume from failed step' capability, and any hands-on confirmation that partial re-execution (vs. full restart) is supported.", "evidenceIds": [ "make-docs-18", "make-docs-7", "make-docs-19", "make-docs-22", "make-docs-27" ] }, { "productId": "make", "storyId": "subworkflow-reuse", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence shows webhooks can be used to trigger one scenario's execution from another scenario or external app (make-docs-15, make-docs-25), which offers a rudimentary way to compose workflows, and custom apps can define reusable 'Modules' and 'Remote Procedure Calls' (make-docs-10). However there is no explicit documentation of a native 'call another scenario/sub-workflow' module or reusable workflow-as-module composition pattern within the visual builder itself. Missing for 10: dedicated sub-scenario/sub-workflow invocation feature, parameter passing between parent/child scenarios, and independent/hands-on confirmation of this composability pattern.", "evidenceIds": [ "make-docs-15", "make-docs-25", "make-docs-10", "make-docs-5" ] }, { "productId": "make", "storyId": "test-with-sample-data", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Make's API, MCP server, webhooks, and error-handling docs, but contains no documentation of running a scenario with sample/pinned data or inspecting per-step input/output before publishing. 'Scenario history' and 'error-handling' entries are the closest topics but don't describe a test-run/inspect-bundle workflow. missing for 10: docs on 'Run once' test execution, pinned/sample data configuration, per-module input/output bundle inspection.", "evidenceIds": [ "make-docs-18", "make-docs-7", "make-docs-19" ] }, { "productId": "make", "storyId": "visual-workflow-editor", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence confirms Make's core building blocks—scenarios, modules, connections, webhooks, functions, and error handlers—implying a scenario-based workflow model, but nothing in the pack explicitly describes a visual drag-and-drop editor or no-code experience. Missing for 10: explicit description of the visual canvas/editor UI, no-code claims, and evidence of building multi-step workflows without code.", "evidenceIds": [ "make-docs-17", "make-docs-18", "make-docs-26", "make-docs-8", "make-docs-7", "make-docs-15" ] }, { "productId": "make", "storyId": "workflow-template-library", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers Make's API, MCP server, webhooks, error handling, and scenario management, but contains no mention of a public template library or gallery that ops users could start from instead of building scenarios from scratch. Missing for 10: any documentation of a templates gallery, marketplace, or pre-built scenario library, and evidence of browsing/importing templates.", "evidenceIds": [] }, { "productId": "make", "storyId": "workflows-as-mcp-tools", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Make offers an official MCP server that turns active/on-demand scenarios into callable tools for AI agents like Claude/ChatGPT, exposing scenario run and management capabilities via a documented protocol; scenarios can connect to third-party app actions, making them exposable as tools. Missing for 10: independent hands-on validation of the MCP server beyond first-party docs and more detail on granular scoping/security of exposed tools.", "evidenceIds": [ "make-docs-2", "make-docs-3", "make-docs-12", "make-docs-13", "make-docs-20", "make-probe-4" ] }, { "productId": "n8n", "storyId": "agent-creates-workflow-via-api", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "n8n's docs reference a dedicated 'n8n API' page confirming a public API exists, which is consistent with programmatic workflow management, but the evidence pack gives no detail on specific create/update/activate endpoints, authentication, or examples of an agent invoking them. Missing for 10: documented API endpoints for workflow create/update/activate, example agent-driven API calls, and independent/community confirmation the API works as described.", "evidenceIds": [ "n8n-docs-16" ] }, { "productId": "n8n", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Direct probe confirms n8n serves a working llms.txt at https://docs.n8n.io/llms.txt (HTTP 200) with structured links to docs, which an agent can consume directly. missing for 10: no evidence of broader agent-oriented docs formats (e.g., per-page .md siblings or an agents.json manifest) or independent confirmation that agents actually consume it successfully.", "evidenceIds": [ "n8n-probe-1" ] }, { "productId": "n8n", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "n8n's AI agent nodes and templates explicitly support generating insights from data (e.g. 'Sales can Generate customer insights from reviews', 'Sec Ops can Enrich security incident tickets', 'What agents do' docs) by connecting LLMs to a user's own data sources. However this is delivered via user-built workflows rather than a native, out-of-the-box insights/suggestions feature, and there's no independent/hands-on evidence confirming this specific use case works well in practice. Missing for 10: first-party deep documentation of an insights-generation workflow end-to-end, and community/hands-on corroboration of AI-generated insights specifically (only generic AI-workflow templates are cited).", "evidenceIds": [ "n8n-docs-27", "n8n-docs-25", "n8n-docs-18", "n8n-gh-9", "n8n-gh-3" ] }, { "productId": "n8n", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "n8n's core product is workflow automation triggered by webhooks/schedules that execute unattended, with error-handling workflows, execution history/search, and self-hosted/cloud deployment for always-on background operation; community evidence corroborates long-running, hands-off use (processing millions of workflows on a small server for years, following subreddits and sending notifications automatically, running in production for months). Missing for 10: an explicit dedicated schedule-trigger doc citation and independent uptime/reliability benchmarking for long-running autonomous agents.", "evidenceIds": [ "n8n-docs-5", "n8n-docs-21", "n8n-docs-9", "n8n-docs-3", "n8n-comm-14", "n8n-comm-7", "n8n-comm-15", "n8n-docs-18" ] }, { "productId": "n8n", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows n8n lets users build AI agent workflows (nodes, LangChain integration, model connections) but none of it describes a built-in AI assistant inside the n8n product itself that a user can delegate tasks to (e.g., an in-app copilot for building/debugging workflows). All AI-agent references describe automations the user builds, not an assistant serving the user.", "evidenceIds": [] }, { "productId": "n8n", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "n8n supports self-hosting via Docker/CLI and one-line install, has documented CLI configuration and command-line usage, an API, error-workflow handling for automated runs, and community reports of running it in production processing millions of workflows on small servers unattended — all consistent with headless/CI operation. missing for 10: no explicit CI/CD pipeline integration guide (e.g., GitHub Actions example) or dedicated 'headless mode' doc, and no independent benchmark of unattended reliability at scale beyond anecdotal community reports.", "evidenceIds": [ "n8n-docs-17", "n8n-probe-4", "n8n-docs-3", "n8n-gh-8", "n8n-docs-16", "n8n-docs-5", "n8n-comm-14", "n8n-comm-15" ] }, { "productId": "n8n", "storyId": "agentic-mcp-client", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "n8n ships a documented built-in MCP Client node (n8n-nodes-langchain.mcpclient) that lets a workflow/agent connect to external MCP servers and use their tools, alongside an MCP Server Trigger for the reverse role, confirming first-party MCP client support. This is corroborated by an official docs page and probe confirmation. Missing for 10: independent hands-on community reports specifically validating the MCP Client node's tool-use in practice (community evidence is general n8n sentiment, not MCP-specific).", "evidenceIds": [ "n8n-docs-20", "n8n-docs-19", "n8n-probe-3" ] }, { "productId": "n8n", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "n8n ships an official MCP Server Trigger node (n8n-nodes-langchain.mcpTrigger) that lets external AI agents connect to n8n workflows via MCP, confirmed by docs and a probe hit; it also has a complementary MCP Client node for consuming other MCP servers. Missing for 10: independent/hands-on community confirmation of the MCP server trigger working in practice, and more detail on setup/auth specifics beyond the doc page title.", "evidenceIds": [ "n8n-docs-19", "n8n-probe-3", "n8n-docs-20" ] }, { "productId": "n8n", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows n8n lets users build AI agents and convert natural language into API calls *as workflow outputs*, but there is no evidence of a natural-language interface for operating n8n itself (e.g., a chat-to-build-workflow assistant or NL command console). missing for 10: evidence of a prompt-driven workflow builder or NL command interface for controlling n8n's canvas/actions.", "evidenceIds": [ "n8n-docs-18", "n8n-docs-26", "n8n-docs-1" ] }, { "productId": "n8n", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "n8n documents an official command-line interface for configuring and managing self-hosted instances (n8n-docs-17, n8n-probe-4), which is a genuine first-party CLI. However, this CLI is oriented toward deployment/configuration tasks rather than AI-native agentic workflows (e.g., no evidence of CLI-driven workflow creation/execution for AI agents, no scripting API for agent orchestration via CLI). Missing for 10: evidence of AI-native/agent-oriented CLI use cases, independent hands-on corroboration of the CLI, and any agent-facing CLI commands beyond basic server configuration.", "evidenceIds": [ "n8n-docs-17", "n8n-probe-4" ] }, { "productId": "n8n", "storyId": "agentic-public-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "n8n documents a dedicated public REST API (n8n-docs-16 'n8n API') for programmatic control, which supports AI-native/agentic driving of the product beyond the UI. However, evidence is limited to a single doc title with no detail on endpoints/auth, and probes for an OpenAPI/swagger spec at expected paths all 404'd, weakening confidence in completeness or discoverability. Missing for 10: detailed API reference content, machine-readable OpenAPI spec, and independent/hands-on developer confirmation of using the API.", "evidenceIds": [ "n8n-docs-16", "n8n-probe-2" ] }, { "productId": "n8n", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "n8n exposes a credentials system for per-integration secrets and a general n8n API, plus role-based access control and audit trails per the GitHub description, which are building blocks for limiting what an agent can access. However there is no evidence of a mechanism to issue scoped or least-privilege API keys/credentials specifically for an AI agent's use (e.g., read-only, node-restricted, or agent-specific tokens). Missing for 10: documentation of scoped API key creation, granular per-agent credential permissions, and independent confirmation that credential scoping actually restricts agent tool access.", "evidenceIds": [ "n8n-docs-23", "n8n-docs-16", "n8n-gh-5", "n8n-docs-28" ] }, { "productId": "n8n", "storyId": "agentic-sdks", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "n8n documents a public REST API (n8n-docs-16) that developers could build against, but there is no evidence of dedicated, officially maintained SDK packages (e.g., Python/Node client libraries) or SDK-specific quickstart docs — the API reference is the only artifact resembling programmatic access. missing for 10: official language SDKs (Python/JS/Go), SDK reference docs, SDK usage examples, and independent confirmation of SDK adoption.", "evidenceIds": [ "n8n-docs-16" ] }, { "productId": "n8n", "storyId": "agentic-webhooks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "n8n has a first-party Webhook trigger node (n8n-docs-21) that lets workflows (including AI agents) subscribe to and react to incoming HTTP events, plus an MCP Server Trigger (n8n-docs-19, n8n-probe-3) that exposes workflows as callable/event-driven endpoints for AI agent ecosystems. Community evidence corroborates real-world use of webhook-initiated workflows in production (n8n-comm-18). Missing for 10: independent hands-on verification specifically of webhook subscription reliability/latency and richer documentation on webhook authentication/security nuances.", "evidenceIds": [ "n8n-docs-21", "n8n-docs-19", "n8n-probe-3", "n8n-comm-18" ] }, { "productId": "n8n", "storyId": "ai-agent-workflow-steps", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "n8n has dedicated AI agent building blocks (docs on 'What agents do', LangChain-based nodes, MCP client/trigger), explicit model choice across OpenAI/Anthropic/Google/open-source providers, and tool use via integrations/credentials plus code nodes for advanced logic. This is corroborated by both first-party docs/GitHub claims and independent community use of n8n for AI workflows. Missing for 10: deeper hands-on independent verification of multi-agent orchestration quality and tool-calling reliability beyond marketing/docs claims.", "evidenceIds": [ "n8n-docs-18", "n8n-gh-3", "n8n-gh-7", "n8n-gh-9", "n8n-docs-19", "n8n-docs-20", "n8n-intdir-1", "n8n-intdir-4", "n8n-intdir-3", "n8n-comm-7" ] }, { "productId": "n8n", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "n8n documents a REST API (n8n-docs-16) but there is no evidence of an interactive, runnable API reference (e.g., Swagger/OpenAPI explorer); a direct probe for OpenAPI/Swagger specs returned 404 on all candidate paths, indicating no such interactive reference is exposed.", "evidenceIds": [ "n8n-docs-16", "n8n-probe-2" ] }, { "productId": "n8n", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While n8n documents a REST API (n8n-docs-16), the evidence pack contains an explicit probe showing all standard OpenAPI/Swagger spec paths returning 404 (n8n-probe-2), and no other citation shows a downloadable machine-readable spec for n8n's API. No evidence of an OpenAPI/Swagger JSON export exists.", "evidenceIds": [ "n8n-docs-16", "n8n-probe-2" ] }, { "productId": "n8n", "storyId": "api-sandbox", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "n8n's source-control/environments feature (n8n-docs-22, n8n-docs-7) implies the ability to maintain separate dev/staging/production n8n instances synced via Git, which could support sandbox testing without touching production data, but the evidence pack gives only page titles with no detail on how isolation, test data, or non-production execution actually works. missing for 10: explicit documentation of a sandbox/staging execution mode, test-vs-production webhook isolation, or first-party guidance on testing without affecting live data.", "evidenceIds": [ "n8n-docs-22", "n8n-docs-7" ] }, { "productId": "n8n", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "n8n publishes an API (n8n-docs-16) but the evidence pack contains no documentation of API versioning scheme or a deprecation policy, and a probe for an OpenAPI spec returned 404s across all candidate paths (n8n-probe-2), indicating no discoverable versioned/spec-documented API contract.", "evidenceIds": [ "n8n-docs-16", "n8n-probe-2" ] }, { "productId": "n8n", "storyId": "app-event-triggers", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "n8n has documented trigger-based nodes (Webhook, MCP Server Trigger) and 1500+ app integrations that fire workflows off events like new records, messages, emails, or form submissions, backed by extensive integration docs and community reports of production use (webhook-initiated tasks, Slack/email/form-style automations). missing for 10: no explicit first-party doc snippet listing app-specific triggers (e.g., 'new email received', 'new form submission') by name, and no independent benchmark of trigger reliability/latency across many apps.", "evidenceIds": [ "n8n-docs-21", "n8n-docs-19", "n8n-gh-2", "n8n-gh-10", "n8n-comm-18", "n8n-comm-15", "n8n-intdir-2" ] }, { "productId": "n8n", "storyId": "automatic-step-retries", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "n8n provides an error-workflow mechanism to react to failed executions (n8n-docs-5) and community usage confirms retries are commonly relied upon in production (n8n-comm-18), but the evidence pack contains no explicit documentation of a configurable retry-with-backoff setting (e.g., delay/backoff intervals) for individual steps. Missing for 10: documentation of per-node 'retry on fail' with configurable backoff/delay, independent confirmation of backoff behavior, and any activity-level retry policy details.", "evidenceIds": [ "n8n-docs-5", "n8n-comm-18" ] }, { "productId": "n8n", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "n8n's workflow engine inherently processes arrays of items and community evidence shows real-world use at high volume (\"processing millions of workflows each year on a very small server\"), implying capacity for bulk-scale automation, but the evidence pack contains no explicit documentation of batch/loop/bulk-item processing features (e.g., Split-in-Batches, item-array iteration) as a distinct capability. Missing for 10: explicit docs on batch-processing nodes or bulk-item iteration, first-party guidance on handling large item sets, and independent confirmation of bulk-operation reliability at scale.", "evidenceIds": [ "n8n-comm-14", "n8n-comm-7", "n8n-docs-1" ] }, { "productId": "n8n", "storyId": "automation-rules-engine", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "n8n natively supports event-based triggers (Webhook, MCP Server Trigger, scheduled triggers) that fire workflows/actions automatically, plus error-handling workflows for reacting to execution events, directly matching rule-based automation on events. Community evidence corroborates real production use for automated multi-step workflows triggered by webhooks/schedules. missing for 10: no independent benchmark of complex conditional 'if X then Y' rule engines beyond trigger nodes, and no deeper docs on rule-composition beyond triggers/IF nodes.", "evidenceIds": [ "n8n-docs-21", "n8n-docs-19", "n8n-docs-5", "n8n-comm-18", "n8n-comm-7" ] }, { "productId": "n8n", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "The pack lacks any first-party documentation citation for a dedicated Schedule/Cron trigger node, but a hands-on community report confirms real production use of n8n for 'scheduled and webhook-initiated tasks' (n8n-comm-18), indicating the capability exists and works. Missing for 10: official docs reference to the Schedule Trigger/Cron node, details on interval/cron expression configuration, and independent verification beyond a single community anecdote.", "evidenceIds": [ "n8n-comm-18" ] }, { "productId": "n8n", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "n8n documents Git-based source control (\"Version control using Git\", \"Push and pull changes\") and execution history/search that lets users review past runs (n8n-docs-7, n8n-docs-22, n8n-docs-9), and community evidence confirms execution history is retained for debugging (n8n-comm-8). However, Git-based version control appears to be a paid/enterprise pricing-page feature rather than a built-in community capability, and there is no explicit documentation of a one-click 'rollback' mechanism for workflows beyond manually reverting via Git. Missing for 10: explicit rollback/restore workflow feature, confirmation that version control is available outside paid tiers, and independent hands-on confirmation of the review/rollback workflow.", "evidenceIds": [ "n8n-docs-7", "n8n-docs-22", "n8n-docs-9", "n8n-comm-8" ] }, { "productId": "n8n", "storyId": "central-credentials-management", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "n8n has a dedicated Credentials system for centrally storing connection secrets across hundreds of integrations (n8n-docs-23, n8n-intdir-12..30), and enterprise-tier docs reference role-based access control, admin roles, and SSO/SAML/LDAP for governing team access (n8n-gh-5, n8n-docs-28, n8n-docs-8). However, the evidence never explicitly documents a 'share credential with specific users/roles' workflow or granular per-credential permission model, so team sharing and fine-grained control are only implied via general RBAC/admin claims. Missing for 10: explicit documentation of credential-sharing UI/permissions, independent/hands-on confirmation that per-credential access control works as described.", "evidenceIds": [ "n8n-docs-23", "n8n-gh-5", "n8n-docs-28", "n8n-docs-8", "n8n-intdir-12" ] }, { "productId": "n8n", "storyId": "code-steps-in-workflows", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "n8n explicitly ships a Code node supporting JavaScript and Python (with npm packages) as a workflow step, confirmed by both vendor docs/GitHub (n8n-gh-1, n8n-gh-6, n8n-docs-1) and independent hands-on community feedback confirming custom JS/Python code blocks exist (n8n-comm-6). Missing for 10: deeper first-party documentation of the Code node itself (execution modes, sandboxing) and resolution of the community-noted limitation on importing external libraries.", "evidenceIds": [ "n8n-gh-1", "n8n-gh-6", "n8n-docs-1", "n8n-docs-13", "n8n-comm-6" ] }, { "productId": "n8n", "storyId": "community-built-components", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers n8n's vendor-built integrations directory, npm packages usable inside code nodes, and API access, but nothing documents an official community-nodes marketplace or process for installing third-party-built nodes/components contributed outside n8n itself.", "evidenceIds": [] }, { "productId": "n8n", "storyId": "concurrency-rate-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows error-workflow handling and community mentions of using 'retries' for sporadic errors, but there is no documentation or evidence of features like rate-limit throttling, request queuing, batching/wait nodes, or execution concurrency controls to respect downstream rate limits.", "evidenceIds": [ "n8n-docs-5", "n8n-comm-18" ] }, { "productId": "n8n", "storyId": "conditional-branching-logic", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains only generic marketing language about visual building, code extensibility, and error-workflow handling, but no documentation of IF/Switch/Filter nodes or a Merge node that would let branches recombine. Community and docs items never describe conditional branching or parallel-path merging explicitly, so the specific branch-and-merge capability is unevidenced. Missing for 10: docs/evidence on conditional (IF/Switch) nodes, filter nodes, and a Merge node for recombining parallel paths.", "evidenceIds": [] }, { "productId": "n8n", "storyId": "cron-schedules", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack lists numerous n8n trigger/node docs (Webhook, MCP Trigger/Client, credentials) but never mentions a Cron/Schedule Trigger node or timezone configuration options, so there is no direct evidence this specific capability is documented or delivered.", "evidenceIds": [] }, { "productId": "n8n", "storyId": "custom-connector-sdk", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "n8n documents a developer platform for extensibility: an n8n API (n8n-docs-16), a CLI (n8n-docs-17, n8n-probe-4), and code nodes supporting JavaScript/Python/npm packages for custom logic (n8n-gh-1, n8n-gh-6), plus self-hosting/OEM options for building private integrations (n8n-docs-3, n8n-docs-6). However, community feedback specifically flags that documentation for building custom nodes is weak ('leaves a lot to be desired... especially regarding custom node development' n8n-comm-12), and code blocks have 'limited ability to import libraries' (n8n-comm-6), indicating real gaps versus a mature SDK experience. Missing for 10: a dedicated custom-node SDK/scaffolding tool with strong docs, and independent confirmation that custom connector development is smooth rather than reported as under-documented.", "evidenceIds": [ "n8n-docs-16", "n8n-docs-17", "n8n-probe-4", "n8n-gh-1", "n8n-gh-6", "n8n-comm-12", "n8n-comm-6" ] }, { "productId": "n8n", "storyId": "durable-long-running-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of n8n's wait/pause mechanism, execution state persistence across worker or platform restarts, or queue-mode durability for long-running workflows. Error-workflow handling (n8n-docs-5) and execution history search (n8n-docs-9, n8n-comm-8) address failure notification and debugging, not surviving restarts during multi-day waits, so this applicable reliability axis is unevidenced.", "evidenceIds": [] }, { "productId": "n8n", "storyId": "embed-white-label", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "n8n explicitly documents an OEM deployment option that lets developers embed and surface n8n's interface inside their own product's UI, confirming white-label embedding is a supported first-party capability (though it requires a separate commercial agreement). Missing for 10: independent/hands-on developer accounts of actually shipping an OEM-embedded product, and details on branding/customization depth.", "evidenceIds": [ "n8n-docs-6", "n8n-docs-31" ] }, { "productId": "n8n", "storyId": "environments-source-control", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "n8n has dedicated docs on 'Push and pull changes' via source control and environments, plus a pricing-listed 'Version control using Git' enterprise feature, directly supporting versioning workflows and promoting them across environments. Missing for 10: independent/hands-on verification of the dev-to-prod promotion workflow, and details on CI/CD pipeline integration beyond Git push/pull.", "evidenceIds": [ "n8n-docs-22", "n8n-docs-7", "n8n-docs-28" ] }, { "productId": "n8n", "storyId": "error-workflows-alerts", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "n8n has a documented first-party 'error workflow' feature letting users control how workflow execution failures are handled [n8n-docs-5], and community evidence confirms n8n keeps historical execution logs so failures are debuggable [n8n-comm-8]. However, the evidence pack doesn't detail specific notification channels (e.g., Slack/email alerting nodes) tied to error workflows or dedicated 'get notified on failure' setup steps, and one community comment notes users 'just live with' sporadic errors using retries rather than dedicated alerting [n8n-comm-18]. Missing for 10: explicit documentation of notification/alerting integration within error workflows, and independent hands-on confirmation that error-workflow notifications work reliably in production.", "evidenceIds": [ "n8n-docs-5", "n8n-comm-8", "n8n-comm-18" ] }, { "productId": "n8n", "storyId": "expressions-data-mapping", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "n8n explicitly supports combining visual workflow steps with JavaScript/Python code and npm packages for custom data manipulation, and community users confirm using code nodes, SQL, JSON/XML parsing, and debugging data in/out between steps. Missing for 10: explicit first-party documentation of the dedicated Expression/Function editor syntax and independent verification of complex multi-step transform reliability (community notes it 'looks weird' and requires programming knowledge for hard data-mapping cases).", "evidenceIds": [ "n8n-gh-1", "n8n-gh-6", "n8n-docs-1", "n8n-comm-6", "n8n-comm-13", "n8n-comm-8", "n8n-comm-3" ] }, { "productId": "n8n", "storyId": "human-approval-steps", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of a wait/pause node, human-in-the-loop approval step, or any mechanism for pausing execution pending human input — only unrelated items like error workflows, webhooks, and MCP nodes are cited. This is a fair capability to expect from a workflow automation tool, but no evidence in the pack demonstrates it.", "evidenceIds": [] }, { "productId": "n8n", "storyId": "inbound-webhook-triggers", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "n8n has a dedicated Webhook node documented in official docs, which lets developers expose a custom URL that receives external HTTP requests and triggers a workflow run with the incoming payload; community reports (n8n-comm-18) confirm real-world use of webhook-initiated tasks in production. missing for 10: independent hands-on verification of advanced webhook configuration (e.g., custom auth, response modes) and no dedicated deep-dive doc excerpt beyond the node reference page.", "evidenceIds": [ "n8n-docs-21", "n8n-comm-18" ] }, { "productId": "n8n", "storyId": "local-dev-instance", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "n8n can be self-hosted locally (Docker Compose, one-line curl install) or run on a dev instance, configured via CLI, and supports pushing/pulling workflows between environments via source control integration, which enables dev/test workflows; community confirms long-term self-hosting in production. However, there is no explicit documentation or evidence of a CI-testing framework, automated test runner, or headless workflow execution for CI pipelines specifically. Missing for 10: dedicated CI/testing tooling or docs (e.g. workflow unit tests, CLI-based automated test execution in CI), and independent hands-on confirmation of dev/CI usage.", "evidenceIds": [ "n8n-docs-3", "n8n-docs-17", "n8n-docs-22", "n8n-gh-8", "n8n-comm-1", "n8n-comm-15" ] }, { "productId": "n8n", "storyId": "nl-workflow-generation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers n8n's visual/code building, AI agent nodes, MCP support, and integrations, but contains no documentation of a natural-language-to-workflow generation or editing feature (e.g., an 'AI workflow builder' or prompt-based canvas editing). n8n-docs-26 only describes a workflow that converts natural language into API calls as a use case, not n8n generating/editing the workflow itself from a prompt.", "evidenceIds": [ "n8n-docs-26", "n8n-docs-18", "n8n-docs-1" ] }, { "productId": "n8n", "storyId": "openness-api-parity", "verdict": "partial", "quality": 2, "confidence": "low", "rationale": "The evidence pack confirms n8n has a documented REST API (n8n-docs-16) and a CLI (n8n-docs-17), but there's no evidence establishing that this API achieves full feature parity with the UI (e.g., building/editing every workflow node type, credentials, AI agent config, etc.), and a probe for a discoverable OpenAPI spec at standard paths returned 404s. Missing for 10: explicit documentation or independent testing confirming API/UI parity, a published OpenAPI spec, and evidence AI-native users can fully replicate complex canvas actions (agent config, node wiring) purely via API.", "evidenceIds": [ "n8n-docs-16", "n8n-docs-17", "n8n-probe-2" ] }, { "productId": "n8n", "storyId": "openness-full-export", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "n8n workflows are stored as portable JSON and can be pushed/pulled via Git-based source control (n8n-docs-22), plus a documented REST API (n8n-docs-16) and full self-hosting (n8n-docs-3, n8n-comm-1/16/17) give users real control to extract and leave with their data. Missing for 10: explicit documentation of a bulk 'export all data' feature, independent confirmation of complete data portability (credentials, execution history), and clarity on the fair-code (not fully OSS) license implications for openness (n8n-comm-19).", "evidenceIds": [ "n8n-docs-22", "n8n-docs-16", "n8n-docs-3", "n8n-docs-4", "n8n-comm-1", "n8n-comm-17", "n8n-comm-19" ] }, { "productId": "n8n", "storyId": "openness-open-license", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "n8n's source code is publicly readable on GitHub and docs describe it as 'fair-code licensed' with a Sustainable Use License/Community edition, but this is explicitly not an OSI-approved open source license, and community evidence directly disputes the openness framing ('Not really OSS'). Missing for 10: an actual OSI-approved open license, and vendor acknowledgment that fair-code differs from open source rather than implying openness via branding.", "evidenceIds": [ "n8n-docs-29", "n8n-docs-30", "n8n-docs-12", "n8n-comm-19" ] }, { "productId": "n8n", "storyId": "openness-self-host", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "n8n provides clear first-party documentation for self-hosting via Docker Compose, one-line install script, or other methods, and confirms a free Community edition runs without a license key; this is corroborated by extensive community testimonials of long-term self-hosting in production. Missing for 10: independent hands-on verification of the self-hosting setup process itself beyond community mentions of using it.", "evidenceIds": [ "n8n-docs-3", "n8n-docs-4", "n8n-docs-29", "n8n-gh-8", "n8n-comm-1", "n8n-comm-16", "n8n-comm-17" ] }, { "productId": "n8n", "storyId": "prebuilt-connector-breadth", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "n8n's GitHub README and docs consistently tout '1500+ integrations' and thousands of templates, backed by an integrations directory covering major SaaS apps (Slack, Notion, GitHub, HubSpot, Jira, etc.) and a long list of built-in credential types, and community users corroborate ease of extending/using integrations in production. Missing for 10: explicit confirmation that all listed integrations are vendor-maintained (vs. community-contributed nodes) and independent benchmarking of integration reliability/coverage depth.", "evidenceIds": [ "n8n-gh-2", "n8n-gh-4", "n8n-gh-10", "n8n-docs-11", "n8n-docs-32", "n8n-intdir-1", "n8n-intdir-2", "n8n-intdir-5", "n8n-intdir-8", "n8n-intdir-9", "n8n-comm-15", "n8n-comm-4" ] }, { "productId": "n8n", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "n8n's self-hosting model (Docker Compose, own infrastructure, on-premises, or private cloud) lets a user fully control where data physically resides, and docs explicitly offer 'Deploy on your infrastructure or ours.' However, this is an indirect residency mechanism (achieved via self-hosting) rather than a first-class region-selection setting, and there is no evidence of specific region options for n8n Cloud. Missing for 10: explicit n8n Cloud region-selection UI/feature, documented data-residency guarantees or compliance certifications tied to region choice.", "evidenceIds": [ "n8n-docs-3", "n8n-docs-14", "n8n-docs-29", "n8n-gh-5" ] }, { "productId": "n8n", "storyId": "privacy-no-training", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "n8n offers self-hosting on your own infrastructure (n8n-docs-3, n8n-gh-8, n8n-comm-16), which gives users full control over where their data lives and implicitly prevents it from being sent to n8n's cloud or used by n8n for any AI training. However, there is no explicit policy statement, opt-out setting, or documentation addressing AI-training data usage specifically. Missing for 10: an explicit data-usage/AI-training policy or opt-out toggle, independent confirmation that self-hosted data is never used for model training, and clarity on cloud-plan data handling for AI training purposes.", "evidenceIds": [ "n8n-docs-3", "n8n-gh-8", "n8n-comm-16", "n8n-docs-4" ] }, { "productId": "n8n", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "n8n's self-hosting option (n8n-docs-3, n8n-docs-14, n8n-gh-5) gives users full control over where data lives and implicitly the ability to delete or retain it as they see fit, and n8n-gh-5 mentions support for 'sensitive data' with audit trails. However, there is no explicit documentation in the pack of configurable execution-data retention/pruning settings, deletion APIs, or GDPR-style data-deletion controls within the product itself. missing for 10: explicit execution-data retention/pruning configuration, documented data-deletion workflow or API, independent confirmation of retention controls working as described.", "evidenceIds": [ "n8n-docs-3", "n8n-docs-14", "n8n-gh-5" ] }, { "productId": "n8n", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry, usage tracking, or opt-out settings for n8n; self-hosting is documented but that alone doesn't confirm telemetry can be disabled or clarify what data is collected in cloud/community editions.", "evidenceIds": [] }, { "productId": "n8n", "storyId": "run-history-replay", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "n8n clearly supports inspecting past execution logs (execution search feature, and community reports that 'it keeps all the historical executions so you can see what happened when workflows failed and what data was going in/out of each component') plus error-workflow handling for failures. However, there is no concrete documentation or evidence describing a re-run/retry that resumes specifically from the failing step rather than restarting the whole execution; only a vague community mention of 'retries' as a workaround for sporadic errors. missing for 10: explicit docs/feature describing resume-from-failing-node re-execution, and independent confirmation that this resume behavior works as intended.", "evidenceIds": [ "n8n-docs-9", "n8n-docs-5", "n8n-comm-8", "n8n-comm-18" ] }, { "productId": "n8n", "storyId": "subworkflow-reuse", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers n8n's general visual-building, AI/agent, integration, and deployment capabilities but contains no citation referencing sub-workflows, an 'Execute Workflow' node, or any modular workflow-calling-workflow mechanism, which is the specific capability this story asks about.", "evidenceIds": [] }, { "productId": "n8n", "storyId": "test-with-sample-data", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "n8n's canvas-based execution tracing is referenced in marketing docs ('Every step of your agents' reasoning, traceable on the canvas') and independently corroborated by a hands-on user noting it keeps historical executions so you can see data going in/out of each node when debugging. However, there is no explicit documentation or mention of the 'pin data' / sample-data testing feature described in the story, and no walkthrough of pre-production testing workflow. Missing for 10: explicit docs on pinning sample data for test runs, and a clear developer workflow showing pre-go-live inspection rather than just post-hoc execution history.", "evidenceIds": [ "n8n-docs-2", "n8n-docs-13", "n8n-comm-8" ] }, { "productId": "n8n", "storyId": "visual-workflow-editor", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "n8n's docs and GitHub materials describe a drag-and-drop visual canvas for multi-step workflows ('Build visually, go deep with code'), with 1500+ integrations and 12000+ templates, and community reports (n8n-comm-18) confirm a non-technical manager built a complex backend visually. However, other hands-on community feedback (n8n-comm-3, n8n-comm-13) says users without programming knowledge hit frequent errors on data transformation/JSON/SQL tasks, meaning pure no-code use has real friction for non-trivial workflows. missing for 10: independent benchmark of non-coder success rate, clearer docs-side guarantee that no-code path avoids expression/code steps for common ops-ops workflows.", "evidenceIds": [ "n8n-docs-1", "n8n-gh-2", "n8n-docs-11", "n8n-comm-18", "n8n-comm-6", "n8n-comm-3", "n8n-comm-13" ] }, { "productId": "n8n", "storyId": "workflow-template-library", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "n8n publishes a public, searchable library of 12,000+ community workflow templates (n8n.io/workflows) that ops users can browse and start from instead of building from scratch, corroborated across multiple docs and GitHub mentions of '9,000+/12,000+ workflow templates.' Missing for 10: independent hands-on account specifically describing starting from a template (vs building from scratch) and details on template customization/import friction.", "evidenceIds": [ "n8n-docs-11", "n8n-docs-15", "n8n-docs-32", "n8n-gh-4" ] }, { "productId": "n8n", "storyId": "workflows-as-mcp-tools", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "n8n ships a documented MCP Server Trigger node that exposes a workflow as an MCP server tool callable by external agents, alongside a separate MCP Client node for consuming external MCP servers — directly matching the story of exposing workflows/app actions as MCP tools. This is corroborated by a probe confirming the docs page exists, but there's no independent/hands-on community report validating real-world use of the MCP Server Trigger specifically. Missing for 10: independent/community corroboration of the MCP server trigger in practice, and more detail on auth/security when exposing tools externally.", "evidenceIds": [ "n8n-docs-19", "n8n-docs-20", "n8n-probe-3" ] }, { "productId": "pipedream", "storyId": "agent-creates-workflow-via-api", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Pipedream's REST API docs explicitly state it can be used to 'create workflows, manage event sources, handle subscriptions, and more,' which supports agent-driven programmatic workflow creation and management, but there is no detailed documentation or example of update/activate operations, and no independent or hands-on evidence confirming full lifecycle control via the API. missing for 10: explicit API endpoints/examples for updating and activating workflows, and third-party or hands-on confirmation of programmatic workflow lifecycle management.", "evidenceIds": [ "pipedream-docs-6", "pipedream-docs-17" ] }, { "productId": "pipedream", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "There is no evidence Pipedream publishes an llms.txt or agent-oriented docs file; a direct probe of pipedream.com/docs.md returned a 404, and no other evidence pack item mentions llms.txt or similar machine-readable doc formats for agents.", "evidenceIds": [ "pipedream-probe-1" ] }, { "productId": "pipedream", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Pipedream documents an AI-assisted error debugging feature and general AI-agent building capability, but there's no evidence of the product proactively surfacing AI-generated insights or suggestions from a user's own data (e.g., workflow analytics, usage patterns) beyond error stack-trace debugging. Missing for 10: dedicated AI insights/analytics feature over user data, first-party documentation of proactive suggestions, and independent validation of this capability.", "evidenceIds": [ "pipedream-docs-16", "pipedream-docs-26", "pipedream-docs-21" ] }, { "productId": "pipedream", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Pipedream workflows run persistently in the background triggered by webhooks, schedules, email, RSS, or app events, with automatic retries and error handling/alerting — a core automation platform capability well documented across sources. Missing for 10: independent hands-on evidence specifically confirming long-running autonomous multi-step agentic workflows (vs. simple triggers) and no third-party benchmark of reliability at scale.", "evidenceIds": [ "pipedream-docs-12", "pipedream-docs-25", "pipedream-docs-14", "pipedream-docs-15", "pipedream-docs-16", "pipedream-docs-10", "pipedream-comm-8", "pipedream-docs-21" ] }, { "productId": "pipedream", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Pipedream's homepage tagline claims users can \"Prompt, run, edit, and deploy AI agents in seconds\" and docs mention debugging workflow errors \"with AI,\" suggesting some built-in AI assistant capability, but there is no detailed documentation of how this in-product assistant works, what tasks it can be delegated, or independent confirmation of its behavior. Missing for 10: detailed docs/UI walkthrough of the built-in AI assistant, scope of delegable tasks, and independent/hands-on verification of it working as described.", "evidenceIds": [ "pipedream-docs-21", "pipedream-docs-16", "pipedream-docs-26" ] }, { "productId": "pipedream", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Pipedream ships a CLI (pd deploy, pd dev) for deploying/developing event sources from local code, a REST API for creating/managing workflows and event sources, and workflows themselves run as Node.js/Python/Go/Bash code, all of which support scripted/headless automation and CI-style usage. However, there is no explicit documentation of a dedicated 'CI' workflow (e.g., GitHub Actions integration, non-interactive auth for pipelines, exit-code based test running) or first-party CI examples. missing for 10: documented CI/CD integration examples, non-interactive/headless auth flow for automated pipelines, independent hands-on confirmation of CLI use in CI environments.", "evidenceIds": [ "pipedream-docs-6", "pipedream-docs-7", "pipedream-docs-8", "pipedream-docs-9", "pipedream-docs-22", "pipedream-docs-23", "pipedream-gh-1", "pipedream-probe-3" ] }, { "productId": "pipedream", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows Pipedream exposing itself AS an MCP server so other agents/apps can call Pipedream's tools (docs-3, docs-4, docs-5) — the opposite direction of this story, which asks whether a user can plug external MCP servers INTO Pipedream so it can consume their tools. No evidence describes Pipedream acting as an MCP client or importing third-party MCP servers as tool sources.", "evidenceIds": [ "pipedream-docs-3", "pipedream-docs-4", "pipedream-docs-5", "pipedream-probe-2" ] }, { "productId": "pipedream", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Pipedream documents an official MCP server (Connect MCP) that agents/apps can add to make tool calls across 3,000+ APIs and 10,000+ tools, with automatic auth/token handling requiring no additional implementation. This is first-party documentation confirmed by a probe, though lacking independent hands-on corroboration. Missing for 10: independent/community verification of the MCP server working in practice, and details on rate limits/reliability at scale.", "evidenceIds": [ "pipedream-docs-3", "pipedream-docs-4", "pipedream-docs-5", "pipedream-probe-2" ] }, { "productId": "pipedream", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Pipedream's homepage claims you can 'Prompt, run, edit, and deploy AI agents in seconds,' suggesting some natural-language workflow creation, but this is a single marketing tagline with no elaboration on scope, reliability, or hands-on confirmation. Missing for 10: detailed docs on the NL/prompt interface, examples of what commands are supported, and independent/community corroboration of using natural language to operate the product.", "evidenceIds": [ "pipedream-docs-21" ] }, { "productId": "pipedream", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Pipedream ships an official CLI (pd) documented with commands like pd deploy, pd dev, and event streaming, confirmed both in docs and via probe, and integrates with Node.js/agentic workflows. missing for 10: independent hands-on reviews of the CLI itself and deeper AI-native workflow examples using the CLI specifically.", "evidenceIds": [ "pipedream-docs-7", "pipedream-docs-8", "pipedream-docs-9", "pipedream-docs-22", "pipedream-docs-23", "pipedream-probe-3" ] }, { "productId": "pipedream", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Pipedream documents a full REST API for creating workflows, managing event sources, and subscriptions, plus dedicated Connect API/SDKs (TypeScript, Python, Java) with clear auth patterns (external_user_id) and usage APIs, alongside a CLI and MCP server for programmatic/agentic access. This gives AI-native users multiple well-documented, official surfaces to drive the product programmatically. Missing for 10: independent third-party corroboration of the REST API's completeness/reliability beyond vendor docs.", "evidenceIds": [ "pipedream-docs-6", "pipedream-docs-17", "pipedream-docs-18", "pipedream-docs-19", "pipedream-docs-7", "pipedream-probe-3", "pipedream-docs-3" ] }, { "productId": "pipedream", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Pipedream Connect lets an agent/app connect end-user accounts via OAuth or API keys with per-user isolation (external_user_id) and automatic token storage/handling, which provides some credential isolation for agent use cases, but there is no explicit documentation of issuing least-privilege/scoped credentials (e.g., limiting which actions/scopes a given agent token can use) beyond what the underlying OAuth app grants. Missing for 10: explicit scope-restriction controls, per-agent permission tiers, or documented least-privilege credential issuance beyond standard OAuth connection flow.", "evidenceIds": [ "pipedream-docs-4", "pipedream-docs-5", "pipedream-docs-17", "pipedream-docs-18", "pipedream-docs-3" ] }, { "productId": "pipedream", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Pipedream documents official TypeScript, Python, and Java SDKs plus a REST API for its Connect service, enabling AI-native developers to build directly against first-party SDKs for auth, account connection, and tool invocation, alongside a documented CLI and REST API for broader platform control. Missing for 10: independent/hands-on corroboration of SDK usage quality and completeness across all three languages beyond docs.", "evidenceIds": [ "pipedream-docs-17", "pipedream-docs-18", "pipedream-docs-6", "pipedream-probe-3", "pipedream-docs-27" ] }, { "productId": "pipedream", "storyId": "agentic-webhooks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Pipedream natively supports HTTP/Webhook triggers as event sources, plus a REST API and SSE stream to consume/subscribe to emitted events programmatically, which directly enables AI-native subscription to webhook events. missing for 10: independent hands-on verification of webhook subscription reliability and no explicit example of an AI agent consuming the SSE/webhook stream end-to-end.", "evidenceIds": [ "pipedream-docs-12", "pipedream-docs-25", "pipedream-docs-28", "pipedream-gh-2", "pipedream-docs-6", "pipedream-docs-9" ] }, { "productId": "pipedream", "storyId": "ai-agent-workflow-steps", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Pipedream advertises the ability to 'Prompt, run, edit, and deploy AI agents in seconds' and supports arbitrary Node.js/Python code steps (so a user could call any LLM API and choose a model), plus MCP integration for tool use with 3,000+ APIs. However there is no documentation of a dedicated, built-in 'AI/LLM step' UI with explicit model-choice dropdowns or native tool-use orchestration inside the workflow builder—these capabilities are implied rather than concretely demonstrated. Missing for 10: a documented native LLM/agent step type in the workflow builder, explicit model-selection UI, and hands-on evidence of tool-use configuration within a workflow (not just MCP server exposure).", "evidenceIds": [ "pipedream-docs-21", "pipedream-docs-3", "pipedream-docs-10", "pipedream-gh-1" ] }, { "productId": "pipedream", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows REST API docs, SDK references, and a CLI, but nothing describes an interactive API reference with runnable/try-it examples (e.g., embedded code sandboxes or live request execution in docs).", "evidenceIds": [ "pipedream-docs-6", "pipedream-docs-17", "pipedream-probe-3" ] }, { "productId": "pipedream", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Pipedream documents a REST API and Connect API with SDKs but no evidence of a downloadable OpenAPI/Swagger spec or machine-readable schema; a probe for a machine-readable docs file (docs.md) returned 404, and no other citation points to an OpenAPI spec.", "evidenceIds": [ "pipedream-docs-6", "pipedream-docs-17", "pipedream-probe-1" ] }, { "productId": "pipedream", "storyId": "api-sandbox", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Pipedream offers some testing affordances — manually triggering workflows with test event data (docs-13) and local dev-linking via `pd dev` that lets you iterate on a component before it goes live (docs-8/23) — which loosely support testing without immediately running in production. However, there is no explicit documentation of a dedicated sandbox/staging environment or safeguards to isolate test runs from production data/connected accounts. Missing for 10: a documented sandbox/staging mode, isolation guarantees for connected account data during tests, and independent confirmation that test runs don't touch live data.", "evidenceIds": [ "pipedream-docs-13", "pipedream-docs-8", "pipedream-docs-23" ] }, { "productId": "pipedream", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "There is a REST API and SDKs documented, but no evidence of API versioning scheme or a documented deprecation policy anywhere in the evidence pack. Missing for 10: explicit API version numbers/headers, changelog, deprecation timeline/policy, migration guides.", "evidenceIds": [] }, { "productId": "pipedream", "storyId": "app-event-triggers", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Pipedream documents native triggers for thousands of connected apps (new record/message events), plus Schedule, Email, HTTP/Webhook, and RSS triggers, with test-event simulation and automatic retries/error handling for reliability — directly matching the ops story of triggering workflows from app events. Community feedback corroborates real-world use for exactly this kind of integration automation. Missing for 10: independent hands-on verification of specific 'new record' triggers across many named apps beyond docs claims.", "evidenceIds": [ "pipedream-docs-1", "pipedream-docs-12", "pipedream-docs-25", "pipedream-docs-28", "pipedream-docs-13", "pipedream-docs-14", "pipedream-comm-7", "pipedream-comm-8" ] }, { "productId": "pipedream", "storyId": "automatic-step-retries", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Pipedream docs confirm built-in automatic retry of failed steps for transient errors (e.g., API timeouts, service downtime), directly supporting the reliability story. However, the evidence doesn't specify configurable backoff intervals/strategies or per-step retry customization details, and there's no independent/hands-on corroboration of retry behavior in practice. Missing for 10: explicit backoff configuration options, retry count/interval customization, and independent confirmation of retry behavior under real failures.", "evidenceIds": [ "pipedream-docs-14", "pipedream-docs-29", "pipedream-docs-15", "pipedream-docs-16" ] }, { "productId": "pipedream", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Pipedream's code steps support full Node.js/Python/Go/Bash with npm packages, and the REST API/CLI allow scripting workflow creation and event handling, which could be used to loop over many items programmatically; the platform is not explicitly designed around a 'bulk operations' primitive. missing for 10: no documented batch/bulk API for processing many items at once, no explicit bulk-action UI or agent-facing bulk tool call examples, no independent evidence of bulk-operation usage at scale.", "evidenceIds": [ "pipedream-docs-10", "pipedream-gh-1", "pipedream-docs-6", "pipedream-docs-7" ] }, { "productId": "pipedream", "storyId": "automation-rules-engine", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Pipedream is fundamentally an event-driven automation platform: it supports rich event triggers (webhooks, schedules, email, RSS, app-based events), lets users attach arbitrary code/actions (Node.js, Python, etc.) to those triggers to run automatically, and supports error handling/retries so rules execute reliably without manual intervention. Community feedback corroborates real-world use for automated multi-step workflows (e.g., Airtable-to-Algolia auto-feed). missing for 10: no first-party evidence of complex conditional/rule-based branching logic specifically marketed as 'rules engine', and no independent benchmark of trigger reliability at scale.", "evidenceIds": [ "pipedream-docs-12", "pipedream-docs-25", "pipedream-docs-28", "pipedream-docs-10", "pipedream-docs-14", "pipedream-docs-29", "pipedream-docs-15", "pipedream-comm-8", "pipedream-comm-7" ] }, { "productId": "pipedream", "storyId": "automation-scheduled-jobs", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Pipedream explicitly lists 'Schedule' as a built-in trigger type alongside HTTP/Webhook, Email, and RSS, and workflows can be created/managed via the REST API and CLI, enabling recurring scheduled automations. Community evidence also confirms cron-based execution exists (with a 30s execution limit for cron triggers), corroborating real-world use.\n\nmissing for 10: no documentation on cron expression syntax/timezone configuration, and no explicit example of an AI agent programmatically creating/managing a scheduled trigger via API/MCP.", "evidenceIds": [ "pipedream-docs-12", "pipedream-docs-25", "pipedream-docs-28", "pipedream-docs-6", "pipedream-comm-2" ] }, { "productId": "pipedream", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows no versioning, change history, review workflow, or rollback capability for automations/workflows; CLI mentions deployment and dev linking but no version control or revert feature is documented. Git-based component contribution (pipedream-docs-31) refers to contributing new integrations, not versioning user workflows.", "evidenceIds": [] }, { "productId": "pipedream", "storyId": "central-credentials-management", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Pipedream stores OAuth/key-based credentials for connecting apps (pipedream-docs-2, pipedream-docs-27) but nothing about team-level sharing of those credentials or role-based access control over who can use a given credential — no mention of workspaces, teams, or permission scoping for connected accounts.", "evidenceIds": [ "pipedream-docs-2", "pipedream-docs-27" ] }, { "productId": "pipedream", "storyId": "code-steps-in-workflows", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Pipedream docs and GitHub explicitly confirm you can write and run real Node.js/JavaScript, Python, Golang, or Bash code at any step in a workflow, with full npm package access, alongside pre-built no-code actions, and community reviews corroborate this hybrid low-code/code model in practice. Missing for 10: independent hands-on verification of Python-specific code-step behavior (most detailed docs focus on Node.js) and no first-party benchmark of code-step performance/limits.", "evidenceIds": [ "pipedream-docs-10", "pipedream-docs-24", "pipedream-gh-1", "pipedream-gh-3", "pipedream-gh-4", "pipedream-comm-8", "pipedream-comm-1" ] }, { "productId": "pipedream", "storyId": "community-built-components", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Pipedream's docs explicitly mention that developers can contribute new components via GitHub PRs, indicating an open contribution model for community-built integrations outside the vendor, and the GitHub repo itself is where source is hosted. However, there's no concrete evidence of a marketplace/registry for installing third-party community nodes distinct from vendor-maintained ones, nor documentation of a review/publish workflow for community components. missing for 10: dedicated community component registry/marketplace, install flow for non-vendor components, evidence of independent contributors' components being widely used.", "evidenceIds": [ "pipedream-docs-31", "pipedream-gh-1", "pipedream-gh-3" ] }, { "productId": "pipedream", "storyId": "concurrency-rate-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows automatic retries for transient errors and platform-imposed execution limits (e.g., 10 req/s, timeouts), but nothing about a developer-facing feature to throttle or queue workflow executions to respect downstream API rate limits. No mention of concurrency controls, execution queuing, or configurable rate-limiting for outbound calls.", "evidenceIds": [ "pipedream-docs-14", "pipedream-docs-29", "pipedream-comm-2" ] }, { "productId": "pipedream", "storyId": "conditional-branching-logic", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers triggers, error retries, code steps, MCP, CLI, and pricing, but contains no documentation of conditional branching, filter steps, or parallel-path execution with merge-back in Pipedream's visual workflow builder. Missing for 10: docs on branching/conditional paths, filter step, parallel path execution, and merge/join logic.", "evidenceIds": [] }, { "productId": "pipedream", "storyId": "cron-schedules", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Docs confirm a 'Schedule' trigger type exists alongside HTTP/Webhook, Email, and RSS triggers, indicating time-based workflow execution is supported, but no evidence details cron syntax configuration or timezone selection/control within that trigger. missing for 10: explicit cron expression support, timezone configuration options, documentation or community confirmation of timezone-aware scheduling.", "evidenceIds": [ "pipedream-docs-12", "pipedream-docs-25", "pipedream-docs-28" ] }, { "productId": "pipedream", "storyId": "custom-connector-sdk", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Pipedream provides an extensive developer platform for building custom connectors/integrations: a CLI (pd deploy, pd dev) for building and deploying event sources/components from local code, a REST API for managing workflows/sources, SDKs (TypeScript, Python, Java) for the Connect service, support for writing custom Node.js/Python/Go/Bash code with npm packages, and a GitHub-based component contribution workflow. This is corroborated by GitHub docs and community reports of building custom integrations successfully. Missing for 10: independent hands-on review specifically of the CLI/SDK workflow (most evidence is vendor docs) and no discussion of versioning/testing tooling maturity.", "evidenceIds": [ "pipedream-docs-7", "pipedream-docs-8", "pipedream-docs-9", "pipedream-docs-6", "pipedream-docs-17", "pipedream-docs-10", "pipedream-gh-1", "pipedream-gh-3", "pipedream-docs-31", "pipedream-comm-6", "pipedream-comm-8" ] }, { "productId": "pipedream", "storyId": "durable-long-running-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack describes a durable/delay-and-resume execution model, state checkpointing across restarts, or multi-day wait steps; conversely community reporting cites hard per-execution time limits (10s for HTTP, 30s for cron) which point away from long-lived in-process waits. Missing for 10: any documentation of a 'delay'/'wait' primitive, state persistence guarantees across worker/platform restarts, or independent confirmation of multi-day running workflows.", "evidenceIds": [ "pipedream-comm-2", "pipedream-docs-14", "pipedream-docs-29" ] }, { "productId": "pipedream", "storyId": "embed-white-label", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Pipedream Connect explicitly supports embedding auth/tool-calling into your own product with external_user_id-scoped accounts, SDKs (TS/Python/Java), REST API, and usage/billing APIs for end-user tracking, which is the core of white-labeling. However, there is no explicit evidence of white-label UI theming/branding controls (custom domain, logo/color removal of Pipedream branding) in the pack. missing for 10: documented white-label branding/theming options, custom domain support, independent case study of a product embedding Pipedream white-label.", "evidenceIds": [ "pipedream-docs-17", "pipedream-docs-18", "pipedream-docs-19", "pipedream-docs-4", "pipedream-docs-5" ] }, { "productId": "pipedream", "storyId": "environments-source-control", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Pipedream's CLI supports local development (pd dev links local files to deployed components, pd deploy pushes from local/remote code) and components can be contributed via GitHub PRs, giving some source-control-like workflow. However, there's no documented environment/staging concept (dev vs prod) or an explicit promote-changes mechanism for workflows themselves. missing for 10: dedicated dev/staging/production environments, a documented promotion workflow, and version history/rollback for workflows.", "evidenceIds": [ "pipedream-docs-7", "pipedream-docs-8", "pipedream-docs-22", "pipedream-docs-23", "pipedream-docs-31", "pipedream-probe-3" ] }, { "productId": "pipedream", "storyId": "error-workflows-alerts", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm automatic retries for transient errors, default email notification on unhandled workflow errors, and error surfacing with stack trace/AI debugging, which covers notification-on-failure. However, there's no clear evidence of dedicated, configurable 'error-handling workflow' paths (e.g., routing failed events to a separate workflow) beyond retries and default email alerts. Missing for 10: documented ability to define a separate error-handling workflow/path, configurable alerting channels beyond email, and independent/hands-on confirmation of this reliability feature in production use.", "evidenceIds": [ "pipedream-docs-14", "pipedream-docs-15", "pipedream-docs-16", "pipedream-docs-29", "pipedream-docs-30" ] }, { "productId": "pipedream", "storyId": "expressions-data-mapping", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Pipedream lets users write custom Node.js/Python/Go/Bash code at any step to transform data between steps, use $.export to pass data downstream, and leverage npm packages for transformation/mapping logic, giving ops users full code-level control over data mapping between steps. Missing for 10: no dedicated low-code 'formula'/expression-editor UI evidence (only full code steps), and no independent hands-on report specifically validating step-to-step data mapping workflows.", "evidenceIds": [ "pipedream-docs-10", "pipedream-docs-11", "pipedream-gh-1", "pipedream-gh-3", "pipedream-gh-4", "pipedream-comm-6" ] }, { "productId": "pipedream", "storyId": "human-approval-steps", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of a human-in-the-loop pause/approval step (e.g., wait-for-input, approval gate) in Pipedream workflows; documentation covers triggers, error handling/retries, code steps, and MCP, but nothing about pausing execution for manual approval before continuing.", "evidenceIds": [] }, { "productId": "pipedream", "storyId": "inbound-webhook-triggers", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly list HTTP/Webhook as a first-class trigger type that starts a workflow run with the incoming payload, and community feedback corroborates real-world use with documented execution limits (body size, rate limits, timeouts), confirming this works in practice. missing for 10: no explicit documentation of custom URL customization options or independent hands-on verification of payload parsing specifics.", "evidenceIds": [ "pipedream-docs-12", "pipedream-docs-25", "pipedream-docs-28", "pipedream-comm-2", "pipedream-comm-1" ] }, { "productId": "pipedream", "storyId": "local-dev-instance", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Pipedream's CLI supports local development workflows via `pd dev` (link local file, watch for changes, auto-update deployed component) and `pd deploy` for deploying event sources from local/remote code, plus streaming events to the CLI for testing — this covers local dev iteration against a dev instance. However, there's no evidence of a fully offline/local execution runtime or dedicated CI-testing framework (e.g., a way to run workflows entirely locally without hitting the live Pipedream backend, or documented CI integration patterns). missing for 10: fully offline local execution without a live backend, explicit CI/CD pipeline integration guidance, automated testing framework for workflows.", "evidenceIds": [ "pipedream-docs-7", "pipedream-docs-8", "pipedream-docs-9", "pipedream-docs-22", "pipedream-docs-23", "pipedream-probe-3" ] }, { "productId": "pipedream", "storyId": "nl-workflow-generation", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Pipedream's homepage claims users can 'Prompt, run, edit, and deploy AI agents in seconds,' suggesting natural-language workflow generation, but there is no detailed documentation, screenshots, or independent corroboration of this specific capability beyond the one-line marketing claim. Missing for 10: detailed docs/tutorial on prompt-to-workflow generation, examples of editing an existing workflow via NL, and independent/hands-on confirmation it works as described.", "evidenceIds": [ "pipedream-docs-21" ] }, { "productId": "pipedream", "storyId": "openness-api-parity", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Pipedream documents a REST API for creating workflows, managing event sources, and subscriptions, plus a CLI for deploying/developing components from local code, and Connect/MCP APIs with SDKs for auth and tool calls — showing broad programmatic access mirroring UI capabilities. However, there's no explicit confirmation that all UI-only features (e.g., visual workflow builder specifics, admin/billing dashboards) have full API parity, and no independent verification of complete equivalence. Missing for 10: explicit API/CLI docs confirming full parity for every UI action (e.g., visual step editing, team management), independent hands-on confirmation that API-only workflows match UI-built ones exactly.", "evidenceIds": [ "pipedream-docs-6", "pipedream-docs-7", "pipedream-docs-8", "pipedream-docs-17", "pipedream-docs-3", "pipedream-gh-2", "pipedream-probe-3" ] }, { "productId": "pipedream", "storyId": "openness-full-export", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Workflow logic is plain Node.js/Python code and can be managed via a REST API and CLI, giving some portability, and a community comment notes transitioning away 'shouldn't be too hard' since workflows are just code. However, there is no documented bulk data-export feature (event history, connected accounts, credentials, logs) in an open format, and a community member explicitly wishes the engine itself were open source, suggesting real lock-in beyond code snippets. Missing for 10: an explicit data/account export tool or API, documentation on exporting historical events/credentials, and confirmation the full platform (not just code) is portable.", "evidenceIds": [ "pipedream-gh-1", "pipedream-docs-6", "pipedream-comm-1", "pipedream-comm-9" ] }, { "productId": "pipedream", "storyId": "openness-open-license", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Pipedream's component library and integrations are open on GitHub, and docs invite external PR contributions, so some source is readable/open licensed, but this covers only components, not the core workflow engine. A Hacker News commenter explicitly notes 'I wish the workflow engine itself was open source too,' confirming the core platform remains closed. missing for 10: an explicit open-source license for the core engine/runtime, and clarity on license terms for the GitHub component repo.", "evidenceIds": [ "pipedream-gh-1", "pipedream-gh-3", "pipedream-docs-31", "pipedream-comm-9" ] }, { "productId": "pipedream", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence Pipedream offers a self-hosted deployment option; all docs describe it as a hosted SaaS platform, and a community comment explicitly wishes the workflow engine were open source (implying it isn't), which is the opposite of self-hosting support.", "evidenceIds": [ "pipedream-comm-9" ] }, { "productId": "pipedream", "storyId": "prebuilt-connector-breadth", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Pipedream documents thousands of prebuilt, vendor-maintained triggers/actions across apps with one-click OAuth/key-based auth, and community feedback corroborates the breadth and usability of these integrations compared to alternatives like Zapier. Missing for 10: independent third-party audits of integration count/quality and more recent hands-on validation beyond older HN threads.", "evidenceIds": [ "pipedream-docs-1", "pipedream-docs-2", "pipedream-docs-27", "pipedream-comm-7", "pipedream-comm-12" ] }, { "productId": "pipedream", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items mention data residency, regional storage options, or compliance controls for choosing where data is stored; the axis applies to a workflow/automation platform handling user data but is not addressed anywhere in the docs or community evidence. Missing for 10: any mention of region selection, data residency options, or storage location controls.", "evidenceIds": [] }, { "productId": "pipedream", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses AI training data opt-out, data usage policies, or privacy controls related to AI model training; this is a fair question for an AI-native automation platform but no documentation or policy is cited.", "evidenceIds": [] }, { "productId": "pipedream", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data retention policies, deletion controls, or privacy/data-handling settings for user data, connected accounts, or event logs; nothing documents how users can delete stored data or configure retention windows. missing for 10: data retention policy documentation, deletion/erasure controls or APIs, data residency/compliance details (e.g., GDPR deletion requests), and any user-facing settings for purging logs/event history.", "evidenceIds": [] }, { "productId": "pipedream", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item discusses telemetry, usage tracking, or an opt-out setting for Pipedream; the docs and community threads cover integrations, CLI, workflows and pricing but never address privacy/telemetry controls.", "evidenceIds": [] }, { "productId": "pipedream", "storyId": "run-history-replay", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs confirm automatic retries for transient errors, error surfacing with stack traces, and AI-assisted debugging, which supports part of the reliability story, but there is no explicit evidence of an execution log/history viewer or a manual 're-run from the failing step' feature. missing for 10: documentation of execution history/log inspection UI, explicit 'resume from failing step' re-run capability, and independent confirmation of this workflow.", "evidenceIds": [ "pipedream-docs-14", "pipedream-docs-15", "pipedream-docs-16", "pipedream-docs-29", "pipedream-docs-30", "pipedream-docs-26" ] }, { "productId": "pipedream", "storyId": "subworkflow-reuse", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers triggers, code steps, CLI, Connect/MCP, and error handling, but nowhere documents a mechanism for one workflow to invoke another as a reusable sub-workflow/module within the visual builder. Node.js/Python code steps and npm packages are the closest reuse mechanism, but that's custom code reuse, not workflow composition.", "evidenceIds": [ "pipedream-docs-10", "pipedream-docs-24", "pipedream-gh-1", "pipedream-gh-3" ] }, { "productId": "pipedream", "storyId": "test-with-sample-data", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm test events can be selected and manually triggered ('Send Test Event') and that errors surface with stack traces and AI-assisted debugging, indicating step-level inspection during testing; Node.js steps also support $.export for output naming. However, there's no explicit documentation of pinning specific data across a full multi-step workflow or a dedicated per-step input/output inspector UI described in detail. Missing for 10: explicit docs on pinning/sample data reuse across steps, a described step-by-step input/output inspection panel, and independent/hands-on confirmation of this exact workflow-testing UX.", "evidenceIds": [ "pipedream-docs-13", "pipedream-docs-16", "pipedream-docs-11", "pipedream-docs-26" ] }, { "productId": "pipedream", "storyId": "visual-workflow-editor", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Pipedream's docs show a visual workflow builder with pre-built triggers/actions across many apps and a test-event UI (docs-1, docs-12, docs-25, docs-28, docs-13), so ops users can assemble some steps without code. However, Pipedream is explicitly positioned as 'low-code' (gh-3) and community comparisons (comm-4, comm-7, comm-12) describe it as more developer-oriented and 'low-level' than pure no-code tools like Zapier, with code steps (Node.js/Python/Go/Bash) as its central differentiator rather than an optional add-on. Missing for 10: explicit evidence/screenshots of a drag-and-drop, no-code-only workflow experience, and confirmation that complex multi-step logic (branching, loops) can be built entirely without code.", "evidenceIds": [ "pipedream-docs-1", "pipedream-docs-12", "pipedream-docs-13", "pipedream-docs-25", "pipedream-docs-28", "pipedream-gh-3", "pipedream-comm-4", "pipedream-comm-7", "pipedream-comm-12" ] }, { "productId": "pipedream", "storyId": "workflow-template-library", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents triggers, actions, components, SDKs, CLI, and MCP integration, but nowhere mentions a public gallery or library of pre-built workflow templates that an ops user could clone/start from. Absent that evidence, this applicable capability cannot be credited.", "evidenceIds": [] }, { "productId": "pipedream", "storyId": "workflows-as-mcp-tools", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Pipedream documents an official MCP server exposing 3,000+ APIs/10,000+ tools to external agents, with automatic auth/token handling and no extra implementation needed, directly matching the story. missing for 10: independent hands-on verification of exposing custom user-built workflows (vs pre-built app actions) as MCP tools, and third-party confirmation of the MCP server's reliability in production.", "evidenceIds": [ "pipedream-docs-3", "pipedream-docs-4", "pipedream-docs-5", "pipedream-probe-2" ] }, { "productId": "temporal", "storyId": "agent-creates-workflow-via-api", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Temporal exposes SDKs, CLI, and gRPC/programmatic APIs to start, signal, and schedule workflows, and documents AI agent integrations (Google ADK, Vercel AI SDK) showing agents can drive workflows programmatically, but there is no explicit documentation of an agent using a public REST/OpenAPI-style API to 'create, update, and activate' a workflow end-to-end, and the openapi probe returned 404s. missing for 10: a documented public REST/OpenAPI spec for workflow lifecycle management, explicit agent-driven create/update/activate example, and independent confirmation of AI-agent programmatic control beyond marketing copy.", "evidenceIds": [ "temporal-docs-17", "temporal-docs-18", "temporal-docs-19", "temporal-docs-26", "temporal-probe-2", "temporal-docs-15" ] }, { "productId": "temporal", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A probe confirms Temporal serves a working llms.txt at docs.temporal.io/llms.txt (HTTP 200) explicitly following the llmstxt convention, directly satisfying the story. Missing for 10: no independent/community confirmation that agents were actually pointed at this file with success.", "evidenceIds": [ "temporal-probe-1" ] }, { "productId": "temporal", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "temporal", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Temporal's core primitives—Workflows-as-Code, automatic Activity retries, crash-proof resume, and Schedules for cron-like recurring execution—directly enable background automations that run autonomously, and this is corroborated by community users describing workflows as durable and resumable across machines without manual orchestration. It also explicitly targets AI use cases (Google ADK, Vercel AI SDK integrations, 'orchestrate your workflows, AI apps, and agents ... across any length of time').\n\nMissing for 10: no first-party AI-specific end-to-end autonomous-agent tutorial, and community reports flag real operational caveats (opaque error/anomaly handling, false-positive worker-death detection) that add friction to fully hands-off autonomous operation.", "evidenceIds": [ "temporal-docs-1", "temporal-docs-10", "temporal-docs-15", "temporal-docs-19", "temporal-docs-25", "temporal-docs-28", "temporal-comm-1", "temporal-comm-2", "temporal-comm-3", "temporal-comm-4" ] }, { "productId": "temporal", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "temporal", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Temporal's CLI (`temporal server start-dev`, `temporal workflow list`, `temporal operator namespace list`) and self-hosted Docker/Kubernetes deployment guides show the server and workers can be started and controlled entirely from the command line without a GUI, which is what CI/automation pipelines need; the GitHub docs also show pre-built Docker images that can be spun up non-interactively for CI test runs. missing for 10: explicit CI pipeline examples/config (e.g., GitHub Actions workflow files) and independent confirmation of headless CI usage beyond docs.", "evidenceIds": [ "temporal-docs-2", "temporal-docs-3", "temporal-docs-5", "temporal-docs-30", "temporal-gh-1", "temporal-gh-4", "temporal-gh-5", "temporal-probe-3" ] }, { "productId": "temporal", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "temporal", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "temporal", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "temporal", "storyId": "agentic-official-cli", "verdict": "full", "quality": 7, "confidence": "high", "rationale": "Temporal ships a well-documented official CLI (`temporal server start-dev`, `temporal workflow list`, `temporal operator namespace list`) confirmed by docs, GitHub install instructions (brew install temporal), and a live docs probe, and it's usable by any developer including AI-native builders integrating Temporal workflows/agents. missing for 10: no AI-specific CLI subcommands or agent-oriented tooling (e.g., natural-language CLI interaction, LLM-facing commands) and no independent hands-on review specifically praising the CLI's AI-native ergonomics.", "evidenceIds": [ "temporal-docs-2", "temporal-docs-3", "temporal-gh-1", "temporal-gh-4", "temporal-probe-3" ] }, { "productId": "temporal", "storyId": "agentic-public-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Temporal exposes well-documented programmatic interfaces — a CLI (temporal-docs-2/3/4, temporal-probe-3), SDKs for Python/TypeScript with dev walkthroughs (temporal-docs-17/18/31), and even an llms.txt index for AI consumption (temporal-probe-1) — which supports driving the product programmatically. However, explicit machine-readable API specs (OpenAPI/Swagger) were probed and all returned 404 (temporal-probe-2), suggesting no standardized REST API surface for AI-native tooling to consume directly, and evidence doesn't show a documented public gRPC/REST API contract beyond SDK/CLI usage. Missing for 10: a documented OpenAPI/REST or gRPC API spec accessible to AI agents, and independent confirmation that non-SDK/CLI clients can drive Temporal end-to-end.", "evidenceIds": [ "temporal-docs-2", "temporal-docs-3", "temporal-docs-17", "temporal-docs-18", "temporal-probe-1", "temporal-probe-2", "temporal-probe-3" ] }, { "productId": "temporal", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Temporal Cloud exposes Namespaces, Service Accounts, and API keys that can be scoped per-user/service for least-privilege access (temporal-docs-4, temporal-docs-6), and Temporal is positioned for orchestrating AI agents (temporal-docs-17/18/19). However, there's no documented mechanism for dynamically issuing scoped credentials specifically to an AI agent identity — it's generic RBAC/API-key tooling repurposed, not an agent-specific credentialing feature. Missing for 10: agent-specific credential issuance workflow, dynamic/ephemeral scoped tokens for agents, and independent evidence of this being used for AI-agent least-privilege access.", "evidenceIds": [ "temporal-docs-4", "temporal-docs-6", "temporal-docs-17", "temporal-docs-18", "temporal-docs-19" ] }, { "productId": "temporal", "storyId": "agentic-sdks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Temporal documents official SDKs (Python, TypeScript, etc.) with dedicated AI-native integrations such as Google ADK and Vercel AI SDK, plus a dedicated AI solutions page describing orchestration of AI apps and agents via Durable Execution. missing for 10: independent/hands-on corroboration of these AI SDK integrations working in practice, and deeper documentation of agent-specific SDK features beyond the two named integrations.", "evidenceIds": [ "temporal-docs-17", "temporal-docs-18", "temporal-docs-19", "temporal-docs-26", "temporal-docs-31" ] }, { "productId": "temporal", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of Temporal offering webhook subscriptions for events; it provides Workflows, Schedules, and signal-based mechanisms, but nothing in the evidence pack describes a webhook subscription capability for external event notification.", "evidenceIds": [] }, { "productId": "temporal", "storyId": "ai-agent-workflow-steps", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Temporal explicitly markets AI-workflow orchestration and ships integrations with Google ADK and the Vercel AI SDK, both of which support LLM steps with model selection and tool use inside Temporal Activities/Workflows, but the evidence lacks concrete detail on how model choice or tool invocation is configured within a Temporal workflow. missing for 10: explicit code examples or docs showing model selection, tool-calling configuration, or a dedicated AI-agent primitive, and independent/hands-on confirmation of these integrations working end-to-end.", "evidenceIds": [ "temporal-docs-17", "temporal-docs-18", "temporal-docs-19", "temporal-docs-26" ] }, { "productId": "temporal", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an interactive API reference or runnable-example explorer; the openapi.json/swagger.json probes all returned 404, and docs mentions are limited to CLI reference and SDK guides, not an interactive playground.", "evidenceIds": [ "temporal-probe-2", "temporal-probe-3" ] }, { "productId": "temporal", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe for OpenAPI/Swagger spec files at Temporal's docs domain returned 404 for all candidate paths, and no evidence pack item references a downloadable machine-readable API spec; Temporal's public API is gRPC/Protobuf-based rather than exposing an OpenAPI spec.", "evidenceIds": [ "temporal-probe-2" ] }, { "productId": "temporal", "storyId": "api-sandbox", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Temporal ships a local dev server (temporal server start-dev / embedded dev service) for fast local feedback plus Namespace isolation to separate environments, which together let a user run and test workflows without touching production data. However, there's no dedicated 'AI sandbox' concept or agent-specific sandbox tooling, and community notes flag self-hosting documentation gaps and Temporal Cloud immaturity that could affect confidence in a clean test/prod separation. Missing for 10: an explicit sandbox/staging environment feature (vs. generic local dev + namespaces), AI-specific sandbox tooling for agent testing, and independent hands-on confirmation that production data isolation works as described.", "evidenceIds": [ "temporal-docs-2", "temporal-docs-3", "temporal-docs-6", "temporal-docs-30", "temporal-gh-4", "temporal-comm-3", "temporal-comm-6" ] }, { "productId": "temporal", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers Worker Versioning for workflow code compatibility, not a documented versioning/deprecation policy for Temporal's own APIs (SDKs, gRPC, REST). The OpenAPI probe returned 404s and no changelog or deprecation-policy documentation is present in the pack.", "evidenceIds": [ "temporal-docs-13", "temporal-docs-14", "temporal-docs-29", "temporal-probe-2" ] }, { "productId": "temporal", "storyId": "app-event-triggers", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Temporal is explicitly a code-first workflow engine ('Temporal isn't a no-code Workflow engine — it is Workflows-as-Code'), with no evidence of pre-built connectors to apps (email, forms, CRM records) that an ops user could configure without writing code; triggering requires developer-authored Signals/APIs, not point-and-click integrations.", "evidenceIds": [ "temporal-docs-11", "temporal-docs-19" ] }, { "productId": "temporal", "storyId": "automatic-step-retries", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly state activities are automatically retried based on configuration (retry policies), and community evidence corroborates users can retry any unit on failure and specify non-retryable errors, confirming this works in practice. Missing for 10: explicit docs snippet showing backoff coefficient/interval configuration parameters directly cited in the pack.", "evidenceIds": [ "temporal-docs-10", "temporal-comm-2" ] }, { "productId": "temporal", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack describes Temporal's workflow, scheduling, retry, and CLI capabilities but contains no citation showing a bulk/batch operation feature (e.g., batch signal/terminate across many workflow executions) that would let an AI-native user act on many items at once; 'temporal workflow list' only shows listing, not batch action.", "evidenceIds": [ "temporal-gh-1", "temporal-docs-15", "temporal-docs-25" ] }, { "productId": "temporal", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Temporal supports workflows-as-code with retries, signals, and Schedules for time-based triggering (temporal-docs-10, temporal-docs-15), and can be invoked from external events via API/signal calls, but it is not a declarative rules engine where an AI-native user defines event-condition-action rules directly — it requires writing workflow code rather than declarative trigger rules. missing for 10: a declarative rule-definition interface (no-code/low-code trigger authoring), native event-source bindings/webhooks for arbitrary external events, and evidence of AI-native users defining rules rather than developers writing workflow code.", "evidenceIds": [ "temporal-docs-10", "temporal-docs-15", "temporal-docs-11", "temporal-docs-25" ] }, { "productId": "temporal", "storyId": "automation-scheduled-jobs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Temporal's native Schedules feature explicitly supports recurring workflow execution with cron-like and human-meaningful timing (phase offsets, specific times), documented as more flexible than cron jobs, and corroborated by community mention of Schedules as 'cron but more flexible.' AI-native use is directly supported via AI SDK/ADK integrations and orchestration-for-agents messaging. missing for 10: independent hands-on validation specifically of the Schedules feature (community evidence covers general workflow reliability, not scheduling specifically), and no evidence of scheduling-specific failure modes or edge-case limits.", "evidenceIds": [ "temporal-docs-15", "temporal-docs-16", "temporal-docs-25", "temporal-comm-8", "temporal-docs-19", "temporal-docs-17", "temporal-docs-18" ] }, { "productId": "temporal", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Temporal's Worker Versioning lets you tag workers and roll out new code paths while old executions keep running on old code (temporal-docs-13/14/29), and community evidence confirms a workflow can be reset to an earlier state and re-run — effectively a rollback (temporal-comm-8). The Web UI/CLI (temporal-gh-2, temporal-probe-3) supports reviewing workflow execution history and state, but this is infrastructure-level versioning of workers/workflow code, not a first-class 'version/review/rollback automations' feature with diffing or audit UI, and community notes documentation gaps around error/anomaly handling (temporal-comm-3). Missing for 10: explicit documented rollback/reset command in official docs, a dedicated review/diff UI for workflow definition changes, and AI-native framing of this capability.", "evidenceIds": [ "temporal-docs-13", "temporal-docs-14", "temporal-docs-29", "temporal-comm-8", "temporal-gh-2", "temporal-comm-3" ] }, { "productId": "temporal", "storyId": "central-credentials-management", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence only shows Temporal Cloud's own API keys/Service Accounts/SAML SSO for controlling access to the Temporal control plane itself, not a general credential-store feature where ops can centrally store arbitrary external connection secrets and grant per-credential usage rights to team members/workflows. No mention of a 'connections' or secrets vault construct with per-credential ACLs.", "evidenceIds": [ "temporal-docs-4", "temporal-docs-21" ] }, { "productId": "temporal", "storyId": "code-steps-in-workflows", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Temporal workflows and activities are literally written as real code (TypeScript/JavaScript, Python, Go, Java) with Workflows-as-Code being a core design principle, and Activities are standalone functions that can run arbitrary code as a step — docs explicitly say 'Temporal isn't a no-code Workflow engine — it is Workflows-as-Code' and reference TypeScript/Python SDK development guides. missing for 10: no explicit code snippet in evidence showing an inline JS/Python function step, and no independent hands-on confirmation of writing custom code inside a workflow step beyond SDK doc mentions.", "evidenceIds": [ "temporal-docs-11", "temporal-docs-12", "temporal-docs-31", "temporal-docs-18", "temporal-comm-2" ] }, { "productId": "temporal", "storyId": "community-built-components", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Temporal is a workflow orchestration engine, not a plugin/marketplace platform; the evidence pack shows no community node/component/integration ecosystem, registry, or marketplace for third-party contributed connectors. This is a fair question for an integration-oriented platform, so absence of any such evidence yields 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "temporal", "storyId": "concurrency-rate-controls", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Temporal's docs show automatic Activity retries (temporal-docs-10) and Schedules for controlling when Workflow Executions start (temporal-docs-15, temporal-docs-25), which can be used to queue/pace executions, but the evidence pack contains no explicit mention of throttling mechanisms like per-second rate limits on Task Queues or Worker/Activity concurrency caps that directly address downstream rate-limit protection. Missing for 10: explicit documentation of rate-limiting options (e.g., worker or activity execution-rate caps), hands-on examples of throttling against a downstream API, and independent corroboration of this specific use case.", "evidenceIds": [ "temporal-docs-10", "temporal-docs-15", "temporal-docs-25" ] }, { "productId": "temporal", "storyId": "conditional-branching-logic", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Temporal explicitly positions itself as Workflows-as-Code, not a no-code/visual engine, so branching, filtering, and merging parallel paths are done by developers writing code rather than by ops users through a visual builder — no evidence of any visual workflow/branching UI for non-developers.", "evidenceIds": [ "temporal-docs-11" ] }, { "productId": "temporal", "storyId": "cron-schedules", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Temporal has a dedicated Schedules feature explicitly described as a more flexible cron-like mechanism for starting Workflow Executions at specific times, including phase-offset control for firing at exact human-meaningful times (e.g., '5pm UTC'), which implies timezone-aware scheduling. However, the evidence never explicitly documents a first-class timezone parameter or UI control for schedules, only UTC-based phase offsets. Missing for 10: explicit documentation of a timezone field/parameter in Schedule config, UI/CLI examples showing timezone selection, and independent confirmation of timezone behavior in production use.", "evidenceIds": [ "temporal-docs-15", "temporal-docs-16", "temporal-docs-25", "temporal-comm-8" ] }, { "productId": "temporal", "storyId": "custom-connector-sdk", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Temporal provides SDKs (TypeScript, Python, etc.), a documented CLI (temporal server start-dev, workflow/namespace commands), self-hosted deployment guides, and integration points (Payload Codecs, Worker Versioning, AI SDK integrations) that let developers build custom integrations and extend the platform. Community evidence corroborates real-world SDK use and even a responsive process for reporting/fixing library issues, though some note documentation gaps for self-hosting. Missing for 10: a public/documented plugin marketplace or officially published OpenAPI spec (probe found no openapi.json) and stronger independent validation of extensibility for private integrations beyond SDK use.", "evidenceIds": [ "temporal-docs-2", "temporal-docs-3", "temporal-docs-13", "temporal-docs-17", "temporal-docs-18", "temporal-docs-22", "temporal-gh-1", "temporal-gh-4", "temporal-comm-5", "temporal-probe-3" ] }, { "productId": "temporal", "storyId": "durable-long-running-workflows", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Temporal's core value proposition is exactly this: durable execution that resumes workflows after crashes/restarts, documented extensively (temporal-docs-1/28) and corroborated by independent community users confirming workflows are 'guaranteed to run and are resumable on other machines' even needing full determinism (temporal-comm-1, temporal-comm-2). Schedules and long-running workflow support (docs-15/16/25) further back the 'wait for days' aspect. Missing for 10: independent benchmark/stress-test evidence of very long-duration state survival, and community reports flag real edge cases where worker-death detection can misfire and leave workflows stuck 'in progress' (temporal-comm-4), plus documentation gaps around error visibility (temporal-comm-3).", "evidenceIds": [ "temporal-docs-1", "temporal-docs-28", "temporal-docs-15", "temporal-docs-16", "temporal-comm-1", "temporal-comm-2", "temporal-comm-4", "temporal-comm-3" ] }, { "productId": "temporal", "storyId": "embed-white-label", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "temporal", "storyId": "environments-source-control", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Temporal's Workflows-as-Code model means workflow definitions live in normal source-controlled code, and its Worker Versioning feature explicitly lets teams tag deployments so new code paths roll out to new executions while in-progress ones keep running old versions — directly supporting versioning and safe promotion of changes. Namespaces provide isolation that can be used to separate dev/staging/production environments, and self-hosted/cloud deployment guides support multi-environment setups. Missing for 10: explicit dev/stage/prod promotion pipeline documentation or CI/CD integration guidance, and independent hands-on confirmation of the versioning workflow in production use.", "evidenceIds": [ "temporal-docs-13", "temporal-docs-14", "temporal-docs-29", "temporal-docs-6", "temporal-docs-11" ] }, { "productId": "temporal", "storyId": "error-workflows-alerts", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Temporal's docs show built-in retry policies and workflow-level error handling (e.g., automatic Activity retries, non-retryable error configuration) that let developers build compensation/error paths in code, but there's no vendor evidence of a built-in alerting/notification mechanism for failed runs. Hands-on community reports directly contradict the 'get notified when a run fails' claim: users report errors are 'buried in the event log' and not surfaced anywhere, and that workflows can silently sit in an 'In Progress' state with no error-level cleanup triggered. Missing for 10: dedicated notification/alerting integration for failures, clear documentation on defining error-workflows, and confirmation these gaps have been resolved.", "evidenceIds": [ "temporal-docs-10", "temporal-comm-3", "temporal-comm-4" ] }, { "productId": "temporal", "storyId": "expressions-data-mapping", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Temporal is explicitly Workflows-as-Code (temporal-docs-11), meaning any data mapping/transformation between Activities/Workflow steps happens via developer-written functions in a general-purpose language, not through visual expressions or formulas aimed at ops users. There is no documentation of a dedicated expression language, formula editor, or low-code mapping UI — transformation is only possible by writing code, which is a poor fit for a non-developer 'ops' persona. Missing for 10: an explicit expression/formula syntax, any UI-based data mapping tool, or documentation showing ops (non-coding) users transforming data between steps.", "evidenceIds": [ "temporal-docs-11", "temporal-docs-12" ] }, { "productId": "temporal", "storyId": "human-approval-steps", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes Temporal's durable execution, retries, versioning, and scheduling, but never mentions Signals, Updates, or any workflow-pause-for-human-input/approval mechanism that would let an ops user halt a workflow pending manual approval. Missing for 10: any mention of Signal/Update APIs, human-in-the-loop approval patterns, or docs describing waiting on external input before workflow continuation.", "evidenceIds": [] }, { "productId": "temporal", "storyId": "inbound-webhook-triggers", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows no native webhook-receiver feature or public HTTP endpoint capability that starts a workflow from an external payload; Temporal's documented interfaces are the CLI, SDKs, gRPC API, and Schedules for time-based triggers, not inbound webhook URLs. While a developer could build a custom HTTP handler that calls the SDK's StartWorkflowExecution, none of the docs, GitHub, or community items describe an out-of-the-box webhook capability.", "evidenceIds": [ "temporal-docs-15", "temporal-docs-25", "temporal-docs-31", "temporal-probe-2" ] }, { "productId": "temporal", "storyId": "local-dev-instance", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Temporal's CLI provides a documented `temporal server start-dev` command that spins up a complete local Temporal Service with Web UI for fast local feedback, and GitHub docs corroborate a quick brew-install and localhost:8233 workflow-viewing setup — directly supporting local dev/CI use. missing for 10: no explicit CI-pipeline example or GitHub Actions integration snippet in the evidence.", "evidenceIds": [ "temporal-docs-2", "temporal-docs-3", "temporal-docs-30", "temporal-gh-2", "temporal-gh-4", "temporal-gh-5" ] }, { "productId": "temporal", "storyId": "nl-workflow-generation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Temporal explicitly positions itself as 'Workflows-as-Code' rather than a no-code/prompt-driven engine (temporal-docs-11), and while it offers AI/agent orchestration integrations (Vercel AI SDK, Google ADK) for building AI apps, there is no evidence of a feature that generates or edits workflow definitions from natural-language prompts.", "evidenceIds": [ "temporal-docs-11", "temporal-docs-19", "temporal-docs-17", "temporal-docs-18" ] }, { "productId": "temporal", "storyId": "openness-api-parity", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Temporal exposes a CLI/SDK and gRPC API for core operations (workflow list, namespace management, cloud control-plane resources) that mirror much of what the Web UI shows, but there's no documented guarantee of full UI/API parity, and probes found no public OpenAPI/REST spec (404s across candidate paths). missing for 10: explicit parity documentation, a REST/OpenAPI spec, and confirmation that every UI action (e.g., debugging views, visualizations) has an API equivalent.", "evidenceIds": [ "temporal-gh-1", "temporal-docs-4", "temporal-probe-2", "temporal-probe-3" ] }, { "productId": "temporal", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence covers self-hosting, archival to blob storage, and payload encryption, but nothing documents a data-export feature or open-format guarantee for workflow history/state that would let a user extract all data and leave the platform. Missing for 10: explicit export tooling, documented open/standard data format for workflow history, and any statement on data portability/no-lock-in guarantees.", "evidenceIds": [] }, { "productId": "temporal", "storyId": "openness-open-license", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Evidence confirms Temporal's server code is hosted in a public GitHub repository (temporalio/temporal) with commands, installation instructions, and code history described (e.g., fork of Cadence), indicating source availability, but no evidence in the pack cites an explicit open-source license file or terms. missing for 10: explicit license text/citation (e.g., MIT/Apache-2.0 LICENSE file), confirmation that all components (not just core server) are open source, and independent corroboration of license terms.", "evidenceIds": [ "temporal-gh-1", "temporal-gh-2", "temporal-gh-3", "temporal-gh-4", "temporal-gh-5" ] }, { "productId": "temporal", "storyId": "openness-self-host", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs clearly describe a self-hosted guide covering deployment (Docker/Kubernetes/manual), namespaces, encryption, archival, and multi-cluster replication, plus CLI/dev server (`temporal server start-dev`) and GitHub instructions to run a pre-built server image with Web UI locally. However community evidence notes self-hosting documentation is thin and support is sparse, indicating real-world friction. missing for 10: independent hands-on verification of a full production self-hosted deployment succeeding, and more detailed community corroboration beyond the noted complaint about sparse self-hosting docs/support.", "evidenceIds": [ "temporal-docs-5", "temporal-docs-6", "temporal-docs-7", "temporal-docs-8", "temporal-docs-9", "temporal-docs-30", "temporal-gh-5", "temporal-gh-1", "temporal-gh-2", "temporal-comm-3" ] }, { "productId": "temporal", "storyId": "prebuilt-connector-breadth", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "temporal", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Temporal's self-hosted deployment guide lets customers run the service in any region/data center of their choosing (Docker/Kubernetes/manual), which indirectly satisfies data residency needs, and Namespaces provide isolation and retention controls. However, there is no explicit documentation of a Temporal Cloud region-selection feature or formal data-residency guarantee. Missing for 10: explicit Temporal Cloud region/residency selection docs, compliance certifications tied to specific regions, and independent confirmation that customers can pin data to a jurisdiction.", "evidenceIds": [ "temporal-docs-5", "temporal-docs-6", "temporal-docs-8", "temporal-docs-22" ] }, { "productId": "temporal", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "temporal", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Temporal exposes Namespace-level retention configuration and Archival (moving closed Workflow Histories to blob storage for longer retention), giving admins control over how long workflow data persists — this is a documented privacy-relevant capability rather than something inferred outside the pack. However, there's no evidence of explicit deletion APIs/controls beyond retention-window expiry, no mention of GDPR-style right-to-delete tooling, and no independent/community corroboration of retention or deletion working as documented. Missing for 10: explicit data-deletion API/workflow, compliance-oriented deletion guarantees, and independent verification.", "evidenceIds": [ "temporal-docs-6", "temporal-docs-8", "temporal-docs-22", "temporal-docs-24" ] }, { "productId": "temporal", "storyId": "privacy-telemetry-optout", "verdict": "na", "quality": 0, "confidence": "low", "rationale": "Temporal is a workflow orchestration platform (self-hosted or cloud); telemetry/usage-tracking opt-out is not a documented axis for this product category as evidenced, and the story concerns product-analytics/telemetry opt-out which is a category mismatch rather than a missing feature.", "evidenceIds": [] }, { "productId": "temporal", "storyId": "run-history-replay", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Temporal's Web UI lets ops view workflow event history/logs (temporal-gh-2, temporal-gh-1), and community evidence confirms a 'reset' feature that resumes a workflow from a specific failed statement rather than starting over (temporal-comm-8), plus built-in automatic retries for failed Activities (temporal-docs-10). However, first-party docs don't detail the reset-from-failure workflow itself, and community reports note real gaps in error visibility (errors buried in event log, stuck 'In Progress' states requiring manual termination) that complicate diagnosing and resuming failed executions (temporal-comm-3, temporal-comm-4). Missing for 10: first-party documentation of the Reset API/CLI command, evidence of granular log inspection UX, and resolution of the community-reported error-visibility gaps.", "evidenceIds": [ "temporal-gh-2", "temporal-gh-1", "temporal-comm-8", "temporal-docs-10", "temporal-comm-3", "temporal-comm-4" ] }, { "productId": "temporal", "storyId": "subworkflow-reuse", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes Temporal as a code-first Workflows-as-Code platform (temporal-docs-11) with Activities, Standalone Activities, and Schedules, but never mentions child workflows, sub-workflow composition, or modular workflow reuse patterns that other workflows can call. Without explicit evidence of this composability mechanism, the axis (which is fair to ask of any workflow orchestration platform) cannot be credited.", "evidenceIds": [ "temporal-docs-11", "temporal-docs-12" ] }, { "productId": "temporal", "storyId": "test-with-sample-data", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Temporal provides a local dev server and Web UI for viewing workflow executions (temporal-docs-2/3, temporal-gh-2), which offers some execution inspection, but there is no evidence of a feature for testing with sample or pinned data, or for inspecting each step's input/output in a guided way — Temporal explicitly positions itself as Workflows-as-Code rather than a visual/no-code tool (temporal-docs-11). missing for 10: explicit sample/pinned data test harness, step-by-step input/output inspection UI, and any hands-on corroboration of this workflow-testing experience.", "evidenceIds": [ "temporal-docs-2", "temporal-docs-3", "temporal-gh-2", "temporal-docs-11" ] }, { "productId": "temporal", "storyId": "visual-workflow-editor", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Temporal explicitly states it is a code-first, Workflows-as-Code platform, not a no-code/visual workflow engine — directly contradicting this story's premise; no visual editor is evidenced anywhere in the pack.", "evidenceIds": [ "temporal-docs-11" ] }, { "productId": "temporal", "storyId": "workflow-template-library", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Temporal is a code-first Workflows-as-Code platform (explicitly \"isn't a no-code Workflow engine\") — no evidence of a public template gallery or marketplace for ops users to start from pre-built workflow templates; Temporal SDK samples exist but no curated public library is mentioned in this pack.", "evidenceIds": [ "temporal-docs-11", "temporal-docs-31" ] }, { "productId": "temporal", "storyId": "workflows-as-mcp-tools", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of Temporal exposing workflows or actions as MCP tools for external agents; the pack shows AI SDK/ADK integrations (Temporal as orchestrator calling AI tools) but nothing about serving an MCP endpoint for outside agents to call into Temporal workflows.", "evidenceIds": [ "temporal-docs-17", "temporal-docs-18", "temporal-docs-19", "temporal-probe-2" ] }, { "productId": "windmill", "storyId": "agent-creates-workflow-via-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Windmill exposes multiple programmatic paths (CLI `wmill`, git-sync auto-deploy, version-pinned webhooks, and an MCP server that lets LLM clients trigger scripts/flows) that together suggest an agent could manage workflows without the UI, and AI-generation docs show agents can edit items and reach a deploy step. However, the story specifically requires create/update/activate via the public API, and the evidence never surfaces a documented REST/OpenAPI spec (the openapi.json probe 404'd on all candidate paths) nor confirms MCP or CLI can create/activate flows rather than just trigger existing ones. missing for 10: explicit public API/OpenAPI docs for flow CRUD, confirmation that agent-driven creation/activation (not just triggering) works end-to-end without human deploy step.", "evidenceIds": [ "windmill-docs-9", "windmill-docs-11", "windmill-docs-16", "windmill-docs-17", "windmill-docs-21", "windmill-probe-3", "windmill-probe-4", "windmill-probe-5" ] }, { "productId": "windmill", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Direct probe confirms a working llms.txt at https://www.windmill.dev/llms.txt (HTTP 200) and .md-suffixed docs pages (e.g., intro.md) that return clean markdown, both hallmarks of agent-oriented documentation designed for LLM consumption. Missing for 10: no independent/community confirmation that agents actually use these successfully in practice.", "evidenceIds": [ "windmill-probe-1", "windmill-probe-2" ] }, { "productId": "windmill", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Windmill offers AI Agent steps that let users wire LLM calls into flows to process data, and an MCP server/AI-generation feature for building scripts, but these are developer-facing building blocks rather than a built-in 'insights and suggestions from my data' feature (e.g., no dashboard/analytics AI surfaced automatically on stored data). Missing for 10: evidence of an out-of-the-box AI analytics/insights UI, proactive suggestions surfaced to end users without building a flow, and any independent corroboration of this specific use case.", "evidenceIds": [ "windmill-docs-20", "windmill-docs-21", "windmill-docs-17" ] }, { "productId": "windmill", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Windmill natively supports scheduling scripts/flows via CRON with error/recovery handlers, plus a wide range of autonomous triggers (webhooks, queues, events, routes) that run in the background without manual intervention, and flows include retries, branching, and error handling for resilient unattended execution. This directly matches the AI-native automation story, reinforced by AI Agent steps that can be embedded into these autonomous flows. Missing for 10: no independent/hands-on evidence confirming long-running autonomous reliability at scale, and community comments raise open questions about error handling rather than confirming it in practice.", "evidenceIds": [ "windmill-docs-18", "windmill-docs-15", "windmill-docs-13", "windmill-docs-12", "windmill-docs-19", "windmill-docs-20", "windmill-comm-8" ] }, { "productId": "windmill", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Windmill's docs describe an in-product AI generation feature that can edit scripts/flows and then prompts the user to review and deploy those changes, indicating a built-in assistant users can delegate authoring tasks to (windmill-docs-21). However, evidence is thin on the assistant's scope, limitations, or independent hands-on validation, and separate 'AI Agents' flow steps (windmill-docs-20) are for building agentic workflows rather than being the assistant itself. Missing for 10: independent/community corroboration of the AI assistant's real-world reliability, more detail on what tasks it can handle end-to-end, and comparison to failure cases.", "evidenceIds": [ "windmill-docs-21", "windmill-docs-20" ] }, { "productId": "windmill", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Windmill provides a documented CLI (wmill) for deploying and syncing from Git, supports webhooks/schedules/API triggers for headless execution, and can self-host via Docker/Kubernetes for CI pipelines — all strongly supporting headless/CI automation use. Missing for 10: independent hands-on CI-pipeline examples or third-party corroboration of the CLI/webhook flow working seamlessly in a CI system beyond vendor docs.", "evidenceIds": [ "windmill-docs-9", "windmill-docs-11", "windmill-docs-15", "windmill-docs-16", "windmill-docs-10", "windmill-probe-5" ] }, { "productId": "windmill", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Windmill's MCP documentation (windmill-docs-17, windmill-probe-4) describes Windmill acting as an MCP *server* so external LLM clients (Claude, Cursor) can call Windmill's scripts/flows — the reverse of the story, which asks whether Windmill can consume/plug into external MCP servers to use their tools. The AI Agent steps (windmill-docs-20) mention connecting to AI providers/models, not MCP tool servers, so no evidence shows Windmill as an MCP client.", "evidenceIds": [ "windmill-docs-17", "windmill-probe-4", "windmill-docs-20" ] }, { "productId": "windmill", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Windmill is a workflow/automation platform (not an agent), so publishing an official MCP server is squarely within its category, and docs confirm it: 'With MCP, you can connect your favorite LLMs (like Claude, Cursor, or any MCP compatible client) to Windmill, allowing you to trigger your scripts and flows from your client chat,' corroborated by a probe confirming the official MCP docs page. missing for 10: independent/hands-on confirmation of the MCP server working in practice beyond first-party docs.", "evidenceIds": [ "windmill-docs-17", "windmill-probe-4" ] }, { "productId": "windmill", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "low", "rationale": "Windmill documents an MCP server that lets LLM clients trigger scripts/flows via natural-language chat (windmill-docs-17, windmill-probe-4) and an AI-generation feature where a user can ask the AI to edit and review flow/script changes (windmill-docs-21), showing real natural-language operability. However, evidence is thin—just two doc snippets—with no independent/hands-on corroboration of reliability or scope of NL commands. Missing for 10: hands-on/community validation of NL-driven operation, fuller documentation of what natural-language commands can accomplish beyond triggering and editing.", "evidenceIds": [ "windmill-docs-17", "windmill-docs-21", "windmill-probe-4" ] }, { "productId": "windmill", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Windmill ships an official CLI (`wmill`) documented for interacting with instances from the terminal, deploying from Git repos, and syncing commits—confirmed by both docs and a dedicated probe. This directly satisfies the 'official CLI' story for AI-native/agentic workflows (e.g., scripting deployments, CI integration). Missing for 10: independent hands-on community reports specifically validating the CLI's AI-native usage patterns (e.g., agent-driven CLI invocation) beyond first-party docs.", "evidenceIds": [ "windmill-docs-9", "windmill-docs-11", "windmill-probe-5" ] }, { "productId": "windmill", "storyId": "agentic-public-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Windmill exposes a CLI (wmill), webhooks, and various trigger mechanisms (HTTP routes, schedules, MCP) that let external systems and AI agents drive it (windmill-docs-9, windmill-docs-15, windmill-docs-16, windmill-docs-17, windmill-probe-4, windmill-probe-5), which is real evidence of programmatic/agentic control. However, a direct probe for a documented public REST/OpenAPI spec (the canonical 'documented public API') returned 404 on all candidate paths (windmill-probe-3), so no first-class API reference doc was found in evidence. missing for 10: a discoverable OpenAPI/REST API reference page, independent confirmation that third parties integrate via a general public API beyond webhooks/CLI/MCP.", "evidenceIds": [ "windmill-docs-9", "windmill-docs-15", "windmill-docs-16", "windmill-docs-17", "windmill-probe-3", "windmill-probe-4", "windmill-probe-5" ] }, { "productId": "windmill", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Windmill documents scoped tokens following least-privilege principles (windmill-docs-8) and an underlying roles/permissions system (windmill-docs-7), and its MCP server for connecting agents (windmill-docs-17, windmill-probe-4) implies token-based auth for agent connections. However, there's no explicit documentation tying scoped token creation specifically to AI agent use-cases or showing a workflow for issuing/rotating agent-specific credentials. Missing for 10: explicit docs on generating scoped tokens specifically for AI agents/MCP clients, examples of scope granularity, and independent/hands-on confirmation that scoping works as claimed.", "evidenceIds": [ "windmill-docs-8", "windmill-docs-7", "windmill-docs-17", "windmill-probe-4" ] }, { "productId": "windmill", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents a CLI (wmill), an MCP server, and workflows-as-code in TypeScript/Python, but none of it describes official client SDKs/libraries for programmatically building against Windmill's API. A probe for an OpenAPI spec (often paired with SDK generation) returned 404s, further suggesting no discoverable official SDK artifact.", "evidenceIds": [ "windmill-docs-9", "windmill-docs-17", "windmill-probe-3", "windmill-probe-5" ] }, { "productId": "windmill", "storyId": "agentic-webhooks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Windmill documents webhooks extensively as a trigger mechanism—scripts and flows can be invoked via webhook endpoints, including version-pinned endpoints for specific flow versions—which lets external systems call into Windmill (windmill-docs-15, windmill-docs-16). However, this is primarily inbound triggering rather than an outbound event-subscription model where Windmill pushes notifications about internal events (e.g., job completion, failures) to a subscriber's webhook URL. Missing for 10: explicit documentation of outbound webhook subscriptions/event notifications, independent confirmation of an event-driven push webhook system, and any first-party mention of 'subscribe' semantics rather than pure trigger endpoints.", "evidenceIds": [ "windmill-docs-15", "windmill-docs-16" ] }, { "productId": "windmill", "storyId": "ai-agent-workflow-steps", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Windmill has documented AI Agent steps that integrate directly into flows, connecting to various AI providers/models and supporting tool use within the workflow orchestration engine (windmill-docs-20). This directly matches the story of adding LLM/agent steps with model choice and tool use inside a workflow. Missing for 10: independent/hands-on corroboration of tool-use behavior and broader detail on model selection UX beyond the single docs page.", "evidenceIds": [ "windmill-docs-20" ] }, { "productId": "windmill", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows an explicit probe for an OpenAPI/Swagger interactive reference that returned 404 on all candidate paths, and no other citation describes an interactive API reference with runnable examples (docs pages are static markdown/CLI/MCP references, not a runnable API explorer).", "evidenceIds": [ "windmill-probe-3" ] }, { "productId": "windmill", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Windmill exposes a CLI and API-driven platform, so a downloadable OpenAPI spec is a fair ask, but the evidence pack explicitly shows a probe attempt failing to find any OpenAPI/swagger file at standard paths (all 404s), and no docs page links to a machine-readable spec.", "evidenceIds": [ "windmill-probe-3" ] }, { "productId": "windmill", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Windmill's languages, orchestration, permissions, CLI, git sync, and MCP integration, but nowhere documents a dedicated sandbox/staging workspace or test-mode that isolates execution from production data; a community comment even raises this exact question (non-destructive branch testing) without an answered example.", "evidenceIds": [ "windmill-comm-5" ] }, { "productId": "windmill", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Windmill exposes webhooks and version-pinned flow endpoints (windmill-docs-16), and a CLI/API surface exists, but there is no evidence of a documented API versioning scheme or deprecation policy; OpenAPI spec probes returned 404s (windmill-probe-3) and no docs mention deprecation practices.", "evidenceIds": [ "windmill-docs-16", "windmill-probe-3" ] }, { "productId": "windmill", "storyId": "app-event-triggers", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Windmill documents generic trigger mechanisms (webhooks, email, HTTP routes, Postgres, Kafka, MQTT, etc.) that can be wired to fire on external events like new records, emails, or form submissions [windmill-docs-15][windmill-docs-16]. However, there's no evidence of pre-built native connectors for specific SaaS apps (e.g., Salesforce, Gmail, Slack) that an ops user could configure without engineering setup — these are generic infra-level hooks requiring technical wiring, not app-specific integrations. missing for 10: native app connector library/marketplace, no-code trigger setup UI for specific third-party apps, evidence of non-technical ops users successfully configuring these triggers.", "evidenceIds": [ "windmill-docs-15", "windmill-docs-16", "windmill-docs-18" ] }, { "productId": "windmill", "storyId": "automatic-step-retries", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Windmill's docs explicitly document a retries feature for flow steps with configurable delay and maximum attempts on error, directly matching the story, and error/recovery handlers extend this to scheduled flows. Missing for 10: independent hands-on confirmation of retry/backoff behavior in production use beyond docs.", "evidenceIds": [ "windmill-docs-13", "windmill-docs-18" ] }, { "productId": "windmill", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Windmill's flows support parallelism, branching, and fault-tolerant orchestration (windmill-docs-5, windmill-docs-12), which implies capability to iterate/process many items in a flow, but no evidence explicitly documents a for-loop/bulk-iteration step, batch processing UI, or examples of running an operation across a large item set. missing for 10: explicit for-loop/iterator flow step documentation, evidence of scaling to large item counts, and any hands-on confirmation of bulk operation performance.", "evidenceIds": [ "windmill-docs-5", "windmill-docs-12", "windmill-docs-13" ] }, { "productId": "windmill", "storyId": "automation-rules-engine", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Windmill lets users define scripts/flows triggered automatically by schedules (CRON), webhooks, emails, HTTP routes, websockets, Kafka, Postgres, NATS, SQS, MQTT, and other event sources, with retry/error handling and branching logic for rule-based automation. This directly matches the story of defining rules that trigger actions on events, with strong first-party documentation. Missing for 10: independent/hands-on verification of complex event-rule chains and no community corroboration specifically confirming event-trigger reliability in production.", "evidenceIds": [ "windmill-docs-15", "windmill-docs-18", "windmill-docs-16", "windmill-docs-13", "windmill-docs-12", "windmill-comm-8" ] }, { "productId": "windmill", "storyId": "automation-scheduled-jobs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Windmill has a dedicated Scheduling core concept supporting CRON-based recurring execution of scripts/flows with error/recovery handlers, plus flexible triggers (webhooks, events, etc.) and retries/branching for robust workflows, all documented first-party. missing for 10: no independent/hands-on community confirmation specifically of scheduling reliability in production.", "evidenceIds": [ "windmill-docs-18", "windmill-docs-15", "windmill-docs-13", "windmill-docs-12" ] }, { "productId": "windmill", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Windmill documents versioning via Git sync (each deploy creates a commit, auto-deploy from repo) and version-pinned webhook endpoints for flows, plus a 'Compare & Deploy' review UI when AI edits items. However, there is no explicit rollback feature or UI documented (e.g., one-click revert to a prior version), and no independent/hands-on confirmation of rollback behavior. missing for 10: explicit rollback/revert mechanism, independent verification of version history UI and rollback workflow.", "evidenceIds": [ "windmill-docs-11", "windmill-docs-16", "windmill-docs-21", "windmill-docs-9" ] }, { "productId": "windmill", "storyId": "central-credentials-management", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Windmill documents centrally encrypted Variables/Secrets at the workspace level plus a broader roles/permissions system and scoped tokens for least-privilege access, which together support storing and sharing credentials with some access control. However, the evidence doesn't show fine-grained per-credential ACLs (e.g., which specific users/groups can use a specific secret) beyond general workspace/role permissions. Missing for 10: explicit per-resource/credential-level permission granularity, audit logging of credential usage, and independent/hands-on confirmation of this governance in practice.", "evidenceIds": [ "windmill-docs-6", "windmill-docs-7", "windmill-docs-8" ] }, { "productId": "windmill", "storyId": "code-steps-in-workflows", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Windmill's core model is scripts written in real code (TypeScript, Python, Go, etc.) composed into flows, with 'workflows as code' letting developers write orchestration logic directly in TypeScript/Python using native functions, conditionals, and loops as flow steps. This is corroborated by GitHub docs describing scripts being composed into flows, and community discussion confirms multi-language script support. Missing for 10: independent hands-on verification beyond community anecdotes and no mention of limitations in language runtime sandboxing raised by a commenter.", "evidenceIds": [ "windmill-docs-1", "windmill-docs-5", "windmill-docs-2", "windmill-gh-1", "windmill-comm-3" ] }, { "productId": "windmill", "storyId": "community-built-components", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "There is no evidence of a marketplace, registry, or community-contributed nodes/integrations/components ecosystem outside the vendor for Windmill—docs describe language support, flows, CLI, git sync, and MCP but nothing about installing third-party/community-built connectors or components. The hub/scripts sharing feature, if any, isn't mentioned in the evidence pack.", "evidenceIds": [] }, { "productId": "windmill", "storyId": "concurrency-rate-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows retries, scheduling, and branching (windmill-docs-13, windmill-docs-18, windmill-docs-12) but contains no mention of concurrency limits, execution throttling, or queuing mechanisms to respect downstream rate limits. Missing for 10: any documentation of concurrency-limit settings, rate-limit-aware queuing, or throttling controls on flows/scripts.", "evidenceIds": [ "windmill-docs-13", "windmill-docs-18" ] }, { "productId": "windmill", "storyId": "conditional-branching-logic", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Windmill's docs explicitly describe flow Branches that split execution based on conditions and a 'branch all' mode that runs all branches in parallel, directly matching conditional and parallel-path branching in a low-code flow builder. Merging paths back together is implied by the flow's linear continuation after a branch block but is not explicitly documented, and 'filters' as a distinct branching primitive isn't named. missing for 10: explicit doc/example of branches merging back into a single downstream step, and confirmation of filter-based branching syntax.", "evidenceIds": [ "windmill-docs-12", "windmill-docs-2", "windmill-comm-8" ] }, { "productId": "windmill", "storyId": "cron-schedules", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Windmill's docs explicitly describe Schedules as a CRON expression tied to a Script or Flow with error/recovery handlers, and schedules are listed among supported trigger types alongside webhooks, HTTP routes, etc. Timezone control is a standard part of Windmill's schedule UI per its core concept documentation, though the evidence pack doesn't explicitly quote a timezone field or independent hands-on confirmation. Missing for 10: explicit citation confirming timezone selector UI, independent/community verification of scheduling reliability.", "evidenceIds": [ "windmill-docs-18", "windmill-docs-15", "windmill-docs-19" ] }, { "productId": "windmill", "storyId": "custom-connector-sdk", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Windmill offers a documented CLI (wmill) for interacting with instances and deploying from Git, multi-language scripting (TypeScript, Python, Go, etc. or custom Docker images), a flows/apps builder, webhooks, triggers, and an MCP server for connecting external clients, all backed by extensive first-party docs. This directly supports building custom integrations/connectors via a developer platform. missing for 10: no public OpenAPI/REST API spec found (probe returned 404s), and no independent third-party SDK usage reports corroborating the CLI/platform beyond vendor docs.", "evidenceIds": [ "windmill-docs-9", "windmill-docs-1", "windmill-docs-4", "windmill-docs-11", "windmill-docs-15", "windmill-docs-17", "windmill-probe-4", "windmill-probe-5", "windmill-probe-3" ] }, { "productId": "windmill", "storyId": "durable-long-running-workflows", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Windmill's docs describe 'workflows as code' with built-in checkpointing, parallelism, and fault tolerance, plus retry/error-handling mechanisms (windmill-docs-5, windmill-docs-13, windmill-docs-19), which imply support for durable, resumable flows. However, there is no explicit documentation or independent confirmation of long-duration (multi-day) sleeps/waits or of state surviving worker/platform restarts specifically. Missing for 10: explicit docs on sleep/suspend semantics for days-long waits, and hands-on/community evidence confirming state survives worker or platform restarts.", "evidenceIds": [ "windmill-docs-5", "windmill-docs-13", "windmill-docs-19", "windmill-docs-18" ] }, { "productId": "windmill", "storyId": "embed-white-label", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of white-labeling capability (custom branding, theming, hiding Windmill identity, or embeddable SDK/iframe for reselling under the developer's own brand) — only self-hosting, app builder, and guest-user features are documented, none of which address white-label embedding for customers.", "evidenceIds": [] }, { "productId": "windmill", "storyId": "environments-source-control", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Windmill documents Git sync where deployments push commits to a repo and can auto-deploy new commits back into workspaces, plus a dedicated CLI (`wmill`) for deploying from a Git repository, enabling version control and promotion across workspaces/environments (dev/staging/prod). This covers source-control-backed versioning and promotion workflows fairly thoroughly, though community feedback questions non-destructive branch validation and local script running pre-merge, indicating some workflow gaps. Missing for 10: independent/hands-on confirmation of a full dev-to-prod promotion pipeline and clearer support for pre-merge branch validation raised by users.", "evidenceIds": [ "windmill-docs-11", "windmill-docs-9", "windmill-docs-4", "windmill-comm-5" ] }, { "productId": "windmill", "storyId": "error-workflows-alerts", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Windmill docs describe dedicated failure-handling mechanisms: retries with configurable delay/attempts (windmill-docs-13), conditional branch execution (windmill-docs-12), custom failure tagging via wm_failure (windmill-docs-19), and explicit 'Error and Recovery Handlers to deal with failed scheduled executions' (windmill-docs-18), which together support building dedicated error paths and being alerted on failure. However, the evidence doesn't detail concrete notification channels (email/Slack/webhook alerts) beyond the handler concept, and error-handler support is documented mainly in the scheduling context rather than as a general flow-level feature; a community question (windmill-comm-8) about error handling/branching is asked but not answered by the evidence pack. missing for 10: explicit documentation of flow-level (not just schedule-level) dedicated error workflows, and concrete notification-channel integration for failure alerts.", "evidenceIds": [ "windmill-docs-13", "windmill-docs-12", "windmill-docs-18", "windmill-docs-19", "windmill-comm-8" ] }, { "productId": "windmill", "storyId": "expressions-data-mapping", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Windmill lets ops users write full scripts in TypeScript, Python, Go, SQL, Bash, etc. to transform data, and 'workflows as code' explicitly supports functions, conditionals, and loops between flow steps, plus branch logic for conditional data flow; the low-code flow builder also composes these steps together for mapping/transformation between steps. Missing for 10: no dedicated documentation on a lightweight expression/formula language for simple inline mappings (e.g., JS expression fields) distinct from full scripts, and no independent/hands-on corroboration of transformation ergonomics.", "evidenceIds": [ "windmill-docs-1", "windmill-docs-2", "windmill-docs-5", "windmill-docs-12", "windmill-gh-1" ] }, { "productId": "windmill", "storyId": "human-approval-steps", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers flow branching, retries, error handling, and scheduling, but contains no mention of a suspend/approval step, human-in-the-loop gate, or 'wait for input' mechanism in Windmill flows. This is a fair axis for a workflow orchestration product, but no evidence in the pack demonstrates it.", "evidenceIds": [ "windmill-docs-12", "windmill-docs-13", "windmill-docs-19" ] }, { "productId": "windmill", "storyId": "inbound-webhook-triggers", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Windmill's docs explicitly list webhooks as a trigger type that starts scripts/flows with the request payload, and even describe version-pinned webhook endpoints for flows, confirming custom webhook URLs can kick off workflow runs. missing for 10: no independent/hands-on confirmation of webhook payload handling or setup walkthrough beyond docs references.", "evidenceIds": [ "windmill-docs-15", "windmill-docs-16" ] }, { "productId": "windmill", "storyId": "local-dev-instance", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Windmill supports self-hosting a dev instance via Docker Compose (windmill-docs-10) and provides a CLI (`wmill`) for interacting with instances and deploying from Git repos (windmill-docs-9, windmill-docs-4, windmill-docs-11), which could support CI workflows against a dev instance. However, there is no explicit documentation of a local/offline execution mode for scripts/flows independent of a running instance, and a community question directly asking 'Can I run my scripts locally without a lot of drama?' (windmill-comm-5) goes unanswered in the evidence, suggesting friction. Missing for 10: explicit docs on local script execution without an instance, CI-specific testing guides, and a direct answer to the community's local-run concern.", "evidenceIds": [ "windmill-docs-9", "windmill-docs-10", "windmill-docs-4", "windmill-docs-11", "windmill-comm-5" ] }, { "productId": "windmill", "storyId": "nl-workflow-generation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "windmill-docs-21 explicitly documents AI-driven editing of scripts/flows/apps followed by a review-and-deploy step, and windmill-docs-20 shows AI agent steps can be embedded in flows, indicating genuine natural-language generation/editing support. However, evidence is purely first-party docs with no independent/hands-on corroboration of prompt-to-workflow quality or reliability. Missing for 10: independent/community validation of AI-generated flow quality, details on scope/limits of NL-to-workflow generation, and confirmation this works for full flow creation (not just editing existing items).", "evidenceIds": [ "windmill-docs-21", "windmill-docs-20" ] }, { "productId": "windmill", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Windmill exposes a CLI (`wmill`) for deployments, webhooks/triggers for running scripts and flows, git-sync for pushing changes, and an MCP server for AI agents to trigger scripts/flows, showing broad programmatic access to core operations. However, there's no explicit documentation or evidence of a comprehensive REST/OpenAPI spec covering all UI actions (e.g., app-builder UI creation, permissions management, variable/secret management) via API, and the openapi probe returned 404s at standard locations. missing for 10: explicit OpenAPI/API reference confirming full UI-API parity, evidence that app-building, permissions, and admin UI actions are all scriptable via API/CLI, independent confirmation of parity.", "evidenceIds": [ "windmill-docs-9", "windmill-docs-11", "windmill-docs-15", "windmill-docs-16", "windmill-docs-17", "windmill-probe-4", "windmill-probe-5", "windmill-probe-3" ] }, { "productId": "windmill", "storyId": "openness-full-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Windmill's CLI (`wmill`) and Git Sync feature let users pull/push all scripts, flows, and apps as plain code (TypeScript/Python/YAML) into a Git repository, giving a genuinely open, portable format for the core workflow logic, and the whole platform is open-source and self-hostable so users are never locked to a vendor. However the evidence pack shows no explicit mechanism for exporting the rest of the instance state — variables, secrets, execution history/logs, users/permissions — as an open-format bundle, only code artifacts.\nmissing for 10: documented full-instance export/backup covering secrets, variables, run history, and permissions, plus independent confirmation that a git-synced workspace can be fully reconstituted elsewhere.", "evidenceIds": [ "windmill-docs-9", "windmill-docs-11", "windmill-docs-5", "windmill-probe-5", "windmill-docs-10" ] }, { "productId": "windmill", "storyId": "openness-open-license", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Windmill is explicitly described as 'open-source and self-hostable' in its own docs/llms.txt, and its source is publicly available on GitHub (windmill-labs/windmill), satisfying the story that an AI-native user can read the source under an open license. missing for 10: no explicit license name (e.g., AGPL/MIT) is cited in the evidence, and no independent confirmation of license terms is provided.", "evidenceIds": [ "windmill-probe-1", "windmill-probe-2", "windmill-gh-1", "windmill-comm-12" ] }, { "productId": "windmill", "storyId": "openness-self-host", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Windmill provides official self-hosting docs with Docker Compose for small setups and Helm/Kubernetes for production, corroborated by independent community reports of successful self-hosting via Caprover and Podman+Docker Compose. Missing for 10: more recent/large-scale production self-host case studies beyond community hobbyist reports.", "evidenceIds": [ "windmill-docs-10", "windmill-comm-1", "windmill-comm-4", "windmill-probe-1" ] }, { "productId": "windmill", "storyId": "prebuilt-connector-breadth", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Windmill offers code-based scripts, flows, and triggers (webhooks, Kafka, Postgres, etc.) but no mention of a prebuilt, vendor-maintained connector/app library comparable to Zapier-style integrations; a community comment even contrasts it with Zapier as a code-first alternative rather than a connector marketplace.", "evidenceIds": [ "windmill-docs-15", "windmill-comm-9", "windmill-comm-3" ] }, { "productId": "windmill", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Windmill is self-hostable via Docker/Docker Compose or Kubernetes Helm chart, which implicitly lets a user control where their data resides by deploying in any region/infrastructure of their choosing. However, there is no explicit documentation of a region-selection feature for Windmill's hosted/cloud offering or any formal data-residency guarantee. Missing for 10: explicit cloud region selection options, formal data residency/compliance statements, and independent confirmation of self-hosted deployments meeting residency requirements.", "evidenceIds": [ "windmill-docs-10", "windmill-comm-1", "windmill-comm-4" ] }, { "productId": "windmill", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses AI-training data usage or opt-out policies for Windmill; the product is a workflow/automation engine and could plausibly publish a data-usage/privacy policy, but none is documented here.", "evidenceIds": [] }, { "productId": "windmill", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers encryption of variables, roles/permissions, and token scoping, but there is no mention of data retention policies, data deletion controls, or export/purge capabilities for an AI-native user's data. No documentation or community evidence addresses this axis.", "evidenceIds": [] }, { "productId": "windmill", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions telemetry, usage tracking, or an opt-out mechanism for Windmill; the docs focus on features, self-hosting, and permissions but never address telemetry data collection. Since Windmill is self-hostable software where telemetry opt-out is a plausible and fair question, absence of evidence yields 'none' rather than 'na'. missing for 10: any mention of telemetry collection, opt-out settings/env vars, or privacy documentation addressing usage tracking.", "evidenceIds": [] }, { "productId": "windmill", "storyId": "run-history-replay", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Windmill's docs describe automatic per-step retries, error handling (wm_failure tagging), and schedule error/recovery handlers, which support reliability around failures, but no evidence explicitly describes an ops user inspecting past execution logs and manually re-running a failed execution resuming from the specific failing step. missing for 10: explicit documentation of log-based re-run/resume-from-step UI or API, and confirmation this is a distinct manual operation rather than automated retry.", "evidenceIds": [ "windmill-docs-13", "windmill-docs-18", "windmill-docs-19" ] }, { "productId": "windmill", "storyId": "subworkflow-reuse", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Windmill's docs and README confirm scripts/functions can be composed together into flows (windmill-gh-1, windmill-docs-2) and workflows-as-code lets you call functions/loops within flows (windmill-docs-5), implying reusable modules, but there is no explicit documentation shown for one flow invoking another flow as a sub-workflow/module across multiple parent workflows. missing for 10: explicit doc/example of a flow calling another flow as a reusable subflow, and independent confirmation of this pattern working in practice.", "evidenceIds": [ "windmill-gh-1", "windmill-docs-2", "windmill-docs-5" ] }, { "productId": "windmill", "storyId": "test-with-sample-data", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes Windmill's flow builder, branching, retries, and error handling, but nothing in the pack documents a test-run mode with sample/pinned data or step-by-step input/output inspection before deployment. A community question (windmill-comm-5) about non-destructive testing/local runs goes unanswered in the evidence. Missing for 10: documentation of a 'test flow' or step-by-step debug/preview feature, evidence of pinning/sample input data per step, and confirmation that outputs of each step can be inspected pre-deployment.", "evidenceIds": [ "windmill-docs-2", "windmill-docs-12", "windmill-docs-13", "windmill-comm-5" ] }, { "productId": "windmill", "storyId": "visual-workflow-editor", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Windmill's docs confirm a visual/low-code flow builder (drag-and-drop orchestration, branches, retries, scheduling) that lets users assemble flows without writing orchestration logic (windmill-docs-2, windmill-docs-12, windmill-docs-13, windmill-docs-18, windmill-comm-8). However, Windmill is fundamentally code-centric — individual flow steps are typically scripts in TypeScript/Python/etc, and the docs repeatedly describe it as 'low-code' rather than no-code, with community feedback confirming it's aimed at developers writing scripts (windmill-docs-1, windmill-comm-3). Missing for 10: evidence of pure no-code step types (e.g. built-in connectors/actions requiring zero code), and hands-on testimony from a non-technical ops user successfully building a flow without touching code.", "evidenceIds": [ "windmill-docs-2", "windmill-docs-12", "windmill-docs-13", "windmill-docs-18", "windmill-docs-1", "windmill-comm-3", "windmill-comm-8" ] }, { "productId": "windmill", "storyId": "workflow-template-library", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a public template library or gallery of pre-built workflows for ops users to start from; docs focus on scripting languages, orchestration, CLI, and roles/permissions but never mention a template marketplace or starter workflow gallery.", "evidenceIds": [] }, { "productId": "windmill", "storyId": "workflows-as-mcp-tools", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Windmill has documented, first-party MCP server support that exposes scripts/flows as tools callable by external MCP-compatible agents (Claude, Cursor, etc.), directly matching the story. missing for 10: independent/hands-on confirmation of the MCP feature working, and more detail on scoping which specific workflows/actions are exposed as tools.", "evidenceIds": [ "windmill-docs-17", "windmill-probe-4" ] }, { "productId": "zapier", "storyId": "agent-creates-workflow-via-api", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Zapier's Developer Platform page mentions a 'Workflow API' that can power AI workflows and integration marketplaces, implying programmatic Zap creation, but the evidence pack has no concrete docs, endpoints, or examples showing an agent creating/updating/activating a Zap via this API — attempts to find an OpenAPI spec returned 404s. The strongest concrete API surface documented is the Platform CLI/SDK (for building app integrations) and MCP (for invoking existing actions), neither of which directly evidences full CRUD+activate control over workflows. Missing for 10: documented Workflow API reference/schema, example of programmatic Zap creation/update/activation, and independent confirmation it works as described.", "evidenceIds": [ "zapier-docs-23", "zapier-docs-24", "zapier-probe-2" ] }, { "productId": "zapier", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A direct probe confirms docs.zapier.com/llms.txt returns HTTP 200 with a structured index for developer documentation (SDK, MCP, integration building), exactly matching the llms.txt agent-discovery pattern, and this is corroborated by docs describing agent-oriented access points (MCP server, CLI, npm skills). Missing for 10: independent/community confirmation of an agent actually consuming llms.txt successfully.", "evidenceIds": [ "zapier-probe-1", "zapier-docs-4", "zapier-docs-11" ] }, { "productId": "zapier", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Zapier shows some AI-generated suggestions tied to user workflow data — AI troubleshooting instructions for Zap errors (docs-10) and Canvas's automatic process mapping/optimization (docs-18), plus Copilot helping build agents (docs-8) — but these are narrow, workflow-centric suggestions rather than broad 'insights from your data' analytics. missing for 10: dedicated data-insights/analytics feature, evidence of AI surfacing patterns or recommendations from connected app data (not just error-fixing or workflow mapping), independent/hands-on corroboration of these AI suggestion features actually working.", "evidenceIds": [ "zapier-docs-10", "zapier-docs-18", "zapier-docs-8" ] }, { "productId": "zapier", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Zapier's core product is trigger-based automations (Zaps) that run autonomously in the background once set up, and this is extended with AI-native features like Zapier MCP and Agents for triggering/running workflows via AI clients. Community evidence confirms real-world autonomous background execution (e.g., a Zap misfiring and sending thousands of invites unattended), corroborating that these run without active user supervision. Missing for 10: independent hands-on verification specifically of AI-triggered autonomous background runs (vs. classic Zaps), and detail on monitoring/error-recovery for unattended agentic runs beyond the troubleshooting-assistant mention.", "evidenceIds": [ "zapier-docs-8", "zapier-docs-24", "zapier-docs-6", "zapier-comm-4", "zapier-docs-10" ] }, { "productId": "zapier", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Zapier ships 'Zapier Agents' and 'Zapier Copilot,' a built-in AI assistant that lets users create agents to perform tasks across 9,000+ apps and even auto-generates troubleshooting help, matching the delegate-to-built-in-assistant story. However, evidence is entirely first-party marketing copy with no independent hands-on verification of how well the assistant performs delegated tasks. Missing for 10: independent/community corroboration of the built-in AI assistant's real-world performance, and more concrete detail on its task scope/limits beyond marketing claims.", "evidenceIds": [ "zapier-docs-8", "zapier-docs-10", "zapier-docs-9", "zapier-docs-18" ] }, { "productId": "zapier", "storyId": "agentic-headless", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Zapier offers a Platform CLI (Node.js) for building/publishing integrations and a Workflow API/Developer Platform, which can be invoked programmatically and scripted in CI for building or deploying integrations, but the core Zap workflow execution itself is primarily triggered by app events/schedules via the visual builder, not designed as a headless batch job runnable in a CI pipeline. There's no documented CLI command or API to 'run a Zap' synchronously in a pipeline the way a CI job would invoke a task. missing for 10: explicit documentation of running/triggering Zap executions headlessly from CI, a CLI/API command to invoke a workflow run synchronously and get results, and independent confirmation of CI-based automation use cases.", "evidenceIds": [ "zapier-docs-3", "zapier-docs-19", "zapier-docs-23", "zapier-probe-4" ] }, { "productId": "zapier", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All evidence describes Zapier acting as an MCP *server* (zapier.com/mcp) that other AI clients (Claude, ChatGPT, Cursor) connect to in order to use Zapier's own tools — not Zapier acting as an MCP *client* that ingests external MCP servers so Zapier itself can call their tools. No documentation shows Zapier importing or connecting to third-party MCP servers as a consumer.", "evidenceIds": [ "zapier-docs-6", "zapier-docs-11", "zapier-docs-15", "zapier-probe-3" ] }, { "productId": "zapier", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Zapier ships an official hosted MCP server (zapier.com/mcp) giving any MCP-compatible client (Claude, ChatGPT, Cursor) governed access to 9,000+ apps/40,000+ actions with SOC2-certified, no-code setup, plus a local MCP server option via the CLI. Missing for 10: independent/hands-on third-party verification of the MCP server's reliability beyond vendor docs.", "evidenceIds": [ "zapier-docs-6", "zapier-docs-11", "zapier-docs-15", "zapier-docs-24", "zapier-probe-3", "zapier-docs-4" ] }, { "productId": "zapier", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Zapier MCP explicitly enables natural-language commands: 'Describe what you need in plain language. Your AI calls the right Zapier action and confirms what it did' and 'Schedule a meeting with one prompt,' letting AI clients trigger 40,000+ actions across 9,000+ apps via governed MCP connections. This is well-documented first-party functionality with a dedicated product page, though independent hands-on validation of the natural-language flow itself is absent. Missing for 10: independent/community corroboration of the MCP natural-language experience specifically (community evidence covers classic Zap-building, not the AI prompt-driven flow).", "evidenceIds": [ "zapier-docs-20", "zapier-docs-7", "zapier-docs-24", "zapier-docs-6", "zapier-docs-15", "zapier-probe-3" ] }, { "productId": "zapier", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Zapier ships an official Platform CLI for building integrations (Node.js, OAuth, webhooks, polling triggers) documented at docs.zapier.com/integrations/build-cli/overview, and docs explicitly note agents can run these as 'a CLI, a typed Node module, or a local MCP server,' tying the CLI directly to AI-agent workflows. Missing for 10: independent/hands-on developer corroboration of CLI usage and more detail on AI-native-specific CLI features beyond general integration building.", "evidenceIds": [ "zapier-docs-3", "zapier-docs-4", "zapier-docs-19", "zapier-probe-4" ] }, { "productId": "zapier", "storyId": "agentic-public-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Zapier documents a Workflow API for building automation experiences (zapier-docs-23) plus a Platform CLI and REST-style integration building (zapier-docs-3, zapier-docs-19), giving AI-native users programmatic ways to drive the product beyond the UI. However, a probe found no discoverable OpenAPI/swagger spec (zapier-probe-2 returned 404s across all candidate paths), suggesting the public API is not machine-discoverable in a standard way, and most current documentation emphasis is on MCP/agent-skill access rather than a general-purpose public API reference. Missing for 10: a published OpenAPI/swagger spec or full public API reference docs, and independent hands-on confirmation of API usage outside MCP/CLI contexts.", "evidenceIds": [ "zapier-docs-23", "zapier-docs-3", "zapier-docs-19", "zapier-probe-2", "zapier-probe-4" ] }, { "productId": "zapier", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Zapier MCP docs mention 'governed access', 'account-level restrictions, managed connections, and workspace controls' applied across agents (zapier-docs-11, zapier-docs-15), implying some control over what an agent can touch, and Zapier explicitly manages OAuth/credentials so users never handle tokens directly (zapier-docs-16). However there is no explicit documentation of issuing distinct, minimal-scope credentials per agent/action (e.g., granular OAuth scopes, per-agent API keys) — missing for 10: explicit per-agent scoped credential/token issuance mechanism, documentation of configurable least-privilege permission sets, and independent confirmation that these restrictions actually enforce least privilege.", "evidenceIds": [ "zapier-docs-11", "zapier-docs-15", "zapier-docs-16", "zapier-docs-24" ] }, { "productId": "zapier", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Zapier documents a typed Node module/CLI SDK (Platform CLI, Node.js v22, OAuth handling) and explicitly positions agents/skills as consuming Zapier as a 'code-file interface' via npm/skill registries, plus a documented MCP server for agent access. However, there's no dedicated 'official SDK' branding beyond the CLI, no OpenAPI spec (probe returned 404s), and no independent/hands-on corroboration of AI-native developers building against these SDKs specifically. missing for 10: independent verification of the AI-native SDK workflow, a published OpenAPI/typed client spec, and community evidence of agents actually building against the SDK rather than just using MCP/no-code paths.", "evidenceIds": [ "zapier-docs-3", "zapier-docs-4", "zapier-docs-13", "zapier-docs-16", "zapier-probe-1", "zapier-probe-2", "zapier-probe-4" ] }, { "productId": "zapier", "storyId": "agentic-webhooks", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Zapier's Platform CLI explicitly supports building webhook triggers (subscribe/unsubscribe patterns) alongside polling triggers, and this is a core, well-documented capability for integration builders including AI-native/agent use cases via CLI, MCP, and Workflow API. Missing for 10: independent hands-on verification of webhook subscription specifically (vs. polling) and no explicit example of an AI agent subscribing to a webhook end-to-end.", "evidenceIds": [ "zapier-docs-3", "zapier-docs-19", "zapier-probe-4", "zapier-docs-23" ] }, { "productId": "zapier", "storyId": "ai-agent-workflow-steps", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Zapier's agents.zapier.com/agents page and Copilot indicate you can create AI agents that connect to business data and perform tasks across thousands of apps, implying LLM+tool-use steps inside workflows, but the evidence never specifies model choice (e.g., selecting GPT vs Claude) or granular tool-use configuration within a Zap step. missing for 10: explicit documentation of selectable LLM/model options inside a workflow step, detailed tool-use/function-calling configuration, and independent hands-on confirmation of the AI-agent-in-workflow experience.", "evidenceIds": [ "zapier-docs-8", "zapier-docs-24", "zapier-docs-10" ] }, { "productId": "zapier", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Zapier has developer docs, CLI docs, and a Workflow API, so an interactive API reference is a fair axis to expect, but the evidence pack shows no such feature—probes explicitly found no OpenAPI/swagger spec (404s across all candidate paths) and no mention of a runnable/interactive API console anywhere in the docs.", "evidenceIds": [ "zapier-probe-2", "zapier-docs-23", "zapier-docs-3" ] }, { "productId": "zapier", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The probe explicitly found all candidate OpenAPI spec paths returning 404, and no evidence pack item shows a downloadable OpenAPI/Swagger spec; Zapier's Workflow API and integration docs are mentioned but no machine-readable spec is provided. Missing for 10: a publicly downloadable OpenAPI/Swagger JSON or YAML file, any documentation page linking to such a spec.", "evidenceIds": [ "zapier-probe-2" ] }, { "productId": "zapier", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions a sandbox environment, staging mode, or test data isolation for Zapier's automation/MCP execution; the docs discuss OAuth, integrations, and MCP connectivity but nothing about testing against non-production data. missing for 10: any documentation of a sandbox/test mode, staging environment, or dry-run capability that avoids touching production data.", "evidenceIds": [] }, { "productId": "zapier", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of API versioning scheme or a documented deprecation policy; the OpenAPI probe returned 404s and no docs mention version numbers, sunset timelines, or breaking-change policy for Zapier's Platform API or Workflow API.", "evidenceIds": [ "zapier-probe-2" ] }, { "productId": "zapier", "storyId": "app-event-triggers", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Zapier's core product is trigger-based workflow automation across 9,000+ connected apps (new record, message, email, form submission triggers), documented extensively via Visual Builder, CLI, and platform docs, and corroborated by community users describing real-world use for webhook/event-based automations. missing for 10: independent hands-on benchmarking of specific trigger types (e.g., form submission) beyond general community anecdotes.", "evidenceIds": [ "zapier-docs-19", "zapier-docs-22", "zapier-docs-9", "zapier-comm-1", "zapier-comm-2", "zapier-comm-6", "zapier-comm-9" ] }, { "productId": "zapier", "storyId": "automatic-step-retries", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions retry policies, backoff configuration, or error-handling settings for failed Zap steps or actions; the only related item is generic AI-generated troubleshooting instructions, which is not the same as configurable retry/backoff.", "evidenceIds": [ "zapier-docs-10" ] }, { "productId": "zapier", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes single-trigger, per-event automation (webhooks, triggers/actions, MCP calls) and even highlights community complaints about execution limits and workflows breaking down at complexity, but nothing documents batch/bulk item processing across many records at once.", "evidenceIds": [ "zapier-comm-3", "zapier-comm-8", "zapier-docs-24" ] }, { "productId": "zapier", "storyId": "automation-rules-engine", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Zapier's core product is trigger-action automation (Zaps): defining rules that fire actions when events occur across 9,000+ apps, with visual builder or CLI, plus AI-native surfaces (MCP, agent skills) that let AI tools invoke these automations. Community evidence corroborates real-world use for exactly this purpose, though some note limits on conditional complexity for advanced logic. Missing for 10: independent benchmarking of reliability/complexity limits and clearer first-party documentation of advanced conditional/multi-step trigger logic.", "evidenceIds": [ "zapier-docs-1", "zapier-docs-9", "zapier-docs-24", "zapier-comm-2", "zapier-comm-3", "zapier-comm-13" ] }, { "productId": "zapier", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Zapier's platform is built around triggers/actions including polling triggers (zapier-docs-3) and Visual Builder/CLI workflow creation (zapier-docs-19, zapier-docs-22), which implicitly support recurring, trigger-based automations, but the evidence pack never shows an AI agent explicitly scheduling or configuring a recurring job through MCP — the MCP examples (zapier-docs-7, zapier-docs-20, zapier-docs-24) describe one-off actions like booking a single meeting or calling an action, not setting up persistent recurring workflows. Missing for 10: explicit documentation of an AI/MCP-driven interface for creating or managing scheduled/recurring Zaps, and independent confirmation that AI agents can configure recurrence intervals rather than just trigger single actions.", "evidenceIds": [ "zapier-docs-3", "zapier-docs-19", "zapier-docs-22", "zapier-docs-7", "zapier-docs-24" ] }, { "productId": "zapier", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers building integrations, MCP/agent access, embedding, and troubleshooting suggestions, but contains no mention of versioning Zaps, reviewing change history, or rolling back automations to a prior state.", "evidenceIds": [] }, { "productId": "zapier", "storyId": "central-credentials-management", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Zapier docs confirm centralized OAuth/credential handling and reuse of existing 'connections' across the org, plus 'managed connections' and 'workspace controls' governing access for MCP/agent use (zapier-docs-15, zapier-docs-12, zapier-docs-16). However, there's no detailed documentation of granular per-credential sharing/permission assignment (e.g., which specific team members can use which specific credential) beyond high-level 'account-level restrictions'. Missing for 10: explicit admin UI/workflow for assigning credential-level permissions, role-based access control details, and independent corroboration of these governance claims.", "evidenceIds": [ "zapier-docs-15", "zapier-docs-12", "zapier-docs-16", "zapier-docs-11" ] }, { "productId": "zapier", "storyId": "code-steps-in-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack never documents a native 'Code by Zapier' step allowing inline JavaScript/Python inside a Zap; the closest content is about the CLI for building whole integrations (zapier-docs-3, zapier-docs-19), not an in-workflow code step. A community comment even complains about the lack of an easy way to 'write some if or just 2 lines of code' inside Zaps (zapier-comm-11, zapier-comm-13), reinforcing that this specific capability isn't evidenced.", "evidenceIds": [ "zapier-docs-19", "zapier-docs-3", "zapier-comm-11", "zapier-comm-13" ] }, { "productId": "zapier", "storyId": "community-built-components", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Zapier's Platform CLI lets third-party developers build integrations and publish them to the App Directory (zapier-docs-1, zapier-docs-3, zapier-docs-19), and docs mention agents discovering Zapier-built skills via npm/skill registries outside Zapier's direct distribution (zapier-docs-4). But these are vendor-reviewed App Directory listings, not an open community-contribution marketplace akin to freely shared nodes; there's no evidence of a permissionless community repository or independent developer reports of installing unofficial/community nodes. Missing for 10: evidence of an open, unreviewed community marketplace, independent confirmation of installing non-vendor-approved integrations, and details on moderation/openness of the contribution process.", "evidenceIds": [ "zapier-docs-1", "zapier-docs-3", "zapier-docs-4", "zapier-docs-19", "zapier-docs-22" ] }, { "productId": "zapier", "storyId": "concurrency-rate-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions throttling, rate-limit handling, queuing, or delay/concurrency controls for workflow executions; docs focus on building integrations, MCP, embedding, and app connections. Community threads even highlight breakage/complexity issues but nothing about rate-limit throttling or queue controls.", "evidenceIds": [] }, { "productId": "zapier", "storyId": "conditional-branching-logic", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Community evidence indicates Zapier supports more than one simultaneous IF/THEN condition compared to IFTTT (zapier-comm-8), implying some conditional/branching capability, but no documentation in the pack describes explicit 'Paths', filters, or parallel-path merge-back features, and users explicitly flag 'the need for escape hatches/conditional logic when needed' and frustration at lacking simple 'if' logic (zapier-comm-11, zapier-comm-13) as ongoing gaps. Missing for 10: first-party docs on Paths/filter steps, evidence of parallel branch execution, and confirmation that branches can merge back into a single continuing workflow.", "evidenceIds": [ "zapier-comm-8", "zapier-comm-11", "zapier-comm-13", "zapier-docs-19", "zapier-docs-22" ] }, { "productId": "zapier", "storyId": "cron-schedules", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses Zapier's Schedule trigger, cron-style scheduling, or timezone configuration for workflows; all citations focus on integrations, MCP, CLI, and general automation marketing rather than scheduling controls.", "evidenceIds": [] }, { "productId": "zapier", "storyId": "custom-connector-sdk", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Zapier documents a full Platform CLI (Node.js, OAuth1/2, webhooks, polling triggers, HTTP middleware) plus a Visual Builder for building and publishing custom integrations to the App Directory, and an embedded/developer platform API for private integrations. Independent probe confirms the CLI docs page and developer llms.txt index exist, corroborating first-party docs. Missing for 10: independent third-party hands-on account of building a CLI integration end-to-end (community evidence covers using Zaps, not building CLI-based custom connectors).", "evidenceIds": [ "zapier-docs-3", "zapier-docs-19", "zapier-docs-22", "zapier-docs-1", "zapier-docs-23", "zapier-probe-4", "zapier-probe-1" ] }, { "productId": "zapier", "storyId": "durable-long-running-workflows", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no documentation or reports about Zapier's workflow execution durability, delay/wait steps surviving restarts, or state persistence guarantees; community comments even suggest complex workflows 'break down' and that Zapier is 'too high level' compared to durable-execution tools like Temporal, but no concrete confirmation or denial of long-lived, restart-surviving workflows is present.", "evidenceIds": [ "zapier-comm-14", "zapier-comm-3" ] }, { "productId": "zapier", "storyId": "embed-white-label", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Zapier's Developer Platform explicitly supports white-label embedding via its Workflow API and Embed offerings, letting customers run automations/workflows under the host product's own brand without leaving the app. missing for 10: independent/hands-on case studies of a real embedded white-label deployment, and pricing/setup details for the embed program.", "evidenceIds": [ "zapier-docs-5", "zapier-docs-14", "zapier-docs-17", "zapier-docs-21", "zapier-docs-23" ] }, { "productId": "zapier", "storyId": "environments-source-control", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers building integrations via CLI/Visual Builder and embedding/MCP access, but nothing describes version control integration, environments (dev/staging/prod), or promotion/deployment pipelines for Zap workflows. No mention of git-based versioning, environment variables per stage, or a promote-to-production workflow mechanism.", "evidenceIds": [] }, { "productId": "zapier", "storyId": "error-workflows-alerts", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "There is a documented troubleshooting help article (AI-generated troubleshooting instructions) for Zap errors, but no evidence of dedicated error-handling paths/branches or configurable failure notifications within a Zap workflow. missing for 10: documentation of built-in error-path branching, retry/fallback steps, and explicit failure-notification configuration (e.g., email/Slack alert on Zap failure) beyond generic troubleshooting help.", "evidenceIds": [ "zapier-docs-10" ] }, { "productId": "zapier", "storyId": "expressions-data-mapping", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Zapier's docs gesture at building and mapping data through the Visual Builder or CLI (zapier-docs-19, zapier-docs-22) and Canvas process mapping (zapier-docs-9, zapier-docs-18), but none of the evidence explicitly documents formula/expression/function-based field mapping between steps. Community evidence directly contradicts smooth delivery of this capability: an engineer using Zapier reports being 'frustrated by the lack of an option to just write some if or just 2 lines of code' and another user 'resonates a lot with the gaps... around the need for escape hatches/conditional logic' (zapier-comm-11, zapier-comm-13), and a separate user notes complex logic 'starts to break down' (zapier-comm-3). Missing for 10: explicit vendor documentation of Formatter/Code-by-Zapier style expression/function steps and independent confirmation they work well for complex transformations.", "evidenceIds": [ "zapier-docs-19", "zapier-docs-22", "zapier-docs-9", "zapier-docs-18", "zapier-comm-11", "zapier-comm-13", "zapier-comm-3" ] }, { "productId": "zapier", "storyId": "human-approval-steps", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions Zapier's built-in 'Human in the Loop'/approval step, delay/pause-for-input actions, or any pause-and-resume-for-approval workflow feature; the pack covers integrations, MCP, canvas, and platform building only.", "evidenceIds": [] }, { "productId": "zapier", "storyId": "inbound-webhook-triggers", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Zapier natively supports the 'Webhooks by Zapier' trigger (documented as part of Platform CLI's webhook trigger support in zapier-docs-3, and the built-in Visual Builder path in zapier-docs-19/22) allowing a custom URL to receive HTTP POSTs and start a Zap run with the payload — this is a long-standing, widely used core Zapier capability. missing for 10: no explicit first-party doc snippet showing the raw webhook URL creation flow or payload-to-trigger-data mapping example, and no independent hands-on confirmation in this pack beyond general webhook-ease commentary in zapier-comm-1.", "evidenceIds": [ "zapier-docs-3", "zapier-docs-19", "zapier-docs-22", "zapier-comm-1" ] }, { "productId": "zapier", "storyId": "local-dev-instance", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only cloud-hosted Zap execution and a CLI for building integration triggers/actions (with local scaffolding), but nothing about running actual Zap workflows locally or against a dedicated dev/staging instance for CI testing. Missing for 10: any documented local workflow execution mode, sandbox/dev environment for Zaps, or CI integration testing tooling.", "evidenceIds": [ "zapier-docs-3", "zapier-docs-19", "zapier-probe-4" ] }, { "productId": "zapier", "storyId": "nl-workflow-generation", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Zapier documents 'Zapier Copilot' as an AI assistant to help create agents/workflows and mentions AI-generated troubleshooting instructions for Zaps, indicating some natural-language workflow creation/editing capability, but the evidence pack lacks concrete detail on how Copilot actually builds or edits a full multi-step Zap from a prompt, and there's no independent/hands-on corroboration of this specific capability. missing for 10: detailed docs/demo of Copilot generating or editing a full Zap workflow, independent user reports confirming it works as described.", "evidenceIds": [ "zapier-docs-8", "zapier-docs-10", "zapier-docs-20" ] }, { "productId": "zapier", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Zapier documents a Workflow API, Platform CLI, and MCP server that let external/agent callers create integrations and trigger actions across 9,000+ apps, suggesting substantial programmatic parity with the UI. However, the probe found no public OpenAPI spec (all candidate URLs 404), and there is no evidence that all UI capabilities (e.g., full Zap editing, Canvas, troubleshooting AI) are exposed via API, so full UI/API parity is unconfirmed. Missing for 10: a documented OpenAPI/complete API reference, explicit statement of full CRUD parity for Zap management via API, and independent confirmation that all UI features are API-accessible.", "evidenceIds": [ "zapier-docs-23", "zapier-docs-3", "zapier-docs-19", "zapier-probe-2", "zapier-docs-24" ] }, { "productId": "zapier", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of any data export feature (e.g., exporting Zaps, workflow configs, or automation history in open/portable formats) that would let a user leave the platform with their data intact; evidence focuses on integrations, MCP, and building automations, not on data portability/export.", "evidenceIds": [] }, { "productId": "zapier", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Zapier is a closed-source, proprietary SaaS platform; no evidence of any open-license source availability (only CLI/SDK docs for building integrations, not the product's own source). This is an applicable axis for judging openness but no evidence supports it, so verdict is none.", "evidenceIds": [] }, { "productId": "zapier", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "zapier", "storyId": "prebuilt-connector-breadth", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Zapier provides thousands of prebuilt, vendor-maintained app integrations (9,000+ apps cited repeatedly) accessible via Visual Builder, MCP, and API, corroborated by independent community reports confirming the breadth (~1,000+) and ease of use of its integration catalog compared to competitors. Missing for 10: no independent third-party audit of the full current app count or comprehensive vendor-maintenance quality across all 9,000+ integrations.", "evidenceIds": [ "zapier-docs-2", "zapier-docs-6", "zapier-docs-12", "zapier-docs-24", "zapier-comm-9", "zapier-comm-2" ] }, { "productId": "zapier", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions data residency, region selection, or storage location controls for Zapier's automation or MCP data; SOC 2 certification is mentioned but that addresses security compliance, not residency choice.", "evidenceIds": [] }, { "productId": "zapier", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item addresses AI training data usage, opt-out controls, or data privacy policies regarding model training; only SOC 2 certification is mentioned in passing without connection to training data opt-out.", "evidenceIds": [] }, { "productId": "zapier", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data retention policies, deletion controls, or data lifecycle management for AI-native users; the pack only covers integrations, MCP connectivity, and automation features. Mentions of SOC 2 certification touch security posture but not retention/deletion controls specifically.", "evidenceIds": [] }, { "productId": "zapier", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry/usage-tracking opt-out settings for Zapier; nothing about privacy controls, data collection preferences, or analytics opt-out is documented. missing for 10: any documentation of telemetry collection, an opt-out toggle/setting, or a privacy policy statement addressing usage tracking controls.", "evidenceIds": [] }, { "productId": "zapier", "storyId": "run-history-replay", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack only mentions AI-generated troubleshooting instructions for Zap errors, with no documentation of inspecting execution logs or re-running/resuming a failed multi-step Zap from the point of failure.", "evidenceIds": [ "zapier-docs-10" ] }, { "productId": "zapier", "storyId": "subworkflow-reuse", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack describes reusable sub-workflows, modules, or the ability for one Zap/workflow to call another as a nested unit; Canvas and Workflow API references cover visual mapping and API access, not composable sub-workflows. missing for 10: documentation of a 'Sub-Zap'/nested-workflow or reusable module feature, evidence of one workflow invoking another as a callable unit, and any hands-on confirmation of this pattern.", "evidenceIds": [] }, { "productId": "zapier", "storyId": "test-with-sample-data", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items describe testing a Zap with sample/pinned data or inspecting per-step input/output before publishing; docs cover integration building, MCP, embedding, and Canvas but not the test/debug workflow in the visual builder. missing for 10: mention of 'test step' or 'pinned sample data' feature, evidence of step-by-step input/output inspection during Zap creation, any documentation of the testing UI.", "evidenceIds": [] }, { "productId": "zapier", "storyId": "visual-workflow-editor", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Zapier's Visual Builder is documented as a no-code interface for building multi-step Zap workflows, explicitly positioned for non-developers ('quick deployment and collaboration across teams, even without developer expertise'), and community evidence corroborates non-technical/ops users successfully building workflows themselves. Missing for 10: independent hands-on review specifically walking through multi-step visual editing (community evidence focuses more on general ease-of-use and pricing than a detailed multi-step build).", "evidenceIds": [ "zapier-docs-19", "zapier-docs-22", "zapier-comm-2", "zapier-comm-6" ] }, { "productId": "zapier", "storyId": "workflow-template-library", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers building integrations, MCP, embedding, and Canvas, but nowhere mentions a public gallery/library of pre-built workflow templates that ops users can start from instead of building from scratch. Without any documentation or community mention of a templates library, this applicable ecosystem capability is unevidenced.", "evidenceIds": [] }, { "productId": "zapier", "storyId": "workflows-as-mcp-tools", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Zapier ships an official MCP server (zapier.com/mcp) that exposes connected app actions across 9,000+ apps as callable MCP tools to any MCP-compatible client (Claude, ChatGPT, Cursor), with governed/managed access and account-level controls documented for IT admins. This directly matches the story of exposing workflow/app actions as MCP tools for external agents. missing for 10: independent hands-on verification of the MCP server's reliability/tool-call behavior beyond first-party docs.", "evidenceIds": [ "zapier-docs-6", "zapier-docs-11", "zapier-docs-15", "zapier-docs-24", "zapier-probe-3" ] } ]