[ { "productId": "bruno", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Direct probe evidence confirms Bruno docs serve a working llms.txt at docs.usebruno.com/llms.txt with structured links to docs pages, which is exactly the agent-oriented docs entry point the story asks for. Missing for 10: no independent/community confirmation of agents actually consuming it, and no broader agent-specific documentation format beyond the single llms.txt file.", "evidenceIds": [ "bruno-probe-1" ] }, { "productId": "bruno", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Bruno's docs describe multiple in-product AI features: ghost-text script suggestions, an AI icon that generates scripts from descriptions, an AI chat sidebar for workspace-wide questions, and AI generation of collections from backend code or full apps — all clearly 'AI-generated insights/suggestions inside the product'. However, all evidence is first-party documentation with no independent/hands-on corroboration of quality or reliability, and it's a BYOK add-on rather than a fully vetted core capability. Missing for 10: independent/community validation of AI feature quality, and evidence of AI 'insights on data' (e.g., response analysis) beyond scripting/chat/generation use cases.", "evidenceIds": [ "bruno-docs-37", "bruno-docs-38", "bruno-docs-39", "bruno-docs-40", "bruno-docs-41", "bruno-docs-42" ] }, { "productId": "bruno", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Bruno's CLI and CI/CD integration (bruno-docs-1,3,18,30) let you run collections unattended in pipelines like GitHub Actions or Docker, which is a form of background automation, but this is traditional scripted/CI automation rather than an AI-native agent autonomously deciding and scheduling actions. Bruno AI features (bruno-docs-37 to 42) are interactive, BYOK assistants for generating scripts/collections, not autonomous background agents. Missing for 10: evidence of scheduled/triggered autonomous runs initiated by AI, agent-driven decision loops, or any 'set it and forget it' automation beyond manual CI/CD wiring.", "evidenceIds": [ "bruno-docs-1", "bruno-docs-3", "bruno-docs-18", "bruno-docs-30", "bruno-docs-37", "bruno-docs-38", "bruno-docs-39" ] }, { "productId": "bruno", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bruno AI offers in-app AI assistance: script generation, ghost-text suggestions, an AI chat sidebar, and generating collections/apps from context, all documented first-party (bruno-docs-37 to 42). However this is BYOK-only, largely scoped to scripts/collections/UI generation rather than broad task delegation, and there is no independent/hands-on corroboration of its effectiveness. missing for 10: independent verification of AI assistant quality, evidence of broader agentic task delegation beyond scripting/UI generation, and confirmation this isn't a limited/beta feature.", "evidenceIds": [ "bruno-docs-37", "bruno-docs-38", "bruno-docs-39", "bruno-docs-40", "bruno-docs-41", "bruno-docs-42" ] }, { "productId": "bruno", "storyId": "agentic-headless", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Bruno CLI is well-documented for running collections headlessly with JSON/JUnit/HTML reports, explicit CI/CD integration, official Docker images, and GitHub Actions support — directly enabling headless/CI automation. Missing for 10: independent hands-on CI pipeline reports beyond vendor docs.", "evidenceIds": [ "bruno-docs-1", "bruno-docs-2", "bruno-docs-3", "bruno-docs-18", "bruno-docs-30", "bruno-probe-3" ] }, { "productId": "bruno", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Bruno's docs describe AI features (BYOK script generation, AI chat sidebar) but no evidence anywhere mentions MCP server support or plugging in MCP tools for Bruno to use.", "evidenceIds": [] }, { "productId": "bruno", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Bruno is an API client/CLI product where an official MCP server is a plausible feature (e.g., for AI agents to run collections), but no evidence pack item mentions an MCP server, MCP protocol, or agent-connection endpoint — only AI-assisted scripting/chat features (BYOK) are documented, which is a different capability.", "evidenceIds": [ "bruno-docs-37", "bruno-docs-38", "bruno-docs-39", "bruno-docs-40" ] }, { "productId": "bruno", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bruno AI lets users describe what they need in natural language to generate scripts, chat with an AI sidebar, and generate full collections/apps from descriptions, which is genuine natural-language operation of parts of the product. However this is scoped to scripts/collections/app generation rather than full natural-language control of all core workflows (running requests, git operations, environment management still require GUI/CLI, not NL commands). missing for 10: evidence of natural-language control over core request execution/CLI operations, independent hands-on validation of Bruno AI's NL features, and confirmation this isn't limited to a paid/BYOK tier.", "evidenceIds": [ "bruno-docs-37", "bruno-docs-38", "bruno-docs-40", "bruno-docs-41", "bruno-docs-42", "bruno-docs-39" ] }, { "productId": "bruno", "storyId": "agentic-official-cli", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Bruno ships an official CLI (`bru`) documented for running collections via simple commands, generating JSON/JUnit/HTML reports, and integrating into CI/CD and Docker pipelines — all features an AI agent or automated workflow could invoke non-interactively. Missing for 10: explicit documentation or examples of AI agents/automation frameworks directly invoking the CLI, and independent hands-on validation of its scripting robustness.", "evidenceIds": [ "bruno-docs-1", "bruno-docs-2", "bruno-docs-3", "bruno-docs-18", "bruno-docs-30", "bruno-probe-3" ] }, { "productId": "bruno", "storyId": "agentic-public-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Bruno ships a well-documented CLI (bru) that lets automation/AI agents run collections, generate reports, and integrate into CI/CD pipelines, and there is a docs llms.txt for AI consumption, but there is no true public HTTP/REST API or OpenAPI spec (probe confirms 404 on all openapi endpoints) that an AI agent could call directly. missing for 10: a documented HTTP/REST API or SDK beyond the CLI, official OpenAPI/swagger spec, and evidence of programmatic access outside the CLI wrapper.", "evidenceIds": [ "bruno-docs-1", "bruno-docs-3", "bruno-docs-18", "bruno-docs-30", "bruno-probe-2", "bruno-probe-3", "bruno-probe-1" ] }, { "productId": "bruno", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Bruno documents secret management (vaults, GCP/Azure Key Vault integration) and a BYOK model for its own AI assistant, but there is no evidence of a feature to mint or scope least-privilege API credentials/tokens specifically for an external AI agent's use. missing for 10: any credential-issuance or scoping mechanism aimed at agents, documentation of granular permission scopes, or evidence of restricting an agent's API access via generated tokens.", "evidenceIds": [ "bruno-docs-33", "bruno-docs-34", "bruno-docs-39" ] }, { "productId": "bruno", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Bruno offers a CLI (bru-cli), Docker images, and a Bru markup format, but no official SDKs (e.g., client libraries in Python/JS/Go) for programmatic integration are documented anywhere in the pack.", "evidenceIds": [] }, { "productId": "bruno", "storyId": "agentic-webhooks", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Bruno is an API client/testing tool for making requests, not an event-emitting service; subscribing to webhooks from Bruno is a category mismatch as there's no evidence of Bruno exposing webhook/event subscription endpoints.", "evidenceIds": [] }, { "productId": "bruno", "storyId": "ai-generate-tests-from-api", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Bruno AI can generate test/assertion scripts via the AI icon in a request's Scripts tab (bruno-docs-38) and can auto-generate whole collections—including tests—from backend source code (bruno-docs-41), which covers 'from spec' generation. However, there's no explicit documentation of generating assertions directly from example responses, and no independent/hands-on confirmation of this AI feature's real-world reliability. Missing for 10: evidence of generating tests specifically from response bodies/OpenAPI specs, and independent verification of output quality.", "evidenceIds": [ "bruno-docs-38", "bruno-docs-41", "bruno-docs-37", "bruno-docs-39" ] }, { "productId": "bruno", "storyId": "ai-llm-traffic-gateway", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Bruno is an API client/collection tool for testing REST/GraphQL APIs; it is not an LLM/MCP gateway or proxy product. Nothing in the evidence relates to proxying or governing LLM/MCP traffic, token limits, prompt guards, or provider routing — this is a category mismatch, not a missing feature.", "evidenceIds": [] }, { "productId": "bruno", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Bruno lets users write documentation alongside their collections and publish it as shareable HTML, and requests/collections are inherently runnable examples (bruno-docs-20, bruno-docs-27), which is the closest match to an 'interactive reference with runnable examples.' However, there is no evidence of an actual interactive API explorer (e.g., OpenAPI/Swagger-style try-it-out UI) — a probe for OpenAPI/Swagger endpoints on Bruno's own docs returned 404s (bruno-probe-2), and no community or hands-on evidence confirms the published HTML docs are truly interactive/executable rather than static. Missing for 10: evidence of a live interactive reference UI, independent confirmation that published docs are runnable (not just readable), and AI-specific tooling around this reference.", "evidenceIds": [ "bruno-docs-20", "bruno-docs-27", "bruno-probe-2" ] }, { "productId": "bruno", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Bruno is a desktop/CLI API client with no public web API of its own, but the axis is still fair to ask (e.g., an OpenAPI export/spec for AI agents to consume). Direct probing found no OpenAPI/Swagger spec (all candidate paths 404) — only an llms.txt docs index exists, which is not a machine-readable API spec. Missing for 10: any OpenAPI/Swagger JSON or equivalent structured spec, first-party docs describing one, and community confirmation of its existence.", "evidenceIds": [ "bruno-probe-2", "bruno-probe-1" ] }, { "productId": "bruno", "storyId": "api-sandbox", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Bruno supports environment variables, global env overrides, and mock responses, which let users point requests at different (e.g. sandbox) environments and separate them from production, and its offline-first, no-cloud-sync design (bruno-docs-8, bruno-docs-9) keeps data local rather than sent to a vendor. However, there is no explicit documentation of a dedicated 'sandbox mode' or built-in safeguards preventing accidental production calls, and no AI-agent-specific sandbox testing workflow is described. Missing for 10: explicit sandbox/staging environment isolation feature, guardrails against hitting production endpoints, and AI-agent-specific evidence of sandboxed test execution.", "evidenceIds": [ "bruno-docs-4", "bruno-docs-8", "bruno-docs-9", "bruno-docs-11" ] }, { "productId": "bruno", "storyId": "api-traffic-observability", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Bruno is an API client/testing tool for building and running requests, not an API gateway or traffic-serving infrastructure; there is no evidence of it proxying or handling live API traffic that could be monitored with metrics/logging/tracing. This gateway-traffic observability story is a category mismatch for this product type.", "evidenceIds": [] }, { "productId": "bruno", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Bruno ships versioned releases/changelogs (e.g., CLI v4.1.0, release notes) but nothing documents an explicit API/CLI deprecation policy or versioning guarantees that an AI-native user could rely on.", "evidenceIds": [] }, { "productId": "bruno", "storyId": "automation-bulk-operations", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Bruno CLI lets you run entire collections (many requests) from the command line with data-driven testing, generate consolidated reports, and integrate into CI/CD pipelines or Docker/GitHub Actions, which is a clear bulk-automation workflow across many API items at once. Missing for 10: independent/hands-on evidence of bulk operations at scale (e.g., large collection performance) and explicit documentation of bulk editing/manipulation of items beyond running them.", "evidenceIds": [ "bruno-docs-1", "bruno-docs-2", "bruno-docs-3", "bruno-docs-18", "bruno-docs-30", "bruno-docs-32", "bruno-probe-3" ] }, { "productId": "bruno", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Bruno supports pre-request and post-response JavaScript that runs automatically on those request lifecycle events, and CI/CD integration lets test suites run automatically on pipeline events, but there is no general-purpose rule/trigger engine for arbitrary events. missing for 10: a configurable rules/event system beyond request/response scripts, support for custom triggers or webhooks, and any evidence of broader automation orchestration.", "evidenceIds": [ "bruno-docs-31", "bruno-docs-32", "bruno-docs-3", "bruno-docs-30" ] }, { "productId": "bruno", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Bruno CLI can be run inside CI/CD pipelines (GitHub Actions, Docker) which could be scheduled externally via cron/CI triggers, but there is no evidence of a native scheduler or built-in recurring-job feature within Bruno itself. missing for 10: any built-in scheduling UI/feature, documentation of cron-like triggers within Bruno, independent confirmation of scheduled runs.", "evidenceIds": [ "bruno-docs-3", "bruno-docs-18", "bruno-docs-30" ] }, { "productId": "bruno", "storyId": "automation-versioned-workflows", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Bruno stores collections/scripts as plain-text files and ships a native Git UI (init, diff, pull, clone, commit) letting users version and review changes to their API automations like code, with community confirmation of the git-collaboration workflow. Explicit rollback is implied via standard git operations (checkout/revert) rather than a dedicated in-app 'rollback' feature. Missing for 10: explicit documented rollback/undo UI beyond generic git commands, independent evidence of reviewing/rolling back script changes specifically.", "evidenceIds": [ "bruno-docs-5", "bruno-docs-6", "bruno-docs-23", "bruno-docs-29", "bruno-docs-46", "bruno-docs-9", "bruno-comm-2", "bruno-comm-7" ] }, { "productId": "bruno", "storyId": "collections-as-files-in-git", "verdict": "full", "quality": 10, "confidence": "high", "rationale": "Bruno stores collections as plain-text .bru files on the filesystem, explicitly designed for git versioning, with native Git UI support and CLI/terminal workflows; this is strongly corroborated by first-party docs and independent community testimonials confirming real-world use.", "evidenceIds": [ "bruno-docs-9", "bruno-docs-28", "bruno-docs-46", "bruno-docs-5", "bruno-docs-23", "bruno-docs-29", "bruno-comm-2", "bruno-comm-7", "bruno-docs-14" ] }, { "productId": "bruno", "storyId": "contract-validation", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Bruno documents general test assertions, data-driven test suites, and CI/CD integration for automated runs (bruno-docs-32, bruno-docs-3, bruno-docs-30), which teams could use to assert response shape/status manually, but there is no evidence of a dedicated schema/contract validation feature (e.g., importing/validating against an OpenAPI spec) to explicitly catch API drift. Missing for 10: explicit OpenAPI/contract schema validation capability, documented drift-detection workflow, and any first-party or community evidence of contract testing use.", "evidenceIds": [ "bruno-docs-32", "bruno-docs-3", "bruno-docs-30", "bruno-docs-22" ] }, { "productId": "bruno", "storyId": "declarative-config-gitops", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Bruno is an API client/testing tool (collections, requests, CLI) not an API gateway or infrastructure control-plane; it has no concept of managing routes, policies, or gateway configuration as declarative infra. The 'API infrastructure' GitOps/APIOps axis targeted at gateway-traffic products doesn't apply to this product category.", "evidenceIds": [] }, { "productId": "bruno", "storyId": "design-export-openapi", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence pack shows Bruno's collections are stored as plain-text .bru files, supports Postman import/export, CLI runs, and Git integration—but nothing about spec-first design workflows or generating/exporting OpenAPI specifications. The probe explicitly found no OpenAPI schema/documentation endpoints exposed.", "evidenceIds": [ "bruno-docs-9", "bruno-docs-24", "bruno-docs-28", "bruno-probe-2" ] }, { "productId": "bruno", "storyId": "environment-variable-sets", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm Bruno has an environment selector with search filtering, drag-and-drop variable management, and CLI overrides via --global-env-var, all indicating a working environment/variable-set system for switching contexts. However, no evidence explicitly documents a dev/staging/prod workflow or environment creation/management UI in depth. Missing for 10: explicit documentation of creating and switching between multiple named environments (e.g. dev/staging/prod), and independent hands-on confirmation of environment-switching UX.", "evidenceIds": [ "bruno-docs-11", "bruno-docs-13", "bruno-docs-16" ] }, { "productId": "bruno", "storyId": "gateway-auth-plugins", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Bruno is an API client/testing tool for developers to make and test API requests; it has no gateway/proxy or policy-enforcement layer to sit in front of upstream APIs. Enforcing authentication on upstream traffic via gateway plugins is a category error for this product type — Bruno's auth support (OAuth2, mTLS certs) is client-side for calling APIs, not for protecting them.", "evidenceIds": [] }, { "productId": "bruno", "storyId": "gateway-rate-limiting", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Bruno is an API client/testing tool, not an API gateway or traffic-management product; enforcing rate limits per consumer/route is a gateway-traffic concern outside Bruno's category of collection-based request building and testing.", "evidenceIds": [] }, { "productId": "bruno", "storyId": "generate-publish-api-docs", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bruno docs explicitly state you can 'Write docs with your collections; publish as shareable HTML' plus a rich text docs editor, directly matching the story of generating/publishing API documentation from collections. However this is a single doc-feature mention with no independent/community corroboration, no detail on hosting/sharing workflow, and no support for generating docs from external specs (e.g. OpenAPI import/export to docs). missing for 10: independent verification of the publish-as-HTML feature, detail on doc customization/publishing workflow, and spec-based (OpenAPI) doc generation.", "evidenceIds": [ "bruno-docs-20", "bruno-docs-12" ] }, { "productId": "bruno", "storyId": "import-export-collections", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Bruno documents importing existing Postman collections and even a 'preserve scripts' option for Postman import/export, plus its own portable plain-text .bru format that lives on the filesystem for easy sharing/versioning. However, evidence only covers Postman interoperability and lacks documentation of exporting to other formats (e.g., OpenAPI, Insomnia) or independent confirmation of import fidelity. Missing for 10: explicit export-to-Postman/OpenAPI support, broader multi-tool interoperability, and hands-on community verification of import/export accuracy.", "evidenceIds": [ "bruno-docs-24", "bruno-docs-7", "bruno-docs-25", "bruno-docs-9", "bruno-docs-28", "bruno-docs-46" ] }, { "productId": "bruno", "storyId": "import-openapi-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents Postman collection import (bruno-docs-24, bruno-docs-25, bruno-docs-7) but contains no mention of importing OpenAPI/Swagger specs to generate requests or collections. This is a fair axis for an API client tool, but no supporting evidence exists in the pack.", "evidenceIds": [] }, { "productId": "bruno", "storyId": "internal-api-catalog", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Bruno lets teams write and publish collection docs as shareable HTML and embed 'open in Bruno' buttons in READMEs, and its Git-native storage enables shared, discoverable collections across a repo/monorepo — a lightweight substitute for a catalog. However there is no dedicated internal developer-portal, no searchable catalog UI, and no cloud-hosted discovery surface (explicitly no cloud sync/account). missing for 10: a true portal/catalog UI, cross-team search/discovery beyond Git repos, hosted publishing beyond static HTML docs.", "evidenceIds": [ "bruno-docs-20", "bruno-docs-27", "bruno-docs-5", "bruno-docs-19", "bruno-docs-8" ] }, { "productId": "bruno", "storyId": "lint-specs-style-rules", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Bruno is an API client/collection tool focused on request execution, testing, and collaboration; it has no evidence of a spec-linting or governance-rules engine (e.g. Spectral-like style guides) for API specs, which is a distinct product category (API design/governance linter). This axis targets spec-design linting tools, not API clients.", "evidenceIds": [] }, { "productId": "bruno", "storyId": "mock-servers-from-examples", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Bruno's marketing site claims it can 'serve realistic mock responses while you build' (bruno-docs-4), which directly supports a mock-server capability, but there is no further documentation on how mocks are configured (from saved examples, specs, or otherwise), no CLI/spec-based generation details, and no community corroboration of this feature in practice. missing for 10: detailed docs on mock configuration/spec-driven generation, independent/hands-on validation of the mocking feature, examples of using saved request examples as mock responses.", "evidenceIds": [ "bruno-docs-4" ] }, { "productId": "bruno", "storyId": "multi-protocol-requests", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No documentation in the evidence pack mentions GraphQL, gRPC, WebSocket, or SSE support; community feedback explicitly states Bruno lacks GraphQL, gRPC, and WebSocket support ('Only missing Websockets functionality', 'Doesn't seem to support advanced stuff like GraphQL, gRPC, WebSockets').", "evidenceIds": [ "bruno-comm-1", "bruno-comm-4" ] }, { "productId": "bruno", "storyId": "offline-local-workspace", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Bruno explicitly markets itself as offline-only with no account/cloud sync required, storing collections as plain text on the local filesystem, corroborated by community users praising the offline-first design after moving from cloud-first tools like Postman/Insomnia. missing for 10: independent audit confirming zero network calls beyond user-initiated ones.", "evidenceIds": [ "bruno-docs-8", "bruno-docs-9", "bruno-docs-48", "bruno-comm-1", "bruno-comm-2" ] }, { "productId": "bruno", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bruno ships a genuine CLI (bru) that can run collections, generate JSON/JUnit/HTML reports, and integrate into CI/CD/Docker pipelines, and collections are stored as plain text files that can be edited/versioned outside the UI — a solid API-like automation path. However, there is no true HTTP/REST API (openapi probe returned only 404s) and many UI-only capabilities (AI script generation/chat, drag-and-drop editing, GUI-based Git diff/commit workflow, app-building) have no documented CLI/API equivalent, so full UI parity is not evidenced. missing for 10: a first-class programmatic API beyond the CLI, CLI equivalents for AI features and collection editing/Git UI actions, independent confirmation of CLI/UI feature parity.", "evidenceIds": [ "bruno-docs-1", "bruno-docs-2", "bruno-docs-3", "bruno-docs-18", "bruno-docs-28", "bruno-docs-30", "bruno-probe-2", "bruno-probe-3" ] }, { "productId": "bruno", "storyId": "openness-full-export", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Bruno stores collections as plain-text .bru files directly on the local filesystem with no cloud lock-in, no account required, and offline-only by design, making export trivial — data is already in an open, human-readable format under user control and versionable via Git. Postman import/export is also supported for migration. missing for 10: no explicit one-click 'export all data' bulk command/tool is documented, and no independent hands-on confirmation of full data portability beyond community praise for the plain-text/git model.", "evidenceIds": [ "bruno-docs-9", "bruno-docs-28", "bruno-docs-46", "bruno-docs-8", "bruno-docs-48", "bruno-docs-24", "bruno-docs-14", "bruno-comm-2" ] }, { "productId": "bruno", "storyId": "openness-open-license", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Evidence repeatedly references the public GitHub repo (usebruno/bruno) and its release history, implying the source is publicly readable, but no citation explicitly states an open-source license (e.g., MIT) or points to a LICENSE file. Missing for 10: explicit license statement/citation, confirmation of license type, and any independent corroboration of licensing terms.", "evidenceIds": [ "bruno-docs-7", "bruno-docs-25", "bruno-docs-9", "bruno-docs-28" ] }, { "productId": "bruno", "storyId": "openness-self-host", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Bruno is explicitly offline-first with no cloud sync, no account/login, and collections stored as plain text files on the local filesystem, and the CLI even ships as an official Docker image for local/CI use — meaning the entire core product runs under user control with no vendor-hosted dependency (bruno-docs-8, bruno-docs-9, bruno-docs-48, bruno-docs-18). Community reports corroborate the offline-first, git-based self-managed workflow.\nmissing for 10: no explicit multi-user self-hosted 'server/sync' deployment guide since the product is architected as a local-only client rather than a hosted service, so there's nothing analogous to a self-hosted backend to document beyond what's given.", "evidenceIds": [ "bruno-docs-8", "bruno-docs-9", "bruno-docs-48", "bruno-docs-18", "bruno-docs-45", "bruno-comm-1", "bruno-comm-2" ] }, { "productId": "bruno", "storyId": "organize-requests-collections", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Bruno stores collections as plain-text files/folders on the filesystem (supporting folder-based organization), integrates with Git for versioning, and provides a CLI plus GUI to rerun collections anytime, corroborated by community testimonials praising collection organization and git-based rerun workflows. missing for 10: explicit documentation of nested folder hierarchies/UI screenshots and independent hands-on verification of large/complex folder structures.", "evidenceIds": [ "bruno-docs-9", "bruno-docs-28", "bruno-docs-46", "bruno-docs-21", "bruno-docs-30", "bruno-comm-2", "bruno-comm-7" ] }, { "productId": "bruno", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Bruno's local-first architecture (\"No cloud sync... stores your collections directly in a folder on your filesystem\", \"offline-only... no plans to add cloud-sync, ever\") means data effectively stays wherever the user's machine/filesystem is located, giving implicit control over data location rather than an explicit region/residency selector. There is no documented feature letting users choose a specific cloud region or residency zone since no cloud storage exists at all. Missing for 10: explicit region-selection UI/settings, enterprise data-residency controls, or documentation addressing multi-region compliance for any hosted/team features.", "evidenceIds": [ "bruno-docs-8", "bruno-docs-9", "bruno-docs-28", "bruno-docs-48" ] }, { "productId": "bruno", "storyId": "privacy-no-training", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Bruno explicitly states it does not train AI models on user data, has no cloud sync/account, and its AI features use a Bring-Your-Own-Key model where no data is routed through Bruno servers, giving clear first-party assurance against AI training use. missing for 10: independent/hands-on confirmation specifically verifying the no-AI-training claim (community evidence corroborates offline/git-first posture but not this specific claim).", "evidenceIds": [ "bruno-docs-8", "bruno-docs-39", "bruno-docs-48", "bruno-comm-1" ] }, { "productId": "bruno", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bruno's offline-first, no-cloud, no-account architecture (local filesystem storage, no telemetry on requests, no AI training on data, BYOK for AI features) inherently gives users full control over data retention since nothing is stored or synced by Bruno servers by default. However, there's no explicit deletion/retention control feature or policy documentation for cases like secret managers, sync features, or team/cloud add-ons. missing for 10: explicit data retention/deletion controls or settings, documentation on how synced/team-tier data (if any) is retained or purged, independent verification of the 'zero visibility' claim.", "evidenceIds": [ "bruno-docs-8", "bruno-docs-9", "bruno-docs-28", "bruno-docs-39", "bruno-docs-48" ] }, { "productId": "bruno", "storyId": "privacy-telemetry-optout", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bruno's marketing emphasizes an offline-first, no-account, no-cloud-sync design with 'zero visibility into your requests' and no AI training on user data, implying no telemetry on API usage by design (bruno-docs-8, bruno-docs-48, bruno-docs-9). However, there is no explicit documentation of a telemetry/analytics toggle or opt-out setting for app-level usage tracking (e.g., crash reporting, feature analytics), so the claim is inferred from architecture rather than an explicit privacy control. missing for 10: explicit telemetry/analytics opt-out setting or documentation, independent verification of no background usage tracking.", "evidenceIds": [ "bruno-docs-8", "bruno-docs-48", "bruno-docs-9" ] }, { "productId": "bruno", "storyId": "request-lifecycle-scripting", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Bruno documents pre-request and post-response JavaScript scripting with a `bru` API for request chaining and data manipulation, plus test suites and data-driven testing to automate runs. Missing for 10: independent hands-on community corroboration specifically validating scripting/chaining behavior (community evidence focuses on other features like git/cookies) and more detail on complex chaining edge cases.", "evidenceIds": [ "bruno-docs-31", "bruno-docs-32", "bruno-docs-33" ] }, { "productId": "bruno", "storyId": "route-transform-requests", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Bruno is an API client/testing tool for building and running requests, not a gateway/proxy product; routing, load-balancing, and transforming traffic between consumers and upstream services is outside its category — this is a wrong-axis question, not a missing capability.", "evidenceIds": [] }, { "productId": "bruno", "storyId": "run-collection-test-suites", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Bruno CLI documented to run entire collections from command line, generate pass/fail-style test reports in JSON/JUnit/HTML formats, and integrate into CI/CD pipelines (Docker, GitHub Actions), plus native assertions/test suites with data-driven testing support. Missing for 10: independent hands-on report specifically confirming CLI test-run reporting in practice (community evidence focuses on desktop app, not CLI runs).", "evidenceIds": [ "bruno-docs-1", "bruno-docs-2", "bruno-docs-3", "bruno-docs-18", "bruno-docs-30", "bruno-docs-32", "bruno-probe-3" ] }, { "productId": "bruno", "storyId": "run-tests-in-ci", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Bruno CLI is well documented for running collections headlessly, generating JUnit/JSON/HTML reports for CI analysis, and integrating with CI/CD pipelines (Docker, GitHub Actions), which directly supports failing builds on test regressions. missing for 10: explicit documentation/example of exit-code-based build failure and independent hands-on CI usage report beyond vendor docs.", "evidenceIds": [ "bruno-docs-1", "bruno-docs-2", "bruno-docs-3", "bruno-docs-18", "bruno-docs-30", "bruno-probe-3" ] }, { "productId": "bruno", "storyId": "scheduled-api-monitors", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Bruno's docs show CLI runs, CI/CD pipeline integration, and JUnit/HTML/JSON reports, but there is no evidence of a built-in monitor/scheduler feature that runs collections on a schedule and alerts on failures — that requires wiring Bruno CLI into an external scheduler (e.g. cron/CI cron jobs) rather than a native monitoring product feature. missing for 10: a documented native scheduling/monitor feature, built-in alerting (email/Slack/webhook) on failed runs, any dashboard for monitor history.", "evidenceIds": [ "bruno-docs-1", "bruno-docs-3", "bruno-docs-18", "bruno-docs-30" ] }, { "productId": "bruno", "storyId": "secrets-masked-or-vaulted", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Bruno docs explicitly state you can 'keep credentials out of version control with secret variables and vaults' and detail concrete external vault integrations (GCP Secret Manager, Azure Key Vault CLI authentication), directly matching the platform-engineer story on masked secrets and vault integration. missing for 10: independent/hands-on confirmation that masked secrets actually hide values in the UI and broader vault provider coverage beyond GCP/Azure.", "evidenceIds": [ "bruno-docs-33", "bruno-docs-17", "bruno-docs-26", "bruno-docs-34" ] }, { "productId": "bruno", "storyId": "shared-team-workspaces", "verdict": "partial", "quality": 5, "confidence": "high", "rationale": "Bruno enables team sharing of collections via native Git integration (plain-text files, built-in Git UI, init/diff/pull/commit) rather than a cloud-synced workspace, and community reports confirm this works well in practice (bruno-comm-2, bruno-comm-7). However, Bruno explicitly states it has no cloud sync and never will, requiring git know-how instead of a true synced shared workspace experience. Missing for 10: real-time/automatic cloud sync, a dedicated shared-workspace UI independent of git, and conflict resolution beyond git merges.", "evidenceIds": [ "bruno-docs-5", "bruno-docs-6", "bruno-docs-23", "bruno-docs-29", "bruno-docs-48", "bruno-docs-8", "bruno-comm-2", "bruno-comm-7" ] }, { "productId": "bruno", "storyId": "workspace-roles-permissions", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Bruno relies on git-based collaboration and filesystem storage rather than any documented role-based access control or permissions system for collections; no evidence pack item mentions roles, permission levels, or access control for viewing/editing collections.", "evidenceIds": [] }, { "productId": "bruno", "storyId": "write-response-assertions", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Bruno's official docs state you can 'Write assertions and test suites, and automate runs with data-driven testing' and generate JSON/JUnit/HTML test reports, confirming built-in test/assertion functionality against API responses. However, the evidence never explicitly breaks down assertions by status code, body, and headers separately. missing for 10: explicit doc examples of status-code/body/header assertion syntax, independent hands-on confirmation of assertion granularity.", "evidenceIds": [ "bruno-docs-32", "bruno-docs-22", "bruno-docs-2" ] }, { "productId": "hoppscotch", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A direct probe confirms llms.txt is live at docs.hoppscotch.io/llms.txt returning HTTP 200 with structured documentation content, and the docs also expose agent-oriented resources like an official MCP server and CLI overview pages that an agent could consume. Missing for 10: no independent/community confirmation of an agent actually consuming llms.txt in practice.", "evidenceIds": [ "hoppscotch-probe-1", "hoppscotch-probe-3", "hoppscotch-probe-4", "hoppscotch-docs-10" ] }, { "productId": "hoppscotch", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Hoppscotch docs explicitly reference AI features including renaming API requests, generating structured request payloads, and scripting for pre-request/test logic, which count as AI-generated suggestions inside the product. However, evidence does not show deeper 'insights' from data (e.g., analyzing response data, anomaly detection, test result summarization) beyond these request-authoring helpers, and there's no independent/hands-on corroboration of these AI features' quality. missing for 10: evidence of AI-generated insights from response/test data (not just request scaffolding), independent user validation of AI feature usefulness.", "evidenceIds": [ "hoppscotch-docs-23" ] }, { "productId": "hoppscotch", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Hoppscotch's CLI supports running API tests/monitoring from the terminal, which can be scheduled via external cron/CI systems, but this requires an external scheduler rather than a native autonomous background automation/trigger system built into the product. There's no evidence of built-in schedulers, webhooks-triggered runs, or agent-orchestrated autonomous workflows. missing for 10: native scheduling/trigger engine, evidence of autonomous background execution without external cron/CI, monitoring automation details, independent confirmation of unattended runs.", "evidenceIds": [ "hoppscotch-docs-8", "hoppscotch-docs-21", "hoppscotch-probe-4" ] }, { "productId": "hoppscotch", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs mention built-in AI features (auto-renaming requests, generating structured request payloads, and scripting for pre-request/test logic), showing some in-product AI delegation of specific tasks, but there's no evidence of a general-purpose conversational AI assistant that a user can delegate broader tasks to. Missing for 10: detailed documentation of assistant scope/UI, examples of multi-step task delegation, and independent/hands-on corroboration of these AI features in use.", "evidenceIds": [ "hoppscotch-docs-23" ] }, { "productId": "hoppscotch", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Hoppscotch ships an official CLI (hopp) explicitly designed to run API tests, automate monitoring, and manage collections from the terminal or automated systems, including CI-friendly JUnit report generation for collection runs — directly enabling headless/CI automation. missing for 10: independent third-party CI pipeline examples/case studies confirming real-world CI usage beyond vendor docs.", "evidenceIds": [ "hoppscotch-docs-8", "hoppscotch-docs-9", "hoppscotch-docs-21", "hoppscotch-probe-4" ] }, { "productId": "hoppscotch", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows Hoppscotch exposing itself AS an MCP server (agents can connect to Hoppscotch to read/change collections, requests, etc.), which is the reverse of the story — Hoppscotch acting as an MCP client that consumes other servers' tools. No evidence of Hoppscotch connecting to or ingesting external MCP servers to use their tools.", "evidenceIds": [ "hoppscotch-docs-10", "hoppscotch-docs-22", "hoppscotch-probe-3" ] }, { "productId": "hoppscotch", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Hoppscotch documents an official MCP server (docs-10, docs-22, probe-3) that exposes collections, requests, environments, and teams to an agent and can execute real HTTP requests, directly matching the story. Missing for 10: independent/hands-on community verification of the MCP server's behavior and any deeper docs on setup/auth scopes beyond the overview page.", "evidenceIds": [ "hoppscotch-docs-10", "hoppscotch-docs-22", "hoppscotch-probe-3" ] }, { "productId": "hoppscotch", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Hoppscotch ships an official MCP server that hands an AI agent read/write access to collections, requests, environments and teams and lets it send real HTTP requests, effectively letting a natural-language-driven agent operate the product (hoppscotch-docs-10, hoppscotch-docs-22, hoppscotch-probe-3). There's also a docs page on in-app 'AI features' covering request renaming, payload generation, and script generation, suggesting some native NL-assisted actions (hoppscotch-docs-23). However, the MCP path requires an external AI client/agent rather than a built-in chat interface, and the AI-features doc is only a single thin line with no concrete examples or independent corroboration. Missing for 10: a first-party in-app natural-language command bar/chat demonstrated end-to-end, richer documentation/examples of the AI features, and independent/hands-on confirmation that these NL capabilities work as described.", "evidenceIds": [ "hoppscotch-docs-10", "hoppscotch-docs-22", "hoppscotch-docs-23", "hoppscotch-probe-3" ] }, { "productId": "hoppscotch", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Hoppscotch ships an official CLI (hopp) documented for running API tests, automating monitoring, managing collections from the terminal, and generating JUnit reports, corroborated by a probe confirming the official docs page. missing for 10: independent/hands-on community confirmation of CLI usage and more detail on full command coverage.", "evidenceIds": [ "hoppscotch-docs-8", "hoppscotch-docs-9", "hoppscotch-docs-21", "hoppscotch-probe-4" ] }, { "productId": "hoppscotch", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Hoppscotch documents a first-party MCP server that lets an agent read and modify collections, requests, environments, and teams and send real HTTP requests, plus a CLI for terminal/automated interaction (running tests, managing collections) — both are officially documented programmatic interfaces suited to AI-native/agentic use. Missing for 10: a general public REST/OpenAPI API for the platform itself (probe found openapi.json/swagger.json all 404), so agentic access is limited to the MCP/CLI surface rather than a broader documented API.", "evidenceIds": [ "hoppscotch-docs-10", "hoppscotch-docs-22", "hoppscotch-docs-8", "hoppscotch-docs-21", "hoppscotch-probe-3", "hoppscotch-probe-4", "hoppscotch-probe-2" ] }, { "productId": "hoppscotch", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Hoppscotch's MCP server docs describe granting an agent full read/write access to all collections, requests, environments, and teams, with no mention of scoped, least-privileged, or permission-limited credentials for agents. No evidence describes issuing restricted API keys/tokens specifically for AI agent use.", "evidenceIds": [ "hoppscotch-docs-10", "hoppscotch-docs-22", "hoppscotch-probe-3" ] }, { "productId": "hoppscotch", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Hoppscotch offers a CLI, an MCP server, and AI-assisted scripting features, but there is no mention of official client SDKs (e.g., language libraries for programmatically building or extending Hoppscotch) anywhere in the docs or GitHub feature list.", "evidenceIds": [] }, { "productId": "hoppscotch", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Hoppscotch is an API client/testing tool; nothing in the evidence shows any webhook subscription or event-notification mechanism for AI agents—the closest agentic features (MCP server, CLI) let an agent send requests and manage collections but do not provide a way to subscribe to incoming events. No evidence of webhook support at all.", "evidenceIds": [ "hoppscotch-docs-10", "hoppscotch-docs-22", "hoppscotch-docs-21" ] }, { "productId": "hoppscotch", "storyId": "ai-generate-tests-from-api", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs mention AI features that include \"scripting capabilities for pre-request logic and test cases,\" implying AI can help generate test/assertion scripts, but there's no detail on generating tests specifically from an endpoint's spec or response data, nor independent corroboration of this feature in use. Missing for 10: detailed documentation on spec/response-driven test generation, examples of AI-generated assertions, and community/hands-on validation of this specific capability.", "evidenceIds": [ "hoppscotch-docs-23" ] }, { "productId": "hoppscotch", "storyId": "ai-llm-traffic-gateway", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Hoppscotch is an API client/testing platform (with an MCP server exposing its own collections to agents), not an LLM/MCP gateway or proxy product designed to govern AI traffic with token limits, prompt guards, or provider routing. This capability is a category error for this product type.", "evidenceIds": [] }, { "productId": "hoppscotch", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Hoppscotch's Embed feature lets users create a mini-version of Hoppscotch embeddable in HTML pages so an audience can interact with API requests, which is a form of interactive, runnable API reference, and OpenAPI import can seed collections for such a reference. However there is no evidence of a dedicated 'API reference/documentation' generator with narrative docs plus embedded runnable examples (like Swagger UI or Postman's published docs), nor independent corroboration of the Embed feature in practice. Missing for 10: auto-generated narrative API reference pages, publishing/sharing workflow for such references, hands-on validation of the Embed feature.", "evidenceIds": [ "hoppscotch-docs-5", "hoppscotch-docs-15", "hoppscotch-docs-16" ] }, { "productId": "hoppscotch", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe for OpenAPI/Swagger specs at the Hoppscotch docs domain returned 404 on all candidate paths, and no other evidence shows a downloadable machine-readable spec of Hoppscotch's own API surface (the only OpenAPI-related mention is about importing external OpenAPI specs into Hoppscotch, not exposing one).", "evidenceIds": [ "hoppscotch-probe-2", "hoppscotch-docs-15" ] }, { "productId": "hoppscotch", "storyId": "api-sandbox", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Hoppscotch supports environment/variable management (including secret variables) and the MCP server exposes environments/collections to agents so a sandbox environment could in principle be selected, but there is no explicit feature or documentation describing isolation, safety checks, or a dedicated 'sandbox vs production' mode that prevents an agent from touching production data — the MCP docs even note it 'sends real HTTP requests.' Missing for 10: explicit sandbox-environment safeguard/isolation feature, documentation of environment-scoped access control for agents, and independent evidence confirming this prevents production impact.", "evidenceIds": [ "hoppscotch-docs-7", "hoppscotch-docs-1", "hoppscotch-docs-22", "hoppscotch-probe-3" ] }, { "productId": "hoppscotch", "storyId": "api-traffic-observability", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Hoppscotch is an API client/testing tool; evidence covers request building, collections, CLI testing, self-hosting, SSO, and audit logs, but nothing describes metrics export, logging pipelines, or tracing integrations for monitoring API traffic (e.g., no OpenTelemetry, Prometheus, or log-shipping integrations mentioned). This axis is plausible for a platform aimed at API teams, so absence of evidence yields 'none' rather than 'na'.", "evidenceIds": [ "hoppscotch-docs-21", "hoppscotch-docs-12", "hoppscotch-docs-30" ] }, { "productId": "hoppscotch", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of API versioning scheme or documented deprecation policy for Hoppscotch's own APIs/products; changelog shows feature releases but no version numbers or deprecation notices. Missing for 10: versioning scheme documentation, deprecation policy, migration guides, version support timeline.", "evidenceIds": [] }, { "productId": "hoppscotch", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "The CLI can execute an entire collection of requests in one command with JUnit reporting, and the MCP server lets an agent read/change collections, requests, environments, and teams programmatically, both of which support automation-style bulk actions across many items. However, there's no explicit documentation of a dedicated bulk-edit/bulk-delete/bulk-update operation spanning many items at once (e.g. batch renaming, batch parameter changes) beyond running/exporting collections. missing for 10: explicit bulk edit/update/delete API or UI across many requests/items simultaneously, independent hands-on confirmation of large-scale automation runs.", "evidenceIds": [ "hoppscotch-docs-8", "hoppscotch-docs-9", "hoppscotch-docs-21", "hoppscotch-docs-22", "hoppscotch-docs-10" ] }, { "productId": "hoppscotch", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Hoppscotch offers pre-request/test scripts (gh-5) and a CLI for scheduled/automated test runs (docs-8, docs-21), but there is no evidence of a rules engine or event-triggered automation (e.g., 'when X happens, do Y') as described in the story.", "evidenceIds": [] }, { "productId": "hoppscotch", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Hoppscotch offers a CLI for running tests/monitoring and an MCP server for agent access, but there is no evidence of any scheduler, cron-like trigger, or recurring workflow orchestration feature — automation still requires an external scheduler (e.g., cron, CI) to invoke the CLI repeatedly. missing for 10: any built-in scheduling/recurrence mechanism, workflow chaining across multiple requests, or trigger-based automation.", "evidenceIds": [ "hoppscotch-docs-8", "hoppscotch-docs-21", "hoppscotch-docs-9", "hoppscotch-docs-22" ] }, { "productId": "hoppscotch", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence Hoppscotch offers versioning, review workflows, or rollback for automations/scripts/collections; a GitHub issue explicitly requests git-friendly storage/sync as an unmet feature, indicating this capability doesn't exist today.", "evidenceIds": [ "hoppscotch-comm-14" ] }, { "productId": "hoppscotch", "storyId": "collections-as-files-in-git", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "There is no evidence Hoppscotch stores collections as plain-text files suitable for git version control; instead, community evidence explicitly shows this is a requested-but-unimplemented feature (an open GitHub issue asking Hoppscotch to make storage git-friendly) and a comparison noting Bruno supports directory-based storage while Hoppscotch does not let you point a workspace to a specific directory.", "evidenceIds": [ "hoppscotch-comm-14", "hoppscotch-comm-10" ] }, { "productId": "hoppscotch", "storyId": "contract-validation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Hoppscotch supports API testing scripts, CLI test runs with JUnit reports, and OpenAPI import for building requests/collections, but nothing describes validating live responses against an API contract (e.g., OpenAPI schema) to detect drift.", "evidenceIds": [] }, { "productId": "hoppscotch", "storyId": "declarative-config-gitops", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Hoppscotch offers a CLI for running collections in CI (docs-8) and a self-host admin dashboard for server settings (docs-6), but there is no evidence of declarative, git-syncable configuration for API infrastructure/gateway routing—in fact a long-standing community request for git-friendly collection storage remains unresolved (comm-14), indicating this GitOps/APIOps capability is not delivered.", "evidenceIds": [ "hoppscotch-docs-8", "hoppscotch-docs-6", "hoppscotch-comm-14" ] }, { "productId": "hoppscotch", "storyId": "design-export-openapi", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Hoppscotch can import OpenAPI specs (e.g., using tags as folders when importing) but there is no mention of designing an API spec-first and exporting it as OpenAPI; the only OpenAPI-related capability documented is import, not export.", "evidenceIds": [ "hoppscotch-docs-15" ] }, { "productId": "hoppscotch", "storyId": "environment-variable-sets", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm the existence of an environments system (with secret variables and request variables that resolve within environments), implying multiple environment sets can be defined and swapped, but no direct evidence describes selecting/switching between dev/staging/prod environments or a dropdown UI for env selection. missing for 10: explicit documentation or screenshot of environment switcher UI, workflow for maintaining separate dev/staging/prod variable sets, and independent/community confirmation of this specific switching behavior.", "evidenceIds": [ "hoppscotch-docs-7", "hoppscotch-docs-1" ] }, { "productId": "hoppscotch", "storyId": "gateway-auth-plugins", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Hoppscotch is an API client/testing tool, not an API gateway; it supports auth methods (OAuth2, mTLS, JWT) for making requests to APIs but does not act as a gateway enforcing authentication policies on upstream APIs via plugins/policies. This is a category mismatch — the gateway-traffic axis does not apply to a request client.", "evidenceIds": [] }, { "productId": "hoppscotch", "storyId": "gateway-rate-limiting", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Hoppscotch is an API client/testing tool, not an API gateway or traffic-management product; enforcing rate limits per consumer or route is a gateway/infra concern outside its category, so the axis does not apply.", "evidenceIds": [] }, { "productId": "hoppscotch", "storyId": "generate-publish-api-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Hoppscotch offers OpenAPI import and an 'Embed' feature to share interactive requests, but there is no evidence of an actual documentation-generation/publishing feature from collections or specs; a community review explicitly notes 'Can't create documentation for a folder/request', directly indicating this capability is absent.", "evidenceIds": [ "hoppscotch-docs-5", "hoppscotch-docs-15", "hoppscotch-comm-10" ] }, { "productId": "hoppscotch", "storyId": "import-export-collections", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs and GitHub evidence show robust import support (cURL import, OpenAPI import with tag-based folder organization, collections/folders organization) but there is no explicit documentation of exporting collections to portable formats, and a community feature request (git-friendly storage sync) suggests export/interop is limited. missing for 10: explicit export-to-file/format documentation, independent confirmation of round-trip export/import between tools like Postman.", "evidenceIds": [ "hoppscotch-gh-1", "hoppscotch-gh-8", "hoppscotch-docs-15", "hoppscotch-gh-11", "hoppscotch-comm-1", "hoppscotch-comm-14" ] }, { "productId": "hoppscotch", "storyId": "import-openapi-spec", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs explicitly mention OpenAPI import ('Use tags as folders when importing from OpenAPI') implying native OpenAPI import into collections, alongside cURL import and collection/folder organization features. However, there's no detailed documentation on the full import workflow (e.g., generating requests from spec, handling auth/schemas) or independent/hands-on confirmation of fidelity. missing for 10: detailed OpenAPI import workflow docs, independent/hands-on validation of spec-to-collection accuracy.", "evidenceIds": [ "hoppscotch-docs-15", "hoppscotch-gh-11", "hoppscotch-gh-1" ] }, { "productId": "hoppscotch", "storyId": "internal-api-catalog", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Hoppscotch offers an 'Embed' feature to expose a mini API-interaction UI in a page and team/collection sharing for collaboration, which can serve as a lightweight internal portal, but there is no dedicated catalog/discovery portal product (no API marketplace, search, or documentation-portal feature) documented. missing for 10: a dedicated API catalog/portal product, discovery/search across published APIs, and evidence of teams actually using it as an internal developer portal.", "evidenceIds": [ "hoppscotch-docs-5", "hoppscotch-gh-6", "hoppscotch-docs-20" ] }, { "productId": "hoppscotch", "storyId": "lint-specs-style-rules", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Hoppscotch is an API client/testing tool with request building, collections, CLI test runner, and MCP server, but there is no evidence of spec linting against configurable style/governance rules (e.g., no Spectral-like ruleset engine or OpenAPI governance checks) in any doc, changelog, or CLI feature listed.", "evidenceIds": [] }, { "productId": "hoppscotch", "storyId": "mock-servers-from-examples", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Hoppscotch is an API request/testing client (with collections, CLI, MCP server) and does not offer mock server generation from examples or specs; this is a category error for this product type — no evidence of such a feature exists.", "evidenceIds": [] }, { "productId": "hoppscotch", "storyId": "multi-protocol-requests", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "GitHub evidence explicitly confirms GraphQL support and full-duplex (WebSocket-like) communication over a single TCP connection, but there's no direct evidence for gRPC or SSE support, and no independent/hands-on corroboration of actually sending requests via these protocols. missing for 10: explicit gRPC support evidence, SSE support evidence, hands-on confirmation of sending GraphQL/WebSocket requests.", "evidenceIds": [ "hoppscotch-gh-9", "hoppscotch-gh-4" ] }, { "productId": "hoppscotch", "storyId": "offline-local-workspace", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Hoppscotch offers a desktop app, installable PWA, and self-hosted deployment with login clearly optional (hoppscotch-docs-29, hoppscotch-docs-13, hoppscotch-comm-9), supporting local/offline use without a cloud account. However, hands-on community evidence shows real limits: 'Only one workspace is available when offline. You can't point your Workspace data to a specific directory like bruno does. Self-hosting login is not possible in the desktop app' (hoppscotch-comm-10), and installing self-hosted locally was reported as unclear/undocumented at one point (hoppscotch-comm-2, hoppscotch-comm-3). Missing for 10: multi-workspace offline support, directory-based local workspace storage, and clearer first-party docs/independent confirmation of a fully-featured offline-only workflow.", "evidenceIds": [ "hoppscotch-docs-29", "hoppscotch-docs-13", "hoppscotch-comm-9", "hoppscotch-comm-10", "hoppscotch-comm-2" ] }, { "productId": "hoppscotch", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Hoppscotch offers a CLI for running tests/monitoring/collection management and an MCP server that lets an agent read and modify collections, requests, environments, and teams and send real HTTP requests, giving substantial programmatic parity with core UI actions. However, there is no documented general-purpose REST/GraphQL API (probe found no OpenAPI/swagger spec), and features like team collaboration, SSO/admin settings, or UI-specific extras (themes, PWA, client certs) aren't shown as scriptable. Missing for 10: a documented public API covering full UI feature parity, evidence for automating admin/enterprise/team-collab features, and independent confirmation of CLI/MCP completeness.", "evidenceIds": [ "hoppscotch-docs-8", "hoppscotch-docs-21", "hoppscotch-docs-22", "hoppscotch-probe-3", "hoppscotch-probe-4", "hoppscotch-probe-2" ] }, { "productId": "hoppscotch", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Hoppscotch is fully open-source (MIT) and can be self-hosted with 'full data ownership' (hoppscotch-docs-11, hoppscotch-docs-14, hoppscotch-docs-24), and its CLI/MCP server can read and manage collections, environments and requests in structured form (hoppscotch-docs-8, hoppscotch-docs-10, hoppscotch-docs-22), implying data is accessible outside the UI. However, there is no explicit documentation of a dedicated 'export' feature or open file format (e.g. JSON export of collections) for a user to take their data and leave, only import (cURL, OpenAPI) is explicitly evidenced. missing for 10: explicit export/download-my-data feature documentation, confirmation of an open interchange format for collections/environments, independent confirmation that self-hosted data is fully portable without vendor lock-in.", "evidenceIds": [ "hoppscotch-docs-11", "hoppscotch-docs-14", "hoppscotch-docs-24", "hoppscotch-docs-8", "hoppscotch-docs-10", "hoppscotch-docs-22", "hoppscotch-gh-1" ] }, { "productId": "hoppscotch", "storyId": "openness-open-license", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Hoppscotch's Community Edition source is publicly hosted on GitHub and documented as MIT-licensed, satisfying the 'read source under open license' story with an active, browsable repo (hoppscotch-docs-11, hoppscotch-gh-1..11). Missing for 10: no explicit clarification of licensing for the separate enterprise/self-host admin features, and community commentary notes it is 'open core' with some contributed functionality restricted (hoppscotch-comm-7, hoppscotch-comm-8), which is a scope caveat rather than a contradiction of the core MIT-licensed repo.", "evidenceIds": [ "hoppscotch-docs-11", "hoppscotch-gh-1", "hoppscotch-comm-7", "hoppscotch-comm-8" ] }, { "productId": "hoppscotch", "storyId": "openness-self-host", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Hoppscotch documents a full self-host path (open-source MIT Community Edition, dedicated self-host getting-started guide, admin dashboard for server config, full data ownership) confirming the core product can run on a user's own infrastructure. Some older community comments note friction in the self-host setup process, but they don't contradict the current documented capability. Missing for 10: independent hands-on verification of a smooth current self-host deployment and more detail on infra requirements/scaling.", "evidenceIds": [ "hoppscotch-docs-11", "hoppscotch-docs-12", "hoppscotch-docs-14", "hoppscotch-docs-24", "hoppscotch-docs-30", "hoppscotch-docs-6", "hoppscotch-comm-2", "hoppscotch-comm-3" ] }, { "productId": "hoppscotch", "storyId": "organize-requests-collections", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Hoppscotch explicitly documents organizing requests into collections and subfolders, reusing them with a single click, and applying shared headers/auth at the collection level, backed by both first-party docs and GitHub feature list; community reviews corroborate collections/saving as a core, easy-to-use feature. Missing for 10: deeper independent hands-on evidence of long-term rerun workflows across large collections.", "evidenceIds": [ "hoppscotch-docs-20", "hoppscotch-docs-4", "hoppscotch-gh-11", "hoppscotch-comm-1" ] }, { "productId": "hoppscotch", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Hoppscotch supports self-hosting on your own infrastructure with 'full data ownership' and on-prem deployment (Enterprise), which indirectly lets users control where their data resides by choosing their own hosting location, rather than offering explicit region-selection within a managed cloud service. missing for 10: an explicit region/residency selector in the hosted Hoppscotch Cloud offering, and independent confirmation of data-location controls.", "evidenceIds": [ "hoppscotch-docs-14", "hoppscotch-docs-24", "hoppscotch-docs-30", "hoppscotch-docs-12" ] }, { "productId": "hoppscotch", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses AI-training data usage, opt-out settings, or a privacy policy statement about model training; self-hosting for data ownership is documented but does not speak to whether cloud/AI features use data for training.", "evidenceIds": [] }, { "productId": "hoppscotch", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Hoppscotch offers self-hosting with 'full data ownership' and is open-source (MIT), which lets an organization control where data lives and effectively manage retention/deletion by owning the infrastructure. However, there is no explicit documentation of account/data deletion tools, configurable retention periods, or export/erasure APIs for the cloud-hosted version. Missing for 10: explicit data retention/deletion controls or APIs, documented GDPR-style erasure workflow, and independent confirmation that self-hosting alone satisfies deletion requirements.", "evidenceIds": [ "hoppscotch-docs-14", "hoppscotch-docs-11", "hoppscotch-docs-12" ] }, { "productId": "hoppscotch", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item addresses telemetry, analytics, or opt-out settings for Hoppscotch; self-hosting is mentioned as giving 'full data ownership' but that is distinct from an explicit telemetry opt-out control.", "evidenceIds": [] }, { "productId": "hoppscotch", "storyId": "request-lifecycle-scripting", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "GitHub feature list explicitly confirms pre-request scripting ('Snippets of code... executed before the request is sent'), and the AI-features doc mentions 'scripting capabilities for pre-request logic and test cases', implying post-response test scripts exist too. However, no dedicated documentation details post-response script syntax, variable-setting APIs, or examples of chaining/massaging data across requests. missing for 10: explicit post-response/test-script documentation, examples of setting/reading environment variables between chained requests, independent hands-on confirmation of scripting workflow.", "evidenceIds": [ "hoppscotch-gh-5", "hoppscotch-docs-23" ] }, { "productId": "hoppscotch", "storyId": "route-transform-requests", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Hoppscotch is an API client/testing tool for building, sending, and testing requests, not an API gateway or load balancer. There is no evidence it provides routing, load balancing, or traffic transformation between consumers and upstream services — this is a category error for this product type.", "evidenceIds": [] }, { "productId": "hoppscotch", "storyId": "run-collection-test-suites", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Hoppscotch's official CLI (hopp test) runs entire collections from the terminal and generates JUnit reports for collection runs, which is a standard pass/fail CI test report format; docs also describe running API tests and automated monitoring via CLI. Missing for 10: no independent/hands-on community corroboration of the CLI test-runner experience or assertion/scripting depth was found in the pack.", "evidenceIds": [ "hoppscotch-docs-8", "hoppscotch-docs-9", "hoppscotch-docs-21", "hoppscotch-probe-4" ] }, { "productId": "hoppscotch", "storyId": "run-tests-in-ci", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Hoppscotch ships an official CLI (hopp) explicitly documented for running API test collections and automated monitoring from the terminal, with JUnit report generation for CI integration (hoppscotch-docs-8, hoppscotch-docs-9, hoppscotch-docs-21, hoppscotch-probe-4). However, the evidence never explicitly documents exit-code/fail-on-regression behavior or shows independent hands-on CI pipeline examples. Missing for 10: explicit documentation of non-zero exit codes on test failure, and independent/community confirmation of CI usage.", "evidenceIds": [ "hoppscotch-docs-8", "hoppscotch-docs-9", "hoppscotch-docs-21", "hoppscotch-probe-4" ] }, { "productId": "hoppscotch", "storyId": "scheduled-api-monitors", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Hoppscotch's CLI docs mention the ability to 'automate monitoring' and 'manage automated API monitoring' via collection runs, implying it can be scripted/scheduled externally (e.g., via cron/CI) to catch API failures, and can output JUnit reports for CI integration. However, there is no evidence of a built-in monitor scheduler or native alerting/notification mechanism (email, Slack, webhook) when a run fails. Missing for 10: dedicated 'Monitors' feature with scheduling UI, built-in alert/notification delivery on failure, independent confirmation of this workflow.", "evidenceIds": [ "hoppscotch-docs-8", "hoppscotch-docs-21", "hoppscotch-docs-9" ] }, { "productId": "hoppscotch", "storyId": "secrets-masked-or-vaulted", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Hoppscotch docs explicitly list 'Secret variables in environments' as a changelog feature, directly matching the masked-secrets need, and self-host docs mention on-prem deployment and audit logs for enterprise-grade control. However, there is no evidence of an external vault integration (e.g., Vault, AWS Secrets Manager) or independent/hands-on confirmation of how secret variables are masked in shared collection files. Missing for 10: external vault/secret-manager integration, hands-on verification that secrets are excluded from exported/shared collection files, and independent corroboration beyond the vendor changelog.", "evidenceIds": [ "hoppscotch-docs-7", "hoppscotch-docs-12", "hoppscotch-docs-14" ] }, { "productId": "hoppscotch", "storyId": "shared-team-workspaces", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Hoppscotch documents Teams/workspace collaboration features (shared collections, headers/auth applied at collection level, admin dashboard, SSO) and GitHub explicitly claims it 'helps you collaborate across your teams to design, develop, and test APIs faster,' with community evidence confirming easy collection sharing. However, evidence also surfaces real gaps: a feature request shows syncing/storage across teams (e.g., git-based sync) is unmet, self-hosted login/workspace sync in the desktop app is limited, and some enterprise-grade sync features (SSO, audit logs) are gated behind paid tiers. Missing for 10: independent verification of real-time sync fidelity across team members, clarity on offline/desktop workspace sync limitations, and confirmation that core sharing works without hitting paywalled features.", "evidenceIds": [ "hoppscotch-gh-6", "hoppscotch-docs-4", "hoppscotch-comm-1", "hoppscotch-comm-10", "hoppscotch-comm-14", "hoppscotch-docs-12" ] }, { "productId": "hoppscotch", "storyId": "workspace-roles-permissions", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Hoppscotch has team collaboration features and an admin dashboard with 'enhanced user management' plus enterprise SSO/audit logs, implying some access-control capability, but there is no explicit documentation of granular roles/permissions for viewing vs editing specific APIs or collections. missing for 10: explicit role/permission model docs (e.g. viewer vs editor roles per collection), evidence of enforcement, independent confirmation of granular access control.", "evidenceIds": [ "hoppscotch-gh-6", "hoppscotch-docs-28", "hoppscotch-docs-6", "hoppscotch-docs-12" ] }, { "productId": "hoppscotch", "storyId": "write-response-assertions", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Hoppscotch docs reference scripting capabilities for 'pre-request logic and test cases' and a CLI 'hopp test' command that can generate JUnit reports for collection runs, implying support for test assertions, but no evidence details how assertions target response status, body, or headers specifically. missing for 10: explicit documentation/examples of assertion syntax (pw.expect-style) for status/body/headers, independent hands-on confirmation of test script execution.", "evidenceIds": [ "hoppscotch-docs-23", "hoppscotch-docs-9", "hoppscotch-docs-21", "hoppscotch-gh-5" ] }, { "productId": "insomnia", "storyId": "agentic-agent-docs", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Kong's developer docs (which host Insomnia's documentation) expose a working llms.txt at the domain root and individual doc pages are directly fetchable as markdown (.md) files, which is agent-friendly, but this is a shared Kong-wide resource rather than an Insomnia-specific llms.txt, and a probe for an Insomnia-specific markdown index (insomnia/.md) returned 404. Missing for 10: a dedicated Insomnia-specific llms.txt or explicit agent-onboarding docs, and confirmation Insomnia itself (the product) references or promotes this file to agents.", "evidenceIds": [ "insomnia-probe-1", "insomnia-probe-2", "insomnia-docs-1", "insomnia-docs-2" ] }, { "productId": "insomnia", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Insomnia has narrow, specific AI features—generating mock servers from prompts/OpenAPI/URL and suggesting Git commit messages—but no evidence of AI analyzing request/response data to surface insights, debugging suggestions, or pattern recognition across a user's API data. Missing for 10: AI-driven analysis of actual request/response payloads, error/anomaly insights, or usage-pattern suggestions beyond mock generation and commit messages.", "evidenceIds": [ "insomnia-docs-10", "insomnia-docs-29", "insomnia-docs-51", "insomnia-docs-35", "insomnia-docs-52" ] }, { "productId": "insomnia", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Insomnia offers CI-triggered test automation via Inso CLI and manual Collection Runner runs, but there is no evidence of autonomous, self-scheduling background automations (e.g., cron-like triggers or agents running unattended) — all automation examples require an external CI pipeline or manual invocation.", "evidenceIds": [ "insomnia-docs-39", "insomnia-docs-53", "insomnia-docs-20" ] }, { "productId": "insomnia", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Insomnia does ship some built-in AI features—generating mock servers from a natural-language prompt/OpenAPI spec (insomnia-docs-10, insomnia-docs-29, insomnia-docs-51) and AI-generated commit message suggestions (insomnia-docs-35, insomnia-docs-52)—but these are narrow, single-purpose AI helpers rather than a general-purpose assistant to which a user can delegate broader tasks (e.g., writing tests, debugging requests, multi-step workflows). Missing for 10: evidence of a general conversational/agentic AI assistant embedded in the app, ability to delegate arbitrary tasks beyond mock/commit generation, and independent/hands-on confirmation of these AI features' quality.", "evidenceIds": [ "insomnia-docs-10", "insomnia-docs-29", "insomnia-docs-51", "insomnia-docs-35", "insomnia-docs-52" ] }, { "productId": "insomnia", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Insomnia ships the Inso CLI, explicitly documented as designed for CI environments — it disables interactive prompts and returns exit codes to pass/fail builds, and supports test running, spec linting, and collection runs from CI/Git workflows without the GUI. This directly satisfies headless/automated execution for CI pipelines. Missing for 10: independent hands-on CI usage reports/case studies and more detail on advanced automation scenarios (e.g., parallelization, complex environment injection) beyond the docs themselves.", "evidenceIds": [ "insomnia-docs-53", "insomnia-docs-39", "insomnia-docs-49", "insomnia-docs-3", "insomnia-probe-4" ] }, { "productId": "insomnia", "storyId": "agentic-mcp-client", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Insomnia has a documented MCP Client feature that connects to external MCP servers to access AI-ready tools, prompts, and resources, discovering and enabling invocation/testing of these elements directly in the app. This directly matches the story of plugging in MCP servers to use their tools. Missing for 10: independent/hands-on corroboration of the MCP client feature beyond first-party docs, and details on limitations or setup friction.", "evidenceIds": [ "insomnia-docs-46", "insomnia-docs-30", "insomnia-docs-14" ] }, { "productId": "insomnia", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Insomnia acts as an MCP *client* (docs-14, docs-30, docs-46) that connects to external MCP servers, not as an MCP server that an agent could connect to. No documentation of Insomnia exposing its own official MCP server endpoint was found in the evidence pack.", "evidenceIds": [ "insomnia-docs-14", "insomnia-docs-30", "insomnia-docs-46" ] }, { "productId": "insomnia", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Insomnia offers targeted AI/natural-language features—generating a mock server from a natural-language prompt and AI-assisted commit message suggestions—but there is no evidence of broader natural-language command control over core operations like creating/running requests, collections, or environments. Missing for 10: NL-driven request creation/execution, NL-based test/collection running, general chat-style control of the app, and independent verification of the AI features' reliability.", "evidenceIds": [ "insomnia-docs-10", "insomnia-docs-29", "insomnia-docs-35", "insomnia-docs-51", "insomnia-docs-52" ] }, { "productId": "insomnia", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Insomnia ships an official 'Inso CLI' documented for automating collection runs, linting OpenAPI specs, and running in CI pipelines with exit codes for pass/fail, which is exactly the kind of scriptable, agent-friendly interface an AI-native workflow would use. missing for 10: independent/hands-on community confirmation of Inso CLI usage and explicit framing of the CLI as AI/agent-oriented rather than just human CI tooling.", "evidenceIds": [ "insomnia-docs-39", "insomnia-docs-53", "insomnia-docs-49", "insomnia-probe-4" ] }, { "productId": "insomnia", "storyId": "agentic-public-api", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Insomnia does not publish a documented public REST/programmatic API for driving the app itself (probe found no OpenAPI/swagger spec for the product), but it does offer the Inso CLI for scripted automation of tests, linting, and collection runs, plus an extensibility plugin system — giving AI-native users some programmatic control, just not a true public API surface. missing for 10: a documented HTTP/SDK-style public API for controlling Insomnia, independent confirmation the Inso CLI/plugin surface is used for AI-agent automation.", "evidenceIds": [ "insomnia-docs-3", "insomnia-docs-39", "insomnia-docs-49", "insomnia-docs-53", "insomnia-docs-36", "insomnia-probe-3", "insomnia-probe-4" ] }, { "productId": "insomnia", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Insomnia offers RBAC for controlling access to its own resources, external vault integration for retrieving secrets, and environment variables for storing credentials, but there is no evidence of a feature to generate or issue scoped, least-privilege API credentials specifically for an AI agent to use. Given Insomnia's MCP client and AI features, this would be a plausible extension, but it's not documented anywhere in the evidence pack.", "evidenceIds": [ "insomnia-docs-11", "insomnia-docs-24", "insomnia-docs-33", "insomnia-docs-46" ] }, { "productId": "insomnia", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Insomnia's automation surface is the Inso CLI and plugin system, plus MCP client support, but no official SDK (e.g., a Node/Python/Go SDK) for programmatically building against Insomnia is documented anywhere in the pack.", "evidenceIds": [ "insomnia-docs-3", "insomnia-docs-39", "insomnia-docs-53", "insomnia-docs-36", "insomnia-probe-4" ] }, { "productId": "insomnia", "storyId": "agentic-webhooks", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Insomnia is an API client/design tool for making requests, testing, and mocking; there is no evidence of a webhook subscription/event-notification feature, and 'subscribing to events via webhooks' is not a natural axis for this product category (it's a client, not a service that emits events).", "evidenceIds": [] }, { "productId": "insomnia", "storyId": "ai-generate-tests-from-api", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Insomnia's AI features documented are limited to mock server generation from prompts/specs and AI commit message suggestions; there is no evidence of AI generating test cases or assertions from an endpoint's spec or responses. Testing capabilities (collection runner, pre-request scripts, chaining) are manual, not AI-driven.", "evidenceIds": [ "insomnia-docs-10", "insomnia-docs-29", "insomnia-docs-51", "insomnia-docs-52", "insomnia-docs-4", "insomnia-docs-20" ] }, { "productId": "insomnia", "storyId": "ai-llm-traffic-gateway", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Insomnia's MCP support is client-side only (docs-30, docs-46) — it lets users discover and invoke external MCP servers' tools/prompts, and a brief mention of connecting to Konnect for 'protecting requests' (docs-14), but there is no evidence of built-in token-limit enforcement, prompt guards, or LLM provider routing/proxying within Insomnia itself. Missing for 10: any documentation of token/rate limiting, prompt injection guards, or multi-provider routing/proxy governance features.", "evidenceIds": [ "insomnia-docs-14", "insomnia-docs-30", "insomnia-docs-46" ] }, { "productId": "insomnia", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Insomnia lets users import OpenAPI/Swagger specs, preview them with linting, auto-generate code snippets in 12+ languages, and directly send/run requests against the spec — effectively turning any imported API spec into an interactive, runnable reference (insomnia-docs-16, insomnia-docs-17, insomnia-docs-18, insomnia-docs-5, insomnia-docs-48). However this requires manually importing a spec rather than offering a built-in, browsable reference experience, and there's no evidence of a dedicated 'try-it-out' documentation portal or AI-native example runner. Missing for 10: a first-party interactive documentation/reference viewer (vs. spec design+testing workflow), and independent/hands-on evidence of this specific 'explore reference with runnable examples' flow.", "evidenceIds": [ "insomnia-docs-16", "insomnia-docs-17", "insomnia-docs-18", "insomnia-docs-5", "insomnia-docs-48", "insomnia-docs-20" ] }, { "productId": "insomnia", "storyId": "api-machine-spec", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Insomnia's Design Hub lets users build/iterate OpenAPI specs, preview and lint them, and export/import in OpenAPI 3.0/3.1 (plus Swagger, Postman, HAR formats), and the Inso CLI can lint specs in CI (`inso lint spec`). This directly delivers a machine-readable, downloadable spec workflow. Missing for 10: no independent/hands-on confirmation of the export UX or format fidelity, and the Kong docs site itself has no discoverable OpenAPI spec (probe-3), slightly muddying 'AI-native' machine-readability of the docs layer.", "evidenceIds": [ "insomnia-docs-5", "insomnia-docs-8", "insomnia-docs-16", "insomnia-docs-17", "insomnia-docs-19", "insomnia-docs-48", "insomnia-docs-49" ] }, { "productId": "insomnia", "storyId": "api-sandbox", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Insomnia supports mock servers (including AI-generated ones from a prompt/OpenAPI spec) and environment variables that can separate dev/staging/production configs, letting users test flows without hitting real production endpoints. However, there is no explicit 'sandbox environment' feature or documented guarantee of production-data isolation — it relies on users manually configuring mock servers/environments rather than a dedicated sandboxed testing mode. Missing for 10: an explicit sandbox/staging isolation feature, documentation guaranteeing no production data exposure, and independent evidence of AI agents using this for safe testing.", "evidenceIds": [ "insomnia-docs-6", "insomnia-docs-10", "insomnia-docs-28", "insomnia-docs-29", "insomnia-docs-9", "insomnia-docs-33", "insomnia-docs-54", "insomnia-docs-51" ] }, { "productId": "insomnia", "storyId": "api-traffic-observability", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Insomnia is an API client/design tool for making and testing requests, not a gateway or traffic-monitoring platform; the evidence pack shows no metrics/logging/tracing integrations for monitoring live API traffic, which is outside its product category.", "evidenceIds": [] }, { "productId": "insomnia", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Insomnia has versioned file/import formats (e.g., Insomnia JSON v4/YAML v5, OpenAPI 3.0/3.1) but there is no documentation of a formal versioning scheme or deprecation policy for any Insomnia-exposed API, plugin API, or CLI interface that an AI agent would rely on for stability guarantees.", "evidenceIds": [ "insomnia-docs-19", "insomnia-docs-8" ] }, { "productId": "insomnia", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Insomnia's Collection Runner and unlimited collection runs let users execute many requests/tests in bulk, and the Inso CLI automates this across CI pipelines, giving solid bulk-execution support (insomnia-docs-4, insomnia-docs-20, insomnia-docs-39, insomnia-docs-53, insomnia-docs-55). However, evidence is limited to running/testing many requests together—there's no documented bulk editing, bulk tagging, or bulk deletion of items, and no independent/hands-on confirmation of this specific bulk workflow. Missing for 10: evidence of bulk editing/management of many items beyond execution, and independent corroboration of the Collection Runner/Inso CLI bulk workflow.", "evidenceIds": [ "insomnia-docs-4", "insomnia-docs-20", "insomnia-docs-39", "insomnia-docs-53", "insomnia-docs-55", "insomnia-docs-38" ] }, { "productId": "insomnia", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Insomnia supports pre-request and after-response scripts that automatically execute logic tied to request lifecycle events, plus chained requests and CI-triggered automation via the Inso CLI, which constitutes a form of automatic action-on-event behavior. However, there is no evidence of a general-purpose rule/trigger system for arbitrary events (e.g., webhooks, scheduled triggers, or cross-tool event listeners) beyond the request-send/response lifecycle. missing for 10: evidence of a broader event-trigger/rule engine beyond request scripting, and independent confirmation of automation reliability.", "evidenceIds": [ "insomnia-docs-2", "insomnia-docs-21", "insomnia-docs-37", "insomnia-docs-38", "insomnia-docs-53" ] }, { "productId": "insomnia", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Insomnia offers Inso CLI for CI-triggered test runs and collection runners, but there is no evidence of any built-in scheduler or recurring-job feature (e.g., cron-like monitors) comparable to what competitors offer; scheduling would have to be bolted on via external CI cron jobs, which isn't documented as part of Insomnia itself.", "evidenceIds": [] }, { "productId": "insomnia", "storyId": "automation-versioned-workflows", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Insomnia's Git Sync connects collections/environments directly to a Git repo, offers a merge view to review and resolve conflicts, and lets you convert projects between Local Vault, Cloud Sync, and Git Sync — giving version history and review workflows for automations (collections, scripts, environments). Rollback is implied via standard Git history rather than a dedicated in-app rollback UI. Missing for 10: explicit in-app rollback/versioned-restore feature beyond relying on external Git tooling, and independent hands-on confirmation of the merge/review workflow in practice.", "evidenceIds": [ "insomnia-docs-22", "insomnia-docs-23", "insomnia-docs-44", "insomnia-docs-42", "insomnia-docs-32", "insomnia-docs-45", "insomnia-gh-1" ] }, { "productId": "insomnia", "storyId": "collections-as-files-in-git", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Insomnia's Git Sync feature explicitly stores project/collection data directly in a user's own Git repository as files (not on Insomnia's servers), supports merge conflict resolution, and can be toggled per-project alongside Cloud/Local storage — directly matching the story. Documentation also confirms this is a longstanding, git-native workflow (git version control mentioned for collaborative API design) and the GitHub README corroborates 'Git Sync: for Git storage using any 3rd party Git repository.' Missing for 10: independent hands-on confirmation of the exact plain-text file format on disk and no community verification of git-diff friendliness.", "evidenceIds": [ "insomnia-docs-22", "insomnia-docs-23", "insomnia-docs-32", "insomnia-docs-44", "insomnia-docs-45", "insomnia-docs-42", "insomnia-gh-1" ] }, { "productId": "insomnia", "storyId": "contract-validation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Insomnia's docs describe general request/response testing, chaining, and OpenAPI spec linting (via 'inso lint spec') for validating the spec file itself, but there is no evidence of a feature that validates live API responses at runtime against an OpenAPI/schema contract to detect drift.", "evidenceIds": [] }, { "productId": "insomnia", "storyId": "declarative-config-gitops", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Insomnia supports Git Sync so projects/specs/collections live as files in a Git repo, and Inso CLI is explicitly designed to run in CI (disabling prompts, providing exit codes) to lint OpenAPI specs and run tests as part of pipelines. This gives a real GitOps-style workflow for API design/testing artifacts, but Insomnia manages API client/design/test config, not actual gateway infrastructure (routes, plugins, gateway resources) — that lives in Kong Gateway/Konnect, not Insomnia. Missing for 10: evidence of declaratively managing live gateway/infrastructure resources (not just specs/tests) from Insomnia, and independent/hands-on confirmation of CI pipelines using Inso CLI successfully.", "evidenceIds": [ "insomnia-docs-22", "insomnia-docs-44", "insomnia-docs-49", "insomnia-docs-53", "insomnia-docs-39", "insomnia-gh-1" ] }, { "productId": "insomnia", "storyId": "design-export-openapi", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Insomnia has a dedicated 'Design' feature for building OpenAPI specs with live preview, automatic linting, and code-snippet generation, and it supports full import/export of OpenAPI 3.0/3.1 specs (including via the inso CLI for CI linting), directly matching a spec-first workflow with OpenAPI export. missing for 10: independent hands-on validation of the spec-design workflow specifically (community evidence only discusses general client usage, not OpenAPI spec-first design).", "evidenceIds": [ "insomnia-docs-5", "insomnia-docs-16", "insomnia-docs-17", "insomnia-docs-18", "insomnia-docs-42", "insomnia-docs-8", "insomnia-docs-19", "insomnia-docs-49", "insomnia-probe-4" ] }, { "productId": "insomnia", "storyId": "environment-variable-sets", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Insomnia explicitly supports environments as groups of variables (base URLs, tokens, credentials) with global, collection, and folder-level scoping, enabling switching between dev/staging/prod configurations; unlimited environments are confirmed in pricing docs. Missing for 10: no explicit hands-on/community example of a dev/staging/prod switching workflow, only first-party docs.", "evidenceIds": [ "insomnia-docs-9", "insomnia-docs-33", "insomnia-docs-54", "insomnia-docs-55" ] }, { "productId": "insomnia", "storyId": "gateway-auth-plugins", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Insomnia is an API client/testing tool for designing, sending, and debugging requests, not an API gateway that enforces authentication/authorization policies on upstream traffic; that role belongs to products like Kong Gateway. The evidence only shows Insomnia's own auth support for making requests (SAML IdP support, vault integration) and RBAC on its collaboration platform, not gateway-plugin-based enforcement on upstream APIs.", "evidenceIds": [] }, { "productId": "insomnia", "storyId": "gateway-rate-limiting", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Insomnia is an API client/design tool for building, testing, and mocking requests, not a gateway or proxy that enforces traffic policies like rate limiting per consumer/route; that capability belongs to gateway products (e.g., Kong Gateway) rather than an API client.", "evidenceIds": [] }, { "productId": "insomnia", "storyId": "generate-publish-api-docs", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Insomnia's design mode lets you build/iterate on OpenAPI specs, view a generated preview, get linting and auto-generated code snippets in 12+ languages, which serves as a form of documentation generation from specs (insomnia-docs-5, insomnia-docs-16, insomnia-docs-17, insomnia-docs-18). However, there is no evidence of a dedicated 'publish' feature (e.g., hosting a documentation site or public API reference) — only in-app preview and collaborative editing via Git are documented. Missing for 10: a documented publish/export-to-hosted-docs capability, evidence of turning collections (not just specs) into shareable documentation, and any independent confirmation of a publishing workflow.", "evidenceIds": [ "insomnia-docs-5", "insomnia-docs-16", "insomnia-docs-17", "insomnia-docs-18", "insomnia-docs-42" ] }, { "productId": "insomnia", "storyId": "import-export-collections", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly detail import/export in multiple portable formats (Insomnia JSON/YAML, Postman v2.0/v2.1, HAR, OpenAPI 3.0/3.1, Swagger, WSDL, cURL) with UI import methods and Inso CLI support for automation/CI, plus Git Sync for version-controlled portability across tools. Missing for 10: no independent hands-on verification of round-trip fidelity between tools beyond vendor docs.", "evidenceIds": [ "insomnia-docs-8", "insomnia-docs-19", "insomnia-docs-48", "insomnia-docs-40", "insomnia-docs-22", "insomnia-docs-53" ] }, { "productId": "insomnia", "storyId": "import-openapi-spec", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Insomnia's docs explicitly list OpenAPI 3.0/3.1 (and Swagger) as supported import formats and describe the UI flow (Import → File/Clipboard/URL) to bring specs into collections, with CI-side linting via `inso lint spec`. missing for 10: independent/hands-on confirmation that OpenAPI import correctly generates requests/collections without manual fixes.", "evidenceIds": [ "insomnia-docs-8", "insomnia-docs-19", "insomnia-docs-48", "insomnia-docs-49" ] }, { "productId": "insomnia", "storyId": "internal-api-catalog", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Insomnia's evidence covers request collaboration, organizations, Git sync, and RBAC for sharing collections among teammates inside the app, but nothing describes a published catalog or developer portal for broader API discovery/consumption by other teams. missing for 10: any mention of a public/internal API catalog, developer portal, or discovery UI outside the Insomnia app itself.", "evidenceIds": [ "insomnia-docs-12", "insomnia-docs-11", "insomnia-docs-42" ] }, { "productId": "insomnia", "storyId": "lint-specs-style-rules", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Insomnia's Design view auto-lints OpenAPI specs showing error type/line/message (insomnia-docs-17), and the Inso CLI can run 'inso lint spec' in CI to fail builds on errors (insomnia-docs-49, insomnia-docs-53), giving a platform-engineer a way to gate specs on validation issues. However, there is no evidence of configurable style/governance rulesets (e.g., custom rule files, organization-wide style guides, or Spectral-like rule configuration) — the linting appears to be basic schema/structural validation rather than customizable governance policy enforcement. Missing for 10: evidence of configurable lint rule files or style-guide customization, evidence of org-level governance policy enforcement, and independent confirmation the lint rules can be tailored beyond basic OpenAPI validity checks.", "evidenceIds": [ "insomnia-docs-17", "insomnia-docs-49", "insomnia-docs-53", "insomnia-docs-16" ] }, { "productId": "insomnia", "storyId": "mock-servers-from-examples", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Insomnia Cloud mock servers can be created directly from OpenAPI definitions, prompts, or live URL responses, auto-scaffolding routes/responses so consumers can build against the API before it's complete, and dynamic mocking supports realistic templated/fake data responses. missing for 10: independent/hands-on verification of mock server reliability and no evidence of generating mocks directly from saved request 'examples' (only prompt/spec/URL inputs documented).", "evidenceIds": [ "insomnia-docs-6", "insomnia-docs-10", "insomnia-docs-29", "insomnia-docs-28", "insomnia-docs-50", "insomnia-docs-51" ] }, { "productId": "insomnia", "storyId": "multi-protocol-requests", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly confirm HTTP, gRPC, GraphQL, and WebSocket request support (insomnia-docs-15, insomnia-docs-7/43, insomnia-docs-26), with GraphQL type-aware autocomplete and gRPC protobuf upload support. SSE is not explicitly named in the docs, though WebSocket/streaming support is documented. missing for 10: explicit SSE (Server-Sent Events) documentation, independent hands-on corroboration of multi-protocol usage.", "evidenceIds": [ "insomnia-docs-15", "insomnia-docs-7", "insomnia-docs-43", "insomnia-docs-26", "insomnia-docs-27", "insomnia-gh-2" ] }, { "productId": "insomnia", "storyId": "offline-local-workspace", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Insomnia's docs describe offline-friendly options — Scratch Pad requires 'No login ever required' (insomnia-docs-31), a 'Local Vault' project-storage mode (insomnia-docs-32/45), and Git Sync that keeps data in the user's own repo without touching Insomnia's cloud (insomnia-gh-1, insomnia-docs-22/44). However, a first-hand community report states the main Insomnia REST client 'now requires an account,' prompting users to call it a 'rug pull' and switch to fully offline-first alternatives like Bruno (insomnia-comm-14), directly contradicting the no-account claim for general (non-Scratch-Pad) usage. missing for 10: clear vendor confirmation that the main workspace (not just Scratch Pad) can be used indefinitely without ever creating a cloud account, and independent hands-on verification resolving the community complaint.", "evidenceIds": [ "insomnia-docs-31", "insomnia-docs-32", "insomnia-docs-45", "insomnia-gh-1", "insomnia-docs-22", "insomnia-docs-44", "insomnia-comm-14" ] }, { "productId": "insomnia", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Insomnia ships an official CLI (Inso CLI) that can run collections, lint OpenAPI specs, and automate tests in CI (insomnia-docs-3, -39, -49, -53, insomnia-probe-4), giving CLI/automation parity for a meaningful subset of workflows. However, there's no evidence of a full programmatic/API surface covering all UI capabilities (e.g., AI-assisted mock server creation, GraphQL design/linting, gRPC/Protobuf uploads, environment/Git-sync management, RBAC) — these remain UI-only per the docs. A community comment even flags weak scripting/glue-code support relative to competitors (insomnia-comm-15). missing for 10: CLI/API coverage for mock servers, GraphQL/gRPC workflows, environment & Git sync management, and independent confirmation that Inso CLI matches full UI functionality.", "evidenceIds": [ "insomnia-docs-3", "insomnia-docs-39", "insomnia-docs-49", "insomnia-docs-53", "insomnia-probe-4", "insomnia-comm-15" ] }, { "productId": "insomnia", "storyId": "openness-full-export", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Insomnia supports export in multiple open formats (Postman v2, HAR, OpenAPI 3.0/3.1, Swagger, WSDL, cURL, plus its own JSON/YAML) and Git Sync keeps project data in a user-controlled repository rather than locked in a proprietary cloud, giving strong data portability. However, there's no explicit 'export everything' bulk/full-account export guarantee (e.g., all environments, secrets, RBAC configs, org settings) and no independent verification that exported data is fully lossless/complete outside vendor docs. missing for 10: evidence of a comprehensive full-account/workspace export (not just collections), independent/hands-on confirmation of lossless round-trip export, and documentation on exporting non-collection data like vault secrets or org settings.", "evidenceIds": [ "insomnia-docs-8", "insomnia-docs-19", "insomnia-docs-22", "insomnia-docs-44", "insomnia-gh-1", "insomnia-docs-32", "insomnia-docs-45" ] }, { "productId": "insomnia", "storyId": "openness-open-license", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "The official GitHub repository explicitly ships an Apache-2.0 LICENSE file, confirming Insomnia's source code is available under a recognized open-source license for anyone (including AI-native users) to read. Missing for 10: independent scrutiny of whether all components (e.g., cloud sync backend) are covered by the same license, and no third-party audit confirming license accuracy beyond the repo itself.", "evidenceIds": [ "insomnia-gh-3" ] }, { "productId": "insomnia", "storyId": "openness-self-host", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Insomnia's core client is Apache-2.0 licensed and open source, and Git Sync explicitly keeps all project data in the user's own Git repo with 'Insomnia does not host or manage it,' letting AI-native users avoid the vendor's cloud entirely. However, there is no documented self-hosted server/backend package (e.g., a self-hosted sync or collaboration server) analogous to on-prem deployments some competitors offer—self-hosting here is effectively 'run the open-source desktop client + your own git remote,' not a full self-hosted service stack. Missing for 10: an official self-hosted server/Docker deployment guide, and independent confirmation that self-built/self-hosted setups work end-to-end.", "evidenceIds": [ "insomnia-gh-3", "insomnia-docs-22", "insomnia-docs-44", "insomnia-docs-34", "insomnia-docs-32" ] }, { "productId": "insomnia", "storyId": "organize-requests-collections", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Insomnia's docs clearly show organizing requests into collections/folders, with Collection Runner to rerun multiple requests in order, unlimited collection runs, environments scoped to collections/folders, and persistent storage via Git Sync/Cloud Sync for rerunning anytime. Community evidence corroborates long-term real-world use of saved/frequently-used requests. Missing for 10: independent hands-on verification specifically of folder-nesting UX and rerun reliability at scale beyond vendor docs.", "evidenceIds": [ "insomnia-docs-20", "insomnia-docs-4", "insomnia-docs-55", "insomnia-docs-9", "insomnia-docs-54", "insomnia-docs-22", "insomnia-comm-6" ] }, { "productId": "insomnia", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Insomnia lets users choose storage mode—Local Vault, Cloud Sync, or Git Sync to a self-hosted/3rd-party repo—giving some control over where data physically resides, and Kong states its cloud servers cannot read synced project data. However there is no evidence of selectable geographic region/data-residency options within Insomnia's cloud offering itself. Missing for 10: explicit region selection for cloud-hosted data, documented data-residency/compliance zones, independent confirmation of storage location guarantees.", "evidenceIds": [ "insomnia-docs-32", "insomnia-docs-22", "insomnia-docs-34" ] }, { "productId": "insomnia", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Insomnia's docs describe AI features (mock generation, commit suggestions) and data-security claims about sync servers not reading project data, but there is no evidence of any control or policy letting users opt out of their data being used for AI model training.", "evidenceIds": [ "insomnia-docs-34", "insomnia-docs-51", "insomnia-docs-52" ] }, { "productId": "insomnia", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Insomnia lets users choose where data lives (Local Vault, Git Sync, Cloud Sync) and states its servers cannot read synced project data, plus a no-login Scratch Pad mode, which gives meaningful control over data storage/retention posture. However there is no explicit documentation of account/data deletion mechanisms, retention periods, or GDPR-style data export/erasure workflows. Missing for 10: explicit data deletion/retention policy documentation, account/data purge controls, independent confirmation of these claims.", "evidenceIds": [ "insomnia-docs-32", "insomnia-docs-45", "insomnia-docs-34", "insomnia-docs-22", "insomnia-docs-44", "insomnia-docs-31" ] }, { "productId": "insomnia", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item discusses telemetry, analytics, or usage-tracking settings, nor any opt-out mechanism, despite this being a reasonable capability for a desktop app to document.", "evidenceIds": [] }, { "productId": "insomnia", "storyId": "request-lifecycle-scripting", "verdict": "disputed", "quality": 6, "confidence": "medium", "rationale": "Docs clearly document pre-request and after-response scripting, chaining responses across requests, environment variables, and CI automation via Inso CLI (insomnia-docs-2, -21, -37, -38, -39, -53), which directly matches the story. However, a hands-on community comment contradicts this, calling Insomnia's scripting/glue-code support 'garbage-tier' compared to Postman (insomnia-comm-15), indicating the scripting capability may be weaker or less mature in practice than the docs suggest. Missing for 10: independent verification of scripting robustness/parity with competitors, and more recent hands-on confirmation resolving the community critique.", "evidenceIds": [ "insomnia-docs-2", "insomnia-docs-21", "insomnia-docs-37", "insomnia-docs-38", "insomnia-docs-39", "insomnia-docs-53", "insomnia-comm-15" ] }, { "productId": "insomnia", "storyId": "route-transform-requests", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Insomnia is an API client/design/testing tool used by developers to build, test, and mock API requests locally; it does not sit in the request path routing, load-balancing, or transforming live traffic between consumers and upstream production services. That gateway/proxy functionality is a different product category (e.g., Kong Gateway), so this axis is a category error for Insomnia.", "evidenceIds": [] }, { "productId": "insomnia", "storyId": "run-collection-test-suites", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Insomnia supports collection runs (Collection Runner, unlimited collection runs), chaining requests, pre/post-request scripts, and the Inso CLI which 'disables prompts and provides exit codes to pass or fail the CI workflow,' enabling automated test suites with pass/fail reporting in CI. Docs also show lint commands failing builds on errors, reinforcing pass/fail semantics. missing for 10: no independent/hands-on report confirming the CLI pass/fail report format or reliability in practice, and no detail on report output formatting (e.g., JUnit/HTML report generation).", "evidenceIds": [ "insomnia-docs-3", "insomnia-docs-4", "insomnia-docs-20", "insomnia-docs-38", "insomnia-docs-39", "insomnia-docs-53", "insomnia-docs-49" ] }, { "productId": "insomnia", "storyId": "run-tests-in-ci", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Insomnia's docs explicitly describe the Inso CLI as designed for CI environments with disabled prompts and pass/fail exit codes, plus commands like 'inso lint spec' to fail builds on errors, and general CLI automation of test collections tied to Git workflows. missing for 10: independent/hands-on confirmation of CI CLI reliability, real-world CI pipeline examples beyond vendor docs", "evidenceIds": [ "insomnia-docs-3", "insomnia-docs-39", "insomnia-docs-49", "insomnia-docs-53", "insomnia-probe-4" ] }, { "productId": "insomnia", "storyId": "scheduled-api-monitors", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Insomnia's Inso CLI can run collection tests in CI with exit codes to pass/fail builds, and can lint OpenAPI specs to fail builds on errors, which developers could wire into a scheduled CI job to approximate a monitor. However, there is no evidence of a native scheduling feature or built-in alerting mechanism (email/Slack/webhook) for failed runs, unlike a dedicated Monitors product — the burden of scheduling and alerting falls on external CI tooling. missing for 10: native monitor scheduling UI, built-in alert/notification channel on failure, and any hands-on confirmation this workflow is used for uptime-style monitoring.", "evidenceIds": [ "insomnia-docs-39", "insomnia-docs-53", "insomnia-docs-49", "insomnia-docs-20", "insomnia-docs-55" ] }, { "productId": "insomnia", "storyId": "secrets-masked-or-vaulted", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Insomnia explicitly documents external vault integration to retrieve secret values automatically when sending requests, and environments are described as a way to store credentials/tokens without hardcoding them in shared collection files. RBAC and organization-level sharing controls further support keeping secrets scoped safely. Missing for 10: explicit documentation of a 'masked' or obfuscated secret display feature in the UI, and independent/hands-on confirmation that vault integration works reliably in practice.", "evidenceIds": [ "insomnia-docs-24", "insomnia-docs-33", "insomnia-docs-9", "insomnia-docs-11" ] }, { "productId": "insomnia", "storyId": "shared-team-workspaces", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Insomnia organizations support sharing collections/environments securely with colleagues, with RBAC, Cloud Sync, and Git Sync as alternate sync backends, plus merge conflict resolution for team workflows. missing for 10: independent/hands-on evidence of team sync experience (community comments focus on rug-pull login requirement and lightweight vs bloat, not team sync workflow specifics), and no detail on real-time collaboration limits or seat/team size constraints.", "evidenceIds": [ "insomnia-docs-12", "insomnia-docs-11", "insomnia-docs-22", "insomnia-docs-23", "insomnia-docs-32", "insomnia-docs-42", "insomnia-docs-34" ] }, { "productId": "insomnia", "storyId": "workspace-roles-permissions", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Insomnia documents RBAC for fine-grained control over Insomnia resources and organizations for secure collaboration/sharing, plus SAML 2.0 SSO for identity, directly matching the platform-engineer's need to control who can view/edit APIs and collections. However, there's no detail on granularity (e.g., per-collection vs per-project roles), no independent/hands-on validation of RBAC in practice, and it's gated behind paid tiers per pricing page rather than shown in a workflow. Missing for 10: detailed documentation on role granularity/permission scopes for collections specifically, independent or community validation that RBAC works as described, and any hands-on walkthrough of setting collection-level permissions.", "evidenceIds": [ "insomnia-docs-11", "insomnia-docs-25", "insomnia-docs-12", "insomnia-docs-47" ] }, { "productId": "insomnia", "storyId": "write-response-assertions", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Insomnia's docs describe a dedicated testing feature set—chaining requests, pre-/after-response scripting to add logic, automatic cookie handling, and CLI-driven automated test runs across collections—strongly implying support for assertions on response status/body/headers, but no evidence pack item shows explicit assertion syntax or examples asserting on status codes, body content, or headers specifically. Missing for 10: explicit assertion API/examples (e.g., expect(response.status).to.equal), documented header/body matcher functions, and independent hands-on confirmation of assertion authoring.", "evidenceIds": [ "insomnia-docs-37", "insomnia-docs-38", "insomnia-docs-39", "insomnia-docs-41", "insomnia-docs-4", "insomnia-docs-21" ] }, { "productId": "kong", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A live probe confirms Kong serves a working llms.txt at developer.konghq.com/llms.txt (HTTP 200) describing its documentation set for Gateway, Konnect, AI Gateway, and Mesh, directly matching the story of pointing an agent at an agent-oriented docs entrypoint. Missing for 10: no evidence of a more extensive llms-full.txt or additional per-product agent-doc indexes, and no independent (non-probe) commentary confirming real-world agent usage of this file.", "evidenceIds": [ "kong-probe-1" ] }, { "productId": "kong", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Kong's evidence focuses on AI Gateway/MCP traffic management (routing LLM calls, semantic security, observability plugins) but none of it shows Kong itself analyzing a user's own API/traffic data to generate AI insights or suggestions inside the product (e.g., an AI copilot summarizing gateway usage or recommending config changes). This is a plausible feature for a management platform like Konnect, so the axis applies, but no evidence supports it.", "evidenceIds": [ "kong-gh-9", "kong-docs-70", "kong-docs-61", "kong-docs-46" ] }, { "productId": "kong", "storyId": "agentic-autonomous-automation", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Kong is an API/AI gateway and traffic-management platform (routing, security, observability for APIs, LLMs, MCP, A2A traffic) — it processes and governs requests, but it is not an automation/workflow engine and no evidence shows users can configure autonomous background-running automations within Kong itself; this axis is a category mismatch for a gateway product.", "evidenceIds": [] }, { "productId": "kong", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Kong's evidence describes AI Gateway/MCP governance features for routing and securing AI traffic, but nothing shows Kong itself embedding a built-in assistant that a user delegates tasks to within the product UI (e.g., Konnect). Missing for 10: any documented in-product AI assistant/chat feature, evidence of task delegation to it, and independent confirmation of its use.", "evidenceIds": [ "kong-docs-2", "kong-docs-3", "kong-gh-9" ] }, { "productId": "kong", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Kong Gateway supports headless/CI operation via DB-less declarative config, decK and kongctl CLI tools for API Lifecycle Automation (APIOps), a RESTful Admin API, and Docker-based deployment—all enabling fully automated, non-interactive pipelines. missing for 10: independent hands-on CI pipeline example or third-party case study demonstrating actual CI/CD integration in production.", "evidenceIds": [ "kong-docs-4", "kong-gh-4", "kong-gh-5", "kong-gh-7", "kong-gh-10", "kong-docs-67", "kong-docs-68", "kong-docs-56" ] }, { "productId": "kong", "storyId": "agentic-mcp-client", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Kong's AI Gateway positions itself as an 'MCP Traffic Gateway' that can govern, secure, and observe traffic to/from MCP servers, and its Catalog can inventory MCP servers alongside APIs and agents (kong-docs-65, kong-docs-22, kong-gh-6). However, the evidence centers on Kong auto-generating/exposing MCP servers from REST APIs and proxying/governing MCP traffic between agents and servers, not on Kong itself acting as an MCP client that plugs in external MCP servers to consume their tools directly. Missing for 10: clear documentation of Kong itself invoking/using tools from a connected MCP server (client-side tool use), independent/hands-on confirmation of this workflow, and detail on how a user 'plugs in' an arbitrary third-party MCP server for Kong's own use.", "evidenceIds": [ "kong-docs-65", "kong-docs-22", "kong-gh-6", "kong-docs-53", "kong-docs-54", "kong-docs-59" ] }, { "productId": "kong", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Kong documents an official Kong Konnect MCP Server (kong-docs-66, kong-probe-3) plus dedicated MCP gateway docs for connecting agents (kong-docs-3, kong-docs-65) and MCP auto-generation/tool composition (kong-docs-53, kong-docs-54). This is first-party documentation of an official MCP server, not just client-side MCP consumption. missing for 10: independent/hands-on third-party verification of the MCP server working end-to-end, and more detail on setup/connection steps beyond marketing copy.", "evidenceIds": [ "kong-docs-66", "kong-probe-3", "kong-docs-3", "kong-docs-65", "kong-docs-53", "kong-docs-54", "kong-gh-6" ] }, { "productId": "kong", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Kong ships an official Konnect MCP Server (kong-docs-66, kong-probe-3), which lets AI-native users operate Kong Konnect via natural-language commands through an MCP client, and Kong also documents AI Gateway support for AI CLIs/agent traffic (kong-docs-58). However, this NL control is scoped to Konnect cloud management rather than the core Gateway, documentation is thin (a one-line stub), and there's no independent/hands-on evidence of the breadth or reliability of natural-language operation. Missing for 10: detailed docs on supported commands/scope, independent user validation, and NL support for the open-source Gateway itself (not just Konnect).", "evidenceIds": [ "kong-docs-66", "kong-probe-3", "kong-docs-58" ] }, { "productId": "kong", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Kong ships official CLIs (decK for declarative APIOps and kongctl for Konnect/Gateway management) documented at developer.konghq.com/deck and /kongctl, which can be scripted/automated by AI-native or agentic workflows. However, there's no evidence these CLIs are explicitly designed or marketed for AI-native/agentic use (e.g., no agent-specific CLI features, no mention of LLM/agent integration in the CLI docs themselves — that capability is described separately for AI Gateway proxying other tools' CLI traffic, not Kong's own CLI being agent-oriented). missing for 10: explicit agentic/AI-native design goals for decK/kongctl, independent hands-on verification of CLI use by AI agents.", "evidenceIds": [ "kong-docs-4", "kong-docs-67", "kong-docs-68", "kong-probe-4" ] }, { "productId": "kong", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Kong exposes a documented public Admin API for configuring the gateway (kong-gh-5, kong-gh-10), a directory of OpenAPI specs for Kong's APIs (kong-docs-9), and RESTful proxying of any API with extensibility via plugins (kong-docs-44), giving AI-native users a clear, documented programmatic surface to drive the product. missing for 10: a live discoverable OpenAPI/swagger machine-readable spec at the docs root (probe found 404s at common openapi.json paths), and independent/hands-on confirmation of API-driven automation beyond vendor docs.", "evidenceIds": [ "kong-gh-5", "kong-gh-10", "kong-docs-9", "kong-docs-44", "kong-docs-11", "kong-probe-2" ] }, { "productId": "kong", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Kong Gateway's core auth plugins (JWT, OAuth2, ACLs, API keys) and Vaults for secret storage provide the building blocks for scoped, least-privilege credentials per consumer, and AI Gateway/Konnect docs mention defining entitlements for agent/MCP/tool consumption, suggesting credential scoping applies to agent traffic too. However, there is no dedicated, documented workflow specifically for 'issuing scoped credentials to an AI agent' — the evidence is generic auth/authz plus general entitlement language rather than a concrete agent-credential-issuance feature or example.\nmissing for 10: a first-party doc/example showing scoped API-key/OAuth-scope issuance specifically for an agent identity, and independent/hands-on confirmation of least-privilege enforcement in agent/MCP contexts.", "evidenceIds": [ "kong-gh-8", "kong-docs-20", "kong-docs-47", "kong-docs-55", "kong-gh-6", "kong-probe-3" ] }, { "productId": "kong", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Kong offers CLIs (decK, kongctl), Admin/OpenAPI API directories, and Kong Manager UI, but no evidence of official client SDKs (e.g., language-specific libraries) for building against Kong Gateway/AI Gateway/Konnect APIs.", "evidenceIds": [ "kong-docs-9", "kong-docs-67", "kong-docs-68", "kong-probe-2" ] }, { "productId": "kong", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Kong is an API/AI gateway product; the evidence covers proxying, MCP, LLM routing, logging, and OpenTelemetry metrics, but there is no mention of webhook subscription or event-driven notification delivery to AI-native consumers.", "evidenceIds": [] }, { "productId": "kong", "storyId": "ai-generate-tests-from-api", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Kong is an API/AI gateway and traffic management platform, not a testing/mocking tool; there is no evidence of any capability to generate test cases or assertions from an API spec or responses. This capability is outside Kong's product category (gateway/governance vs. test generation), so the axis does not apply.", "evidenceIds": [] }, { "productId": "kong", "storyId": "ai-llm-traffic-gateway", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Kong's AI Gateway explicitly proxies and governs LLM traffic (multi-provider routing, rate limiting, semantic security, DLP/prompt guards, cost control) and dedicated MCP Gateway capabilities cover MCP traffic governance, security, observability, and auto-generation, with docs on rate-limiting plugins, data governance/prompt safety, provider routing, and OTel metrics for GenAI/MCP/A2A traffic. Missing for 10: independent hands-on validation of the AI Gateway/MCP governance features specifically (community evidence only covers unrelated core gateway routing bugs).", "evidenceIds": [ "kong-docs-2", "kong-docs-3", "kong-docs-57", "kong-docs-58", "kong-docs-60", "kong-docs-61", "kong-gh-6", "kong-gh-9", "kong-gh-15", "kong-docs-64", "kong-docs-65" ] }, { "productId": "kong", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Kong provides a Dev Portal that lets users browse/search API documentation and try API operations, and has an OpenAPI directory for its own APIs, which supports interactive exploration with runnable examples; however, the probe for standard OpenAPI/Swagger UI endpoints on the docs site 404'd, and there's no direct evidence of a live 'try it out' interactive console with runnable examples specifically for AI-native users. missing for 10: independent/hands-on confirmation of a runnable 'try it now' console, explicit AI-native tailoring of the interactive reference, and successful OpenAPI/swagger endpoint discovery.", "evidenceIds": [ "kong-docs-21", "kong-docs-49", "kong-docs-9", "kong-probe-2" ] }, { "productId": "kong", "storyId": "api-machine-spec", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Kong docs explicitly reference a 'Directory of OpenAPI specifications for various Kong APIs' (developer.konghq.com/api/), and Dev Portal supports publishing OpenAPI/AsyncAPI specs, indicating machine-readable specs exist and are downloadable. However, a direct probe for standard OpenAPI file locations (openapi.json, swagger.json, etc.) returned 404s, showing the spec isn't trivially discoverable at conventional endpoints. Missing for 10: a confirmed, directly downloadable OpenAPI/AsyncAPI file link or independent verification that the /api/ directory serves raw machine-readable spec files.", "evidenceIds": [ "kong-docs-9", "kong-docs-49", "kong-probe-2" ] }, { "productId": "kong", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Kong's evidence covers DB-less/declarative deployments, Konnect control-plane/data-plane separation, and a Docker quickstart script, but none of this documents a dedicated 'sandbox environment' or test/staging mode explicitly isolated from production data for AI-native testing — the closest hint is Dev Portal's 'try API operations' feature (kong-docs-21), which is not framed as a sandbox against production. Missing for 10: explicit sandbox/staging environment documentation, mock/test data isolation for AI agent calls, and any first-party guidance on safely testing AI Gateway/MCP flows without touching production traffic.", "evidenceIds": [ "kong-docs-21", "kong-docs-45", "kong-docs-56", "kong-gh-7" ] }, { "productId": "kong", "storyId": "api-traffic-observability", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Kong ships built-in plugins for traffic controls, logging, and monitoring (kong-gh-3), Konnect provides status monitoring of control/data planes (kong-docs-46), and AI Gateway explicitly supports OpenTelemetry (OTLP) metrics export for AI/MCP/A2A traffic plus file/docker logs (kong-docs-61, kong-docs-69). However, a community GitHub issue reports a real production case where Kong logging failed to surface 404 errors during cache rebuilding (kong-comm-1, kong-comm-2), showing logging gaps in practice, and the evidence pack lacks explicit documentation of a dedicated distributed-tracing integration (e.g., Zipkin/Jaeger) beyond OTel metrics. missing for 10: explicit tracing plugin/integration docs, and stronger independent confirmation that logging reliably surfaces all traffic issues.", "evidenceIds": [ "kong-gh-3", "kong-docs-46", "kong-docs-61", "kong-docs-69", "kong-comm-1", "kong-comm-2" ] }, { "productId": "kong", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While Kong publishes OpenAPI spec directories for its various APIs (kong-docs-9) and has extensive product documentation, no evidence in the pack describes API versioning conventions or any documented deprecation policy for Kong's own APIs (Admin API, Konnect API, AI Gateway API, etc.). The OpenAPI probe even returned 404s, and no changelog/deprecation-notice pages are cited.", "evidenceIds": [ "kong-docs-9", "kong-probe-2" ] }, { "productId": "kong", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "decK and kongctl provide declarative, bulk configuration management (sync entire API/gateway state from files) and the Admin API allows scripted bulk changes, which supports AI-native automation across many entities (routes, services, plugins) at once. However, evidence doesn't show bulk operations specifically framed for AI-native workflows (e.g., bulk operations on AI Models, MCP tools, or catalog entries) beyond general declarative config sync. missing for 10: explicit bulk operation APIs/examples for AI-specific entities (LLMs, MCP servers, agents), independent hands-on validation of bulk operation performance/reliability at scale.", "evidenceIds": [ "kong-docs-4", "kong-docs-67", "kong-docs-68", "kong-gh-10", "kong-docs-11" ] }, { "productId": "kong", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Kong's plugin architecture (rate-limiting, DLP/safety policies, logging, transformations) lets users define declarative rules that automatically trigger actions on gateway request/response events, and AI Gateway policies (e.g., blocking toxic content) act as automated triggers on AI traffic events. However, there's no evidence of a generic, user-authored event-trigger/automation framework (e.g., 'when X happens do Y' rules across arbitrary events) beyond the fixed set of built-in plugins/policies. Missing for 10: a flexible custom event-trigger rule engine, documentation of arbitrary event types beyond traffic/API-call lifecycle, and independent evidence of this automation working end-to-end.", "evidenceIds": [ "kong-gh-3", "kong-docs-60", "kong-docs-64", "kong-docs-57" ] }, { "productId": "kong", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Kong is an API/AI gateway and traffic management platform; scheduling recurring jobs or workflows is outside its product category (it manages request routing, security, and observability, not job/workflow scheduling). This is a category mismatch rather than a missing feature.", "evidenceIds": [] }, { "productId": "kong", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Kong's decK tool is described as enabling 'API Lifecycle Automation (APIOps)' with a toolkit to 'orchestrate and automate' configuration processes, which implies config-as-code that could be version-controlled and reapplied, but the evidence never explicitly describes diffing, review workflows, or rollback of automations. kongctl is also mentioned only by name with no detail. missing for 10: explicit documentation of versioning/diff review, rollback commands or history, and confirmation this applies to AI agent 'automations' rather than just gateway config.", "evidenceIds": [ "kong-docs-4", "kong-docs-67", "kong-docs-68", "kong-docs-11" ] }, { "productId": "kong", "storyId": "collections-as-files-in-git", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Kong's decK tool supports declarative configuration files (YAML/JSON) that represent Kong's gateway config as plain text, enabling APIOps-style workflows and 'Declarative Databaseless Deployment' which is commonly stored and versioned in git repos. However, the evidence never explicitly shows a 'collections' concept (like Postman-style API collections) or an explicit git-based workflow/example — it's inferred from decK's declarative config model. Missing for 10: explicit documentation of a 'collections' artifact, an explicit git-versioning workflow/example, and confirmation this applies beyond gateway config to broader workspace collections.", "evidenceIds": [ "kong-docs-4", "kong-docs-67", "kong-gh-4", "kong-gh-10" ] }, { "productId": "kong", "storyId": "contract-validation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Kong is an API gateway/traffic management platform with plugins for routing, security, rate limiting, and AI/MCP governance, but there is no evidence of a contract-testing or response-validation-against-spec feature to catch API drift; this is a testing/mocking capability that a full lifecycle toolchain could offer but Kong's evidence pack shows no such capability.", "evidenceIds": [] }, { "productId": "kong", "storyId": "declarative-config-gitops", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "decK and kongctl are dedicated CLIs for declarative APIOps, explicitly designed for CI-driven sync of Kong config (declarative databaseless deployment, Admin API/decK configuration), which directly matches the GitOps/APIOps story. missing for 10: no explicit worked CI pipeline example (e.g., GitHub Actions) or independent case study confirming production GitOps workflows.", "evidenceIds": [ "kong-docs-4", "kong-docs-11", "kong-docs-67", "kong-docs-68", "kong-gh-4", "kong-gh-10", "kong-gh-20" ] }, { "productId": "kong", "storyId": "design-export-openapi", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Kong's docs show it can import/publish existing OpenAPI specs (e.g., Dev Portal publishing APIs using OpenAPI/AsyncAPI, decK declarative config) but there is no evidence of a spec-first design/authoring workflow or an OpenAPI export feature for specs created within Kong. missing for 10: a spec-design/editor capability, evidence of generating/exporting an OpenAPI document from a user-authored spec.", "evidenceIds": [ "kong-docs-49", "kong-docs-9", "kong-docs-4" ] }, { "productId": "kong", "storyId": "environment-variable-sets", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Kong's evidence covers secrets storage via Vaults (including environment-variable-backed secrets) and multi-control-plane management via Konnect/decK, but there is no documented feature for developers to define and switch between named 'dev/staging/prod' environment variable sets when making requests, as in a client-testing workflow. Missing for 10: explicit environment/variable-set switching UI or config, evidence of per-environment variable collections, and any workflow tying such sets to request execution.", "evidenceIds": [ "kong-docs-47", "kong-docs-48", "kong-docs-67", "kong-docs-45" ] }, { "productId": "kong", "storyId": "gateway-auth-plugins", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Kong Gateway natively supports authentication plugins (JWT, basic auth, OAuth2, ACLs) as documented in its GitHub README and plugin hub, plus a quickstart demonstrating adding auth to an API in minutes; Vaults further support securing credentials/secrets referenced by these plugins. mTLS is a well-known Kong plugin capability implied by 'and more' but not explicitly cited in this pack. Missing for 10: explicit first-party documentation/example citing mTLS plugin configuration and independent hands-on validation of OAuth2/mTLS enforcement specifically.", "evidenceIds": [ "kong-gh-8", "kong-gh-1", "kong-gh-13", "kong-gh-14", "kong-gh-18", "kong-docs-20", "kong-docs-47", "kong-docs-48" ] }, { "productId": "kong", "storyId": "gateway-rate-limiting", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Kong ships a dedicated Rate Limiting plugin (kong-docs-64) and its plugin architecture explicitly supports 'enforcing traffic controls, rate limiting' (kong-gh-3), configurable per consumer or route via Admin API/decK. This is a well-documented core gateway capability with first-party docs and consistent GitHub descriptions. Missing for 10: independent hands-on verification of per-consumer vs per-route configuration nuances.", "evidenceIds": [ "kong-docs-64", "kong-gh-3", "kong-gh-5", "kong-gh-10" ] }, { "productId": "kong", "storyId": "generate-publish-api-docs", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Kong's Dev Portal explicitly lets developers publish APIs using OpenAPI or AsyncAPI specifications plus Markdown documentation, with configurable public/private visibility, and lets consumers browse/search that documentation and manage credentials — directly matching the story of generating and publishing docs from specs. Missing for 10: explicit support for importing arbitrary 'collections' (e.g., Postman-style) rather than just OpenAPI/AsyncAPI specs, and independent/hands-on corroboration beyond first-party docs.", "evidenceIds": [ "kong-docs-49", "kong-docs-21", "kong-docs-50", "kong-docs-62", "kong-docs-9" ] }, { "productId": "kong", "storyId": "import-export-collections", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Kong is an API gateway/management platform, not an API-client tool built around request 'collections' (like Postman/Insomnia). Its decK tool exports/imports declarative gateway *configuration* state, not portable request collections meant to move between different client tools, so this axis is a category mismatch for Kong's product type.", "evidenceIds": [] }, { "productId": "kong", "storyId": "import-openapi-spec", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Kong's Dev Portal can publish APIs from OpenAPI/AsyncAPI specs and lets developers 'browse and search API documentation, try API operations' (kong-docs-21, kong-docs-49), which is loosely analogous to generating requests from a spec, but there's no evidence of a Postman-like 'collections' concept or explicit request/collection generation workflow. missing for 10: explicit OpenAPI-import-to-request/collection generation feature, evidence of a collections concept, and any hands-on demonstration of this workflow.", "evidenceIds": [ "kong-docs-21", "kong-docs-49", "kong-docs-9" ] }, { "productId": "kong", "storyId": "internal-api-catalog", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Kong ships a dedicated Dev Portal for publishing API docs (OpenAPI/AsyncAPI, Markdown) with public or authenticated visibility so teams can browse, try operations and self-manage credentials, plus a Catalog feature that acts as a platform-level inventory across APIs, API packages, AI Models, MCP servers and agents for org-wide discovery. This directly matches the platform-engineer story of publishing an internal catalog/portal for API discovery and consumption. missing for 10: independent/hands-on evidence of the Dev Portal or Catalog in real use (only first-party docs), and more detail on cross-team governance/workflow beyond the docs descriptions.", "evidenceIds": [ "kong-docs-21", "kong-docs-22", "kong-docs-29", "kong-docs-40", "kong-docs-49", "kong-docs-50", "kong-docs-62", "kong-docs-63", "kong-docs-51" ] }, { "productId": "kong", "storyId": "lint-specs-style-rules", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Kong is an API gateway/runtime and management platform for proxying, securing, and governing API traffic (and AI/MCP traffic); it manages APIs at runtime and via a catalog, but nothing in the evidence shows spec-linting or design-time governance rule enforcement against OpenAPI/AsyncAPI style rules. This capability belongs to a spec-design/linting tool category, not a gateway product, so the axis does not apply.", "evidenceIds": [ "kong-docs-1", "kong-docs-9", "kong-docs-49", "kong-docs-63" ] }, { "productId": "kong", "storyId": "mock-servers-from-examples", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Kong is an API gateway/management platform for proxying, securing, and governing real API traffic — it is not a mock-server or API-design tool for simulating not-yet-built APIs from examples/specs. This capability is a category error for Kong's product type, not a missing feature.", "evidenceIds": [] }, { "productId": "kong", "storyId": "multi-protocol-requests", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Kong is an API gateway/proxy runtime, not an API client or request-testing tool with collections/workspaces; the story concerns a developer authoring and firing test requests (as in a REST client), which is not a role Kong fills. Evidence shows Kong routes/proxies REST, gRPC, and AI/LLM traffic as infrastructure, but nothing suggests a client UI for composing/sending GraphQL, gRPC, WebSocket, or SSE requests — this axis is a category mismatch for a gateway product.", "evidenceIds": [] }, { "productId": "kong", "storyId": "offline-local-workspace", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Kong Gateway supports declarative, DB-less deployment and Docker installation that can run entirely locally without Konnect (kong-gh-4, kong-gh-7, kong-gh-20), and decK/kongctl let you manage configuration as local files (kong-docs-67, kong-docs-68). However, the documented quickstart path explicitly creates a Konnect cloud control plane and ties licensing to it (kong-docs-45), and there's no dedicated 'local workspace' concept or explicit confirmation that all features work fully offline without any Konnect account. Missing for 10: a documented fully offline quickstart/workflow that never touches Konnect, and independent confirmation of offline usage without a cloud account.", "evidenceIds": [ "kong-gh-4", "kong-gh-7", "kong-gh-20", "kong-docs-45", "kong-docs-67", "kong-docs-68" ] }, { "productId": "kong", "storyId": "openness-api-parity", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Kong Gateway's Admin API is the canonical configuration interface, with Kong Manager UI and decK/kongctl CLI explicitly built on top of it (kong-gh-10, kong-gh-11, kong-docs-67/68), implying API-UI parity by design, and Kong also publishes OpenAPI specs for its APIs (kong-docs-9). Missing for 10: an explicit first-party statement guaranteeing 100% UI/API feature parity (especially for Konnect SaaS UI features) and independent confirmation that no UI-only features exist.", "evidenceIds": [ "kong-gh-10", "kong-gh-11", "kong-docs-9", "kong-docs-67", "kong-docs-68", "kong-docs-11" ] }, { "productId": "kong", "storyId": "openness-full-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Kong's decK tool lets users declaratively dump and sync Gateway/Konnect configuration (routes, services, plugins) as open YAML/JSON files, and Kong explicitly touts 'databaseless deployment... without any vendor lock-in,' supporting a form of open, portable data export. However this only covers gateway configuration, not broader platform data (analytics, logs, catalog, dev-portal content), and there's no explicit documented full-account data export/deletion workflow. Missing for 10: a documented full data-export/account-closure process covering all Konnect/Catalog/Dev Portal data, and independent confirmation that decK exports are complete and lossless.", "evidenceIds": [ "kong-docs-4", "kong-docs-67", "kong-gh-4", "kong-docs-11" ] }, { "productId": "kong", "storyId": "openness-open-license", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "The GitHub repo (Kong/kong) is publicly visible and used for docs, issues, and quickstart instructions, indicating the core gateway source is readable in the open, but no evidence pack item explicitly states the license (e.g., Apache-2.0) or confirms all components (AI Gateway, Konnect, plugins) are open-sourced rather than proprietary SaaS add-ons. missing for 10: explicit license statement/file, confirmation that AI Gateway/Konnect/enterprise pieces are also open-source, independent corroboration of license terms.", "evidenceIds": [ "kong-gh-1", "kong-gh-7", "kong-gh-13", "kong-gh-20" ] }, { "productId": "kong", "storyId": "openness-self-host", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Kong Gateway is open-source (GitHub repo) and explicitly supports self-hosted deployment models including Docker, DB-less, and Hybrid deployments 'without any vendor lock-in', with admin API/decK for configuration and a quickstart script for local setup. This covers the core gateway/AI Gateway functionality, not just the SaaS Konnect offering. Missing for 10: no independent third-party confirmation of large-scale self-hosted production use, and some newer AI/MCP capabilities (Catalog, Konnect MCP Server) appear tied to the Konnect SaaS platform rather than self-hosted core.", "evidenceIds": [ "kong-gh-4", "kong-gh-7", "kong-gh-10", "kong-docs-44", "kong-docs-56", "kong-gh-17" ] }, { "productId": "kong", "storyId": "organize-requests-collections", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Kong is an API gateway/management platform, not an API client tool for testing requests; organizing requests into collections/folders for rerun is a Postman-like client feature, which is a category error for this product type.", "evidenceIds": [] }, { "productId": "kong", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Kong is self-hostable (Gateway can be deployed anywhere, giving implicit data-location control), but the evidence pack contains no explicit region/residency selection feature, no Konnect region options, and no documentation letting a user choose where data is stored.", "evidenceIds": [] }, { "productId": "kong", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Kong's AI Gateway evidence covers traffic routing, semantic security/caching, observability, and MCP governance, but nothing in the pack addresses controlling or opting out of AI model training on user data. Missing for 10: any documented feature or policy for preventing data from being used in model training (e.g., zero-retention enforcement, provider opt-out headers, data-use contracts).", "evidenceIds": [ "kong-docs-2", "kong-gh-9", "kong-gh-15" ] }, { "productId": "kong", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Kong's AI Gateway docs cover logging locations, DLP/PII redaction, and secrets vaulting, but there is no documented capability for AI-native users to set data retention windows or explicitly delete stored logs/request data. missing for 10: retention policy configuration, data deletion/purge mechanism, user-facing controls over stored AI traffic data.", "evidenceIds": [ "kong-docs-60", "kong-docs-69", "kong-docs-47" ] }, { "productId": "kong", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry opt-out or usage tracking controls for Kong products; nothing documents a privacy/telemetry setting.", "evidenceIds": [] }, { "productId": "kong", "storyId": "request-lifecycle-scripting", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Kong's plugin architecture includes request/response transformation plugins (kong-gh-3) that let developers modify ('massage') request and response data, which partially supports the scripting/data-massaging part of the story. However, there is no evidence of an explicit scripting mechanism (e.g., pre-function/post-function or serverless script plugins) or of chaining multiple requests together in sequence. Missing for 10: documented custom-script (pre/post function) plugin capability, explicit request-chaining support, and independent confirmation of these features in practice.", "evidenceIds": [ "kong-gh-3", "kong-docs-64" ] }, { "productId": "kong", "storyId": "route-transform-requests", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Kong Gateway is explicitly documented as providing advanced routing, load balancing, health checking, and request/response transformation plugins, configurable via Admin API or declarative config, positioning it as the central layer for orchestrating microservices traffic between consumers and upstreams. This is corroborated by both first-party docs and the GitHub README's feature list. missing for 10: independent hands-on benchmark or case study specifically validating load-balancing/transformation behavior at scale beyond vendor-provided descriptions.", "evidenceIds": [ "kong-gh-5", "kong-gh-14", "kong-gh-18", "kong-gh-3", "kong-docs-1", "kong-docs-64" ] }, { "productId": "kong", "storyId": "run-collection-test-suites", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Kong is an API gateway/AI gateway product, not an API client/testing tool with collections and test suites; running a 'collection' as an automated test suite with pass/fail reporting is a Postman/Insomnia-style feature, not applicable to a gateway's category.", "evidenceIds": [] }, { "productId": "kong", "storyId": "run-tests-in-ci", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Kong is an API gateway/management platform (plus decK/kongctl CLIs for config management), not an API testing tool; there's no evidence of test-collection running or CI regression-testing CLI functionality, which is a wrong axis for this product category.", "evidenceIds": [] }, { "productId": "kong", "storyId": "scheduled-api-monitors", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Kong is an API gateway/AI gateway platform, not a Postman-style API testing tool; scheduled collection runs as monitors that alert on API breakage is a Postman-specific ci-scripting concept with no analog in Kong's evidence (gateway health checks, observability, and rate limiting are not the same as request-collection monitors). This axis is a category error for a gateway product.", "evidenceIds": [] }, { "productId": "kong", "storyId": "secrets-masked-or-vaulted", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Kong Gateway's Vaults feature explicitly lets platform engineers store secrets outside plaintext config and reference them via environment variables, Konnect Config Store, or third-party vault backends (e.g., HashiCorp Vault), ensuring secrets aren't visible in plaintext throughout the platform. This directly matches the story of masking secrets and integrating with external vaults. Missing for 10: independent/hands-on validation of the vault integration beyond first-party docs, and more detail on which third-party backends are supported.", "evidenceIds": [ "kong-docs-20", "kong-docs-23", "kong-docs-47", "kong-docs-48" ] }, { "productId": "kong", "storyId": "shared-team-workspaces", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Kong is an API/AI gateway and traffic management platform, not a collaboration tool for sharing 'collections' (e.g., API request collections like Postman); this story's axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "kong", "storyId": "workspace-roles-permissions", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence covers Dev Portal visibility settings (public/private), Vaults for secrets, and Catalog for API inventory, but nowhere describes role-based access control or granular view/edit permissions for APIs or collections. No mention of teams, roles, or RBAC constructs in Konnect or Gateway for collaborative access management.", "evidenceIds": [] }, { "productId": "kong", "storyId": "write-response-assertions", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Kong is an API gateway/traffic management product, not a testing/mocking framework; there is no evidence of a test-assertion framework for response status, body, or headers. This story targets test-authoring tools, which is a different product category than a gateway.", "evidenceIds": [] }, { "productId": "postman", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Postman's docs site serves a live llms.txt (HTTP 200) and supports appending .md to any docs page for clean Markdown content, directly enabling an agent to be pointed at agent-oriented docs; this is confirmed hands-on via probes rather than just claimed. missing for 10: independent third-party corroboration that an agent successfully consumed llms.txt end-to-end in a real workflow.", "evidenceIds": [ "postman-probe-1", "postman-probe-2" ] }, { "productId": "postman", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Postman ships an in-product AI layer (Agent Mode/Postbot) that generates suggestions and takes actions on your API data via natural language — sending requests, fixing errors, updating tests, visualizing responses, and writing documentation — which matches the 'AI-generated insights and suggestions from data' story. Missing for 10: independent/hands-on corroboration of Agent Mode's suggestion quality and no evidence of deeper analytics-driven insights (e.g., AI-derived patterns from usage/monitoring data) beyond action-oriented suggestions.", "evidenceIds": [ "postman-docs-8", "postman-docs-33", "postman-docs-48", "postman-docs-9" ] }, { "productId": "postman", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Postman offers automation building blocks — CLI collection runs in CI/CD pipelines, webhooks, and a programmatic API (postman-docs-1, postman-docs-6, postman-docs-19, postman-docs-24, postman-docs-32) — and an MCP Generator/Agent Mode for AI interaction (postman-docs-8, postman-docs-9), but these are schedule/CI-triggered or interactive assistant features rather than evidence of self-directed, unattended AI agent automations running continuously in the background. Missing for 10: documented scheduled/monitor-style autonomous jobs, evidence of AI agents independently triggering and managing Postman workflows without human or CI trigger, and any hands-on validation of such autonomous runs.", "evidenceIds": [ "postman-docs-1", "postman-docs-6", "postman-docs-19", "postman-docs-24", "postman-docs-8", "postman-docs-9", "postman-docs-32" ] }, { "productId": "postman", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Postman ships Agent Mode, a built-in AI assistant that can send requests, fix errors, update tests, and write documentation using natural language (postman-docs-8, postman-docs-33), plus Postbot, another in-app AI assistant with defined usage limits (postman-docs-48). This is a first-party, in-product delegation capability. Missing for 10: independent/hands-on user reports validating Agent Mode's real-world reliability and depth beyond vendor docs.", "evidenceIds": [ "postman-docs-8", "postman-docs-33", "postman-docs-48" ] }, { "productId": "postman", "storyId": "agentic-headless", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Postman CLI (and Newman) explicitly support headless execution of collections, mocks, and governance checks in CI/CD pipelines, and this is documented alongside the Postman API for programmatic integration into toolchains. Missing for 10: independent/hands-on CI benchmark or third-party case study confirming real-world CI reliability beyond vendor docs.", "evidenceIds": [ "postman-docs-1", "postman-docs-19", "postman-docs-6", "postman-docs-32", "postman-docs-42", "postman-probe-5" ] }, { "productId": "postman", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence shows Postman can generate/register MCP servers (docs-7, docs-9) and has an AI 'Agent Mode' (docs-8), but there is no documentation or hands-on evidence that Postman itself acts as an MCP client, letting users plug in external MCP servers so its AI/tools can consume them. Missing for 10: explicit MCP-client integration inside Postman's AI/Agent Mode, UI/config for adding external MCP servers, and any hands-on proof of tool invocation from a connected MCP server.", "evidenceIds": [ "postman-docs-7", "postman-docs-8", "postman-docs-9" ] }, { "productId": "postman", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Postman publishes an official MCP server (postman-mcp-server) and documents both registering MCP servers with its Fabric Gateway and generating MCP servers from Postman API collections, giving AI agents a standard way to connect. Missing for 10: independent/hands-on corroboration that the official MCP server works reliably with third-party agents beyond first-party docs.", "evidenceIds": [ "postman-probe-4", "postman-docs-7", "postman-docs-9" ] }, { "productId": "postman", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Postman's Agent Mode explicitly lets users 'turn words into action across the API lifecycle' — sending requests, fixing errors, updating tests — using natural language, plus Postbot for adding tests/docs via natural-language prompts, both first-party documented features. Missing for 10: independent hands-on validation of natural-language command reliability/accuracy beyond vendor docs.", "evidenceIds": [ "postman-docs-8", "postman-docs-33", "postman-docs-48" ] }, { "productId": "postman", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Postman ships an official Postman CLI with detailed docs covering running collections, linting, security/governance checks, mock servers, and CI/CD integration, and explicitly documents giving coding agents CLI context for API discovery, code generation, and maintenance tasks, directly matching an AI-native CLI story. A probe confirms the CLI docs page is live and first-party. Missing for 10: independent/hands-on community validation of AI-agent CLI usage beyond vendor docs.", "evidenceIds": [ "postman-docs-3", "postman-docs-32", "postman-probe-5" ] }, { "productId": "postman", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Postman documents a public Postman API for programmatically managing assets (postman-docs-5,41), an OpenAPI spec is live (postman-probe-3), and the API can be combined with CLI/CI-CD workflows (postman-docs-6,19). This is a well-documented, first-party public API surface for driving Postman programmatically. Missing for 10: no independent/hands-on corroboration of API robustness or rate limits from community sources.", "evidenceIds": [ "postman-docs-5", "postman-docs-6", "postman-docs-19", "postman-docs-41", "postman-probe-3" ] }, { "productId": "postman", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Postman has team/workspace role permissions and a Vault for secrets, but nothing documents issuing scoped or least-privilege API credentials specifically for an AI agent's use (e.g., agent-specific API keys, granular scopes, or token restrictions tied to Agent Mode or MCP usage).", "evidenceIds": [] }, { "productId": "postman", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Postman's own API (postman-docs-5, postman-docs-41) and a CLI (postman-probe-5) for programmatic access, and a feature to generate SDKs *from* user collections (postman-docs-4), but there is no evidence of officially maintained Postman SDKs (e.g., Node/Python/Go client libraries) that an AI-native developer could build against to integrate with Postman itself.", "evidenceIds": [ "postman-docs-4", "postman-docs-5", "postman-docs-41" ] }, { "productId": "postman", "storyId": "agentic-webhooks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Postman CLI docs explicitly mention creating, managing, and forwarding webhooks for event-driven integrations and local development, which supports webhook subscription/handling. However, this is a thin CLI feature mention rather than a full first-class webhook subscription API/product surface with detailed setup, event types, or independent corroboration. Missing for 10: dedicated webhook subscription API docs, event catalog/type list, hands-on/independent verification of webhook reliability, and integration examples beyond a single line mention.", "evidenceIds": [ "postman-docs-24", "postman-docs-32" ] }, { "productId": "postman", "storyId": "ai-generate-tests-from-api", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Postman's Agent Mode/Postbot is documented to generate and update tests via natural language directly from requests/responses ('ask Agent Mode to add tests to requests', 'fix errors, update tests'), directly matching the AI-native test/assertion generation story. Missing for 10: independent/hands-on community corroboration of AI-generated test quality, and explicit documentation of generating tests directly from an OpenAPI/API spec rather than just live requests.", "evidenceIds": [ "postman-docs-8", "postman-docs-33", "postman-docs-48" ] }, { "productId": "postman", "storyId": "ai-llm-traffic-gateway", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Postman's Fabric Gateway lets teams register and expose MCP servers (postman-docs-7), which touches the 'govern MCP traffic' idea, but there is no evidence of token limits, prompt/prompt-guard policies, or LLM provider routing/proxying features. missing for 10: token-limit controls, prompt guard/safety policies, LLM provider routing configuration, and any traffic-proxy/rate-limiting documentation for LLM calls.", "evidenceIds": [ "postman-docs-7" ] }, { "productId": "postman", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Postman's quick-start docs show a real, runnable example (send a request to postman-echo.com/get and view the response) and the API Network lets users 'find any API' with published collections/docs, plus response visualization and Spec Hub for API structure exploration — all consistent with an interactive, runnable API reference. However, there is no explicit evidence of a dedicated 'published documentation with Run-in-Postman' interactive reference page or AI-specific reference browsing feature. missing for 10: explicit documentation of a generated public API reference page with embedded runnable/try-it examples, and any AI-native framing of browsing that reference.", "evidenceIds": [ "postman-docs-28", "postman-docs-29", "postman-docs-31", "postman-docs-44", "postman-docs-45", "postman-docs-38" ] }, { "productId": "postman", "storyId": "api-machine-spec", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A probe confirms Postman publishes a machine-readable OpenAPI spec at learning.postman.com/openapi.json (valid 'openapi' key), and Postman's own developer API docs and Spec Hub further support spec-based API design/consumption. This directly satisfies the AI-native need to fetch a structured API spec. Missing for 10: no independent/community corroboration of using this spec file in real workflows, and no explicit versioning/documentation of the spec's completeness.", "evidenceIds": [ "postman-probe-3", "postman-docs-5", "postman-docs-16", "postman-docs-38" ] }, { "productId": "postman", "storyId": "api-sandbox", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Postman's mock servers and environment/variable management let users simulate API behavior and separate sandbox/staging config from production, directly supporting isolated testing without touching real production data, and these are usable from Agent Mode/CLI workflows that an AI-native user might drive. However, no evidence explicitly frames this as an AI-agent-specific 'sandbox mode' or guarantees isolation from production systems beyond the mock-server abstraction. Missing for 10: explicit documentation tying mock/sandbox environments to AI-agent workflows, an explicit no-production-impact guarantee, and independent/hands-on corroboration of safe sandbox testing.", "evidenceIds": [ "postman-docs-10", "postman-docs-25", "postman-docs-35", "postman-docs-36", "postman-docs-23", "postman-docs-47", "postman-docs-8" ] }, { "productId": "postman", "storyId": "api-traffic-observability", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Postman's evidence pack focuses on API testing, collections, mock servers, collaboration, and an Analytics API for Postman's own usage reporting, but there is no evidence of metrics/logging/tracing integrations (e.g., OpenTelemetry, Prometheus, Datadog) for monitoring live API gateway traffic, which is what a platform-engineer would need. Missing for 10: any documented observability/tracing integration, gateway traffic metrics dashboard, or logging pipeline connecting Postman to APM/observability tools.", "evidenceIds": [ "postman-docs-27", "postman-docs-7" ] }, { "productId": "postman", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows the Postman API exists and is documented for programmatic use (postman-docs-5, postman-docs-41) and an OpenAPI spec is publicly served (postman-probe-3), but nothing in the pack references API versioning conventions or a documented deprecation policy for that API. Since a platform API is a reasonable place to expect version/deprecation documentation, absence of such evidence yields 'none' rather than 'na'.", "evidenceIds": [ "postman-docs-5", "postman-docs-41", "postman-probe-3" ] }, { "productId": "postman", "storyId": "automation-bulk-operations", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Postman lets users run entire collections (many requests) at once via the CLI/Newman and lets AI-native workflows programmatically manage many Postman assets in bulk through the Postman API (postman-docs-1, postman-docs-5, postman-docs-6, postman-docs-19, postman-docs-42). This directly supports bulk operations across many items (requests, collections, assets) in an automatable, scriptable way suited to AI-native/CI use. missing for 10: no explicit documentation of bulk-specific batch limits/throughput, and no independent hands-on evidence confirming large-scale bulk runs work reliably.", "evidenceIds": [ "postman-docs-1", "postman-docs-5", "postman-docs-6", "postman-docs-19", "postman-docs-42" ] }, { "productId": "postman", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Postman supports pre-request/post-response scripts that execute automatically around request events (docs-12), webhook creation for event-driven integrations (docs-24), and CI/CD-triggered collection runs (docs-19), which together provide some event-triggered automation. However there is no documented general-purpose rules/trigger engine (e.g., monitors, scheduled alerts, conditional if-event-then-action builder) that lets a user declaratively define arbitrary event-based rules. Missing for 10: a dedicated rule/trigger builder UI or API, conditional logic mapping arbitrary events to arbitrary actions, and independent/hands-on confirmation of this automation working reliably.", "evidenceIds": [ "postman-docs-12", "postman-docs-24", "postman-docs-19" ] }, { "productId": "postman", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows CI/CD integration via the Postman CLI and API, but nothing about a native scheduler or recurring job/workflow trigger within Postman itself; scheduling would depend on external CI/CD orchestration, not documented here.", "evidenceIds": [ "postman-docs-6", "postman-docs-19", "postman-docs-5" ] }, { "productId": "postman", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Postman documents workspace 'versioning' and team review mechanisms (comments, live sessions, roles/permissions) that could support reviewing automations, but there is no explicit documented rollback/version-history feature for collections or automations, and a community report shows a case where a user's collections were deleted and had to be manually restored via support rather than self-service rollback. missing for 10: explicit version-history/rollback UI or API for collections/automations, first-party docs describing rollback workflow, and independent confirmation that rollback works reliably.", "evidenceIds": [ "postman-docs-18", "postman-docs-39", "postman-docs-13", "postman-comm-4" ] }, { "productId": "postman", "storyId": "collections-as-files-in-git", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Postman's docs describe collections living in cloud workspaces with sharing/versioning via Postman's own platform, but there is no documentation of a native git-friendly, plain-text file storage workflow for collections. Community evidence even shows developers explicitly switching to Bruno specifically because it stores collections as files that can be put in git, implying Postman does not offer this as a first-class workflow.", "evidenceIds": [ "postman-docs-18", "postman-docs-21", "postman-comm-5", "postman-comm-14" ] }, { "productId": "postman", "storyId": "contract-validation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Postman supports writing test scripts that assert on live response status/body/schema, contract/governance checks against API specs via the CLI, and mock servers to compare expected vs actual behavior, which together enable a platform engineer to catch drift between live responses and a contract. However, there's no dedicated 'contract testing' feature that explicitly diffs live responses against an OpenAPI/spec definition — it relies on manually written test scripts or governance rule checks rather than automated schema validation of live traffic. missing for 10: a first-party automated schema-diff/contract-validation feature (e.g., built-in OpenAPI response validation against live traffic), and independent/hands-on evidence confirming this workflow catches real drift in practice.", "evidenceIds": [ "postman-docs-22", "postman-docs-12", "postman-docs-17", "postman-docs-19", "postman-docs-29" ] }, { "productId": "postman", "storyId": "declarative-config-gitops", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Postman's API (postman-docs-5/6/41) lets teams programmatically manage collections/specs, and the CLI can run governance/lint checks and tests inside CI/CD pipelines (postman-docs-19, postman-docs-22, postman-docs-32), which supports an APIOps-style workflow. However there's no explicit gateway/infrastructure declarative-config model, no Git-native sync mechanism, or IaC-style (Terraform/K8s) integration described—this is closer to CI-triggered testing/governance than full GitOps management of API gateway infra. Missing for 10: explicit gateway-infrastructure-as-code support, native Git-sync workflow, and evidence of managing routing/gateway policies (not just collections/specs) declaratively from CI.", "evidenceIds": [ "postman-docs-5", "postman-docs-6", "postman-docs-19", "postman-docs-22", "postman-docs-32" ] }, { "productId": "postman", "storyId": "design-export-openapi", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Postman's docs explicitly mention 'Spec Hub' for designing an API's structure before building requests/collections, supporting spec-first design (postman-docs-16, postman-docs-38). However, the evidence pack never explicitly documents exporting that spec as an OpenAPI file/format, only that specs can be created and CLI can 'lint API schemas' (postman-docs-32), leaving the export step unconfirmed. Missing for 10: explicit documentation of OpenAPI export/download from Spec Hub, and independent/hands-on confirmation of round-tripping a spec-first design to OpenAPI.", "evidenceIds": [ "postman-docs-16", "postman-docs-38", "postman-docs-32" ] }, { "productId": "postman", "storyId": "environment-variable-sets", "verdict": "full", "quality": 6, "confidence": "medium", "rationale": "Postman docs explicitly reference an 'Environments' feature alongside Variables, and community evidence confirms teams use Postman to test/validate APIs 'across environments,' supporting the dev/staging/prod switching workflow via environment variable sets. However, the evidence pack lacks a detailed first-party doc excerpt walking through creating multiple environments and switching between them. Missing for 10: a direct doc excerpt describing environment creation/switching UI, and independent hands-on confirmation of variable-set switching specifically.", "evidenceIds": [ "postman-docs-47", "postman-comm-19" ] }, { "productId": "postman", "storyId": "gateway-auth-plugins", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Postman is an API client, testing, collaboration, and design platform, not an API gateway product; the evidence pack shows no gateway plugin/policy engine for enforcing upstream authentication schemes (API keys, OAuth2, JWT, mTLS). The one 'gateway' reference (postman-docs-7) is about registering MCP servers with Postman's Fabric Gateway for AI agent access, not upstream API auth enforcement — a different axis entirely.", "evidenceIds": [ "postman-docs-7" ] }, { "productId": "postman", "storyId": "gateway-rate-limiting", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Postman is an API client/testing/collaboration platform, not an API gateway; enforcing consumer/route-based rate limits on live traffic is a gateway/proxy capability outside Postman's product category, and no evidence shows Postman shipping such traffic-control enforcement (mocking/simulating disruptions is not the same as enforcing rate limits).", "evidenceIds": [] }, { "productId": "postman", "storyId": "generate-publish-api-docs", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers Postman's collections, mock servers, Spec Hub, CLI, and API, but no citation describes generating or publishing API documentation from collections/specs — a well-known Postman feature not evidenced here. missing for 10: explicit documentation generation/publish feature evidence, public doc hosting/sharing links, generated reference examples.", "evidenceIds": [] }, { "productId": "postman", "storyId": "import-export-collections", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Evidence only indirectly touches this story: docs reference the 'Postman Collection format' as the basis for API design (postman-docs-16, postman-docs-38), and community comments recall Postman's original goal of 'simple JSON based data formats which can be exported and shared' (postman-comm-14) and a user manually restoring collections from a local backup file (postman-comm-16). None of these are explicit first-party documentation of an import/export UI/CLI workflow for moving collections between tools. Missing for 10: explicit docs on collection import/export dialogs or CLI commands, confirmation of cross-tool interoperability (e.g., OpenAPI/Postman format conversion), and independent hands-on verification of round-trip fidelity.", "evidenceIds": [ "postman-docs-16", "postman-docs-38", "postman-comm-14", "postman-comm-16" ] }, { "productId": "postman", "storyId": "import-openapi-spec", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs reference a 'Spec Hub' for designing API structure and using Postman Collection format types, which implies API-spec-driven authoring, but no evidence explicitly describes importing an OpenAPI spec file to auto-generate requests/collections. missing for 10: explicit OpenAPI import feature documentation, generated request/collection examples, and any independent confirmation of the import workflow.", "evidenceIds": [ "postman-docs-16", "postman-docs-38", "postman-docs-15" ] }, { "productId": "postman", "storyId": "internal-api-catalog", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Postman supports internal workspaces for sharing/discovering collections and APIs, an API Network for publishing APIs, and Spec Hub/collection design docs, which together enable an internal catalog-like experience. However, there is no explicit evidence of a dedicated 'developer portal' product or public-facing catalog publishing feature distinct from workspaces. missing for 10: dedicated API portal/catalog publishing feature, evidence of external-facing discovery UI, governance/catalog curation tooling beyond workspace sharing.", "evidenceIds": [ "postman-docs-18", "postman-docs-21", "postman-docs-44", "postman-docs-16", "postman-docs-38" ] }, { "productId": "postman", "storyId": "lint-specs-style-rules", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Postman CLI explicitly supports checking API specs against configured governance rules ('Check API specifications against configured API governance rules' and 'run security and governance checks') and can be wired into CI/CD pipelines, matching the platform-engineer's need to lint specs against configurable style/governance rules. Missing for 10: no independent/hands-on evidence of custom rule authoring depth or third-party validation of linting accuracy.", "evidenceIds": [ "postman-docs-22", "postman-docs-32", "postman-docs-19", "postman-docs-6" ] }, { "productId": "postman", "storyId": "mock-servers-from-examples", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Postman's docs explicitly document creating mock servers from collections/examples or specs to simulate a real API server that's always available, including dynamic mock responses and deployment guidance. missing for 10: independent hands-on corroboration of mock server reliability/consumer workflows beyond first-party docs.", "evidenceIds": [ "postman-docs-10", "postman-docs-25", "postman-docs-35", "postman-docs-36", "postman-docs-23" ] }, { "productId": "postman", "storyId": "multi-protocol-requests", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm Postman's API client supports HTTP, GraphQL, and gRPC requests explicitly (postman-docs-31, postman-docs-34 mentions 'multi-protocol requests'), but there is no explicit documented evidence in this pack of WebSocket or SSE request support. missing for 10: explicit documentation of WebSocket request support, explicit documentation of SSE request support, independent hands-on corroboration of multi-protocol usage.", "evidenceIds": [ "postman-docs-31", "postman-docs-34" ] }, { "productId": "postman", "storyId": "offline-local-workspace", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Docs mention a 'lightweight API client' usable without signing in (postman-docs-34), suggesting some offline capability, but multiple hands-on community reports directly contradict full offline/local-only operation: users report Postman wiping all collections when declining account creation (postman-comm-15), requiring telemetry/online connectivity even for local secrets (postman-comm-11, postman-comm-12), and pushing forced cloud sign-in for data migration (postman-comm-13), plus lost workspaces after account issues (postman-comm-4). This is a concrete, documented conflict between vendor claims and real-world failures, not mere skepticism. Missing for 10: first-party documentation of a persistent fully-offline local workspace mode, and any official acknowledgment/fix of the reported data-loss-on-offline-use bugs.", "evidenceIds": [ "postman-docs-34", "postman-comm-15", "postman-comm-11", "postman-comm-12", "postman-comm-13", "postman-comm-4" ] }, { "productId": "postman", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Postman ships a documented Postman API for programmatic management of assets (collections, workspaces, environments, mocks, analytics) plus a CLI and an official OpenAPI spec, showing broad but not exhaustive API parity with the UI. There's no evidence that UI-only features like Live Sessions, Postbot/Agent Mode, or real-time collaboration have full API equivalents. missing for 10: explicit documentation confirming 1:1 API coverage of collaboration/AI features, independent verification of parity, and any gaps/limitations list.", "evidenceIds": [ "postman-docs-5", "postman-docs-41", "postman-docs-27", "postman-probe-3", "postman-probe-5", "postman-docs-32" ] }, { "productId": "postman", "storyId": "openness-full-export", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Postman's Collection format is JSON-based and documented as exportable (postman-docs-16, comm-14 recalls Postman's stated 'core goal' of JSON export for full data control), but multiple hands-on community reports concretely contradict free portability: users had all collections/workspaces wiped when declining a forced account creation (postman-comm-15), had to dig into hidden config folders to manually recover a backup (postman-comm-16), were pushed to a paid cloud subscription just to migrate data to a new machine (postman-comm-13), and reported secrets/environment data being silently uploaded to Postman's servers (postman-comm-6, postman-comm-12). missing for 10: an official bulk/full-account data export tool, evidence of successful account-free clean exits, and a rebuttal to the documented forced-login data-wipe incidents.", "evidenceIds": [ "postman-docs-16", "postman-comm-14", "postman-comm-15", "postman-comm-16", "postman-comm-13", "postman-comm-6", "postman-comm-12" ] }, { "productId": "postman", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Postman is a closed-source, proprietary SaaS/desktop product; no evidence of an open-source license for its core codebase is provided, and community comments actually reflect frustration with lock-in and lack of local/offline data control, not openness of source. Since the axis (open license to read source) applies to any product category, absence of evidence means 'none'.", "evidenceIds": [ "postman-comm-3", "postman-comm-14", "postman-comm-17" ] }, { "productId": "postman", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Postman is a cloud-based SaaS product with a desktop client that syncs to Postman's servers; there is no evidence of a self-hostable core product (e.g., on-prem/enterprise server package). Community evidence even highlights forced cloud sync and telemetry requirements, but no self-hosting option is documented anywhere in the pack.", "evidenceIds": [ "postman-comm-6", "postman-comm-11", "postman-comm-12", "postman-comm-15" ] }, { "productId": "postman", "storyId": "organize-requests-collections", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Postman's core docs confirm collections/folders (postman-docs-15,17,18,29) and rerunning via Postman CLI/Newman or CI/CD (postman-docs-19,32,42), directly matching the story. Missing for 10: independent hands-on verification of folder-level reruns and no first-party mention of nested folder organization specifics; some community reports (postman-comm-4, postman-comm-15) describe collection data loss tied to account/cloud sync issues, which slightly dents reliability confidence though they don't contradict the core organize/rerun feature.", "evidenceIds": [ "postman-docs-15", "postman-docs-17", "postman-docs-18", "postman-docs-19", "postman-docs-32", "postman-docs-42", "postman-comm-4", "postman-comm-15" ] }, { "productId": "postman", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, region selection, or geographic storage controls for Postman workspaces/collections. Community evidence instead raises concerns about data being uploaded to Postman's servers and telemetry, but this doesn't address region choice specifically. missing for 10: any documentation of regional data residency options, EU/US data center selection, or enterprise data-locality controls.", "evidenceIds": [] }, { "productId": "postman", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item addresses AI training data opt-out or a data-use policy control for AI features (Agent Mode/Postbot); in fact community reports raise concerns about data/telemetry being sent to Postman's servers without any documented opt-out for AI training use. Missing for 10: any documentation of an AI-training opt-out setting, privacy policy statement on model training, or admin control to restrict data use for AI training.", "evidenceIds": [ "postman-comm-6", "postman-comm-12" ] }, { "productId": "postman", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No documentation describes explicit data retention or deletion controls (e.g., configurable retention windows, GDPR-style delete API, data export/purge tools) for Postman. Community reports actually describe the opposite experience — data being wiped without consent, forced cloud account creation, and secrets/telemetry being uploaded to Postman's servers — reinforcing the absence of user-controlled retention/deletion.", "evidenceIds": [ "postman-comm-4", "postman-comm-13", "postman-comm-15", "postman-comm-16", "postman-comm-12" ] }, { "productId": "postman", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No documentation in the evidence pack describes a telemetry/usage-tracking opt-out setting for Postman, and community reports indicate the opposite — users note Postman 'needs to be online in order to send telemetry' and that it uploads secrets/environment data without an apparent way to disable this (postman-comm-11, postman-comm-12, postman-comm-6). No first-party privacy/opt-out control is documented anywhere in the pack.", "evidenceIds": [ "postman-comm-11", "postman-comm-12", "postman-comm-6" ] }, { "productId": "postman", "storyId": "request-lifecycle-scripting", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Postman docs explicitly confirm pre-request and post-response scripting, passing data between requests, and dynamic parameters, plus CI/CD integration via CLI/Newman to chain and automate these scripted workflows. Missing for 10: independent hands-on corroboration of complex data-massaging scripts beyond vendor docs.", "evidenceIds": [ "postman-docs-12", "postman-docs-17", "postman-docs-19", "postman-docs-42", "postman-docs-47" ] }, { "productId": "postman", "storyId": "route-transform-requests", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Postman is an API client/testing/collaboration platform for designing, testing, mocking, and documenting APIs; it is not an API gateway or traffic-management product. There is no evidence of routing, load-balancing, or request/response transformation between consumers and upstream services — mock servers and Postman Flows are for simulation/prototyping, not production traffic routing. This capability is a category error for this type of product.", "evidenceIds": [] }, { "productId": "postman", "storyId": "run-collection-test-suites", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Postman documents running collections with the Postman CLI (and Newman) which includes built-in reporters for pass/fail output, plus writing test scripts (pre-request/post-response) that assert on responses, and explicit CI/CD integration guidance for automated test suite runs. This covers the full workflow: writing tests, running a collection as a suite, and getting a pass/fail report. Missing for 10: independent/hands-on confirmation of the reporter output format or a third-party account of CI pass/fail usage.", "evidenceIds": [ "postman-docs-1", "postman-docs-19", "postman-docs-42", "postman-docs-32", "postman-docs-12", "postman-docs-6" ] }, { "productId": "postman", "storyId": "run-tests-in-ci", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Postman CLI is well-documented for running collections, linting schemas, and running security/governance checks from the command line, and docs explicitly describe combining it with the Postman API to integrate into CI/CD pipelines (fail-on-regression behavior is implicit in collection test run exit codes, standard for such CLIs). Newman is also cited as a longstanding CLI runner for CI use. Missing for 10: explicit documentation/example of exit-code-based build failure and a third-party CI pipeline case study confirming this in practice.", "evidenceIds": [ "postman-docs-1", "postman-docs-6", "postman-docs-19", "postman-docs-32", "postman-docs-42", "postman-probe-5" ] }, { "productId": "postman", "storyId": "scheduled-api-monitors", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Postman CLI, CI/CD integration, mock servers, and collaboration features, but contains no mention of a 'Monitors' capability, scheduling, or automated alerting when API tests fail. Without any documentation of scheduled runs or alerting, this specific story is unevidenced.", "evidenceIds": [] }, { "productId": "postman", "storyId": "secrets-masked-or-vaulted", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Postman advertises 'Postman Vault' to secure credentials (postman-docs-15), but multiple independent community reports claim Postman actually syncs/uploads or logs secrets and environment variables to its servers via telemetry, undermining the 'keep credentials out of shared files' promise (postman-comm-6, postman-comm-12). Missing for 10: detailed docs on secret masking mechanics, external vault (e.g., HashiCorp) integration specifics, and a vendor response addressing the telemetry/secret-leak claims.", "evidenceIds": [ "postman-docs-15", "postman-comm-6", "postman-comm-12" ] }, { "productId": "postman", "storyId": "shared-team-workspaces", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "First-party docs clearly document shared team workspaces with roles/permissions, live sessions, and comments for collaborating on collections (postman-docs-18, postman-docs-21, postman-docs-14, postman-docs-13, postman-docs-39, postman-docs-40). However, community reports describe real friction: syncing/sharing collections is gated behind per-seat cloud subscriptions, and forced account-login flows have in some cases wiped local collections/workspaces (postman-comm-17, postman-comm-4, postman-comm-15), which are concrete caveats around the exact 'shared workspace sync' experience. missing for 10: independent hands-on confirmation that team-shared workspaces sync reliably without data loss, and evidence the feature works well outside the paid/cloud-gated tier.", "evidenceIds": [ "postman-docs-18", "postman-docs-21", "postman-docs-14", "postman-docs-13", "postman-docs-39", "postman-docs-40", "postman-comm-17", "postman-comm-4", "postman-comm-15" ] }, { "productId": "postman", "storyId": "workspace-roles-permissions", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Postman docs explicitly state roles and permissions can be defined at team, workspace, and element levels, directly matching the platform-engineer's need to control view/edit access on APIs and collections. Missing for 10: no independent/hands-on corroboration of granular permission enforcement in practice, and no detail on specific role types (admin/editor/viewer) or audit logging.", "evidenceIds": [ "postman-docs-14", "postman-docs-18", "postman-docs-21" ] }, { "productId": "postman", "storyId": "write-response-assertions", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Postman's docs confirm scripting-based test assertions via pre-request/post-response scripts (postman-docs-12, postman-docs-17), explicit test writing against API responses (postman-docs-29, postman-docs-33), and CI/CD execution of these tests via CLI/Newman (postman-docs-19, postman-docs-42). This directly covers assertions against status, body, and headers, which is Postman's classic 'Tests' tab functionality referenced across multiple docs. Missing for 10: no explicit doc snippet showing pm.test()/pm.response.to.have.status/header assertion syntax or independent hands-on confirmation of assertion behavior.", "evidenceIds": [ "postman-docs-12", "postman-docs-17", "postman-docs-29", "postman-docs-33", "postman-docs-19", "postman-docs-42" ] } ]