[ { "productId": "cloudflare-ai-gateway", "storyId": "agent-spend-guardrails", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "AI Gateway's dynamic routing explicitly supports 'restricting each user/project/team with budget/rate limits' and its rate-limiting feature lets you cap requests per time window (fixed/sliding), which maps to giving an agent bounded spend/rate guardrails when configured as its own route/project. However, the evidence never describes issuing a distinct API key per agent tied to a dollar-denominated budget cap — the mechanism shown is request-count rate limiting plus routing-level quotas, not a native 'agent key + $ budget' guardrail. missing for 10: explicit per-agent key issuance, dollar-based spend caps (not just request counts), and independent confirmation the budget/rate restriction in dynamic routing enforces hard spend caps.", "evidenceIds": [ "cloudflare-ai-gateway-docs-13", "cloudflare-ai-gateway-docs-5", "cloudflare-ai-gateway-docs-6", "cloudflare-ai-gateway-docs-19", "cloudflare-ai-gateway-docs-30", "cloudflare-ai-gateway-docs-11", "cloudflare-ai-gateway-docs-32" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "agent-switches-models-policy", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Cloudflare AI Gateway explicitly supports policy-driven model switching via Dynamic Routing (visual/JSON config for segment-based routing, budget/rate limits, A/B rollouts) and Fallbacks (automatic switch to another model/provider on error or timeout, with cf-aig-step header indicating which model handled the request). These directly satisfy switching by cost, capability, or availability without app code changes. missing for 10: independent/hands-on evidence of this working in production agent workflows, and explicit documentation of capability-based (not just error/availability-based) routing logic.", "evidenceIds": [ "cloudflare-ai-gateway-docs-7", "cloudflare-ai-gateway-docs-8", "cloudflare-ai-gateway-docs-11", "cloudflare-ai-gateway-docs-12", "cloudflare-ai-gateway-docs-13", "cloudflare-ai-gateway-docs-17", "cloudflare-ai-gateway-docs-20", "cloudflare-ai-gateway-docs-24", "cloudflare-ai-gateway-docs-32", "cloudflare-ai-gateway-docs-33" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "agentic-agent-docs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Cloudflare's developer docs (which cover AI Gateway) expose a working llms.txt at developers.cloudflare.com/llms.txt, letting an agent discover agent-oriented documentation; an OpenAPI spec is also available. However, the AI Gateway-specific docs page itself returns 404 for the .md variant, so agent-native markdown access is inconsistent at the product-specific level. Missing for 10: a dedicated llms.txt or .md endpoint scoped to AI Gateway docs specifically, and confirmation the top-level llms.txt indexes AI Gateway pages.", "evidenceIds": [ "cloudflare-ai-gateway-probe-1", "cloudflare-ai-gateway-probe-2", "cloudflare-ai-gateway-probe-3" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Cloudflare AI Gateway is an infrastructure/observability layer for routing and monitoring calls to AI providers, not a product that surfaces AI-generated insights or suggestions from a user's own data. This story is a category error for this type of product.", "evidenceIds": [] }, { "productId": "cloudflare-ai-gateway", "storyId": "agentic-autonomous-automation", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Cloudflare AI Gateway is a request-routing/observability proxy layer sitting between an application and LLM providers (caching, rate limiting, fallbacks, logging) — it processes calls reactively as they occur rather than running independent background automations. This axis (autonomous background automation) fits agent/automation platforms, not a gateway product, so it does not apply to this product category.", "evidenceIds": [] }, { "productId": "cloudflare-ai-gateway", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Cloudflare AI Gateway is an infrastructure/proxy layer for routing, caching, logging and observability of AI API calls — it is not itself an AI assistant product with a built-in agent persona to delegate tasks to. This story applies to conversational/agentic assistant products, not to a gateway/middleware tool, making it a category mismatch.", "evidenceIds": [] }, { "productId": "cloudflare-ai-gateway", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "AI Gateway is API/header-driven by design (OpenAI-compatible endpoint, JSON-based dynamic routing config, header-based controls like cf-aig-cache-status and cf-aig-collect-log), which naturally supports headless/programmatic use in scripts or CI pipelines. However, there is no explicit documentation of CI/CD integration, automation examples, or a CLI for pipeline use. Missing for 10: explicit CI/CD workflow examples, dedicated CLI/SDK for automation, documented non-interactive auth/setup flow for pipelines.", "evidenceIds": [ "cloudflare-ai-gateway-docs-18", "cloudflare-ai-gateway-docs-11", "cloudflare-ai-gateway-docs-22", "cloudflare-ai-gateway-docs-25", "cloudflare-ai-gateway-docs-31" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Cloudflare AI Gateway is an LLM proxy/observability layer for routing, caching, rate-limiting and logging requests to model providers — it is not an agent that consumes tools, and there is no evidence of MCP server integration in the pack. This axis is a category error for this type of product.", "evidenceIds": [] }, { "productId": "cloudflare-ai-gateway", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Cloudflare AI Gateway is a proxy/gateway for routing and observing LLM API calls; the evidence pack contains no mention of an MCP server, MCP protocol, or agent connectivity via MCP. This is an applicable axis (a gateway product could plausibly ship an MCP server for agentic access) but no evidence supports it.", "evidenceIds": [] }, { "productId": "cloudflare-ai-gateway", "storyId": "agentic-nl-commands", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Cloudflare AI Gateway is an infrastructure/proxy layer for routing, caching, logging and rate-limiting LLM API calls, configured via dashboard, JSON, or headers — it is not an interface a user interacts with via natural-language commands. This axis is a category error for a gateway/proxy product, not a missing feature.", "evidenceIds": [] }, { "productId": "cloudflare-ai-gateway", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions an official CLI for AI Gateway (e.g., Wrangler or a dedicated AI Gateway CLI); all evidence covers dashboard, API headers, and configuration features. missing for 10: any mention of a CLI tool, command-line usage, or CLI-based configuration for AI Gateway.", "evidenceIds": [] }, { "productId": "cloudflare-ai-gateway", "storyId": "agentic-public-api", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Cloudflare AI Gateway exposes a documented public API surface: an OpenAI-compatible /chat/completions endpoint, native provider passthroughs, a Universal endpoint for fallbacks, control headers (cf-aig-*), and a patchLog method for feedback/metadata, all described in first-party docs, and Cloudflare's platform-wide openapi.json confirms a machine-readable API schema exists. missing for 10: independent/hands-on third-party corroboration of API usage and a dedicated AI-Gateway-specific OpenAPI reference beyond the platform-wide one.", "evidenceIds": [ "cloudflare-ai-gateway-docs-18", "cloudflare-ai-gateway-docs-22", "cloudflare-ai-gateway-docs-23", "cloudflare-ai-gateway-docs-26", "cloudflare-ai-gateway-docs-29", "cloudflare-ai-gateway-probe-3" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers AI Gateway's routing, caching, rate limiting, logging, and fallback features, but there is no mention of issuing scoped or least-privilege API credentials/tokens for agents (e.g., per-agent API keys with restricted permissions). Rate limiting and per-user/project quotas are traffic controls, not credential scoping mechanisms.", "evidenceIds": [] }, { "productId": "cloudflare-ai-gateway", "storyId": "agentic-sdks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "AI Gateway is compatible with OpenAI-style SDKs and providers' native APIs (via the OpenAI-compatible /chat/completions endpoint and native API pass-through), and Workers AI bindings imply SDK-like integration, but the evidence pack never explicitly names or documents an official Cloudflare AI Gateway SDK for major languages (e.g., a dedicated 'cloudflare-ai-gateway' npm/python package) with API references. missing for 10: explicit official SDK documentation/package listing, multi-language SDK examples, independent confirmation of SDK reliability/maintenance.", "evidenceIds": [ "cloudflare-ai-gateway-docs-18", "cloudflare-ai-gateway-docs-22", "cloudflare-ai-gateway-docs-23", "cloudflare-ai-gateway-docs-9", "cloudflare-ai-gateway-docs-36" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions webhooks or any event-subscription mechanism; AI Gateway's documented features are logging, caching, rate limiting, fallbacks, and dynamic routing, none of which describe outbound webhook notifications for events. This is a plausible axis for a gateway product (it could offer webhook callbacks on events like errors or budget thresholds), so absence of evidence yields 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "cloudflare-ai-gateway", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only static documentation pages with code snippets and an openapi.json file, but no interactive API reference, playground, or runnable-example explorer is described anywhere in the pack.", "evidenceIds": [ "cloudflare-ai-gateway-probe-3", "cloudflare-ai-gateway-docs-1", "cloudflare-ai-gateway-docs-9" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "api-machine-spec", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Cloudflare publishes a machine-readable OpenAPI spec at a public, verifiable endpoint (openapi.json containing an 'openapi' key), covering the Cloudflare API including AI Gateway configuration/management endpoints, plus an llms.txt for AI-native discovery. missing for 10: no dedicated AI-Gateway-specific OpenAPI file (it's part of the broader Cloudflare API spec) and no independent/community confirmation of completeness.", "evidenceIds": [ "cloudflare-ai-gateway-probe-3", "cloudflare-ai-gateway-probe-1" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of a sandbox/test environment feature separate from production; AI Gateway docs focus on routing, caching, rate limiting, logging, and fallbacks, with no mention of a sandbox mode or test-data isolation from production.", "evidenceIds": [] }, { "productId": "cloudflare-ai-gateway", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence covers AI Gateway's features (caching, rate limiting, fallbacks, logging) but nowhere mentions API versioning schemes or a documented deprecation policy for the AI Gateway API itself; the openapi.json probe confirms an API spec exists but says nothing about version lifecycle or deprecation commitments. Missing for 10: explicit versioning scheme documentation, deprecation policy/timeline, changelog or migration guides for breaking changes.", "evidenceIds": [ "cloudflare-ai-gateway-probe-3" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "automatic-fallbacks", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Cloudflare AI Gateway explicitly documents automatic fallback to another model or provider on request errors or timeouts via the Universal endpoint, with clear examples (e.g., Workers AI failing over to OpenAI) and a response header (cf-aig-step) to identify which step succeeded. This is a well-documented, first-party feature with configuration details and dynamic routing support for fallbacks. Missing for 10: independent/hands-on verification beyond vendor docs, and no evidence of fallback behavior nuances (e.g., latency impact, partial failure handling) from real-world usage.", "evidenceIds": [ "cloudflare-ai-gateway-docs-7", "cloudflare-ai-gateway-docs-8", "cloudflare-ai-gateway-docs-20", "cloudflare-ai-gateway-docs-33", "cloudflare-ai-gateway-docs-17" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers per-request features (rate limiting, caching, fallbacks, logging, dynamic routing) but contains no mention of bulk or batch operations across multiple items (e.g., batch API calls, bulk config management, or multi-item processing).", "evidenceIds": [] }, { "productId": "cloudflare-ai-gateway", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "AI Gateway supports rule-based automation like dynamic routing that evaluates conditions and triggers fallback/model-switching actions, rate limiting, and cache overrides via headers, which are event-triggered automated behaviors. However, this is narrowly scoped to request routing/traffic control rather than a general-purpose 'define custom rules that trigger arbitrary actions on events' system (e.g., no evidence of user-defined triggers for alerts, webhooks, or custom workflows beyond routing/fallback/caching/rate-limits). missing for 10: evidence of arbitrary custom event-action rule definitions (e.g., webhook/alert triggers on thresholds), independent/hands-on corroboration of dynamic routing behavior, and confirmation of a general automation/rules engine beyond routing, caching, and rate limiting.", "evidenceIds": [ "cloudflare-ai-gateway-docs-11", "cloudflare-ai-gateway-docs-12", "cloudflare-ai-gateway-docs-13", "cloudflare-ai-gateway-docs-32", "cloudflare-ai-gateway-docs-7", "cloudflare-ai-gateway-docs-8", "cloudflare-ai-gateway-docs-20", "cloudflare-ai-gateway-docs-5", "cloudflare-ai-gateway-docs-19" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Cloudflare AI Gateway is a proxy/observability layer for AI API requests (caching, routing, rate limiting, fallbacks, logging); it is not a workflow/job scheduling product, so scheduling recurring jobs is a category error for this product type.", "evidenceIds": [] }, { "productId": "cloudflare-ai-gateway", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs mention that dynamic routing lets you 'publish a new route version' without touching application code, implying some versioning of routing configs, but there is no documented review/diff interface or explicit rollback mechanism for automations. missing for 10: explicit rollback/undo functionality, change-review or diff UI, versioning of prompts/logic beyond routing config.", "evidenceIds": [ "cloudflare-ai-gateway-docs-32", "cloudflare-ai-gateway-docs-11", "cloudflare-ai-gateway-docs-24" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "budgets-limits", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "AI Gateway supports rate limiting per app/gateway and dynamic routing docs mention 'restricting each user/project/team with budget/rate limits,' but there's no dedicated documentation on setting a hard dollar spend cap or budget limit per API key, team, or user that blocks requests once exceeded — the budget/rate-limit reference is a single bullet without configuration detail. missing for 10: dedicated budget-limit configuration docs (dollar-amount caps), per-key/per-user budget enforcement examples, and evidence of automatic cutoff/alerting when a hard budget is reached.", "evidenceIds": [ "cloudflare-ai-gateway-docs-13", "cloudflare-ai-gateway-docs-19", "cloudflare-ai-gateway-docs-5", "cloudflare-ai-gateway-docs-30" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "byok", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs confirm AI Gateway proxies requests to providers using the developer's own API key (e.g. 'Switch providers by changing the model and apiKey parameters' and passthrough pricing with no markup, meaning the developer's own provider account/key is billed directly), and it connects to native provider APIs and OpenAI-compatible endpoints for many providers. missing for 10: no explicit first-party doc section titled 'bring your own key' explaining storage/security of BYO keys, and no independent/hands-on confirmation of this exact flow.", "evidenceIds": [ "cloudflare-ai-gateway-docs-22", "cloudflare-ai-gateway-docs-23", "cloudflare-ai-gateway-docs-16", "cloudflare-ai-gateway-docs-10", "cloudflare-ai-gateway-docs-18" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "cost-tracking", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "AI Gateway logs per-request cost, tokens, provider, and model, and supports dynamic routing with per-user/project/team budget and rate limits, giving a foundation for spend tracking across providers in one dashboard. However, there is no explicit evidence of built-in aggregated spend dashboards or reports broken down by 'team' or 'user' as first-class filters/rollups, or per-API-key spend attribution beyond raw logs. missing for 10: dedicated cost-aggregation/reporting UI by team/user/key, evidence of budget alerts tied to spend thresholds, independent/hands-on confirmation of cross-provider spend rollups.", "evidenceIds": [ "cloudflare-ai-gateway-docs-2", "cloudflare-ai-gateway-docs-13", "cloudflare-ai-gateway-docs-34", "cloudflare-ai-gateway-docs-26" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "edge-low-latency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents caching, rate limiting, fallbacks, and analytics features but contains no benchmarks, latency figures, or explicit claims about the gateway's own processing overhead added to provider calls. Caching (docs-4, docs-31) improves speed only for cache hits and says nothing about baseline pass-through latency. Missing for 10: any latency benchmark, p50/p99 overhead numbers, or first-party/independent claim that the proxy adds minimal delay to non-cached requests.", "evidenceIds": [ "cloudflare-ai-gateway-docs-4", "cloudflare-ai-gateway-docs-31" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "load-balancing-routing", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "AI Gateway supports dynamic routing (conditions, quotas, A/B/gradual rollouts, segment-based routing) and fallback chains across providers/models with sequential retry-on-error, but the evidence describes rule-based/conditional routing and ordered fallback sequences rather than true weighted, latency-based, or cost-based load balancing across providers/keys. missing for 10: explicit weighted traffic splitting, latency-based routing decisions, cost-based routing logic, and any independent/hands-on verification of these mechanisms.", "evidenceIds": [ "cloudflare-ai-gateway-docs-7", "cloudflare-ai-gateway-docs-8", "cloudflare-ai-gateway-docs-11", "cloudflare-ai-gateway-docs-12", "cloudflare-ai-gateway-docs-13", "cloudflare-ai-gateway-docs-20", "cloudflare-ai-gateway-docs-24", "cloudflare-ai-gateway-docs-32", "cloudflare-ai-gateway-docs-33" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "model-catalog-discovery", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows AI Gateway supports multiple providers, per-token pricing pass-through, logging, caching, and routing, but there is no mention of a browsable/queryable catalog of models with pricing and context-window metadata—developers must know provider/model names themselves and check pricing docs separately (docs-16 only states pricing pass-through policy, not a queryable catalog).", "evidenceIds": [ "cloudflare-ai-gateway-docs-10", "cloudflare-ai-gateway-docs-16" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "openai-compatible-endpoint", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs confirm an OpenAI-compatible /chat/completions endpoint that allows switching providers by only changing model/apiKey parameters against a single URL, directly matching the story. missing for 10: independent/hands-on developer confirmation outside Cloudflare's own docs, and explicit example showing literal drop-in replacement of OpenAI SDK base_url with no other code changes.", "evidenceIds": [ "cloudflare-ai-gateway-docs-18", "cloudflare-ai-gateway-docs-22", "cloudflare-ai-gateway-docs-9", "cloudflare-ai-gateway-docs-26" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "AI Gateway configuration such as caching, rate limiting, fallbacks, and dynamic routing can be set via headers, API/JSON config, and the OpenAI-compatible endpoint, and an OpenAPI spec exists confirming broad API coverage. However, some features like dynamic routing are explicitly described as configurable via a 'visual interface' as an alternative to JSON, and there's no explicit confirmation that every dashboard capability (e.g., A/B rollouts, log feedback via patchLog, full analytics views) is fully API-exposed with parity to the UI. missing for 10: explicit statement of full UI/API parity, evidence that dashboard-only features (analytics visualizations, visual routing builder) have equivalent API endpoints, independent confirmation of complete parity.", "evidenceIds": [ "cloudflare-ai-gateway-docs-11", "cloudflare-ai-gateway-docs-18", "cloudflare-ai-gateway-docs-14", "cloudflare-ai-gateway-docs-25", "cloudflare-ai-gateway-docs-29", "cloudflare-ai-gateway-probe-3" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence pack covers logging, caching, fallback, rate-limiting, and routing features, but nowhere mentions a data export capability or open-format export/data portability for users leaving the platform. No documentation of bulk export, data download, or migration tooling is present.", "evidenceIds": [] }, { "productId": "cloudflare-ai-gateway", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Cloudflare AI Gateway is a closed, proprietary hosted service; the evidence pack contains only documentation and probes, with no mention of an open-source license or public source repository for the product itself.", "evidenceIds": [] }, { "productId": "cloudflare-ai-gateway", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Cloudflare AI Gateway is a managed, hosted cloud service built into Cloudflare's edge network; there is no evidence of, or plausible path to, self-hosting the core product outside Cloudflare's infrastructure. Self-hosting is a category error for this SaaS/edge-service product type.", "evidenceIds": [] }, { "productId": "cloudflare-ai-gateway", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack mentions data residency, region selection, or data storage location controls for AI Gateway; only logging opt-out toggles for payload/prompt data are documented. This applicable privacy axis lacks any supporting evidence. missing for 10: region/residency selection settings, data localization guarantees, documentation on where logs/cache/metadata are stored.", "evidenceIds": [] }, { "productId": "cloudflare-ai-gateway", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows AI Gateway can control its own logging/storage of prompts (e.g., disabling payload logging) but nothing indicates it can prevent downstream AI providers from using submitted data to train their models — that policy is controlled by the provider (OpenAI, Anthropic, etc.), not by the gateway. No documentation ties Cloudflare's logging controls to opting out of provider-side training use.", "evidenceIds": [ "cloudflare-ai-gateway-docs-15", "cloudflare-ai-gateway-docs-27", "cloudflare-ai-gateway-docs-35" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "AI Gateway offers concrete controls relevant to data retention: per-request headers to bypass logging (cf-aig-collect-log) or exclude storing raw prompt/completion payloads (cf-aig-collect-log-payload), letting users limit what sensitive data is persisted. However, there's no documented data deletion mechanism, retention period/TTL settings, or bulk/account-level data purge capability. missing for 10: explicit data retention duration controls, a documented deletion/purge workflow for stored logs, and independent confirmation that opt-out headers fully prevent persistence.", "evidenceIds": [ "cloudflare-ai-gateway-docs-15", "cloudflare-ai-gateway-docs-25", "cloudflare-ai-gateway-docs-35", "cloudflare-ai-gateway-docs-34" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "privacy-telemetry-optout", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "AI Gateway exposes per-request headers (cf-aig-collect-log, cf-aig-collect-log-payload) that let a user bypass logging or avoid storing raw prompt/completion payloads, and docs note this is 'useful when you want to maintain visibility into usage metrics... without persisting sensitive prompt or completion data.' However, these are opt-outs for log/payload storage only, not for the underlying usage/metrics tracking (tokens, cost, request counts) that the gateway inherently collects for billing and rate limiting. missing for 10: a global/account-level telemetry opt-out toggle, documentation of disabling usage-metrics tracking entirely, independent confirmation that opting out actually stops Cloudflare-side usage tracking.", "evidenceIds": [ "cloudflare-ai-gateway-docs-25", "cloudflare-ai-gateway-docs-35", "cloudflare-ai-gateway-docs-15", "cloudflare-ai-gateway-docs-34" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "provision-via-api", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs show gateways can be identified/configured programmatically (e.g., gateway id in code) and budgets/rate limits can be set via JSON-based dynamic routing config, suggesting some programmatic provisioning, and Cloudflare's general openapi.json exists. However there is no explicit documentation of an admin API for creating gateways, issuing/rotating API keys, or setting budgets outside the dashboard/JSON routing config. Missing for 10: explicit AI Gateway REST/GraphQL API endpoints for gateway CRUD, key provisioning, and budget management with request/response examples.", "evidenceIds": [ "cloudflare-ai-gateway-docs-13", "cloudflare-ai-gateway-docs-32", "cloudflare-ai-gateway-docs-36", "cloudflare-ai-gateway-probe-3" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "rate-limit-smoothing", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "AI Gateway provides rate limiting (fixed/sliding, request-per-time-window) and fallback/retry to alternate providers/models, which helps smooth provider limits and errors, and dynamic routing can enforce per-user/team quotas. However, there is no explicit documentation of key rotation/spreading traffic across multiple API keys for the same provider, nor of a request queuing mechanism (rate limiting appears to reject/throttle rather than queue). missing for 10: explicit multi-key load balancing/rotation across provider keys, documented request queuing behavior (vs. simple throttling/rejection), independent evidence of this working at scale.", "evidenceIds": [ "cloudflare-ai-gateway-docs-5", "cloudflare-ai-gateway-docs-6", "cloudflare-ai-gateway-docs-19", "cloudflare-ai-gateway-docs-30", "cloudflare-ai-gateway-docs-7", "cloudflare-ai-gateway-docs-8", "cloudflare-ai-gateway-docs-20", "cloudflare-ai-gateway-docs-33", "cloudflare-ai-gateway-docs-13", "cloudflare-ai-gateway-docs-32" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "request-logging", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly state the dashboard shows per-request logs including user prompt, model response, provider, timestamp, status, token usage, cost, duration, and user agent, plus aggregate metrics for requests/tokens/cost. This directly matches the story's requirement to inspect logged requests/responses with latency, token counts, and cost. Missing for 10: independent/hands-on corroboration beyond first-party docs.", "evidenceIds": [ "cloudflare-ai-gateway-docs-34", "cloudflare-ai-gateway-docs-2", "cloudflare-ai-gateway-docs-28" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "response-caching", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs clearly describe gateway-level caching that serves responses from Cloudflare's cache instead of calling the model provider (cost/latency savings), with per-request header overrides and a cf-aig-cache-status HIT/MISS indicator for verification. Missing for 10: independent/hands-on benchmark corroborating actual latency/cost savings.", "evidenceIds": [ "cloudflare-ai-gateway-docs-4", "cloudflare-ai-gateway-docs-14", "cloudflare-ai-gateway-docs-31" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "retry-policies", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Cloudflare AI Gateway explicitly supports request retries and fallbacks triggered by errors or timeouts (docs-7, docs-20, docs-33), with response headers indicating which step/model succeeded (docs-17). This directly satisfies automatic retry policies for transient provider errors. missing for 10: no evidence of configurable retry counts/backoff intervals specifically for 'retry' (vs fallback) and no independent/hands-on corroboration beyond first-party docs.", "evidenceIds": [ "cloudflare-ai-gateway-docs-7", "cloudflare-ai-gateway-docs-8", "cloudflare-ai-gateway-docs-17", "cloudflare-ai-gateway-docs-20", "cloudflare-ai-gateway-docs-33" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "streaming-passthrough", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes AI Gateway as a proxy supporting multiple providers, an OpenAI-compatible /chat/completions endpoint, and native API pass-through, but nowhere explicitly confirms token-by-token streaming support or documents how streaming responses are handled through the gateway. missing for 10: explicit documentation or example of streaming/SSE support, confirmation that streaming works uniformly across all connected providers.", "evidenceIds": [ "cloudflare-ai-gateway-docs-18", "cloudflare-ai-gateway-docs-23", "cloudflare-ai-gateway-docs-9" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "structured-outputs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Cloudflare AI Gateway is a proxy/observability layer that passes requests through to native or OpenAI-compatible provider APIs; the evidence describes routing, caching, fallbacks, logging, and rate limiting, but nothing about enforcing or normalizing structured JSON-schema output across providers. Missing for 10: any mention of JSON-schema/response_format enforcement, schema validation, or cross-provider structured-output normalization.", "evidenceIds": [ "cloudflare-ai-gateway-docs-18", "cloudflare-ai-gateway-docs-22", "cloudflare-ai-gateway-docs-23" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "tool-call-fidelity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "AI Gateway provides an OpenAI-compatible /chat/completions endpoint and unified switching between providers via model/apiKey parameters, plus native API pass-through, which supports a consistent schema for chat-style requests across providers. However, the evidence never explicitly discusses tool/function-calling schema normalization (e.g., translating tool_call formats between Anthropic, Gemini, OpenAI, etc.) — it only covers general chat completion routing, caching, logging, and fallback headers. missing for 10: explicit documentation or examples of function/tool-call schema normalization across providers, evidence of consistent tool-call response parsing, and independent/hands-on confirmation that function calling works uniformly across all supported providers.", "evidenceIds": [ "cloudflare-ai-gateway-docs-9", "cloudflare-ai-gateway-docs-18", "cloudflare-ai-gateway-docs-22", "cloudflare-ai-gateway-docs-23", "cloudflare-ai-gateway-docs-10" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "tracing-integrations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence describes Cloudflare's own dashboard logging, metrics, and header-based controls for log collection (e.g., cf-aig-collect-log, patchLog), but there is no mention of exporting logs/traces to an external observability stack (e.g., via Logpush, webhook, or third-party integration).", "evidenceIds": [] }, { "productId": "cloudflare-ai-gateway", "storyId": "unified-api-many-providers", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs clearly show a unified OpenAI-compatible /chat/completions endpoint and a Universal endpoint that route to many providers (Workers AI, Anthropic, Gemini, OpenAI, Replicate, etc.) via consistent APIs, with model/provider switching via simple parameter changes and pass-through pricing. Missing for 10: independent/hands-on developer corroboration beyond first-party docs.", "evidenceIds": [ "cloudflare-ai-gateway-docs-18", "cloudflare-ai-gateway-docs-10", "cloudflare-ai-gateway-docs-9", "cloudflare-ai-gateway-docs-22", "cloudflare-ai-gateway-docs-23", "cloudflare-ai-gateway-docs-26" ] }, { "productId": "cloudflare-ai-gateway", "storyId": "virtual-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence covers rate limiting, dynamic routing, fallbacks, caching, and logging, but nowhere describes a mechanism for minting gateway-managed keys that abstract or hide raw provider API keys for teams/apps.", "evidenceIds": [] }, { "productId": "kong-ai-gateway", "storyId": "agent-spend-guardrails", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Kong AI Gateway supports per-consumer/agent authentication (AI Auth Strategy, AI Vault for keys) plus AI Rate Limiting Advanced and model cost calculation/tracking that matches provider billing, which together let an operator scope an agent's key and cap its request rate and see accrued spend. However, evidence shows cost *calculation and observability* (docs-9, docs-10, docs-18) rather than an explicit hard budget cap/kill-switch that automatically cuts off an agent once a spend threshold is reached, so the 'cannot run away on spend' guarantee is only partially evidenced. Missing for 10: explicit documented budget-threshold enforcement/auto-cutoff tied to a specific agent key, and a worked example of issuing a scoped key to an autonomous agent with combined rate+spend limits.", "evidenceIds": [ "kong-ai-gateway-docs-12", "kong-ai-gateway-docs-18", "kong-ai-gateway-docs-22", "kong-ai-gateway-docs-23", "kong-ai-gateway-docs-9" ] }, { "productId": "kong-ai-gateway", "storyId": "agent-switches-models-policy", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Kong AI Gateway documents load balancing across multiple LLM models (covering availability-based routing) and per-request cost calculation, which together imply some policy-driven model selection, but there is no explicit documentation of switching models based on capability or triggering mid-task, dynamic policy switches tied to cost or availability thresholds. Missing for 10: explicit cost/capability-based routing policy documentation, mid-task dynamic model switching evidence, and independent/hands-on corroboration of this routing-resilience behavior.", "evidenceIds": [ "kong-ai-gateway-docs-17", "kong-ai-gateway-docs-18", "kong-ai-gateway-docs-24" ] }, { "productId": "kong-ai-gateway", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A probe confirms developer.konghq.com/llms.txt returns HTTP 200 with a valid llms.txt manifest listing Kong's docs (including AI Gateway), and individual doc pages are also served as .md files (e.g. ai-providers.md, a2a.md), making the docs directly agent-consumable. missing for 10: no explicit vendor statement instructing agents to use llms.txt, and the generic /.md probe for the ai-gateway index returned 404, showing coverage is per-page rather than fully uniform.", "evidenceIds": [ "kong-ai-gateway-probe-1", "kong-ai-gateway-docs-1", "kong-ai-gateway-probe-2" ] }, { "productId": "kong-ai-gateway", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Kong AI Gateway provides usage analytics, logs, and cost/latency metrics (docs-3, docs-9, docs-10), but these are raw operational metrics/dashboards, not AI-generated insights or suggestions derived from the user's own data. No evidence shows the product itself using AI to analyze a customer's traffic/data and proactively surface recommendations or insights.", "evidenceIds": [ "kong-ai-gateway-docs-3", "kong-ai-gateway-docs-9", "kong-ai-gateway-docs-10" ] }, { "productId": "kong-ai-gateway", "storyId": "agentic-autonomous-automation", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Kong AI Gateway is a proxy/control-plane for governing LLM, MCP, and A2A traffic (rate limiting, observability, auth, routing) — it does not itself host or execute autonomous background automations/agents; that is a different product category (agent orchestration/runtime platforms).", "evidenceIds": [] }, { "productId": "kong-ai-gateway", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Kong AI Gateway is an infrastructure/proxy layer for routing, governing, and observing AI/LLM/agent traffic — it is not itself a product with a built-in assistant a user delegates tasks to; that axis doesn't apply to a gateway product category.", "evidenceIds": [] }, { "productId": "kong-ai-gateway", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Kong AI Gateway exposes a CLI (kongctl) and Konnect APIs (list-ai-gateways, config conversion) plus a quickstart script, implying it can be driven headlessly and scripted rather than only via UI. However there is no explicit CI or CD pipeline example, declarative config-as-code workflow, or GitOps documentation demonstrating automated headless operation in CI. Missing for 10: explicit CI or CD pipeline documentation, dedicated headless-mode guide, independent CI usage report.", "evidenceIds": [ "kong-ai-gateway-docs-14", "kong-ai-gateway-docs-25", "kong-ai-gateway-docs-2", "kong-ai-gateway-probe-4" ] }, { "productId": "kong-ai-gateway", "storyId": "agentic-mcp-client", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Kong AI Gateway documents an 'AI MCP Proxy' capability that explicitly can front upstream MCP servers (in addition to converting REST APIs into MCP tools), meaning it can plug in external MCP servers so the traffic/tools flow through the gateway. However, the evidence is a single brief concept mention with no detailed configuration walkthrough, no example of registering a specific upstream MCP server, and no independent/hands-on corroboration. Missing for 10: a full docs page on connecting/registering upstream MCP servers, concrete config examples, and third-party verification.", "evidenceIds": [ "kong-ai-gateway-docs-13", "kong-ai-gateway-docs-19", "kong-ai-gateway-docs-15" ] }, { "productId": "kong-ai-gateway", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Kong AI Gateway ships an official 'AI MCP Server' entity and 'AI MCP Proxy' that converts REST APIs into MCP tools or fronts upstream MCP servers, letting agents connect via a first-party MCP interface, plus governance of MCP traffic (auth, OTel metrics, rate limiting) alongside it. Missing for 10: independent/hands-on verification of an agent successfully connecting through the MCP server and no example configuration walkthrough shown in the pack.", "evidenceIds": [ "kong-ai-gateway-docs-13", "kong-ai-gateway-docs-19", "kong-ai-gateway-docs-15", "kong-ai-gateway-docs-26" ] }, { "productId": "kong-ai-gateway", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Kong AI Gateway's documentation covers proxying LLM/CLI/A2A/MCP traffic, observability, and cost tracking, but there is no evidence that the gateway itself can be configured or operated via natural-language commands (its control plane relies on kongctl CLI and declarative config, not NL commands).", "evidenceIds": [ "kong-ai-gateway-probe-4", "kong-ai-gateway-docs-25" ] }, { "productId": "kong-ai-gateway", "storyId": "agentic-official-cli", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Kong ships an official CLI, kongctl, documented at a dedicated docs URL and explicitly used for AI Gateway tasks (e.g., the 'kongctl AI Gateway conversion extension' for migrating configurations). This confirms a real, first-party CLI exists for AI Gateway workflows. Missing for 10: deeper documentation of kongctl's full command surface for AI-native tasks (agents, MCP, A2A) and independent/hands-on corroboration beyond the docs probe.", "evidenceIds": [ "kong-ai-gateway-docs-25", "kong-ai-gateway-probe-4" ] }, { "productId": "kong-ai-gateway", "storyId": "agentic-public-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Kong documents a Konnect API for managing AI Gateway resources (e.g., list-ai-gateways endpoint) and a dedicated kongctl CLI, showing programmatic control exists and is documented. However, probes for a discoverable OpenAPI/swagger spec at standard locations all returned 404, so the breadth/openness of the API surface is not independently confirmed. Missing for 10: a publicly discoverable OpenAPI/swagger spec, broader endpoint coverage beyond the single example, and independent/hands-on confirmation of API usage.", "evidenceIds": [ "kong-ai-gateway-docs-14", "kong-ai-gateway-probe-4", "kong-ai-gateway-probe-3" ] }, { "productId": "kong-ai-gateway", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Kong AI Gateway provides building blocks relevant to credential control for agents — AI Auth Strategy entities can authenticate via enterprise IdPs (Okta/Azure AD/OIDC) 'without managing keys manually,' and AI Vault lets model providers, auth strategies, agents, and MCP servers reference secrets from external backends instead of storing literal keys, plus Azure Managed Identity support for scoped cloud authentication. However, none of this evidence explicitly describes issuing scoped/least-privilege credentials per agent (e.g., fine-grained permission scopes, per-agent token minting) — it's centralized secret referencing and IdP auth rather than a dedicated least-privilege credential-issuance mechanism. missing for 10: explicit documentation of scoped/least-privilege credential issuance per agent, granular permission/scope controls tied to individual agent identities, and independent/hands-on verification of this behavior.", "evidenceIds": [ "kong-ai-gateway-docs-22", "kong-ai-gateway-docs-23", "kong-ai-gateway-docs-4" ] }, { "productId": "kong-ai-gateway", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of official SDKs for building against Kong AI Gateway (only a CLI 'kongctl' and REST API references), and probes explicitly show no OpenAPI/SDK artifacts (404s for openapi.json, swagger.json, etc.). While an API gateway could plausibly offer official client SDKs, none are documented here.", "evidenceIds": [ "kong-ai-gateway-probe-3", "kong-ai-gateway-probe-4", "kong-ai-gateway-docs-14" ] }, { "productId": "kong-ai-gateway", "storyId": "agentic-webhooks", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Kong AI Gateway is an API/AI traffic gateway product, not an event-driven system for end users to subscribe to via webhooks; its role is proxying, governing, and observing LLM/MCP/A2A traffic, not offering webhook-based event subscription as a user-facing feature. No evidence in the pack even mentions webhooks or event subscription, and this axis is a category mismatch for a gateway of this kind.", "evidenceIds": [] }, { "productId": "kong-ai-gateway", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Kong publishes API reference content (e.g., list-ai-gateways endpoint docs) but there is no indication of an interactive reference with runnable/try-it-out examples; probes for OpenAPI/swagger specs on the docs site all returned 404, suggesting no such interactive tooling is exposed.", "evidenceIds": [ "kong-ai-gateway-docs-14", "kong-ai-gateway-probe-3" ] }, { "productId": "kong-ai-gateway", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence includes API reference pages (e.g. Konnect AI Gateway API docs) but no downloadable OpenAPI/Swagger spec file is evidenced; a direct probe for standard OpenAPI/swagger.json locations returned 404 at all candidate paths, indicating no machine-readable spec is exposed there.", "evidenceIds": [ "kong-ai-gateway-probe-3", "kong-ai-gateway-docs-14" ] }, { "productId": "kong-ai-gateway", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack only shows a quickstart script for spinning up a demo instance and general AI Gateway features (proxying, observability, policies) but no dedicated sandbox/staging environment or test-mode capability that isolates from production data. No mention of environment separation, mock upstreams, or a documented sandbox mode.", "evidenceIds": [] }, { "productId": "kong-ai-gateway", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows an API reference exists (e.g., 'v1' Konnect AI Gateway API) but there is no documentation of a versioning scheme or deprecation policy for the AI Gateway APIs, and OpenAPI spec probes returned 404s.", "evidenceIds": [ "kong-ai-gateway-docs-14", "kong-ai-gateway-probe-3" ] }, { "productId": "kong-ai-gateway", "storyId": "automatic-fallbacks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Kong AI Gateway documents load balancing across multiple LLM models/providers, which implies routing resilience, but the evidence never explicitly describes an automatic fallback mechanism triggered by provider failure (e.g., retry-on-error, circuit breaking, or failover behavior). Missing for 10: explicit fallback-on-failure documentation, failure detection/health-check mechanism, and any hands-on confirmation that fallback actually occurs when a provider errors out.", "evidenceIds": [ "kong-ai-gateway-docs-17", "kong-ai-gateway-docs-1" ] }, { "productId": "kong-ai-gateway", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence covers proxying, observability, security, and MCP/A2A support but never describes any bulk or batch operation capability (e.g., batch creating/updating routes, plugins, or AI entities across many items at once). The kongctl CLI and Konnect API are mentioned but only for single-entity or config migration tasks, not bulk item processing.", "evidenceIds": [ "kong-ai-gateway-docs-14", "kong-ai-gateway-docs-25", "kong-ai-gateway-probe-4" ] }, { "productId": "kong-ai-gateway", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Kong AI Gateway supports policy-based automatic actions triggered by request/response events—e.g., Prompt Guards and content-safety policies acting on chat/completion requests, AI Rate Limiting Advanced enforcing limits automatically, AWS Guardrails validating requests/responses before forwarding, and semantic caching/routing acting on similarity triggers—which functions as rule-based automation on traffic events. However, this is scoped to gateway traffic policies rather than a general-purpose event/trigger automation framework for arbitrary conditions and actions. Missing for 10: evidence of a generic conditional rule engine (if-this-then-that style), user-defined custom triggers beyond built-in policies, and any workflow/orchestration automation beyond request-level policy enforcement.", "evidenceIds": [ "kong-ai-gateway-docs-8", "kong-ai-gateway-docs-12", "kong-ai-gateway-docs-20", "kong-ai-gateway-docs-24", "kong-ai-gateway-docs-28" ] }, { "productId": "kong-ai-gateway", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Kong AI Gateway is an API/traffic gateway product for proxying, governing, and observing AI/LLM/A2A/MCP traffic; it is not a workflow/job scheduling or automation-orchestration product. Scheduling recurring jobs or workflows is outside its category — no evidence pack item relates to cron-like scheduling or workflow orchestration, confirming this is a wrong-axis question rather than a missing feature.", "evidenceIds": [] }, { "productId": "kong-ai-gateway", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers AI Gateway's traffic proxying, observability, security, and provider-agnostic API features, but contains no mention of versioning, reviewing, or rolling back gateway configurations or 'automations' (e.g., no changelog, approval workflow, or rollback mechanism is documented). The only tangential item is a config migration extension (kongctl conversion tool), which addresses format conversion, not version control or rollback. missing for 10: version history/changelog for AI Gateway configs, review/approval workflow for changes, rollback mechanism for policies or entities.", "evidenceIds": [ "kong-ai-gateway-docs-25" ] }, { "productId": "kong-ai-gateway", "storyId": "budgets-limits", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Kong AI Gateway provides cost calculation, tracking across teams, and rate limiting plugins (AI Rate Limiting Advanced) which can enforce request/token limits per consumer, but evidence shows rate limiting and cost tracking as separate capabilities rather than a unified hard 'budget/spend limit' enforcement tied directly to cost thresholds per key/team/user. missing for 10: explicit documentation of setting a dollar-based spend cap or budget that auto-blocks requests once exceeded, per-key/team/user budget configuration UI or API, and independent confirmation of budget enforcement in production.", "evidenceIds": [ "kong-ai-gateway-docs-12", "kong-ai-gateway-docs-18", "kong-ai-gateway-docs-21", "kong-ai-gateway-docs-9" ] }, { "productId": "kong-ai-gateway", "storyId": "byok", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show AI Model Providers are configured with credentials, and an AI Vault entity lets AI Model Providers reference externally-stored secrets (AWS Secrets Manager, HashiCorp Vault, env vars) rather than storing keys as literal values, and Azure-specific managed identity auth is also supported — implying developers can supply and manage their own provider keys/secrets. However, no explicit doc snippet shows the exact 'paste your own API key per provider' config example or per-request key override behavior. Missing for 10: an explicit example of configuring a provider with a literal/bring-your-own API key, and confirmation of per-consumer or per-request key overrides.", "evidenceIds": [ "kong-ai-gateway-docs-1", "kong-ai-gateway-docs-4", "kong-ai-gateway-docs-22", "kong-ai-gateway-docs-23" ] }, { "productId": "kong-ai-gateway", "storyId": "cost-tracking", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Kong AI Gateway computes exact per-request LLM costs matching provider billing, logs cost metrics with token/model details, exports cost-related OTel metrics, and documentation explicitly states cost tracking 'across teams'; since all providers route through the same gateway, aggregation across providers is plausible. However, evidence never shows a consolidated dashboard or breakdown specifically by model, API key, or individual user — only 'teams' is explicitly named as a tracked dimension. Missing for 10: explicit per-user and per-API-key cost attribution/dashboard, and a documented single-pane cross-provider spend view (vs. raw logs/metrics that would need external aggregation).", "evidenceIds": [ "kong-ai-gateway-docs-18", "kong-ai-gateway-docs-10", "kong-ai-gateway-docs-9", "kong-ai-gateway-docs-21", "kong-ai-gateway-docs-26" ] }, { "productId": "kong-ai-gateway", "storyId": "edge-low-latency", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Kong documents latency-related tooling (usage analytics and OTel metrics that track LLM request latency and upstream processing time, semantic caching, load balancing across models) which implies attention to performance, but there is no explicit claim or benchmark quantifying the gateway's own added latency overhead as 'minimal'. missing for 10: quantified latency-overhead benchmarks, independent performance tests, explicit vendor claim of low proxy overhead.", "evidenceIds": [ "kong-ai-gateway-docs-9", "kong-ai-gateway-docs-11", "kong-ai-gateway-docs-17", "kong-ai-gateway-docs-3" ] }, { "productId": "kong-ai-gateway", "storyId": "load-balancing-routing", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Kong AI Gateway explicitly documents load-balancing across multiple LLM models/providers (docs-17), and it also tracks and calculates per-request LLM costs (docs-18) and latency metrics (docs-9), which are the raw signals needed for cost- or latency-aware routing. However, the evidence pack never confirms the specific load-balancing algorithms (e.g., weighted, latency-based, cost-based) are selectable options, so it's unclear whether these criteria can actually be used as balancing strategies. Missing for 10: explicit documentation of weight-based, latency-based, and cost-based load-balancing algorithms/configuration options, and independent confirmation these modes work as described.", "evidenceIds": [ "kong-ai-gateway-docs-17", "kong-ai-gateway-docs-18", "kong-ai-gateway-docs-9" ] }, { "productId": "kong-ai-gateway", "storyId": "model-catalog-discovery", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Kong AI Gateway supports multi-provider routing and calculates per-request costs, but there is no documented catalog/endpoint for browsing or querying available models with pricing and context-window metadata — the closest evidence (model-cost-management, list-ai-gateways API) covers cost calculation per request and listing gateway instances, not a model metadata catalog.", "evidenceIds": [ "kong-ai-gateway-docs-18", "kong-ai-gateway-docs-14", "kong-ai-gateway-docs-1" ] }, { "productId": "kong-ai-gateway", "storyId": "openai-compatible-endpoint", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Kong AI Gateway is documented as a provider-agnostic API layer that proxies LLM requests (chat/completions endpoints) with streaming support, which implies OpenAI-style compatibility, but no evidence explicitly confirms that existing OpenAI SDK code works unmodified by only swapping base URL and API key. missing for 10: explicit doc/example showing an OpenAI SDK client redirected via base_url with no code changes, and confirmation that request/response schemas exactly mirror OpenAI's format.", "evidenceIds": [ "kong-ai-gateway-docs-1", "kong-ai-gateway-docs-16", "kong-ai-gateway-docs-27" ] }, { "productId": "kong-ai-gateway", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Kong exposes AI Gateway configuration via the Konnect API (e.g. list-ai-gateways endpoint) and the kongctl CLI, and docs explicitly note that 'the same capabilities' configurable via plugins can be set on Services/Routes rather than only the UI, implying strong API-driven configuration. However there is no explicit statement of full UI/API parity, and probes for a discoverable OpenAPI spec returned 404s, leaving completeness unverified. Missing for 10: an explicit parity statement or comprehensive OpenAPI spec confirming every UI action (e.g. dashboards, analytics views) is also API-accessible, and independent confirmation of no UI-only features.", "evidenceIds": [ "kong-ai-gateway-docs-14", "kong-ai-gateway-docs-27", "kong-ai-gateway-probe-4", "kong-ai-gateway-probe-3" ] }, { "productId": "kong-ai-gateway", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Kong AI Gateway/Konnect is a SaaS-hosted control plane with configuration, logs, and analytics data, so data portability/export-and-leave is a fair axis to ask, but the evidence pack contains no mention of a bulk data export feature, open-format export of configs/logs/analytics, or a documented migration-out path in open standards. Evidence only covers observability (metrics, logs) and a CLI conversion extension for migrating configs between versions, not exporting all user data to leave the platform.", "evidenceIds": [ "kong-ai-gateway-docs-25", "kong-ai-gateway-docs-9", "kong-ai-gateway-docs-10" ] }, { "productId": "kong-ai-gateway", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Kong AI Gateway is a commercial/enterprise product built on Kong Gateway; the evidence pack contains only product documentation, API references, and probes for docs/CLI—no mention of source code availability or an open-source license for AI Gateway itself. Kong Gateway OSS is a separate product; nothing here confirms AI Gateway's source is published under an open license.", "evidenceIds": [] }, { "productId": "kong-ai-gateway", "storyId": "openness-self-host", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs reference configuring AI Gateway capabilities 'on-prem' via plugins on Services and Routes, and a quickstart script to spin up a demo instance, implying self-hosted deployment is possible. However, there's no detailed self-hosting guide, licensing/open-source clarification, or infrastructure requirements documented in the evidence pack. Missing for 10: explicit self-hosting/installation documentation, open-source license details, infrastructure/deployment guides beyond a demo quickstart.", "evidenceIds": [ "kong-ai-gateway-docs-27", "kong-ai-gateway-docs-2" ] }, { "productId": "kong-ai-gateway", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Kong AI Gateway documentation mentions an on-prem/self-hosted configuration option (configure-on-prem.md), which implies some ability to control where the gateway (and thus data in transit) runs, but there is no explicit documentation of region selection, data residency guarantees, or storage location controls for logs/analytics/cost data. missing for 10: explicit region/residency configuration options, documented data storage location controls, compliance statements about data residency.", "evidenceIds": [ "kong-ai-gateway-docs-27", "kong-ai-gateway-docs-10" ] }, { "productId": "kong-ai-gateway", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Kong AI Gateway is an enterprise routing/governance layer for LLM traffic, so control over how provider data is used is a fair question, but the evidence pack contains no mention of training-data opt-out, zero-retention guarantees, or contractual terms preventing model providers from using proxied data for training — only general DLP/PII redaction and content-safety features are documented, which don't address this specific claim.", "evidenceIds": [] }, { "productId": "kong-ai-gateway", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Kong AI Gateway logs request/response payloads, token usage, and costs (docs-10), and offers DLP/PII removal (docs-20) and semantic caching (docs-11), but none of the evidence describes configurable data retention periods or deletion/purge controls for logs, caches, or audit trails. Missing for 10: explicit retention policy settings, log/cache deletion or expiry controls, data subject deletion workflow.", "evidenceIds": [ "kong-ai-gateway-docs-10", "kong-ai-gateway-docs-20", "kong-ai-gateway-docs-11" ] }, { "productId": "kong-ai-gateway", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items mention any telemetry opt-out, privacy settings, or usage-tracking controls for Kong AI Gateway itself; all evidence concerns the product's own traffic logging/analytics features rather than user opt-out of vendor telemetry.", "evidenceIds": [] }, { "productId": "kong-ai-gateway", "storyId": "provision-via-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Kong exposes a Konnect API for listing/managing AI Gateways (list-ai-gateways) and entity-based configuration (AI Auth Strategy, AI Vault) that implies programmatic provisioning of gateways and credentials, plus a kongctl CLI for gateway configuration/migration. However, there is no explicit documentation of an API or CLI for creating/managing spend budgets or quotas, and no end-to-end example of provisioning gateway+keys+budget via API. Missing for 10: explicit budget/quota provisioning API, a full API reference for creating AI Auth Strategies/keys programmatically, and independent confirmation of the admin API's completeness for this workflow.", "evidenceIds": [ "kong-ai-gateway-docs-14", "kong-ai-gateway-docs-22", "kong-ai-gateway-docs-23", "kong-ai-gateway-docs-25", "kong-ai-gateway-docs-18" ] }, { "productId": "kong-ai-gateway", "storyId": "rate-limit-smoothing", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Kong AI Gateway documents load balancing across multiple LLM models/providers and a dedicated 'AI Rate Limiting Advanced' plugin for enforcing rate limits, which together support spreading traffic and throttling. However, there is no explicit documentation of spreading traffic across multiple API keys for the same provider or of request queuing behavior when limits are hit. Missing for 10: explicit key-rotation/multi-key load balancing, request queuing mechanics, and any hands-on/independent verification of these features working together.", "evidenceIds": [ "kong-ai-gateway-docs-17", "kong-ai-gateway-docs-12", "kong-ai-gateway-docs-18" ] }, { "productId": "kong-ai-gateway", "storyId": "request-logging", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Kong AI Gateway docs explicitly describe audit logs capturing request/response payloads, token usage, model details, latency, and cost metrics, plus usage analytics dashboards and OTel metrics for latency, tokens, and cost. missing for 10: independent/hands-on corroboration of the logging UI in practice and detail on log retention/query interface.", "evidenceIds": [ "kong-ai-gateway-docs-3", "kong-ai-gateway-docs-9", "kong-ai-gateway-docs-10", "kong-ai-gateway-docs-18", "kong-ai-gateway-docs-26" ] }, { "productId": "kong-ai-gateway", "storyId": "response-caching", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Kong ships a dedicated AI Semantic Cache plugin explicitly designed to cache LLM responses at the gateway, and separate docs confirm gateway-level semantic caching to reduce redundant calls and improve latency/cost. This is first-party documentation with a named product feature, though no independent benchmark or hands-on cost/latency measurement is provided. Missing for 10: independent/hands-on validation of actual cost/latency savings, and exact-match (non-semantic) cache configuration details.", "evidenceIds": [ "kong-ai-gateway-docs-11", "kong-ai-gateway-docs-24" ] }, { "productId": "kong-ai-gateway", "storyId": "retry-policies", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Kong AI Gateway documents load balancing across multiple LLM models/providers (docs-17), which implies some failover capability, but the evidence never explicitly describes configurable automatic retry policies for transient provider errors (e.g., retry counts, backoff, error-code triggers). Missing for 10: explicit retry-policy documentation, retry configuration options, and evidence of behavior on transient errors specifically.", "evidenceIds": [ "kong-ai-gateway-docs-17", "kong-ai-gateway-docs-1" ] }, { "productId": "kong-ai-gateway", "storyId": "streaming-passthrough", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Kong AI Gateway has a dedicated streaming.md doc confirming token-by-token streaming from models is supported and rendered in real time, layered on top of its provider-agnostic API supporting multiple LLM providers. Missing for 10: independent/hands-on confirmation of streaming behavior across every provider and any details on streaming with proxy-added policies (e.g., guardrails) applied mid-stream.", "evidenceIds": [ "kong-ai-gateway-docs-16", "kong-ai-gateway-docs-1" ] }, { "productId": "kong-ai-gateway", "storyId": "structured-outputs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers provider-agnostic proxying, streaming, load balancing, cost tracking, and safety policies, but no document mentions structured/JSON-schema output enforcement or normalization across providers. This is a fair capability to expect from an AI gateway, but no evidence supports it.", "evidenceIds": [] }, { "productId": "kong-ai-gateway", "storyId": "tool-call-fidelity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Kong AI Gateway documents a provider-agnostic API for LLM traffic and an AI MCP Proxy/Server that converts REST APIs into MCP tools, suggesting a consistent tool-calling interface across providers and models. However, there is no explicit documentation showing normalization of function/tool-call schemas (e.g., translating OpenAI-style function calls to Anthropic tool-use format) for the same request across different providers. Missing for 10: explicit cross-provider function-call schema translation examples, hands-on/independent verification of consistent tool-call behavior across providers.", "evidenceIds": [ "kong-ai-gateway-docs-1", "kong-ai-gateway-docs-13", "kong-ai-gateway-docs-19", "kong-ai-gateway-docs-15" ] }, { "productId": "kong-ai-gateway", "storyId": "tracing-integrations", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Kong AI Gateway documents exporting OpenTelemetry (OTLP) metrics for generative AI, MCP, and A2A traffic via an OpenTelemetry AI Policy, plus detailed audit logs capturing request/response payloads, token usage, latency, and cost — enabling integration with a developer's own observability stack. Missing for 10: explicit trace export (spans/tracing) documentation and independent/hands-on confirmation of exporting to third-party backends like Datadog or Grafana.", "evidenceIds": [ "kong-ai-gateway-docs-26", "kong-ai-gateway-docs-9", "kong-ai-gateway-docs-10" ] }, { "productId": "kong-ai-gateway", "storyId": "unified-api-many-providers", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Kong AI Gateway explicitly provides a provider-agnostic API to route to many LLM providers, with load balancing and unified logging/observability across providers, matching the core unified-API story. Missing for 10: independent/hands-on developer corroboration of consistent request/response schema across providers, and no explicit list of exact supported providers in evidence.", "evidenceIds": [ "kong-ai-gateway-docs-1", "kong-ai-gateway-docs-17", "kong-ai-gateway-docs-9", "kong-ai-gateway-docs-18" ] }, { "productId": "kong-ai-gateway", "storyId": "virtual-keys", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Kong AI Gateway's AI Vault lets provider keys be stored in external secret backends and referenced by AI Model Providers/Agents instead of literal values, and AI Auth Strategy lets client apps authenticate via an IdP or Kong-managed auth without handling provider keys directly — both point toward the platform-engineer being able to shield raw provider keys from consumers. However, the evidence never explicitly describes a workflow for minting distinct gateway-managed keys per team/app, rotating them, or a consumer/key-auth flow tied to AI traffic. Missing for 10: explicit documentation of issuing/rotating per-team or per-app gateway-managed API keys, and any admin UI/API walkthrough for that key-minting process.", "evidenceIds": [ "kong-ai-gateway-docs-22", "kong-ai-gateway-docs-23", "kong-ai-gateway-docs-7" ] }, { "productId": "litellm", "storyId": "agent-spend-guardrails", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "LiteLLM's virtual key system explicitly supports per-key budgets (budget_duration, max_budget), rate limits (rpm_limit), and personal/team-scoped keys, directly enabling an autonomous agent to get its own key with spend and rate guardrails. Docs also show spend tracking by key/user/tag to monitor and enforce these limits. Missing for 10: independent/hands-on verification that budget caps hard-stop overspend in practice, and more detail on rate-limit enforcement behavior under load.", "evidenceIds": [ "litellm-docs-7", "litellm-docs-8", "litellm-docs-25", "litellm-docs-26", "litellm-docs-6", "litellm-docs-24", "litellm-docs-15" ] }, { "productId": "litellm", "storyId": "agent-switches-models-policy", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "LiteLLM's proxy documents concrete routing policies matching the story: load balancing across deployments, automatic fallback to a healthy backup model/provider on failure, and content-policy-specific fallbacks mapped across providers — all confirmed by community commentary on its origin as a fix for manual multi-provider if/else logic. Missing for 10: explicit cost-triggered routing (switching to cheaper model based on budget threshold) and any hands-on demonstration of live mid-task switching rather than static failover/fallback configuration.", "evidenceIds": [ "litellm-docs-3", "litellm-docs-4", "litellm-docs-21", "litellm-docs-18", "litellm-docs-22", "litellm-comm-12" ] }, { "productId": "litellm", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A direct probe confirms LiteLLM serves a valid llms.txt file at https://docs.litellm.ai/llms.txt returning HTTP 200 with an agent-readable overview and links, exactly matching the story's request. Missing for 10: no independent/community corroboration of agents actually consuming this file, and the related docs-as-markdown endpoint (.md) returns 404, suggesting the agent-oriented docs surface is only partially built out.", "evidenceIds": [ "litellm-probe-1", "litellm-probe-2" ] }, { "productId": "litellm", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "LiteLLM is an LLM gateway/proxy for routing, cost tracking, and reliability across model providers — it is infrastructure, not a data-facing application that surfaces AI-generated insights or suggestions 'inside the product' from a user's own data. This axis targets end-user-facing analytics/insight features, which is outside LiteLLM's category as a developer-facing gateway.", "evidenceIds": [] }, { "productId": "litellm", "storyId": "agentic-autonomous-automation", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "LiteLLM is an LLM gateway/proxy for routing and managing API calls to models, not an automation/agent orchestration platform; it has no scheduling, triggers, or background workflow execution features. Setting up autonomous background automations is a category error for this product type.", "evidenceIds": [] }, { "productId": "litellm", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "LiteLLM is an LLM gateway/proxy infrastructure product, not an end-user assistant product; it provides an API/proxy for routing calls to LLMs rather than a built-in assistant persona to delegate tasks to. This axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "litellm", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "LiteLLM is a Python SDK and proxy server with a documented CLI (worker processes, uvicorn/gunicorn/Granian) that can run headlessly in scripts or containers, and community evidence shows it running via Docker in unattended homelab/gateway setups. However there is no explicit CI/CD pipeline example, GitHub Actions integration, or automation-specific documentation in the evidence pack. Missing for 10: explicit CI pipeline examples, automation/scripting docs beyond CLI flags, evidence of non-interactive auth/config for CI environments.", "evidenceIds": [ "litellm-docs-13", "litellm-docs-20", "litellm-probe-4", "litellm-comm-6", "litellm-docs-5" ] }, { "productId": "litellm", "storyId": "agentic-mcp-client", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "LiteLLM Proxy has a documented MCP Gateway feature letting users add MCP servers via the UI and use a fixed endpoint to expose their tools, controllable by key/team — directly matching the story. missing for 10: independent/hands-on corroboration of MCP tool usage beyond first-party docs, and detail on tool-calling behavior once connected.", "evidenceIds": [ "litellm-docs-14", "litellm-docs-27" ] }, { "productId": "litellm", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "LiteLLM Proxy ships an official MCP Gateway feature exposing a fixed endpoint for all configured MCP tools, with access controlled by Key/Team, and a UI flow to add MCP servers — this is exactly the 'connect an agent via an official MCP server' capability for a gateway product. Missing for 10: independent/hands-on confirmation that external agents (e.g. Claude, Cursor) successfully connect via this gateway, and deeper documentation of the MCP protocol compliance/auth flow beyond the brief doc mentions.", "evidenceIds": [ "litellm-docs-14", "litellm-docs-27" ] }, { "productId": "litellm", "storyId": "agentic-nl-commands", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "LiteLLM is an LLM gateway/proxy for routing API calls between providers, not a user-facing interface meant to be operated via natural-language commands; this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "litellm", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "LiteLLM ships an official CLI for its proxy (litellm docs-proxy-cli, litellm-probe-4) covering server startup and worker config, but the docs shown focus narrowly on process/worker management rather than a broad AI-native workflow CLI (e.g., chat, model testing, key/budget management from CLI). missing for 10: evidence of CLI commands beyond process startup (e.g., testing prompts, managing virtual keys/budgets, MCP servers via CLI), independent/hands-on confirmation of CLI usability.", "evidenceIds": [ "litellm-docs-13", "litellm-docs-20", "litellm-probe-4" ] }, { "productId": "litellm", "storyId": "agentic-public-api", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "LiteLLM's core interface — the completion() function and the OpenAI-compatible proxy REST API — is extensively documented as a public, drop-in API that AI-native users can call directly, corroborated by community Show HN discussion confirming streaming/function-calling parity with the OpenAI SDK. Missing for 10: a discoverable machine-readable OpenAPI/swagger spec (probe found only 404s) and independent hands-on confirmation of API stability/versioning.", "evidenceIds": [ "litellm-docs-1", "litellm-docs-2", "litellm-gh-1", "litellm-docs-16", "litellm-docs-17", "litellm-comm-13", "litellm-probe-3" ] }, { "productId": "litellm", "storyId": "agentic-scoped-keys", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "LiteLLM's proxy virtual keys let admins scope credentials by model access, budget, team, and rate limits (rpm_limit), directly enabling least-privilege API keys for agents. This is well documented across multiple first-party docs (virtual_keys, users, cost_tracking) showing budgets, per-key model restrictions, and team-based limits. Missing for 10: independent/hands-on verification of the scoping granularity in practice, and one community comment (litellm-comm-3) noted vague 'issues with creating tokens for other users' suggesting rough edges.", "evidenceIds": [ "litellm-docs-8", "litellm-docs-7", "litellm-docs-25", "litellm-docs-26", "litellm-docs-6", "litellm-comm-3" ] }, { "productId": "litellm", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "LiteLLM ships an official Python SDK (the `completion()` interface) that mirrors the OpenAI SDK format across 100+ providers, letting AI-native developers build without learning per-provider APIs; this is corroborated by community Show HN feedback praising it for standardizing APIs and confirming SDK parity for streaming/function-calling. Missing for 10: evidence of official SDKs in other languages (e.g., JS/TS) and independent benchmarking of SDK completeness beyond community praise.", "evidenceIds": [ "litellm-docs-1", "litellm-docs-2", "litellm-gh-1", "litellm-docs-16", "litellm-docs-17", "litellm-comm-13", "litellm-comm-14" ] }, { "productId": "litellm", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions webhooks or event subscription mechanisms; LiteLLM's documented features cover logging integrations, cost tracking, and MCP gateway, but nothing about outbound webhook events for subscribers.", "evidenceIds": [] }, { "productId": "litellm", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows explicit probe failures for an OpenAPI/Swagger spec (litellm-probe-3) and no documented interactive API reference or runnable examples in the docs; only static markdown-style docs and code snippets are cited (litellm-docs-1/2/16/17). Since LiteLLM's proxy is an API gateway, this axis clearly applies, but there's no evidence of an interactive/try-it-out API reference.", "evidenceIds": [ "litellm-probe-3", "litellm-probe-2", "litellm-docs-1", "litellm-docs-16" ] }, { "productId": "litellm", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack includes explicit probes for OpenAPI/swagger endpoints on LiteLLM's docs site, all returning 404, and no other citation shows a downloadable OpenAPI spec (e.g., from the proxy's FastAPI docs). Since LiteLLM's proxy is a gateway with an admin UI and CLI, this axis plausibly applies, but no evidence confirms a machine-readable spec is available.", "evidenceIds": [ "litellm-probe-3", "litellm-probe-2" ] }, { "productId": "litellm", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack of a dedicated sandbox/test mode, mock-response capability, or staging environment for validating integrations without touching production data — the docs cover routing, budgets, virtual keys, logging, and reliability, but nothing about isolated test environments. missing for 10: sandbox/test mode, mock LLM responses, staging deployment guidance, docs mentioning safe non-production testing.", "evidenceIds": [] }, { "productId": "litellm", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of API versioning scheme or a documented deprecation policy anywhere in docs; OpenAPI/spec discovery probes returned 404s, suggesting no formal versioned API contract is published.", "evidenceIds": [ "litellm-probe-3", "litellm-probe-2" ] }, { "productId": "litellm", "storyId": "automatic-fallbacks", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "LiteLLM's proxy reliability docs explicitly describe configuring fallbacks so a failing model/provider automatically fails over to a healthy backup, including model-group fallback chains and content-policy-specific fallback handling, corroborated by a maintainer's stated motivation for reliable model switching across providers. Missing for 10: independent hands-on validation of fallback behavior in production beyond docs/maintainer statements.", "evidenceIds": [ "litellm-docs-4", "litellm-docs-21", "litellm-docs-22", "litellm-docs-18", "litellm-comm-12" ] }, { "productId": "litellm", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes LiteLLM's unified completion interface, routing, fallbacks, cost tracking, and virtual keys, but contains no mention of batch/bulk operations (e.g., batch completions across many prompts, bulk key/user management, or bulk import/export) that would let a user act on many items at once. Since LiteLLM manages many models, keys, and users, this axis is plausible for the product, but no supporting evidence exists.", "evidenceIds": [] }, { "productId": "litellm", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "LiteLLM ships several built-in event-triggered automations (automatic fallback to another model/provider on call failure or content-policy violation, budget resets on a schedule, rate limits per key/team) but these are fixed, pre-defined behaviors rather than a general user-definable rule/trigger-action engine (e.g. no webhook/if-this-then-that automation builder is evidenced). Missing for 10: a general-purpose custom rule/condition-action automation system, evidence of user-defined triggers beyond fallback/budget/rate-limit, and any webhook or external-action firing mechanism.", "evidenceIds": [ "litellm-docs-4", "litellm-docs-18", "litellm-docs-21", "litellm-docs-22", "litellm-docs-25", "litellm-docs-26" ] }, { "productId": "litellm", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "LiteLLM is an LLM gateway/proxy for routing, cost tracking, and reliability across providers, not a workflow/job scheduling tool; scheduling recurring jobs is outside its product category.", "evidenceIds": [] }, { "productId": "litellm", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "LiteLLM is an LLM gateway/proxy for routing, cost tracking, and reliability across providers; it has no concept of versioning, reviewing, or rolling back 'automations' (workflows/prompts/pipelines). This is a category mismatch, not a missing feature.", "evidenceIds": [] }, { "productId": "litellm", "storyId": "budgets-limits", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly cover per-key, per-team, and per-user budgets/spend tracking (virtual keys with cost tracking, personal budgets, team rpm_limit, budget_duration for resets, tag-based spend tracking) plus marketing copy on capping spend before it runs. Missing for 10: independent hands-on verification that hard limits actually block calls once exceeded (only docs, no community confirmation of enforcement behavior).", "evidenceIds": [ "litellm-docs-5", "litellm-docs-6", "litellm-docs-7", "litellm-docs-8", "litellm-docs-15", "litellm-docs-24", "litellm-docs-25", "litellm-docs-26" ] }, { "productId": "litellm", "storyId": "byok", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "LiteLLM's core design lets developers call any provider through one interface and issue virtual keys that map to underlying provider credentials, with per-key spend/model access control ('Put your full AI stack behind one key', virtual key docs) — implying BYOK usage. However, the evidence pack lacks an explicit walkthrough of adding a personal provider API key to the proxy config and having it used for a developer's own traffic, and there is no independent confirmation of this specific flow. Missing for 10: explicit BYOK config example, independent/hands-on confirmation of per-developer key routing.", "evidenceIds": [ "litellm-docs-1", "litellm-docs-8", "litellm-docs-15", "litellm-docs-5" ] }, { "productId": "litellm", "storyId": "cost-tracking", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "LiteLLM's docs directly document tracking spend per key, user, team, and by tags across 100+ providers via the proxy's cost tracking feature, plus budgets, virtual keys, and admin UI to view spend — matching the story closely. Missing for 10: independent hands-on verification of the cost dashboard's accuracy/UX and explicit per-model spend breakdown evidence beyond the general 'across 100+ LLMs' claim.", "evidenceIds": [ "litellm-docs-6", "litellm-docs-8", "litellm-docs-24", "litellm-docs-25", "litellm-docs-5", "litellm-docs-15" ] }, { "productId": "litellm", "storyId": "edge-low-latency", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "LiteLLM docs describe features that reduce or mitigate latency (response caching, configurable worker processes for uvicorn/gunicorn/Granian) and fast failover, but there is no explicit benchmark or vendor claim quantifying the proxy's own added latency overhead versus direct provider calls. Community evidence includes a report of the proxy being 'incredible flaky and causing outages,' which is a reliability complaint rather than a direct latency measurement, so it doesn't rise to a concrete dispute of a latency claim.\nmissing for 10: explicit latency-overhead benchmarks/numbers, independent performance testing of proxy overhead, direct vendor claim about added latency being minimal.", "evidenceIds": [ "litellm-docs-9", "litellm-docs-13", "litellm-docs-20", "litellm-docs-4", "litellm-comm-4" ] }, { "productId": "litellm", "storyId": "load-balancing-routing", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm load-balancing across multiple deployments and automatic failover/fallback to healthy backups, which covers reliability-based routing, but the evidence pack never cites explicit weight-based, latency-based, or cost-based routing strategies by name (LiteLLM's known routing options aren't documented here). missing for 10: explicit documentation of weight-based routing strategy, latency-based routing strategy, and cost-based routing strategy configuration.", "evidenceIds": [ "litellm-docs-3", "litellm-docs-4", "litellm-docs-21", "litellm-docs-22", "litellm-docs-18" ] }, { "productId": "litellm", "storyId": "model-catalog-discovery", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While LiteLLM claims support for 100+ LLMs and tracks spend/cost, the evidence pack contains no mention of a browsable/queryable catalog listing models with pricing and context-window metadata (e.g., no model_cost table, /model/info endpoint, or docs page referencing context window sizes). Cost-tracking docs (litellm-docs-6) cover spend tracking, not a metadata catalog.", "evidenceIds": [] }, { "productId": "litellm", "storyId": "openai-compatible-endpoint", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "LiteLLM's docs and GitHub explicitly advertise drop-in OpenAI compatibility with consistent Chat Completions format across providers, and the self-hosted proxy exposes a single gateway endpoint with virtual keys, matching the exact story of only swapping base URL and key. Community evidence (Show HN, HN threads) corroborates real-world use as a drop-in gateway for existing OpenAI-style code. Missing for 10: independent hands-on verification of literal 'change only base_url+key' migration and no explicit code diff example in the pack.", "evidenceIds": [ "litellm-docs-1", "litellm-docs-2", "litellm-gh-1", "litellm-docs-5", "litellm-docs-23", "litellm-comm-12", "litellm-comm-13" ] }, { "productId": "litellm", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "LiteLLM's proxy exposes virtual keys, budgets, spend tracking, MCP server management, and routing/fallback config via REST API endpoints and CLI, and the admin UI is largely a wrapper over these same proxy APIs, suggesting strong API/UI parity. However, no OpenAPI spec was found at expected locations (404s), and some UI-specific flows like adding MCP servers are only documented via UI clicks rather than confirmed equivalent API calls, and no independent confirmation exists that every UI action has a documented API equivalent. Missing for 10: a published OpenAPI/swagger spec confirming full endpoint coverage, explicit API equivalents for every UI workflow (e.g., MCP server add via API), and independent/hands-on verification that API and UI are truly at parity.", "evidenceIds": [ "litellm-docs-5", "litellm-docs-8", "litellm-docs-14", "litellm-docs-27", "litellm-docs-13", "litellm-probe-3", "litellm-probe-4" ] }, { "productId": "litellm", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "LiteLLM stores spend, logs, virtual-key and config data (self-hosted proxy with admin UI), so an explicit 'export all data in open formats' capability is a fair ask, but no evidence pack item documents any export/download feature, open data format, or data-portability tooling — only logging integrations to third-party sinks (Langfuse, S3, GCS) are mentioned, which is not the same as user-initiated full data export. missing for 10: documented export/download feature, open-format data export (CSV/JSON) of spend/logs/keys, any data-portability or 'leave the platform' guidance.", "evidenceIds": [ "litellm-docs-5", "litellm-docs-6", "litellm-docs-10" ] }, { "productId": "litellm", "storyId": "openness-open-license", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "The product's source is publicly hosted on GitHub (BerriAI/litellm), confirmed by community references to 'public repos' being vandalized, showing the codebase is openly readable. However, the evidence pack never states or cites an actual license (e.g., MIT/Apache) for the repository, so the 'open license' claim itself is unconfirmed. Missing for 10: explicit license documentation/citation, first-party statement of licensing terms.", "evidenceIds": [ "litellm-gh-1", "litellm-comm-2" ] }, { "productId": "litellm", "storyId": "openness-self-host", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "LiteLLM explicitly ships a self-hosted LLM Gateway/Proxy (docs-5, docs-23) with configurable worker processes and CLI (docs-13/20), and this is corroborated by hands-on community use running it via Docker in a homelab (litellm-comm-6) and widespread adoption as a self-hosted gateway (litellm-comm-7). missing for 10: no independent benchmark or enterprise-scale self-hosting case study beyond a single homelab anecdote.", "evidenceIds": [ "litellm-docs-5", "litellm-docs-23", "litellm-docs-13", "litellm-docs-20", "litellm-comm-6", "litellm-comm-7" ] }, { "productId": "litellm", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "LiteLLM is self-hostable (which lets users control physical data location by choosing their own deployment region), but no evidence pack item documents any explicit data residency/region-selection feature, region-pinning configuration, or compliance-region controls offered by the product itself.", "evidenceIds": [] }, { "productId": "litellm", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows LiteLLM can disable logging of prompts/responses to its own logging providers (litellm-docs-12), but nothing indicates it offers a mechanism to opt out of model-training use by the underlying LLM providers (e.g., passing zero-retention/no-train flags to OpenAI/Anthropic/etc.). As a gateway, this is a plausible capability, but no evidence supports it.", "evidenceIds": [ "litellm-docs-12" ] }, { "productId": "litellm", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "LiteLLM lets admins disable logging of request/response content to reduce data retention while still tracking metadata (litellm-docs-12), and being self-hosted gives users infrastructure-level control over where data lives. However, there is no documented explicit data-deletion API, retention-period settings, or data lifecycle/export controls in the evidence pack. Missing for 10: explicit deletion/purge API, configurable retention windows, documented data lifecycle policy, independent confirmation these logging controls work as described.", "evidenceIds": [ "litellm-docs-12", "litellm-docs-5" ] }, { "productId": "litellm", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence covers logging integrations (Langfuse, OTel, S3, etc.) and a way to disable message/response logging to those *external* logging providers (litellm-docs-12), but nothing in the pack addresses opting out of LiteLLM's own product telemetry/usage tracking sent back to the vendor. As a self-hosted gateway this axis plausibly applies, but there is no documented telemetry opt-out flag or setting in the evidence pack. Missing for 10: any mention of a telemetry/usage-tracking toggle (e.g., an env var or config flag disabling anonymized usage reporting to the vendor), independent confirmation that no phone-home occurs.", "evidenceIds": [ "litellm-docs-12", "litellm-docs-10" ] }, { "productId": "litellm", "storyId": "provision-via-api", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Docs describe programmatic key/team/budget management (virtual keys, personal and team budgets, rpm_limit, team_alias via API/config) which supports the story's core claim, but no OpenAPI/swagger spec was found (probe shows 404s) and a community user reported concrete hands-on issues 'creating tokens for other users' via the proxy admin functions, directly contradicting the smoothness implied by the docs. Missing for 10: verified OpenAPI/admin-API reference, evidence of 'gateway' provisioning (multiple proxy instances) via API, and independent confirmation that key/budget creation works reliably at scale.", "evidenceIds": [ "litellm-docs-7", "litellm-docs-8", "litellm-docs-25", "litellm-docs-26", "litellm-probe-3", "litellm-comm-3" ] }, { "productId": "litellm", "storyId": "rate-limit-smoothing", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "LiteLLM's Router explicitly supports load-balancing across multiple deployments/keys (e.g., multiple Azure/OpenAI keys) and virtual keys carry per-key rpm_limit fields, directly enabling spreading traffic across provider keys to smooth rate limits, with retries/fallbacks for failures. Missing for 10: explicit documentation of a request queuing/throttling mechanism beyond rpm_limit and load balancing, and independent hands-on confirmation that this smooths real provider rate-limit errors under load.", "evidenceIds": [ "litellm-docs-3", "litellm-docs-26", "litellm-docs-4", "litellm-docs-21", "litellm-docs-22" ] }, { "productId": "litellm", "storyId": "request-logging", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show LiteLLM logs each request with a unique call_id, tracks spend/cost by key/user/team, and integrates with Langfuse, OpenTelemetry, S3, GCS, etc. for logging proxy input/output/exceptions, all viewable via the admin UI — a reasonable basis for inspecting requests with cost attached. However, the evidence never explicitly confirms that latency or token counts are captured/displayed alongside these logs, and there's no independent/hands-on confirmation of the inspection UI itself. Missing for 10: explicit documentation of per-request latency metrics, explicit token-count logging, and independent verification of the UI/log inspection experience.", "evidenceIds": [ "litellm-docs-5", "litellm-docs-6", "litellm-docs-10", "litellm-docs-11", "litellm-docs-24" ] }, { "productId": "litellm", "storyId": "response-caching", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "LiteLLM proxy has a documented caching system that explicitly stores and reuses LLM responses to save costs and reduce latency, directly matching the story. Missing for 10: independent/hands-on benchmarks quantifying latency/cost savings, and detail on cache backend configuration options in the evidence pack.", "evidenceIds": [ "litellm-docs-9" ] }, { "productId": "litellm", "storyId": "retry-policies", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs explicitly describe a `num_retries` setting on the proxy, after which LiteLLM falls back to another model/group, directly supporting configurable retry policies for transient failures (docs-4, docs-21, docs-22). Missing for 10: detailed backoff/jitter configuration, granular retry-per-error-type mapping (only content policy fallback is separately documented), and independent hands-on verification of retry behavior under real transient provider errors.", "evidenceIds": [ "litellm-docs-4", "litellm-docs-21", "litellm-docs-22", "litellm-docs-18" ] }, { "productId": "litellm", "storyId": "streaming-passthrough", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "LiteLLM's unified completion() interface and consistent output format across providers is well documented, and a maintainer explicitly confirmed the proxy 'supports streaming and function-calling in the same way as the openai-python SDK' across 50+ providers, matching the streaming-tools story directly. Missing for 10: a dedicated first-party streaming docs page/code snippet (e.g., stream=True) and independent hands-on confirmation beyond the single maintainer HN comment.", "evidenceIds": [ "litellm-comm-13", "litellm-docs-1", "litellm-docs-2", "litellm-docs-16", "litellm-docs-17", "litellm-comm-12" ] }, { "productId": "litellm", "storyId": "structured-outputs", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers LiteLLM's unified completion() interface, consistent OpenAI-format responses, retries/fallbacks, and proxy features, but nowhere mentions structured/JSON-schema output support (e.g. response_format, json_schema mode) across providers. This is a fair axis for a multi-provider LLM gateway, but no citation substantiates the specific capability.", "evidenceIds": [] }, { "productId": "litellm", "storyId": "tool-call-fidelity", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "LiteLLM documents a unified completion() interface that normalizes tool/function calls and outputs to the OpenAI Chat Completions schema across 100+ providers, and a maintainer confirms function-calling parity with the openai-python SDK. missing for 10: independent hands-on verification of tool-call schema consistency across many specific providers, and no dedicated docs excerpt specifically on function-calling parameter translation edge cases.", "evidenceIds": [ "litellm-docs-1", "litellm-docs-2", "litellm-docs-16", "litellm-docs-17", "litellm-gh-1", "litellm-comm-13" ] }, { "productId": "litellm", "storyId": "tracing-integrations", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly confirm proxy logging integrations to external observability stacks (Langfuse, OpenTelemetry, GCS, S3, Azure Blob), plus a unique call_id for tracing requests across systems, directly matching the story of exporting logs/traces to one's own stack. Missing for 10: independent hands-on verification of these integrations working reliably, and more detail on trace format/OpenTelemetry span structure.", "evidenceIds": [ "litellm-docs-10", "litellm-docs-11", "litellm-docs-19", "litellm-docs-12" ] }, { "productId": "litellm", "storyId": "unified-api-many-providers", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "LiteLLM's core value proposition—calling 100+ providers through one OpenAI-compatible completion() interface with consistent output format—is well documented (litellm-docs-1, litellm-docs-2, litellm-docs-16, litellm-docs-17, litellm-gh-1) and corroborated by independent Show HN community feedback praising it as a way to standardize disparate AI APIs and quickly add new providers (litellm-comm-12, litellm-comm-13, litellm-comm-14, litellm-comm-15). Some community sentiment flags proxy flakiness/outages, but that concerns operational reliability, not whether the unified-API abstraction itself works. Missing for 10: no independent benchmark verifying identical output format across all supported providers, and no public OpenAPI spec was found for the proxy (litellm-probe-3).", "evidenceIds": [ "litellm-docs-1", "litellm-docs-2", "litellm-docs-16", "litellm-docs-17", "litellm-gh-1", "litellm-comm-12", "litellm-comm-13", "litellm-comm-14", "litellm-comm-15" ] }, { "productId": "litellm", "storyId": "virtual-keys", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "LiteLLM's proxy docs explicitly support minting scoped 'virtual keys' for teams/users/apps with budget and rate limits (litellm-docs-5, litellm-docs-6, litellm-docs-7, litellm-docs-8, litellm-docs-26), which is exactly the gateway-managed-key model that hides raw provider keys behind the proxy. However, one community report describes hands-on 'issues with creating tokens for other users' (litellm-comm-3), and there is no independent corroboration of the key-isolation guarantees or a walkthrough of the actual key-generation API beyond first-party docs. Missing for 10: independent/hands-on confirmation that key scoping reliably prevents provider-key leakage, and resolution of the reported token-creation friction.", "evidenceIds": [ "litellm-docs-5", "litellm-docs-6", "litellm-docs-7", "litellm-docs-8", "litellm-docs-26", "litellm-comm-3" ] }, { "productId": "openrouter", "storyId": "agent-spend-guardrails", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "OpenRouter supports programmatic API key management (creation, per-key limits/expiry) and workspace-level budget caps (daily/weekly/monthly/lifetime) that auto-block requests once reached, directly enabling an agent to be given its own scoped key with hard spend guardrails; community confirms billing caps and per-key limits are a valued differentiator. missing for 10: explicit per-key rate-limit (requests/sec) controls beyond spend caps, and independent hands-on verification that runaway agent spend is actually blocked in practice.", "evidenceIds": [ "openrouter-docs-8", "openrouter-docs-10", "openrouter-comm-11", "openrouter-comm-12" ] }, { "productId": "openrouter", "storyId": "agent-switches-models-policy", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "OpenRouter's docs and community evidence confirm model fallback lists (priority order, automatic retry on error/rate-limit/down), provider-selection routing rules, load balancing across providers, a 'latest' alias for staying current, and an auto-routing 'meta' model that picks capability-appropriate models to control cost — directly matching policy-based mid-task model switching by cost/capability/availability. missing for 10: no first-party documentation or hands-on evidence of dynamically switching models *within* a single in-flight task/conversation (vs. per-request fallback/config), and no independent benchmark validating routing-rule reliability in production.", "evidenceIds": [ "openrouter-docs-3", "openrouter-docs-19", "openrouter-docs-4", "openrouter-docs-21", "openrouter-docs-2", "openrouter-comm-2", "openrouter-comm-13", "openrouter-docs-17" ] }, { "productId": "openrouter", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "OpenRouter serves a working llms.txt at openrouter.ai/llms.txt (HTTP 200, agent-oriented summary), plus an OpenAPI spec and MCP server that expose docs/data programmatically to agents. This directly satisfies pointing an agent at llms.txt or agent-oriented docs. Missing for 10: independent/community confirmation that agents successfully consume llms.txt in practice.", "evidenceIds": [ "openrouter-probe-1", "openrouter-probe-2", "openrouter-probe-3", "openrouter-docs-14" ] }, { "productId": "openrouter", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "OpenRouter is an API/routing gateway for accessing LLMs, not a data product that holds a user's own data and surfaces AI-generated insights from it — the axis is a category mismatch for this type of infrastructure tool.", "evidenceIds": [] }, { "productId": "openrouter", "storyId": "agentic-autonomous-automation", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "OpenRouter is a unified API/routing layer for LLM inference, not an automation/orchestration platform with scheduling or background-run capabilities; setting up autonomous background automations is outside its product category (wrong axis) rather than a missing feature.", "evidenceIds": [] }, { "productId": "openrouter", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "OpenRouter is a model-routing/API aggregation platform for developers to access LLMs, not a product with its own built-in AI assistant that end-users delegate tasks to; its MCP server lets *other* assistants pull OpenRouter data, which is the inverse of this story's axis. Task delegation to an in-product assistant is a category mismatch for an API routing service.", "evidenceIds": [] }, { "productId": "openrouter", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "OpenRouter is fundamentally a headless REST API (single endpoint, OpenAI-SDK compatible, streaming, programmatic API key management) that requires no UI and can be called from any script or pipeline, making it inherently suited to CI/automation use; the keyless models endpoint and management API keys reinforce programmatic, non-interactive use. Missing for 10: no explicit CI/CD pipeline examples (e.g., GitHub Actions), no dedicated 'headless/automation' documentation section, and no independent report of someone running it in a CI pipeline.", "evidenceIds": [ "openrouter-docs-8", "openrouter-docs-15", "openrouter-docs-18", "openrouter-docs-11", "openrouter-probe-4", "openrouter-probe-2" ] }, { "productId": "openrouter", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows OpenRouter publishing its own MCP server so external AI tools/editors can pull OpenRouter data (docs-14, docs-26, docs-34, probe-3) — this is OpenRouter acting as an MCP server for other clients, not OpenRouter itself consuming/plugging in external MCP servers to gain their tools. There is no evidence that OpenRouter's API or platform lets a user attach third-party MCP servers so OpenRouter-routed models can call their tools; the standardized tool-calling docs (docs-5) refer to generic function-calling, not MCP integration specifically.", "evidenceIds": [ "openrouter-docs-14", "openrouter-docs-26", "openrouter-docs-34", "openrouter-probe-3", "openrouter-docs-5" ] }, { "productId": "openrouter", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "OpenRouter publishes an official MCP server that lets AI assistants/agents connect and pull live OpenRouter data (models, prices, credits, rankings, docs) directly from within the editor, confirmed both in docs and via a live probe of the documentation page. Missing for 10: independent/hands-on community validation of the MCP server's real-world reliability beyond first-party docs.", "evidenceIds": [ "openrouter-docs-14", "openrouter-docs-26", "openrouter-docs-34", "openrouter-probe-3" ] }, { "productId": "openrouter", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "OpenRouter is an API/routing platform, not a natural-language interface itself, but it does provide an official MCP server that lets an AI assistant issue natural-language commands to pull model data, pricing, credits, and send test messages without leaving the editor — this is the main evidence of NL-command operability. Beyond MCP, there's no chat-style NL command surface for managing keys, budgets, or routing directly. missing for 10: a native conversational/NL interface for account and routing operations, independent hands-on validation of the MCP server's NL command handling, and evidence of NL commands controlling core features like BYOK, budgets, or provider selection.", "evidenceIds": [ "openrouter-docs-14", "openrouter-docs-26", "openrouter-docs-34", "openrouter-probe-3" ] }, { "productId": "openrouter", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "OpenRouter is an API/model-routing platform for which an official CLI would be a plausible ecosystem tool, but the evidence pack contains no mention of any official OpenRouter CLI — only SDK compatibility, REST API, MCP server, and web UI are documented.", "evidenceIds": [] }, { "productId": "openrouter", "storyId": "agentic-public-api", "verdict": "full", "quality": 10, "confidence": "high", "rationale": "OpenRouter is fundamentally an API product: extensive first-party docs cover the unified chat completions endpoint, streaming, tool calling, model routing/fallbacks, key management, and an OpenAPI spec plus a live, keyless /api/v1/models endpoint confirm a documented, programmatically accessible public API. Community evidence corroborates real-world programmatic use (switching models, key minting, billing caps) reinforcing that it is driven via API in production.", "evidenceIds": [ "openrouter-docs-1", "openrouter-docs-11", "openrouter-docs-5", "openrouter-docs-35", "openrouter-probe-1", "openrouter-probe-2", "openrouter-probe-4", "openrouter-comm-2", "openrouter-comm-12" ] }, { "productId": "openrouter", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "OpenRouter offers a documented Management API for creating/rotating API keys, workspace spend budgets, and per-key limits/expiry, which supports issuing distinct, capped credentials for agents (openrouter-docs-8, openrouter-docs-10, openrouter-comm-12, openrouter-comm-11). However, there's no explicit documentation of fine-grained scoping (e.g., restricting a key to specific models, tools, or actions) beyond spend caps, so 'least-privilege' is only partially realized. Missing for 10: explicit per-key model/tool/action scoping, granular permission controls beyond spend limits, and independent hands-on verification of the key-management API's robustness.", "evidenceIds": [ "openrouter-docs-8", "openrouter-docs-10", "openrouter-comm-11", "openrouter-comm-12" ] }, { "productId": "openrouter", "storyId": "agentic-sdks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "OpenRouter documents an OpenAI-compatible API and explicitly recommends using the OpenAI SDK as a drop-in replacement, plus offers an OpenAPI spec and request builder for generating client code, which supports building against it programmatically. However, there is no evidence of dedicated first-party OpenRouter SDK packages (e.g., official Python/Node/Go libraries) beyond OpenAI SDK compatibility. Missing for 10: dedicated official multi-language SDKs, independent developer corroboration of SDK usage (only OpenAI-SDK compatibility and generic API docs are cited).", "evidenceIds": [ "openrouter-docs-15", "openrouter-docs-18", "openrouter-probe-2", "openrouter-docs-29" ] }, { "productId": "openrouter", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions webhooks or event subscription mechanisms of any kind for OpenRouter; the documented features (routing, caching, key management, MCP server) do not include webhook support.", "evidenceIds": [] }, { "productId": "openrouter", "storyId": "api-interactive-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "OpenRouter's docs explicitly mention an interactive Request Builder for generating API requests in any language, plus a live OpenAPI spec is served and confirmed by probe, and the docs site is extensively organized with runnable-style examples across guides. Missing for 10: X, Y — no independent/hands-on account of actually running the interactive Request Builder or evidence of embedded runnable code sandboxes beyond the Request Builder claim.", "evidenceIds": [ "openrouter-docs-29", "openrouter-probe-2", "openrouter-docs-35" ] }, { "productId": "openrouter", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A probe confirms https://openrouter.ai/openapi.json returns HTTP 200 with an actual \"openapi\" key, giving a downloadable machine-readable OpenAPI spec, and docs also mention an interactive Request Builder and llms.txt for AI-native consumption. missing for 10: independent/community confirmation that the spec is actively used or kept in sync with the API by third-party developers.", "evidenceIds": [ "openrouter-probe-2", "openrouter-probe-1", "openrouter-docs-29" ] }, { "productId": "openrouter", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "OpenRouter's evidence describes production routing, billing, BYOK, and caching features but nothing about a sandbox/test environment isolated from production data or billing. The keyless models endpoint (openrouter-probe-4) only exposes a public catalog, not an isolated test mode for requests. No documentation of test/staging keys, sandbox mode, or non-production data isolation exists in the pack.", "evidenceIds": [] }, { "productId": "openrouter", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers routing, fallbacks, caching, BYOK, key management, and privacy settings, but nowhere documents API versioning (e.g., v1/v2 endpoints) or a formal deprecation policy/timeline for endpoints or models. The 'latest alias' for models is about model freshness, not API version stability guarantees.", "evidenceIds": [] }, { "productId": "openrouter", "storyId": "automatic-fallbacks", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "OpenRouter documents explicit model fallback via a priority-ordered `models` array that auto-retries the next model on error, plus provider-level fallback/load balancing across top providers, and community confirms this works ('They also support fallback by default so you don't have to write wrappers'). This directly matches the platform engineer's story of configuring automatic fallback across models/providers on failure. Missing for 10: independent quantitative reliability data or a documented edge-case failure of the fallback mechanism itself.", "evidenceIds": [ "openrouter-docs-19", "openrouter-docs-3", "openrouter-docs-4", "openrouter-docs-17", "openrouter-comm-2" ] }, { "productId": "openrouter", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows OpenRouter provides a single-request API, model routing/fallbacks, and programmatic key management, but there is no documentation of a batch/bulk inference endpoint or any mechanism to submit many items/prompts in one bulk operation. missing for 10: batch API or bulk job submission endpoint, evidence of processing many items in a single call, bulk data/export operations.", "evidenceIds": [] }, { "productId": "openrouter", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "OpenRouter is an LLM routing/API gateway; nothing in the evidence describes user-defined event-driven rules or triggers (e.g., webhooks, if-this-then-that automations) that fire actions automatically. Model fallback and provider routing are static request-time configs, not event-triggered automation rules.", "evidenceIds": [] }, { "productId": "openrouter", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "OpenRouter is an API routing/aggregation layer for LLM inference, not a workflow/job scheduler; nothing in its evidence pack relates to scheduling recurring jobs or workflows. This axis is a category error for this kind of product.", "evidenceIds": [] }, { "productId": "openrouter", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "OpenRouter is an API routing/aggregation layer for LLM inference, not an automation-builder product with workflows to version, review, or roll back; this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "openrouter", "storyId": "budgets-limits", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "OpenRouter's docs clearly show hard, auto-enforced 'workspace budgets' with configurable daily/weekly/monthly/lifetime dollar caps that block requests once reached (openrouter-docs-10), covering team-level budgets. Per-key limits are only supported by an informal community mention of 'expiry/limits per key' (openrouter-comm-12, openrouter-comm-11) rather than first-party documentation, and per-user budget controls are not evidenced at all. Missing for 10: official docs confirming per-key hard spend limits, explicit per-user budget/quota controls, and independent verification of enforcement.", "evidenceIds": [ "openrouter-docs-10", "openrouter-docs-8", "openrouter-comm-11", "openrouter-comm-12" ] }, { "productId": "openrouter", "storyId": "byok", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "OpenRouter's docs explicitly describe BYOK (bring your own provider keys), letting developers use their own provider API keys for direct control over rate limits and costs, and this is corroborated by community testimony citing consolidated billing/key management as a core benefit. Missing for 10: independent hands-on verification specifically of BYOK traffic routing (most community evidence discusses OpenRouter credits/key management generally rather than BYOK specifically).", "evidenceIds": [ "openrouter-docs-9", "openrouter-docs-24", "openrouter-docs-32", "openrouter-comm-12" ] }, { "productId": "openrouter", "storyId": "cost-tracking", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "OpenRouter provides key management (creation, limits per key), workspace budgets with dollar caps by interval, and BYOK cost control, plus community confirmation of billing caps and per-key limits — but the evidence pack lacks explicit documentation of a unified spend dashboard broken down by model, team, or user across providers in one place. missing for 10: consolidated analytics/dashboard showing spend segmented by model, by team, and by individual user; documentation of team-level spend attribution; independent confirmation of cross-provider spend reporting granularity.", "evidenceIds": [ "openrouter-docs-8", "openrouter-docs-10", "openrouter-docs-9", "openrouter-comm-11", "openrouter-comm-12" ] }, { "productId": "openrouter", "storyId": "edge-low-latency", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "OpenRouter's docs describe latency-reducing features (response caching bypasses billing and reduces latency, sticky routing to maximize cache hits, load-balanced provider selection, streaming support) that imply a gateway designed for low overhead, but there is no explicit benchmark or first-party claim quantifying the gateway's own latency overhead versus direct provider calls. Community feedback is mixed—some praise ease of switching models and reliability (comm-2), while others report unexplained inconsistency and debugging difficulty (comm-8)—but none of this directly measures gateway latency overhead. Missing for 10: quantified latency-overhead benchmarks (vendor or independent), explicit performance comparison vs. direct provider calls, and hands-on latency testing evidence.", "evidenceIds": [ "openrouter-docs-6", "openrouter-docs-7", "openrouter-docs-4", "openrouter-docs-11", "openrouter-comm-2", "openrouter-comm-8" ] }, { "productId": "openrouter", "storyId": "load-balancing-routing", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs show default load balancing across top providers, customizable provider routing (sort by price/throughput), and ordered model fallbacks, plus BYOK for provider-key control, corroborated by community mentions of built-in fallback behavior. Missing for 10: explicit documentation of weighted routing across specific API keys and independent benchmarks validating latency/cost-based routing performance in practice.", "evidenceIds": [ "openrouter-docs-4", "openrouter-docs-19", "openrouter-docs-21", "openrouter-docs-30", "openrouter-docs-9", "openrouter-comm-2" ] }, { "productId": "openrouter", "storyId": "model-catalog-discovery", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "OpenRouter documents a browsable model catalog at openrouter.ai/models and a programmatic GET /api/v1/models endpoint listing model slugs, and a live probe confirms this endpoint returns 426 models with pricing/context-window metadata unauthenticated. Community feedback corroborates ease of comparing/switching models via the unified catalog. Missing for 10: no independent third-party review specifically validating completeness/accuracy of pricing or context-window fields beyond the probe.", "evidenceIds": [ "openrouter-docs-35", "openrouter-probe-4", "openrouter-docs-16", "openrouter-comm-2" ] }, { "productId": "openrouter", "storyId": "openai-compatible-endpoint", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly state the OpenAI SDK can be pointed at OpenRouter as a drop-in replacement, and community users corroborate using 'the same api key and same request' across models, confirming base-URL/key swap works in practice. Missing for 10: no explicit first-party code snippet showing the exact base_url/api_key change in the evidence pack, only the general drop-in claim.", "evidenceIds": [ "openrouter-docs-15", "openrouter-docs-18", "openrouter-comm-2", "openrouter-probe-1" ] }, { "productId": "openrouter", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "OpenRouter's core capabilities—model routing, fallbacks, provider selection, key management, workspace budgets, privacy settings, and even the model catalog—are all documented as programmatically accessible via the API (openrouter-docs-8, 10, 12, 21, 35, openapi.json probe), suggesting strong API/UI parity for a product that is API-first. However, there's no explicit vendor claim of full UI-API parity, and dashboard-only features like rankings/analytics visualizations aren't confirmed as API-exposed. Missing for 10: explicit parity statement, evidence that UI-exclusive features (e.g., rankings, usage analytics dashboards) are also fully API-accessible, and independent confirmation from hands-on API users of complete feature parity.", "evidenceIds": [ "openrouter-docs-8", "openrouter-docs-10", "openrouter-docs-12", "openrouter-docs-21", "openrouter-docs-35", "openrouter-probe-2", "openrouter-probe-4", "openrouter-comm-12" ] }, { "productId": "openrouter", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers API routing, model access, MCP server, and billing features, but contains no mention of any user data export capability (usage logs, account data, chat history) in open formats or account portability/closure process — the axis is applicable to a SaaS platform storing usage/billing data, but no supporting evidence exists.", "evidenceIds": [] }, { "productId": "openrouter", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence that OpenRouter's own source code (the API service, routing platform, or website) is published under an open license; evidence pack only covers API docs, features, and OpenAPI spec, not source availability. missing for 10: any repository link, open-source license file, or statement that OpenRouter's codebase is publicly available.", "evidenceIds": [] }, { "productId": "openrouter", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "OpenRouter is a hosted API routing/aggregation service, not open-source software designed to be self-hosted; no evidence of a self-hostable core product exists. Self-hosting is a category mismatch for this kind of cloud aggregation service.", "evidenceIds": [] }, { "productId": "openrouter", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "OpenRouter documents EU/US in-region routing for enterprise customers and per-request/account-wide data-policy restrictions, giving users some control over where data is processed, but this is gated to enterprise accounts rather than available to all AI-native users, and community comments note there's no true ZDR/data-sovereignty guarantee in practice. missing for 10: general-tier (non-enterprise) region selection, independent verification that in-region routing works as claimed, and clarity on residency guarantees beyond enterprise contracts.", "evidenceIds": [ "openrouter-docs-13", "openrouter-docs-12", "openrouter-docs-25", "openrouter-comm-16" ] }, { "productId": "openrouter", "storyId": "privacy-no-training", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "OpenRouter lets users restrict requests/account-wide to providers with certain data policies and opt out of routing to providers that may train on data, plus EU/US in-region processing for enterprise, which directly supports preventing training-data use; however community evidence notes there is 'no ZDR' (zero data retention) and 'no true guarantees of ZDR/privacy/data sovereignty,' indicating limits to how absolute this protection is. missing for 10: independent verification that opt-out settings are consistently honored by all providers, clarity on default behavior, and resolution of the ZDR gap raised by users.", "evidenceIds": [ "openrouter-docs-12", "openrouter-docs-25", "openrouter-docs-33", "openrouter-docs-13", "openrouter-comm-4", "openrouter-comm-16" ] }, { "productId": "openrouter", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "OpenRouter lets users restrict routing to providers with certain data policies (e.g. no-training, zero-retention) both per-request and account-wide, and offers EU/US in-region processing for enterprise, which are real retention/data-policy controls. However there is no documented mechanism for deleting already-retained data or setting explicit retention periods, and community comments (e.g. 'No ZDR. No dice') indicate the zero-data-retention toggle has real-world limitations/gaps rather than full guaranteed retention control. Missing for 10: explicit data deletion/erasure endpoint, documented retention-period settings, and independent confirmation that ZDR/data-policy filters reliably cover all providers.", "evidenceIds": [ "openrouter-docs-12", "openrouter-docs-13", "openrouter-docs-25", "openrouter-docs-33", "openrouter-comm-4", "openrouter-comm-6", "openrouter-comm-16" ] }, { "productId": "openrouter", "storyId": "privacy-telemetry-optout", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "OpenRouter documents privacy controls—restricting requests to providers with certain data policies, disallowing providers that may train on your data, and EU/US in-region routing for enterprise—which give users some control over how their prompt data is handled downstream, but this is about provider-side data handling, not an explicit opt-out of OpenRouter's own telemetry/usage tracking. Community evidence directly notes 'No ZDR. No dice' and 'no true guarantees of ZDR/privacy/data sovereignty,' indicating real limits on data-handling assurances. Missing for 10: an explicit telemetry/usage-tracking opt-out toggle for OpenRouter's own platform analytics, and independent confirmation that opting out actually stops tracking.", "evidenceIds": [ "openrouter-docs-12", "openrouter-docs-25", "openrouter-docs-13", "openrouter-comm-4", "openrouter-comm-16" ] }, { "productId": "openrouter", "storyId": "provision-via-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "OpenRouter documents a management API for provisioning and rotating API keys programmatically (openrouter-docs-8) and community users confirm the ability to mint/limit keys (openrouter-comm-12, openrouter-comm-11). Workspace budgets are documented as a feature to cap spend (openrouter-docs-10), but the evidence doesn't explicitly confirm budgets are settable via the same admin API, and there's no mention of provisioning 'gateways' as a distinct programmable resource. Missing for 10: explicit API endpoint documentation for setting/adjusting workspace budgets programmatically, and any concept of provisioning separate 'gateways' via API.", "evidenceIds": [ "openrouter-docs-8", "openrouter-docs-10", "openrouter-comm-11", "openrouter-comm-12" ] }, { "productId": "openrouter", "storyId": "rate-limit-smoothing", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "OpenRouter's default load balancing across providers and automatic model/provider fallbacks (docs-3,4,19) inherently smooths provider-side rate limits, and its key-management API (docs-8) plus BYOK (docs-9,24,32) let engineers create/distribute multiple keys with per-key limits (echoed by comm-12). But there is no explicit documentation of client-side request queuing or throttling, or of deliberately spreading traffic across multiple OpenRouter keys for load smoothing — the evidence covers provider-side resilience, not key-pool traffic shaping. Missing for 10: explicit queuing/throttling mechanism, documented multi-key load-balancing pattern, independent confirmation of rate-limit smoothing at scale.", "evidenceIds": [ "openrouter-docs-3", "openrouter-docs-4", "openrouter-docs-8", "openrouter-docs-9", "openrouter-docs-19", "openrouter-comm-12" ] }, { "productId": "openrouter", "storyId": "request-logging", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers routing, caching, BYOK, budgets, and key management, but contains no mention of a request/response log viewer, latency dashboards, or per-request token/cost breakdowns that a platform engineer could inspect. missing for 10: activity/logs dashboard documentation, per-request latency metrics, per-request token count and cost attribution evidence.", "evidenceIds": [] }, { "productId": "openrouter", "storyId": "response-caching", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "OpenRouter's docs explicitly describe gateway-level response caching that returns cached results with zero billable usage (cutting cost) and near-instant latency, plus provider sticky routing specifically designed to maximize cache hit rates on subsequent requests. This directly matches the story's ask for gateway-side caching to reduce cost/latency on repeated calls. Missing for 10: independent or hands-on community confirmation that caching actually reduces cost/latency in practice, and more detail on cache TTL/configurability beyond the sticky-routing mechanism.", "evidenceIds": [ "openrouter-docs-6", "openrouter-docs-22", "openrouter-docs-7", "openrouter-docs-23", "openrouter-docs-31" ] }, { "productId": "openrouter", "storyId": "retry-policies", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "OpenRouter's model-fallbacks and provider load-balancing features automatically retry a request against alternate providers/models when the primary is down, rate-limited, or errors out, and this is corroborated by community reports of built-in fallback behavior (openrouter-docs-3, openrouter-docs-4, openrouter-docs-17, openrouter-docs-19, openrouter-comm-2). However, this is provider/model-level fallback rather than a granular, configurable retry policy (e.g., retry count, backoff intervals, specific transient-error codes) that a platform engineer might expect. Missing for 10: explicit retry-count/backoff configuration options, granular control over which transient error codes trigger retries, and independent verification of retry behavior under real outages.", "evidenceIds": [ "openrouter-docs-3", "openrouter-docs-4", "openrouter-docs-17", "openrouter-docs-19", "openrouter-comm-2" ] }, { "productId": "openrouter", "storyId": "streaming-passthrough", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "OpenRouter's docs explicitly state streaming is supported for any model through the gateway API, and community feedback corroborates using a single API/key across many providers with model switching in production. missing for 10: independent hands-on confirmation specifically of token-by-token streaming behavior (e.g., latency/chunking quality) and any provider-specific streaming caveats or limitations.", "evidenceIds": [ "openrouter-docs-11", "openrouter-docs-20", "openrouter-docs-1", "openrouter-comm-2" ] }, { "productId": "openrouter", "storyId": "structured-outputs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers tool-calling standardization, streaming, routing, fallbacks, and key management, but contains no mention of structured JSON-schema/response_format output support across providers. missing for 10: any documentation or example of JSON-schema response_format, schema validation, or structured output enforcement across providers.", "evidenceIds": [] }, { "productId": "openrouter", "storyId": "tool-call-fidelity", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "OpenRouter explicitly documents a standardized tool-calling interface across models/providers (docs-5), consistent with its single-API model access across 500+ models and providers (docs-1, docs-16), and community confirms using the same API key/request format to switch models seamlessly (openrouter-comm-2). Streaming is also documented as working across any model (docs-11/20), supporting the streaming-tools theme. Missing for 10: no independent hands-on verification specifically of function/tool-call schema consistency across multiple providers, and no detailed example showing identical tool-call JSON working unchanged across e.g. OpenAI vs Anthropic vs Google models.", "evidenceIds": [ "openrouter-docs-5", "openrouter-docs-1", "openrouter-docs-11", "openrouter-docs-20", "openrouter-comm-2" ] }, { "productId": "openrouter", "storyId": "tracing-integrations", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence pack shows OpenRouter's API gateway features (routing, fallbacks, caching, key management, privacy settings) but contains no mention of exporting logs/traces, webhooks, log streaming, or integration with observability stacks like Datadog, OpenTelemetry, or similar. missing for 10: log/trace export mechanism, OpenTelemetry or webhook integration, documentation of any observability-stack connector.", "evidenceIds": [] }, { "productId": "openrouter", "storyId": "unified-api-many-providers", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "OpenRouter's core value proposition is a single API endpoint (OpenAI-SDK compatible) that routes to hundreds of models across 80+ providers, with fallback, load-balancing, standardized tool calling, and streaming for any model — corroborated by live probes (426-model catalog, openapi.json) and community testimonials confirming 'same API key and same request' across providers with automatic fallback. Minor gaps: community notes some inconsistencies (missing sampling params, occasional reliability complaints), missing for 10: fully independent benchmark of API consistency across all providers and resolution of reported debugging/latency complaints.", "evidenceIds": [ "openrouter-docs-1", "openrouter-docs-15", "openrouter-docs-19", "openrouter-docs-5", "openrouter-docs-20", "openrouter-probe-1", "openrouter-probe-4", "openrouter-comm-2", "openrouter-comm-10" ] }, { "productId": "openrouter", "storyId": "virtual-keys", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "OpenRouter's docs explicitly describe programmatic key management endpoints for creating/managing/rotating API keys for apps without exposing provider keys (docs-8), backed by BYOK for provider key control (docs-9/24/32) and workspace budgets to cap spend per workspace/team (docs-10). Community corroborates ease of minting and limiting keys (comm-12, comm-11). Missing for 10: independent verification of team/app-scoped key isolation guarantees and no first-hand platform-engineer account of using the management API at scale.", "evidenceIds": [ "openrouter-docs-8", "openrouter-docs-9", "openrouter-docs-10", "openrouter-comm-12", "openrouter-comm-11" ] }, { "productId": "portkey", "storyId": "agent-spend-guardrails", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Portkey docs explicitly support issuing API keys with budget and rate limits (spend thresholds, hourly/daily/per-minute request/token limits) that automatically block further usage when exceeded, directly matching the guardrail need for an autonomous agent's key. Fine-grained control at org/workspace level and per-key configuration is documented. Missing for 10: no explicit mention of a per-agent key issuance workflow or independent/hands-on verification that the cutoff actually stops an agent mid-run.", "evidenceIds": [ "portkey-docs-13", "portkey-docs-25", "portkey-docs-38", "portkey-docs-33", "portkey-docs-35" ] }, { "productId": "portkey", "storyId": "agent-switches-models-policy", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Portkey's gateway supports policy-based model switching via conditional fallbacks (status-code triggers), load balancing/weighted routing, gradual migration and canary rollout, plus budget/rate-limit enforcement that can trigger automatic switching for cost/availability reasons - all configurable via routing configs applied at request time. This directly matches mid-task, policy-driven model switching for cost, capability, or availability. Missing for 10: independent/hands-on corroboration of dynamic mid-task capability-based switching (vs. failure-triggered fallback), and explicit example of capability-based routing rules (e.g., routing by task type).", "evidenceIds": [ "portkey-docs-3", "portkey-docs-21", "portkey-docs-7", "portkey-docs-22", "portkey-docs-27", "portkey-docs-13", "portkey-docs-25", "portkey-docs-38", "portkey-docs-18" ] }, { "productId": "portkey", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Portkey hosts a working llms.txt file (HTTP 200) with a description of the product, confirming agents can be pointed at it for agent-oriented docs. Missing for 10: no docs.md fallback (404) or OpenAPI spec confirmation, and no independent/community corroboration of agents actually using it.", "evidenceIds": [ "portkey-probe-1", "portkey-probe-2" ] }, { "productId": "portkey", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Portkey's evidence pack covers gateway routing, fallbacks, caching, logging, and budget controls, but there is no mention of AI-generated insights, summaries, or suggestions derived from a user's data within the product itself — logs support manual feedback tagging (portkey-docs-34), not AI-driven analysis. missing for 10: any AI-generated insight/summary feature, anomaly detection or recommendation engine over logs/usage data, in-product AI assistant surfacing suggestions.", "evidenceIds": [ "portkey-docs-11", "portkey-docs-34", "portkey-docs-13" ] }, { "productId": "portkey", "storyId": "agentic-autonomous-automation", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Portkey is an AI gateway/observability layer for routing, caching, and governing LLM requests, not a workflow/automation platform for running autonomous background tasks; the evidence covers gateway features (fallbacks, retries, caching, load balancing, logs, budgets) with no scheduler or autonomous-agent execution capability. This story targets a different product category and does not fit Portkey's axis.", "evidenceIds": [] }, { "productId": "portkey", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Portkey is an AI gateway/observability platform for routing and managing LLM traffic, not a product with a built-in chat/assistant UI that a user delegates tasks to; this axis is a category error for its product type.", "evidenceIds": [] }, { "productId": "portkey", "storyId": "agentic-headless", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Portkey's gateway can be run headlessly via a CLI command (npx @portkey-ai/gateway) and REST API with a base URL, and it is open source and self-hostable, which supports CI/automation usage. However, there is no explicit documentation of CI pipeline integration, GitHub Actions examples, or headless automation testing workflows. missing for 10: explicit CI/CD examples, headless automation guides, non-interactive scripted usage documentation, independent verification of CI usage.", "evidenceIds": [ "portkey-docs-2", "portkey-docs-37", "portkey-docs-15", "portkey-docs-16" ] }, { "productId": "portkey", "storyId": "agentic-mcp-client", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Portkey's docs explicitly state it can 'Connect to Remote MCP servers, allowing you to connect external tools and data sources' via the AI Gateway, directly supporting the story. However, evidence lacks detail on setup steps, supported MCP transport, or independent/hands-on confirmation of this working in practice. Missing for 10: detailed configuration docs for adding MCP servers, independent verification of MCP tool invocation, and examples of tool-calling via MCP in a live agent workflow.", "evidenceIds": [ "portkey-docs-17" ] }, { "productId": "portkey", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Portkey's docs describe connecting *to* remote MCP servers as a client-side feature (portkey-docs-17), but there is no evidence that Portkey itself exposes an official MCP server endpoint that external agents can connect to. Missing for 10: any documentation of Portkey hosting/serving an MCP endpoint, MCP server setup instructions, or third-party confirmation of agents connecting via Portkey's MCP server.", "evidenceIds": [ "portkey-docs-17" ] }, { "productId": "portkey", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Portkey's evidence describes an API/SDK-based AI gateway, dashboard configs, logs, and MCP server connectivity for tools/data sources, but nothing shows an interface where a user issues natural-language commands to operate Portkey itself (e.g., a chat-based admin/control plane). The axis is plausible for a platform like this, but no evidence supports it.", "evidenceIds": [] }, { "productId": "portkey", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The only CLI-like evidence is running the open-source gateway locally via `npx @portkey-ai/gateway`, which self-hosts the gateway rather than offering a full-featured CLI for managing configs, logs, or other Portkey resources. Missing for 10: a documented dedicated CLI tool/command set, CLI-based workflow examples, and independent corroboration of CLI usage.", "evidenceIds": [ "portkey-docs-2", "portkey-docs-37" ] }, { "productId": "portkey", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Portkey exposes a documented REST API (base URL https://api.portkey.ai/v1) with API-reference docs for its inference API, OpenAI-compatible integration, and llms.txt discovery confirms API-first positioning; gateway is also open-sourced and npx-runnable. Missing for 10: a discoverable OpenAPI/swagger spec (probe found all candidate paths 404) and independent/third-party corroboration of API usage.", "evidenceIds": [ "portkey-docs-15", "portkey-docs-16", "portkey-docs-26", "portkey-docs-36", "portkey-probe-1", "portkey-probe-3", "portkey-docs-37" ] }, { "productId": "portkey", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Portkey lets admins issue API keys scoped with budgets, rate limits, and model allow-lists at org/workspace level, and virtual keys keep provider credentials out of code — this maps well to least-privilege credential issuance. However, the docs frame this around human/org governance rather than explicitly per-agent scoped credentials, and there's no independent corroboration of enforcement in practice. Missing for 10: explicit agent-specific credential scoping workflow, independent/hands-on validation of enforcement.", "evidenceIds": [ "portkey-docs-13", "portkey-docs-25", "portkey-docs-33", "portkey-docs-38", "portkey-docs-32" ] }, { "productId": "portkey", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Portkey documents SDK-style integration (base URL swap for OpenAI Python/Node SDKs, REST API reference, open-source gateway installable via npx) supporting AI-native builders, but the evidence pack lacks explicit mention of dedicated first-party SDKs beyond OpenAI-compatible drop-in usage, and OpenAPI spec probes returned 404s. missing for 10: dedicated official multi-language SDK docs (e.g., Python/JS Portkey SDK reference pages), working OpenAPI/spec discovery, independent developer corroboration of SDK usage.", "evidenceIds": [ "portkey-docs-15", "portkey-docs-16", "portkey-docs-26", "portkey-docs-36", "portkey-docs-37", "portkey-probe-3" ] }, { "productId": "portkey", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions webhooks or event subscriptions of any kind; Portkey's docs cover gateway routing, caching, retries, budgets, and logs but nothing about outbound webhook notifications for events.", "evidenceIds": [] }, { "productId": "portkey", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Portkey documents a REST API reference (base URL, headers, SDK compatibility) at /docs/api-reference, but the evidence pack contains no mention of an interactive explorer or runnable code examples, and probes for a discoverable OpenAPI/Swagger spec all returned 404s. Missing for 10: evidence of an interactive 'try it' console, runnable code snippets, and a published OpenAPI/Swagger schema.", "evidenceIds": [ "portkey-docs-15", "portkey-docs-16", "portkey-docs-26", "portkey-docs-36", "portkey-probe-3" ] }, { "productId": "portkey", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Portkey has an API reference (base URL, inference API docs) but no evidence of a downloadable OpenAPI/Swagger spec; a direct probe for common OpenAPI paths returned 404s, and docs.md also 404s.", "evidenceIds": [ "portkey-probe-3", "portkey-probe-2", "portkey-docs-16" ] }, { "productId": "portkey", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Portkey's evidence covers gateway routing, fallbacks, caching, logging, and gradual/canary rollout of models in production, but there is no mention of a distinct sandbox/staging environment or test mode that isolates a user from production data. The axis is fair for a gateway/dev-tool product, but no evidence supports it.", "evidenceIds": [] }, { "productId": "portkey", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of API versioning scheme (e.g., v1/v2 in URLs beyond a single 'v1' base path) or any documented deprecation policy/changelog; OpenAPI spec probe also returned 404s. Missing for 10: versioning scheme documentation, explicit deprecation policy, changelog/migration guides, and any independent corroboration.", "evidenceIds": [ "portkey-docs-16", "portkey-probe-3" ] }, { "productId": "portkey", "storyId": "automatic-fallbacks", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Portkey's docs explicitly describe configuring a prioritized list of providers/models with automatic fallback when the primary fails, customizable trigger status codes, plus complementary retries and load balancing for resilience. missing for 10: independent/hands-on corroboration beyond vendor docs.", "evidenceIds": [ "portkey-docs-3", "portkey-docs-21", "portkey-docs-5", "portkey-docs-18" ] }, { "productId": "portkey", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Portkey's evidence covers gateway routing, fallbacks, caching, logging, and budget/rate-limit controls, but nothing describes a bulk/batch operation capability (e.g., batch inference calls, bulk editing of configs/keys, or bulk log actions) that lets a user act on many items at once. Missing for 10: any documentation of a batch API, bulk config/key management, or bulk log/annotation actions.", "evidenceIds": [] }, { "productId": "portkey", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Portkey ships several built-in rule/event mechanisms — fallback triggers on non-2xx status codes, automatic retries with backoff, circuit-breaker/failure handling per strategy, and budget/rate-limit thresholds that automatically send notifications or block further usage — which are automatic actions triggered by defined conditions. However, these are fixed, product-defined automations (reliability/spend rules) rather than a general user-authorable rule engine where an AI-native user can define arbitrary custom triggers/actions. Missing for 10: general-purpose custom rule/trigger builder, arbitrary event types beyond status codes/spend thresholds, user-defined action targets (e.g. webhooks, custom scripts) beyond built-in notifications/fallback/retry.", "evidenceIds": [ "portkey-docs-3", "portkey-docs-21", "portkey-docs-5", "portkey-docs-18", "portkey-docs-14", "portkey-docs-25", "portkey-docs-38" ] }, { "productId": "portkey", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Portkey is an AI gateway/observability platform for routing, caching, and monitoring LLM requests, not a workflow/job scheduling or orchestration product; nothing in the evidence pack relates to scheduling recurring jobs or workflows, making this a category mismatch rather than a missing feature.", "evidenceIds": [] }, { "productId": "portkey", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Portkey's evidence covers gateway configs, logs, retries, caching, and budgets, but nothing describes versioning, review workflows, or rollback of automations/configs — the closest hint is 'Config ID' filtering in logs, which is not version control. Missing for 10: any documentation of config/version history, diff/review process, or rollback mechanism for automations.", "evidenceIds": [ "portkey-docs-4", "portkey-docs-11", "portkey-docs-12" ] }, { "productId": "portkey", "storyId": "budgets-limits", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Portkey's docs explicitly describe setting budget limits (max spend or token usage) and rate limits on API keys, with org/workspace-level fine-grained control, automatic enforcement when limits are reached, notification thresholds, and weekly resets — directly matching per-key/team/user budget/spend controls. Missing for 10: no independent/hands-on corroboration beyond first-party docs.", "evidenceIds": [ "portkey-docs-13", "portkey-docs-14", "portkey-docs-25", "portkey-docs-33", "portkey-docs-35", "portkey-docs-38" ] }, { "productId": "portkey", "storyId": "byok", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Portkey's virtual keys and model catalog explicitly let developers add/store their own provider API keys ('Add a provider, then use @provider-slug/model-name', 'Provider credentials stored securely, never exposed in code'), and the gateway routes traffic through these keys with governance/rate-limit controls layered on top. Missing for 10: independent hands-on verification of the BYO-key flow and no explicit mention of per-key traffic isolation/audit beyond docs claims.", "evidenceIds": [ "portkey-docs-9", "portkey-docs-24", "portkey-docs-32", "portkey-docs-10", "portkey-docs-33" ] }, { "productId": "portkey", "storyId": "cost-tracking", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Portkey's docs describe budget/rate limits and spend caps on API keys, a unified model catalog with org/workspace-level governance and allow-lists, and logs filterable by Config ID for auditing usage, which together give spend visibility across providers/keys/teams. Missing for 10: explicit per-user spend breakdown dashboards and independent/hands-on confirmation of a unified cost dashboard beyond docs claims.", "evidenceIds": [ "portkey-docs-13", "portkey-docs-25", "portkey-docs-33", "portkey-docs-10", "portkey-docs-4", "portkey-docs-9", "portkey-docs-38" ] }, { "productId": "portkey", "storyId": "edge-low-latency", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Portkey's docs claim caching can serve responses '20x faster' and market the gateway as 'performant,' but there is no direct benchmark or first-party claim quantifying the gateway's own added latency overhead per provider call, nor independent corroboration of low overhead. Missing for 10: explicit latency/overhead benchmarks for the proxy itself, third-party performance testing, and architecture details (e.g., edge/WASM deployment) that would substantiate minimal added latency beyond the cache-hit speedup claim.", "evidenceIds": [ "portkey-docs-8", "portkey-docs-19", "portkey-docs-29", "portkey-docs-37" ] }, { "productId": "portkey", "storyId": "load-balancing-routing", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Portkey's docs explicitly document load balancing to distribute traffic across multiple LLMs/providers by weight, plus gradual migration (percentage-based rollout) and fallback on failure/latency-driven retries, all configurable via Configs. This directly matches weight-based and failure/latency-based routing across providers/deployments/keys, backed by first-party docs on load-balancing, fallbacks, and virtual keys. Missing for 10: explicit cost-based routing strategy documentation and independent/hands-on verification beyond vendor docs.", "evidenceIds": [ "portkey-docs-7", "portkey-docs-22", "portkey-docs-31", "portkey-docs-3", "portkey-docs-21", "portkey-docs-9", "portkey-docs-18" ] }, { "productId": "portkey", "storyId": "model-catalog-discovery", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Portkey documents a 'Model Catalog' providing a single pane to view/manage all providers and models with discovery and governance controls, and claims access to 250+ models, but no evidence shows the catalog surfaces pricing or context-window metadata for models. missing for 10: explicit documentation of per-model pricing info, explicit documentation of context-window size metadata, and any UI/query example showing these fields.", "evidenceIds": [ "portkey-docs-10", "portkey-docs-20", "portkey-docs-24", "portkey-docs-33" ] }, { "productId": "portkey", "storyId": "openai-compatible-endpoint", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly state that developers using OpenAI's Python/Node SDKs can integrate Portkey by only changing the base URL and adding Portkey-specific headers, with the base URL documented as https://api.portkey.ai/v1. This is a first-party documented, purpose-built migration path. Missing for 10: independent/hands-on third-party confirmation beyond vendor docs.", "evidenceIds": [ "portkey-docs-15", "portkey-docs-16", "portkey-docs-26", "portkey-docs-36" ] }, { "productId": "portkey", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Portkey's core AI gateway (chat/completions, fallbacks, caching, retries, load balancing) is clearly exposed via a REST API with a stable base URL, and the open-source gateway can be run locally — showing strong API-first design for request routing. However there is no evidence of API access to the administrative/UI-only features such as budget & rate-limit configuration, log viewing/sharing, feedback tagging, or virtual-key management, and probes show no public OpenAPI spec, so full UI-API parity isn't demonstrated. missing for 10: API endpoints for budget/rate-limit admin, log retrieval/sharing via API, feedback API, and a public OpenAPI spec confirming complete parity.", "evidenceIds": [ "portkey-docs-15", "portkey-docs-16", "portkey-docs-26", "portkey-docs-37", "portkey-docs-13", "portkey-docs-11", "portkey-probe-3" ] }, { "productId": "portkey", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Portkey's docs cover logging, gateway configuration, and self-hosting the open-source gateway, but there is no mention of exporting stored logs/configs/data in open formats or facilitating a full data export for migration away from the platform. Shareable log URLs (portkey-docs-12) are not a bulk export/portability feature.", "evidenceIds": [ "portkey-docs-11", "portkey-docs-12", "portkey-docs-37" ] }, { "productId": "portkey", "storyId": "openness-open-license", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Portkey explicitly states it open-sourced its AI gateway component ('We've open sourced our battle-tested AI gateway to the community... run it locally with a single command'), which supports reading some source code. However, this only covers the gateway, not the full product (observability, admin console, model catalog, etc.), and no explicit license name or repository link is provided in the evidence. Missing for 10: explicit open-source license identification, confirmation that the broader Portkey platform (not just the gateway) is open source, and a linked public repository.", "evidenceIds": [ "portkey-docs-37", "portkey-docs-2" ] }, { "productId": "portkey", "storyId": "openness-self-host", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm the AI Gateway — described as the 'battle-tested' core routing component — is open-sourced and can be run locally via a single npx command, which is genuine self-hosting evidence. However, other core parts of the product (observability/logs UI, admin budget/rate-limit controls, model catalog governance) are described only as hosted SaaS features with no mention of self-hosting. Missing for 10: evidence that the full platform (observability, admin, model catalog) can be self-hosted, not just the gateway component, and any independent confirmation of self-hosting in practice.", "evidenceIds": [ "portkey-docs-2", "portkey-docs-37", "portkey-docs-10", "portkey-docs-13" ] }, { "productId": "portkey", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack items mention data residency, regional storage options, or geographic hosting choices; Portkey's docs focus on gateway routing, caching, retries, budgets, and observability rather than data residency controls.", "evidenceIds": [] }, { "productId": "portkey", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence pack covers gateway routing, caching, retries, logging, budgeting, etc., but contains no mention of a data-training opt-out, zero-retention policy, or any privacy control preventing use of data for AI model training.", "evidenceIds": [] }, { "productId": "portkey", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers gateway routing, caching, retries, budgets, and logging, but contains no mention of data retention policies, log/data deletion controls, or configurable retention windows.", "evidenceIds": [] }, { "productId": "portkey", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Portkey is essentially a telemetry/observability platform (logs, monitoring of all LLM requests) with no evidence of any opt-out mechanism for its own tracking or usage telemetry; the evidence pack only documents features for logging, caching, retries, and governance, never a privacy/telemetry opt-out control.", "evidenceIds": [] }, { "productId": "portkey", "storyId": "provision-via-api", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence shows budget/rate-limit and key management as dashboard-configurable features (docs-13, docs-25, docs-35, docs-38) but never describes a programmatic Admin API for provisioning gateways, keys, or budgets; the only API reference documented is the inference API (docs-15, docs-16, docs-36), and probes for an OpenAPI/admin API spec returned 404 (portkey-probe-3, portkey-probe-2).", "evidenceIds": [ "portkey-docs-13", "portkey-docs-25", "portkey-docs-35", "portkey-docs-16", "portkey-probe-3", "portkey-probe-2" ] }, { "productId": "portkey", "storyId": "rate-limit-smoothing", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Portkey documents load balancing across multiple provider keys/models to avoid bottlenecks, automatic retries with exponential backoff or provider retry-after headers, and configurable per-key/org rate limits (hourly/daily/per-minute) alongside budget limits — directly addressing spreading traffic and throttling/queuing around rate limits. Multiple virtual keys per provider further support distributing load to smooth rate limits. Missing for 10: explicit first-party or independent evidence of a request queue (vs. retry/backoff) and hands-on/independent validation of these features working under real rate-limit pressure.", "evidenceIds": [ "portkey-docs-7", "portkey-docs-5", "portkey-docs-6", "portkey-docs-30", "portkey-docs-38", "portkey-docs-13", "portkey-docs-9", "portkey-docs-33" ] }, { "productId": "portkey", "storyId": "request-logging", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Portkey's docs confirm a Logs section showing a chronological list of all requests with unique shareable URLs and manual feedback tagging, supporting log inspection generally, and repeated claims of 'monitoring all LLM requests' and 'visibility' across the gateway. However, the evidence pack never explicitly documents latency, token count, or cost fields being displayed per log entry. Missing for 10: explicit documentation or screenshots showing latency/tokens/cost columns in the logs UI, independent/hands-on confirmation of these fields.", "evidenceIds": [ "portkey-docs-11", "portkey-docs-12", "portkey-docs-34", "portkey-docs-1", "portkey-docs-19" ] }, { "productId": "portkey", "storyId": "response-caching", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Portkey's docs explicitly describe gateway-level caching (simple and semantic) that serves responses up to 20x faster and cheaper, including semantic matching via cosine similarity for near-duplicate queries. This directly matches the story's cost/latency reduction on repeated requests. missing for 10: independent/hands-on benchmark corroboration beyond vendor docs.", "evidenceIds": [ "portkey-docs-8", "portkey-docs-23" ] }, { "productId": "portkey", "storyId": "retry-policies", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Portkey's docs explicitly describe automatic retries with exponential backoff and support for provider retry-after headers, directly matching the story of configuring retry policies for transient errors. missing for 10: independent/hands-on corroboration beyond vendor docs, and configuration details on per-error-code or max-retry-count customization.", "evidenceIds": [ "portkey-docs-5", "portkey-docs-6", "portkey-docs-30" ] }, { "productId": "portkey", "storyId": "streaming-passthrough", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes Portkey's gateway, fallbacks, retries, load balancing, caching, and OpenAI-compatible base URL integration, but none of the provided docs mention streaming or token-by-token response support. Streaming is a fair and expected capability for an LLM gateway product, so its absence in the evidence pack means this axis applies but is unproven.", "evidenceIds": [ "portkey-docs-15", "portkey-docs-16", "portkey-docs-26", "portkey-docs-36" ] }, { "productId": "portkey", "storyId": "structured-outputs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers gateway features like fallbacks, retries, load balancing, caching, virtual keys, and logging, but contains no mention of structured JSON-schema output support, response_format normalization, or schema validation across providers. Since Portkey acts as a universal AI gateway, this capability would be plausible, but no documentation cites it.", "evidenceIds": [] }, { "productId": "portkey", "storyId": "tool-call-fidelity", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Portkey's gateway offers a unified OpenAI-SDK-compatible API (base URL + headers) across 250+ models, which implies a consistent request format including function/tool call fields, and it supports connecting external tools via remote MCP servers. However, there is no explicit documentation describing how tool/function-calling schemas are normalized or translated across different providers' native formats. Missing for 10: explicit docs on function/tool-call schema normalization, examples of tool-calling requests across multiple providers, and independent verification that tool calls work consistently across providers.", "evidenceIds": [ "portkey-docs-20", "portkey-docs-26", "portkey-docs-15", "portkey-docs-17" ] }, { "productId": "portkey", "storyId": "tracing-integrations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence describes Portkey's own logs UI, sharing log URLs, and manual feedback tagging, but there is no mention of exporting logs/traces to third-party observability stacks (e.g., OpenTelemetry, Datadog, Grafana) or any API/webhook for bulk export.", "evidenceIds": [] }, { "productId": "portkey", "storyId": "unified-api-many-providers", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Portkey documents a unified API/gateway (single base URL, single API key) that supports 250+ providers/models via consistent SDK/REST calls, with virtual keys and model catalog abstracting provider differences. Missing for 10: independent third-party benchmark or hands-on confirmation of the claimed 250+ provider coverage beyond vendor docs.", "evidenceIds": [ "portkey-docs-9", "portkey-docs-15", "portkey-docs-16", "portkey-docs-20", "portkey-docs-24", "portkey-docs-26" ] }, { "productId": "portkey", "storyId": "virtual-keys", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Portkey's virtual keys/model catalog explicitly let platform engineers mint provider-agnostic API keys that map to real provider credentials, which are stored securely and never exposed in code, while budgets, rate limits, and model allow-lists can be enforced per key at org/workspace level for teams and apps. This directly matches minting gateway-managed keys without exposing raw provider keys. Missing for 10: independent/hands-on verification beyond vendor docs, and detail on per-team/app key issuance workflow (e.g., self-service or RBAC-scoped key creation) rather than admin-level config.", "evidenceIds": [ "portkey-docs-9", "portkey-docs-32", "portkey-docs-13", "portkey-docs-25", "portkey-docs-33", "portkey-docs-38", "portkey-docs-24" ] }, { "productId": "requesty", "storyId": "agent-spend-guardrails", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Requesty is an LLM gateway product where issuing scoped API keys with spend/rate limits would be a natural and expected capability, but the evidence pack only shows BYOK (using your own provider keys), content-safety 'guardrails' (sensitive-data scanning, not spend limits), and usage analytics dashboards for visibility — none of which describe issuing an agent its own key with a budget cap or rate limit to prevent runaway spend.", "evidenceIds": [ "requesty-docs-8", "requesty-docs-10", "requesty-docs-11" ] }, { "productId": "requesty", "storyId": "agent-switches-models-policy", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Requesty's gateway supports fallback policies (retry with different models on failure/rate limit/error), load-balancing policies (distribute across models by weight, sticky routing by trace/user), and access to 300+/600+ models via a single API, directly enabling policy-based mid-task model switching for cost, capability, or availability reasons. missing for 10: no independent/hands-on corroboration of dynamic policy switching in a live agent task, and no explicit example combining cost+capability+availability rules in one policy config.", "evidenceIds": [ "requesty-docs-3", "requesty-docs-4", "requesty-docs-5", "requesty-docs-6", "requesty-docs-13", "requesty-docs-16" ] }, { "productId": "requesty", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Requesty's docs site serves a working llms.txt (HTTP 200) with a structured index, plus .md variants of docs pages that explicitly point agents to llms.txt for the full documentation index, directly enabling an agent to be pointed at agent-oriented docs. missing for 10: no independent/third-party confirmation of agents actually consuming this successfully, and no dedicated docs page describing the llms.txt feature itself.", "evidenceIds": [ "requesty-probe-1", "requesty-probe-2" ] }, { "productId": "requesty", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Requesty's docs describe a real-time analytics dashboard for usage/cost/latency tracking, but nowhere is there evidence of AI-generated insights, recommendations, or suggestions derived from that data — the dashboard is purely observational reporting.", "evidenceIds": [ "requesty-docs-11" ] }, { "productId": "requesty", "storyId": "agentic-autonomous-automation", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Requesty is an LLM gateway/router product for API traffic management (routing, fallback, load balancing, caching), not an automation or agent-orchestration platform; there is no concept of user-defined background-running autonomous automations in this product category.", "evidenceIds": [] }, { "productId": "requesty", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Requesty is an LLM routing/gateway product (unified API, fallback, load balancing, caching, analytics) rather than an end-user product with a built-in AI assistant persona to delegate tasks to; this axis is a category error for its type.", "evidenceIds": [] }, { "productId": "requesty", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Requesty is an OpenAI-compatible API gateway accessed via simple base_url swap, meaning any script or CI pipeline using the OpenAI SDK can call it headlessly without a UI, and BYOK/API-key auth supports non-interactive use. However, there is no explicit documentation or example showing CI/automation setup (e.g., GitHub Actions, headless auth flow, or CLI for pipelines). Missing for 10: explicit CI/automation docs or examples, headless auth/token provisioning guidance, independent confirmation of CI usage.", "evidenceIds": [ "requesty-docs-1", "requesty-docs-2", "requesty-docs-8" ] }, { "productId": "requesty", "storyId": "agentic-mcp-client", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Requesty's MCP Gateway explicitly lets AI coding assistants (Claude Code, Cursor, Roo Code) connect to MCP servers through Requesty's unified API, directly matching the story. missing for 10: deeper documentation on configuring specific MCP servers and independent/hands-on corroboration of the gateway working in practice.", "evidenceIds": [ "requesty-docs-9" ] }, { "productId": "requesty", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Requesty is a gateway/router (not itself an agent), and it documents an official 'MCP Gateway' that lets AI coding assistants like Claude Code, Cursor, and Roo Code securely connect to MCP servers through Requesty's unified API, directly matching the story. Missing for 10: independent/hands-on corroboration beyond first-party docs and more detail on setup/config specifics.", "evidenceIds": [ "requesty-docs-9", "requesty-docs-13" ] }, { "productId": "requesty", "storyId": "agentic-nl-commands", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Requesty is an LLM routing/gateway API product configured via SDKs, config files, and a dashboard, not an interactive assistant meant to be operated via natural-language commands; this axis is a category mismatch for an infrastructure/API product.", "evidenceIds": [] }, { "productId": "requesty", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Requesty is presented as a unified LLM gateway/router with SDK compatibility and integrations (Claude Code, Cursor, etc.), but no evidence mentions an official Requesty CLI tool. Evidence only covers API routing, policies, MCP gateway, and dashboard features—no CLI is documented or probed.", "evidenceIds": [] }, { "productId": "requesty", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Requesty exposes a documented, OpenAI-compatible public API (base_url swap, quickstart docs) plus a rich set of documented API-level features (fallback policies, load balancing, caching, structured outputs, BYOK) that let an AI-native user drive the product programmatically. missing for 10: an actual OpenAPI/swagger spec (probe found all candidate spec paths 404) and independent third-party corroboration of API robustness.", "evidenceIds": [ "requesty-docs-1", "requesty-docs-2", "requesty-docs-3", "requesty-docs-5", "requesty-docs-14", "requesty-probe-1", "requesty-probe-3" ] }, { "productId": "requesty", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Requesty's docs mention BYOK (bringing your own provider keys), analytics, and 'centralized governance,' but there is no evidence of issuing scoped or least-privilege Requesty API keys/credentials specifically for individual agents. No mention of key scoping, permission levels, or per-agent credential issuance appears anywhere in the evidence pack.", "evidenceIds": [ "requesty-docs-8", "requesty-docs-16" ] }, { "productId": "requesty", "storyId": "agentic-sdks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Requesty documents OpenAI-SDK compatibility (point the OpenAI SDK's base_url at Requesty) so developers can build using the OpenAI SDK's standard interfaces, but there is no evidence of a dedicated first-party Requesty SDK package (e.g., a requesty-python or requesty-node client) or SDKs for other languages beyond OpenAI-compatible clients. missing for 10: dedicated official Requesty-branded SDKs, multi-language SDK coverage, and independent confirmation of SDK maintenance/versioning.", "evidenceIds": [ "requesty-docs-1", "requesty-docs-2", "requesty-probe-1" ] }, { "productId": "requesty", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack of webhook subscription support for events (e.g., request completion, budget alerts, usage thresholds); the docs cover routing, fallback, caching, analytics, MCP gateway, etc., but no webhook/event-subscription feature is mentioned.", "evidenceIds": [] }, { "productId": "requesty", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows only markdown-based quickstart docs and code snippets; explicit probes for an OpenAPI/swagger spec all returned 404, and there is no mention of an interactive, runnable API reference or playground.", "evidenceIds": [ "requesty-probe-3", "requesty-docs-1", "requesty-docs-2" ] }, { "productId": "requesty", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Requesty is an API/gateway product where a downloadable OpenAPI spec would be a natural and expected artifact, so the axis applies, but a direct probe found all candidate OpenAPI/swagger paths returning 404 and no docs page references a spec file. No first-party mention of an OpenAPI spec exists anywhere in the docs pack.", "evidenceIds": [ "requesty-probe-3" ] }, { "productId": "requesty", "storyId": "api-sandbox", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Requesty is an LLM routing/gateway product for production API traffic, not a sandboxed testing/data environment; the sandbox-vs-production-data story is a category error for this kind of product.", "evidenceIds": [] }, { "productId": "requesty", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of API versioning scheme or a documented deprecation policy; OpenAPI spec probes all returned 404, and none of the docs mention version numbers, changelogs, or deprecation timelines.", "evidenceIds": [ "requesty-probe-3" ] }, { "productId": "requesty", "storyId": "automatic-fallbacks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Requesty's Fallback Policies feature explicitly documents automatic retry to a different model/provider when the primary fails (timeout, rate limit, error), directly matching the story. Missing for 10: independent/hands-on corroboration beyond vendor docs and detail on configuration granularity (e.g., how fallback chains are set up in the dashboard/API).", "evidenceIds": [ "requesty-docs-3", "requesty-docs-4" ] }, { "productId": "requesty", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Requesty's docs cover routing, fallback, load balancing, caching, and analytics for individual requests, but there is no evidence of a batch/bulk API or mechanism for processing many items in a single operation.", "evidenceIds": [] }, { "productId": "requesty", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Requesty provides several automatic policy mechanisms that act as event-triggered rules: Fallback Policies automatically retry with different models on failure events, Load Balancing Policies route by defined weights, and Guardrails automatically scan/act on content before reaching a provider. These are rule-based automations tied to conditions/events, but they are fixed policy types rather than a general-purpose custom event/action rule engine that an AI-native user could freely define. Missing for 10: a general rules/automation builder allowing arbitrary custom triggers and actions beyond the predefined fallback/load-balancing/guardrail policy types, and independent evidence of this automation working in production.", "evidenceIds": [ "requesty-docs-3", "requesty-docs-4", "requesty-docs-5", "requesty-docs-10" ] }, { "productId": "requesty", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Requesty is an LLM routing/gateway API, not a workflow orchestration or job-scheduling product; scheduling recurring jobs is outside its category's scope.", "evidenceIds": [] }, { "productId": "requesty", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Requesty is an LLM gateway/router product, not an automation-building tool with workflows to version, review, or roll back; this story is a category error for its product type.", "evidenceIds": [] }, { "productId": "requesty", "storyId": "budgets-limits", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows analytics/usage tracking and BYOK/governance mentions, but no documentation of setting hard budgets or spend limits per API key, team, or user. This is a plausible and expected axis for an LLM gateway product like Requesty, so absence of evidence yields 'none'.", "evidenceIds": [ "requesty-docs-11", "requesty-docs-16" ] }, { "productId": "requesty", "storyId": "byok", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Requesty explicitly documents a BYOK feature allowing developers to use personal API keys from various providers with the gateway, alongside the standard OpenAI-compatible routing setup. Missing for 10: independent/hands-on confirmation of BYOK working in practice and details on key storage/security guarantees.", "evidenceIds": [ "requesty-docs-8", "requesty-docs-1", "requesty-docs-2" ] }, { "productId": "requesty", "storyId": "cost-tracking", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Requesty's analytics dashboard tracks costs, requests, tokens, cache savings, and latency across all models/providers in real-time, and trace_id/user_id tagging plus BYOK support per-key tracking, which covers most of the story. missing for 10: explicit documentation of per-team breakdowns/team-level grouping, per-user cost attribution dashboards, and independent/hands-on corroboration of the analytics dashboard's granularity beyond vendor docs.", "evidenceIds": [ "requesty-docs-11", "requesty-docs-6", "requesty-docs-8", "requesty-docs-16" ] }, { "productId": "requesty", "storyId": "edge-low-latency", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Requesty is a gateway/router with caching, fallback, and analytics features that imply performance-conscious design, and analytics dashboard tracks latency, but there is no explicit benchmark or claim quantifying gateway-added latency overhead. missing for 10: published latency benchmarks or overhead numbers, independent performance testing, and any explicit low-latency architecture documentation.", "evidenceIds": [ "requesty-docs-7", "requesty-docs-11", "requesty-docs-16" ] }, { "productId": "requesty", "storyId": "load-balancing-routing", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Requesty documents weight-based Load Balancing Policies that distribute requests across multiple models/providers, plus session affinity via trace_id/user_id, and separate Fallback Policies for failure-driven rerouting. However, there is no evidence of latency-based or cost-based load balancing—only weight is explicitly supported. missing for 10: latency-based routing/balancing, cost-based routing/balancing, explicit per-key (not just per-model/provider) balancing.", "evidenceIds": [ "requesty-docs-5", "requesty-docs-6", "requesty-docs-3", "requesty-docs-4" ] }, { "productId": "requesty", "storyId": "model-catalog-discovery", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence confirms Requesty routes across 300–600+ models via a unified API, but nothing in the pack documents a browsable/queryable model catalog exposing pricing or context-window metadata (no /models endpoint, no models page, and the OpenAPI probe returned 404s). missing for 10: a models catalog page or API endpoint, per-model pricing data, per-model context-window metadata.", "evidenceIds": [ "requesty-docs-16", "requesty-docs-13", "requesty-probe-3" ] }, { "productId": "requesty", "storyId": "openai-compatible-endpoint", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly show swapping only base_url (to router.requesty.ai/v1) and API key while keeping OpenAI SDK unchanged, directly matching the story. Missing for 10: independent/hands-on third-party confirmation beyond vendor docs and a formal OpenAPI spec (probe found 404s).", "evidenceIds": [ "requesty-docs-1", "requesty-docs-2", "requesty-probe-1" ] }, { "productId": "requesty", "storyId": "openness-api-parity", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Requesty's core inference path (chat completions, model routing) is fully API-driven via an OpenAI-compatible endpoint, and features like fallback/load-balancing policies, BYOK, and guardrails are described as usable at request time. However, there's no evidence of a management/admin API for configuring these policies, keys, or viewing analytics programmatically, and the OpenAPI spec probe returned 404s across all candidate paths, suggesting dashboard-only configuration for policy setup and analytics viewing. missing for 10: a documented admin/management API for configuring fallback policies, load balancing, guardrails, BYOK keys, and pulling analytics data programmatically; a published OpenAPI spec confirming full API parity with the dashboard.", "evidenceIds": [ "requesty-docs-3", "requesty-docs-5", "requesty-docs-8", "requesty-docs-10", "requesty-docs-11", "requesty-probe-3" ] }, { "productId": "requesty", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers routing, caching, guardrails, analytics dashboard, and pricing, but nothing describes exporting usage data, logs, or configuration in open formats or a data-portability/account-deletion path. This is a fair question for a gateway that stores analytics/usage data, so absence of evidence yields 'none' rather than 'na'.", "evidenceIds": [ "requesty-docs-11" ] }, { "productId": "requesty", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an open-source license or public source repository for Requesty; it appears to be a closed commercial SaaS gateway with only documentation and API access exposed. Missing for 10: any mention of open-source licensing, public code repository, or license file.", "evidenceIds": [] }, { "productId": "requesty", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Requesty is presented as a hosted cloud gateway/router service (EU routing, hosted dashboard, per-usage pricing) with no evidence of a self-hosted deployment option, open-source repo, or on-prem package. No documentation mentions self-hosting the core routing product.", "evidenceIds": [] }, { "productId": "requesty", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Requesty offers an explicit EU-routing option (Frankfurt, AWS eu-central-1) keeping processing and storage within the EU, giving users some data-residency choice. However, evidence only shows one alternative region (EU vs default), not a broader set of selectable regions/residency options. Missing for 10: documentation of additional region choices (e.g., US-only, APAC), enterprise data-residency controls, and independent verification of enforcement.", "evidenceIds": [ "requesty-docs-12" ] }, { "productId": "requesty", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers routing, caching, guardrails, EU data residency, and BYOK, but contains no statement about a no-training policy or data opt-out for model training, so this claim is unevidenced.", "evidenceIds": [] }, { "productId": "requesty", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows EU data-residency routing and guardrails for sensitive content scanning, but nothing about data retention periods, deletion controls, or user-initiated data purge/export mechanisms. Missing for 10: documented retention policy, deletion/erasure API or dashboard control, retention configuration options.", "evidenceIds": [ "requesty-docs-12", "requesty-docs-10" ] }, { "productId": "requesty", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows analytics/usage dashboards and EU data routing but contains no mention of a telemetry/usage-tracking opt-out setting or privacy control for disabling data collection. As a hosted API gateway that logs requests, this is a fair axis to expect documentation on, but none is present.", "evidenceIds": [ "requesty-docs-11", "requesty-docs-12" ] }, { "productId": "requesty", "storyId": "provision-via-api", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an admin API for programmatically provisioning gateways, keys, or budgets; docs cover BYOK (manual key entry), analytics dashboard, and routing policies, but nothing about API-driven account/key/budget provisioning. The OpenAPI/swagger probe returned 404 for all candidate paths, further indicating no discoverable admin API surface.", "evidenceIds": [ "requesty-docs-8", "requesty-docs-11", "requesty-probe-3" ] }, { "productId": "requesty", "storyId": "rate-limit-smoothing", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Requesty's Fallback Policies explicitly retry on rate-limit errors by switching models, and Load Balancing Policies distribute requests across multiple models/providers by weight, which can smooth rate-limit pressure; BYOK lets users supply multiple provider keys. However, there is no documented queuing/throttling mechanism or explicit per-key round-robin/rate-limit-aware routing across multiple keys for the same provider. Missing for 10: explicit request queuing/throttling controls, documented multi-key rotation within a single provider, and rate-limit-specific load-balancing logic (vs. general failure retries).", "evidenceIds": [ "requesty-docs-3", "requesty-docs-4", "requesty-docs-5", "requesty-docs-8" ] }, { "productId": "requesty", "storyId": "request-logging", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Requesty's analytics dashboard docs confirm tracking of costs, requests, tokens, cache savings, and latency in real-time, which covers most of the story, but there is no evidence of inspecting individual logged request/response payload content (bodies) rather than aggregate metrics. missing for 10: evidence of per-request log inspection with full request/response bodies, screenshots or independent corroboration of the dashboard UI, and confirmation of per-call cost/latency drill-down rather than aggregate analytics.", "evidenceIds": [ "requesty-docs-11" ] }, { "productId": "requesty", "storyId": "response-caching", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Requesty documents automatic gateway-level caching of long system prompts and repeated content to reduce costs, with usage analytics tracking cache savings and latency in real-time, directly matching the story's intent. missing for 10: independent/hands-on benchmarks confirming actual latency/cost reduction, and detail on cache TTL/configurability.", "evidenceIds": [ "requesty-docs-7", "requesty-docs-11" ] }, { "productId": "requesty", "storyId": "retry-policies", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Requesty's Fallback Policies explicitly retry requests on a different model when the primary fails due to timeout, rate limit, or error, which directly addresses transient provider errors. However, this is model-fallback rather than a configurable retry policy (e.g., retry counts, backoff intervals, retry-same-provider-first) explicitly for transient errors. missing for 10: documentation of configurable retry counts/backoff on the same provider before failover, independent/hands-on confirmation of retry behavior, and explicit mention of 'transient error' handling distinct from full model fallback.", "evidenceIds": [ "requesty-docs-3", "requesty-docs-4" ] }, { "productId": "requesty", "storyId": "streaming-passthrough", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Requesty claims full OpenAI-SDK compatibility with no code changes (drop-in base_url swap), which implies standard SDK streaming (stream=True) would work across any of its 300+ models, but no evidence pack item explicitly confirms streaming/token-by-token behavior or documents a streaming example. missing for 10: explicit streaming documentation/example, confirmation across multiple providers, and any independent/hands-on verification of streaming behavior.", "evidenceIds": [ "requesty-docs-1", "requesty-docs-2", "requesty-docs-16" ] }, { "productId": "requesty", "storyId": "structured-outputs", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Requesty explicitly documents structured outputs, stating it makes 'every supported model speak structured JSON — from simple json_object mode to strict, schema-enforced json_schema mode,' directly addressing cross-provider JSON-schema support via its unified OpenAI-compatible API. Missing for 10: independent/hands-on verification across multiple providers and no explicit list of which of the 300+ models fully honor strict schema enforcement.", "evidenceIds": [ "requesty-docs-14", "requesty-docs-16", "requesty-docs-1" ] }, { "productId": "requesty", "storyId": "tool-call-fidelity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Requesty's OpenAI-compatible unified API (docs-1, docs-2, docs-16) implies that developers can use the same OpenAI SDK request format—including tool/function-call schema—across 300+/600+ models, and structured-outputs docs (docs-14) show JSON/schema-enforced output support across providers. However, there is no explicit documentation or example showing tool/function-call payloads being translated or normalized for non-OpenAI-native providers, nor any hands-on confirmation that function calling works uniformly across all providers. Missing for 10: explicit function/tool-calling schema documentation, cross-provider tool-call examples, and independent verification of consistent behavior across providers.", "evidenceIds": [ "requesty-docs-1", "requesty-docs-2", "requesty-docs-14", "requesty-docs-16" ] }, { "productId": "requesty", "storyId": "tracing-integrations", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Requesty documents an internal analytics dashboard for usage visibility, but there is no evidence of any export mechanism (webhooks, OpenTelemetry, log/trace export API, or third-party observability integrations like Datadog/Grafana) that would let a developer send gateway logs/traces to their own observability stack.", "evidenceIds": [ "requesty-docs-11" ] }, { "productId": "requesty", "storyId": "unified-api-many-providers", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Requesty is explicitly documented as an OpenAI-compatible unified API/gateway supporting 300+ (or 600+) models via a single base_url swap, with routing, fallback, load balancing, and BYOK across providers, all first-party documented and corroborated by an llms.txt probe describing it as a unified LLM gateway. Missing for 10: independent/hands-on third-party verification beyond docs, and no public OpenAPI spec found (404s on probe).", "evidenceIds": [ "requesty-docs-1", "requesty-docs-2", "requesty-docs-8", "requesty-docs-16", "requesty-probe-1" ] }, { "productId": "requesty", "storyId": "virtual-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers BYOK (users supplying their own provider keys into Requesty) and general governance/analytics claims, but nothing describes an admin/platform-engineer flow for minting distinct gateway-managed keys per team or app that shield the underlying provider keys. Missing for 10: documentation of virtual/scoped key issuance, team/app-level key scoping, or key rotation/management APIs.", "evidenceIds": [ "requesty-docs-8", "requesty-docs-16" ] }, { "productId": "vercel-ai-gateway", "storyId": "agent-spend-guardrails", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm you can create per-API-key budgets and view spend/usage broken out by API key (docs-16, docs-12, docs-5), which supports giving an agent its own key with a spend cap. However, there is no explicit documentation of rate-limit guardrails (only spend budgets), and community evidence shows real friction around balance-based enforcement even with BYOK keys, so the 'cannot run away on spend' guarantee is only partially evidenced. Missing for 10: explicit per-key rate-limiting controls, clear docs on key-level rate guardrails (not just budget caps), and independent confirmation that budget enforcement reliably halts an autonomous agent before overspend.", "evidenceIds": [ "vercel-ai-gateway-docs-16", "vercel-ai-gateway-docs-12", "vercel-ai-gateway-docs-5", "vercel-ai-gateway-comm-1" ] }, { "productId": "vercel-ai-gateway", "storyId": "agent-switches-models-policy", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs show routing rules via providerOptions.gateway (order, only, sort, timeouts) that let requests fail over across providers by availability/latency, plus cost/spend visibility and budgets, which supports policy-driven switching. However, there's no explicit evidence of dynamic mid-task capability-based switching or agent-driven runtime policy changes beyond static config, and community evidence flags BYOK requests being blocked once account balance hits $0, undermining pure availability-based routing in some cases. missing for 10: explicit support for capability-based (not just cost/availability) routing rules, evidence of an agent dynamically changing models mid-task at runtime, and independent confirmation that failover/ordering works reliably in production.", "evidenceIds": [ "vercel-ai-gateway-docs-9", "vercel-ai-gateway-docs-10", "vercel-ai-gateway-docs-29", "vercel-ai-gateway-docs-3", "vercel-ai-gateway-docs-16", "vercel-ai-gateway-comm-1" ] }, { "productId": "vercel-ai-gateway", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Vercel confirms a live llms.txt (probe-1) plus per-page .md documentation endpoints (probe-2) and an OpenAPI spec (probe-3), all machine-readable for agent consumption, and docs explicitly mention coding-agent setup guidance (docs-8, docs-24). Missing for 10: no independent/community confirmation that agents actually use these llms.txt/docs successfully in practice.", "evidenceIds": [ "vercel-ai-gateway-probe-1", "vercel-ai-gateway-probe-2", "vercel-ai-gateway-probe-3", "vercel-ai-gateway-docs-8", "vercel-ai-gateway-docs-24" ] }, { "productId": "vercel-ai-gateway", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Vercel AI Gateway is developer infrastructure that routes and manages LLM API calls between an application and model providers; it has no end-user product surface or stored user data of its own from which to surface insights. This capability belongs to applications built on top of the gateway, not the gateway itself, so the axis is a category error here.", "evidenceIds": [] }, { "productId": "vercel-ai-gateway", "storyId": "agentic-autonomous-automation", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Vercel AI Gateway is a model-routing/API gateway product (unified API, failover, spend monitoring, observability) — it is not an automation/orchestration platform with schedulers, triggers, or persistent background execution. Building or running autonomous background automations is outside its product category; that capability would live in a separate agent/orchestration tool that might use the Gateway for model calls.", "evidenceIds": [] }, { "productId": "vercel-ai-gateway", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Vercel AI Gateway is an infrastructure/routing layer for LLM API calls, not a product with a built-in AI assistant UI for users to delegate tasks to; it enables developers to build agents (e.g., ToolLoopAgent) but does not itself ship a first-party assistant persona for end users.", "evidenceIds": [] }, { "productId": "vercel-ai-gateway", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "AI Gateway is an API/HTTP gateway with OpenAI-compatible endpoints, unauthenticated model listing, OIDC token auth for deployed apps, and a keyless models endpoint verified live — all of which support pure headless/CI/automation use without any UI dependency. missing for 10: no explicit CI pipeline example (e.g., GitHub Actions) or independent third-party report of CI usage.", "evidenceIds": [ "vercel-ai-gateway-docs-7", "vercel-ai-gateway-docs-14", "vercel-ai-gateway-docs-15", "vercel-ai-gateway-docs-22", "vercel-ai-gateway-docs-28", "vercel-ai-gateway-probe-3", "vercel-ai-gateway-probe-4" ] }, { "productId": "vercel-ai-gateway", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes Vercel AI Gateway as a model-routing/proxy layer (provider switching, fallbacks, spend monitoring, caching, coding-agent connections) but contains no mention of MCP server integration or tool-use via MCP anywhere in the docs pack.", "evidenceIds": [] }, { "productId": "vercel-ai-gateway", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The docs mention connecting 'supported coding agents' and generic gateway integration, but there is no evidence of an official MCP (Model Context Protocol) server or endpoint that agents can connect to. Missing for 10: explicit MCP server documentation, MCP endpoint URL, and demonstration of an agent connecting via MCP.", "evidenceIds": [ "vercel-ai-gateway-docs-8", "vercel-ai-gateway-docs-24" ] }, { "productId": "vercel-ai-gateway", "storyId": "agentic-nl-commands", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Vercel AI Gateway is developer infrastructure (API routing/failover/billing for LLM providers) that is configured via code, dashboard, and API calls — not an interactive assistant meant to be 'operated' via natural-language commands. Evidence about coding-agent connectivity and ToolLoopAgent describes building agents that use the gateway, not commanding the gateway itself in natural language, so this axis is a category mismatch for this product type.", "evidenceIds": [] }, { "productId": "vercel-ai-gateway", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The only evidence of a CLI is the fragment 'vercel ai-gateway coding-agents setup', implying an official `vercel` CLI subcommand for configuring the AI Gateway with coding agents, but there is no detailed command reference, install instructions, or broader CLI documentation for AI-native workflows beyond this single mention. Missing for 10: full CLI command reference/docs, examples of CLI usage for model routing/config, independent confirmation of CLI functionality.", "evidenceIds": [ "vercel-ai-gateway-docs-8" ] }, { "productId": "vercel-ai-gateway", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "AI Gateway exposes a documented, OpenAI-compatible public API (Chat Completions, models endpoint) with a verified OpenAPI spec, unauthenticated models endpoint, and probe confirming a live JSON API response, enabling programmatic/agentic control via standard SDKs or raw HTTP. missing for 10: independent third-party developer writeups explicitly praising the API's completeness/stability beyond Vercel's own docs and probes.", "evidenceIds": [ "vercel-ai-gateway-docs-7", "vercel-ai-gateway-docs-15", "vercel-ai-gateway-docs-25", "vercel-ai-gateway-docs-28", "vercel-ai-gateway-probe-3", "vercel-ai-gateway-probe-4" ] }, { "productId": "vercel-ai-gateway", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs show budget caps that can be set per API key, per project, or per team member (docs-16), and per-key usage breakdown in Requests logs (docs-12), plus short-lived OIDC tokens for Vercel-deployed apps that avoid static secrets (docs-14/22). This supports issuing distinct, budget-limited keys for an agent, but there is no documentation of fine-grained permission scoping (e.g. restricting a key to specific models/providers/actions) beyond spend limits. missing for 10: explicit least-privilege scoping controls (model/provider/action restrictions per key), role-based permission model, first-party or independent confirmation of true least-privilege agent credentials", "evidenceIds": [ "vercel-ai-gateway-docs-16", "vercel-ai-gateway-docs-12", "vercel-ai-gateway-docs-14", "vercel-ai-gateway-docs-22" ] }, { "productId": "vercel-ai-gateway", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "AI Gateway is designed to be used via existing official SDKs (OpenAI client libraries, Anthropic SDK, Vercel AI SDK) with just a base URL/model-string change, documented in openai-compat and getting-started guides, and confirmed by a live OpenAI-compatible endpoint probe. It also supports agent-building primitives like ToolLoopAgent for agentic workflows. missing for 10: no independent/hands-on developer confirmation beyond vendor docs and one API probe.", "evidenceIds": [ "vercel-ai-gateway-docs-7", "vercel-ai-gateway-docs-18", "vercel-ai-gateway-docs-28", "vercel-ai-gateway-docs-31", "vercel-ai-gateway-docs-26", "vercel-ai-gateway-probe-4" ] }, { "productId": "vercel-ai-gateway", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack of webhook subscriptions or event-driven notifications from AI Gateway; the product exposes REST APIs, logs, and observability dashboards, but nothing about webhooks for events like request completion, budget alerts, or failures.", "evidenceIds": [] }, { "productId": "vercel-ai-gateway", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "There's a documented OpenAPI spec and a live, keyless models endpoint that returns JSON model data, showing some interactive/runnable API surface, but there's no evidence of an interactive API reference UI (like Swagger/Redoc) with embedded runnable examples or a try-it-now console. missing for 10: an actual interactive API explorer/playground page, runnable code snippets embedded in docs, independent confirmation of a docs UI with live request execution.", "evidenceIds": [ "vercel-ai-gateway-probe-3", "vercel-ai-gateway-probe-4", "vercel-ai-gateway-docs-15", "vercel-ai-gateway-docs-25" ] }, { "productId": "vercel-ai-gateway", "storyId": "api-machine-spec", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "A probe confirms a live OpenAPI JSON spec at vercel.com/openapi.json (contains the 'openapi' key) and the docs also expose a keyless models JSON endpoint and OpenAI-compatible API, all of which are machine-readable and programmatically consumable. Missing for 10: explicit confirmation the openapi.json spec covers AI Gateway endpoints specifically (vs. general Vercel API) and independent third-party validation of the spec's completeness.", "evidenceIds": [ "vercel-ai-gateway-probe-3", "vercel-ai-gateway-probe-4", "vercel-ai-gateway-docs-15", "vercel-ai-gateway-docs-25" ] }, { "productId": "vercel-ai-gateway", "storyId": "api-sandbox", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "AI Gateway is a model-routing/API proxy layer, not a data-plane product with production datasets or sandbox/staging environments to isolate from production data; the story's premise (sandbox vs production data) doesn't map onto its category. No evidence describes environment separation or sandbox testing modes, confirming this axis is a category mismatch rather than a gap.", "evidenceIds": [] }, { "productId": "vercel-ai-gateway", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows an OpenAI-compatible endpoint with a 'v1' path and an openapi.json spec, but nowhere does the pack document an explicit API versioning scheme or a deprecation policy for endpoints/models. Missing for 10: explicit versioning strategy documentation, deprecation/sunset timelines, migration guidance for breaking changes.", "evidenceIds": [ "vercel-ai-gateway-probe-3", "vercel-ai-gateway-probe-4", "vercel-ai-gateway-docs-25" ] }, { "productId": "vercel-ai-gateway", "storyId": "automatic-fallbacks", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly describe automatic retries to other providers on failure, plus fine-grained control via `order`, `only`, and `sort` in providerOptions.gateway, per-provider timeouts for fast failover, and even BYOK-to-system-credential fallback — directly matching the fallback/routing-resilience story. This is well-documented first-party capability with concrete configuration options, though missing for 10: independent hands-on verification that fallback behaves correctly in production scenarios.", "evidenceIds": [ "vercel-ai-gateway-docs-3", "vercel-ai-gateway-docs-9", "vercel-ai-gateway-docs-10", "vercel-ai-gateway-docs-13", "vercel-ai-gateway-docs-20", "vercel-ai-gateway-docs-24" ] }, { "productId": "vercel-ai-gateway", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers routing, failover, BYOK, observability, and embeddings, but there is no mention of any batch/bulk request API or capability to process many items in a single operation. Since a gateway product could plausibly offer batch endpoints (e.g., batch inference or bulk embeddings), the axis applies, but no evidence supports it being delivered.", "evidenceIds": [] }, { "productId": "vercel-ai-gateway", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "The Gateway supports narrow built-in automation triggers — failover/retry on provider failure, per-provider timeouts triggering fast failover, and budget caps that block spend once a threshold is hit — which are rule-like actions triggered by events (docs-3, docs-10, docs-13, docs-16). However, there's no general-purpose, user-defined rules/event system (e.g., custom conditions, webhooks, arbitrary trigger-action pairs) for AI-native users to configure; the 'rules' are limited to routing, caching, and spend-cap logic. Missing for 10: a configurable rules engine for arbitrary events/actions, webhook or external-trigger support, and evidence of user-defined conditional automation beyond routing/budget defaults.", "evidenceIds": [ "vercel-ai-gateway-docs-3", "vercel-ai-gateway-docs-10", "vercel-ai-gateway-docs-13", "vercel-ai-gateway-docs-16", "vercel-ai-gateway-docs-20" ] }, { "productId": "vercel-ai-gateway", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Vercel AI Gateway is a request-routing/proxy layer for LLM API calls (unified API, retries, budgets, observability) — it has no scheduler, cron, or workflow orchestration feature for recurring jobs. This is a category mismatch, not a missing capability of this product type.", "evidenceIds": [] }, { "productId": "vercel-ai-gateway", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Vercel AI Gateway is a model-routing/API gateway product, not an automation-building platform with a concept of versionable 'automations' to review or roll back — this axis is a category mismatch for its product type.", "evidenceIds": [] }, { "productId": "vercel-ai-gateway", "storyId": "budgets-limits", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly state budgets can be set to cap spend for a team, project, API key, or team member, directly matching the story, and spend monitoring/usage dashboards support enforcement. missing for 10: independent/hands-on confirmation that hard budgets actually block spend at the limit (community evidence only discusses BYOK balance blocking, not per-key/team/user budget enforcement specifics) and more detail on granularity/configuration UI.", "evidenceIds": [ "vercel-ai-gateway-docs-16", "vercel-ai-gateway-docs-5", "vercel-ai-gateway-docs-12", "vercel-ai-gateway-docs-30" ] }, { "productId": "vercel-ai-gateway", "storyId": "byok", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Vercel's docs clearly describe BYOK support — pass provider credentials per-request via providerOptions.gateway.byok, with no markup on tokens, and even fallback to system credentials on failure — suggesting bring-your-own-key traffic should run on the developer's own credentials (docs-6, docs-13, docs-19, docs-27). However, a concrete community report describes BYOK requests being blocked by Vercel's account balance reaching $0 even when the user's own provider key has available usage, directly contradicting the expectation that BYOK traffic is unaffected by platform account balance (comm-1). missing for 10: independent confirmation resolving the balance-dependency complaint, clearer documentation on how billing/balance interacts with BYOK requests.", "evidenceIds": [ "vercel-ai-gateway-docs-6", "vercel-ai-gateway-docs-13", "vercel-ai-gateway-docs-19", "vercel-ai-gateway-docs-27", "vercel-ai-gateway-comm-1" ] }, { "productId": "vercel-ai-gateway", "storyId": "cost-tracking", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs show request/usage summaries broken down by project and API key, budgets settable per team/project/API key/team member, and detailed logs filterable by model/provider/status with CSV/JSON export, directly matching the story's cost-tracking dimensions. Missing for 10: explicit per-user (not just per-team-member/API key) breakdowns and independent third-party confirmation of the dashboard's granularity in practice.", "evidenceIds": [ "vercel-ai-gateway-docs-5", "vercel-ai-gateway-docs-12", "vercel-ai-gateway-docs-16", "vercel-ai-gateway-docs-23", "vercel-ai-gateway-docs-30" ] }, { "productId": "vercel-ai-gateway", "storyId": "edge-low-latency", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs show performance-related controls (per-provider timeouts for fast failover, sort-by-performance metric, automatic caching) that imply attention to latency, but there is no direct benchmark or explicit claim quantifying the gateway's own added latency overhead. Missing for 10: published latency benchmarks or first-party/independent measurements showing the gateway adds minimal overhead versus direct provider calls, and no community corroboration of low added latency.", "evidenceIds": [ "vercel-ai-gateway-docs-10", "vercel-ai-gateway-docs-29", "vercel-ai-gateway-docs-11", "vercel-ai-gateway-docs-21" ] }, { "productId": "vercel-ai-gateway", "storyId": "load-balancing-routing", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm ordering/sorting by performance or cost metric (`order`, `only`, `sort`), per-provider timeouts for fast failover, and automatic retries/fallbacks to other providers — covering latency- and cost-based routing and failover. However, explicit weight-based load balancing across providers/deployments/keys is not documented, and there's no evidence of key-level load balancing beyond BYOK fallback to system credentials. missing for 10: explicit weighted load-balancing configuration, per-key routing/balancing control, independent verification of routing behavior under load.", "evidenceIds": [ "vercel-ai-gateway-docs-9", "vercel-ai-gateway-docs-10", "vercel-ai-gateway-docs-29", "vercel-ai-gateway-docs-3", "vercel-ai-gateway-docs-13" ] }, { "productId": "vercel-ai-gateway", "storyId": "model-catalog-discovery", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly describe a models endpoint returning model IDs, context windows, and pricing with no authentication required, and a probe independently confirms the live unauthenticated /v1/models endpoint returns a full JSON model catalog. This directly satisfies browsing/querying a model catalog with pricing and context-window metadata. Missing for 10: no evidence of a dedicated visual browsing UI/dashboard for the catalog beyond the API and docs page.", "evidenceIds": [ "vercel-ai-gateway-docs-15", "vercel-ai-gateway-docs-25", "vercel-ai-gateway-probe-4" ] }, { "productId": "vercel-ai-gateway", "storyId": "openai-compatible-endpoint", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Vercel explicitly documents OpenAI-compatible Chat Completions API where developers just swap the base URL and key with existing OpenAI SDKs, with no code rewrites, and a probe confirms the gateway serves an OpenAI-compatible /v1/models JSON response. Community evidence around BYOK balance behavior is unrelated to base-URL compatibility, so it doesn't dispute this. Missing for 10: independent third-party hands-on confirmation of a full migration beyond Vercel's own docs/probes.", "evidenceIds": [ "vercel-ai-gateway-docs-7", "vercel-ai-gateway-docs-18", "vercel-ai-gateway-docs-28", "vercel-ai-gateway-docs-31", "vercel-ai-gateway-probe-4" ] }, { "productId": "vercel-ai-gateway", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "The gateway is largely API-first: model access, provider routing, caching, BYOK, and even a keyless public models endpoint are confirmed to work programmatically (docs-1,2,9,15,25,27; probe-4). However, budgets, spend caps, and the detailed Requests/Logs dashboard are described only as UI features (docs-5,12,16,23,30) without documented API endpoints to set budgets or pull logs programmatically. Missing for 10: budgets/spend-cap management API, log/usage export API parity, team/key management API parity.", "evidenceIds": [ "vercel-ai-gateway-docs-1", "vercel-ai-gateway-docs-2", "vercel-ai-gateway-docs-9", "vercel-ai-gateway-docs-15", "vercel-ai-gateway-docs-25", "vercel-ai-gateway-docs-27", "vercel-ai-gateway-docs-5", "vercel-ai-gateway-docs-16", "vercel-ai-gateway-docs-23", "vercel-ai-gateway-probe-4" ] }, { "productId": "vercel-ai-gateway", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Vercel documents that the Logs page lets users export request/usage data as CSV or JSON, and its OpenAI-compatible API means you can point clients elsewhere with minimal rewrite, reducing lock-in. However, there's no evidence of a comprehensive account/data export (billing history, configs, keys) beyond request logs, and BYOK behavior tied to Vercel balance (comm-1) suggests some platform dependency remains. Missing for 10: full account/config data export, evidence of exporting all historical usage/billing data, and confirmation that BYOK/API keys are fully portable without gateway dependency.", "evidenceIds": [ "vercel-ai-gateway-docs-23", "vercel-ai-gateway-docs-7", "vercel-ai-gateway-docs-18", "vercel-ai-gateway-docs-28", "vercel-ai-gateway-comm-1" ] }, { "productId": "vercel-ai-gateway", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Vercel AI Gateway is a closed, proprietary hosted service; no evidence in the pack points to any open-source repository or open license for its source code — all evidence is documentation and API endpoints, not source availability.", "evidenceIds": [] }, { "productId": "vercel-ai-gateway", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Vercel AI Gateway is a hosted cloud service with no evidence of a self-hostable core product or open-source release; nothing in the docs suggests an on-prem or self-managed deployment option.", "evidenceIds": [] }, { "productId": "vercel-ai-gateway", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of region/data-residency controls or storage location choices in the AI Gateway docs; the pack covers routing, failover, pricing, and observability but nothing about where data is stored or processed. Missing for 10: any mention of regional data residency options, storage location controls, or compliance certifications addressing data locality.", "evidenceIds": [] }, { "productId": "vercel-ai-gateway", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data-training opt-out policies, zero-retention agreements, or any explicit statement that prompts/outputs are excluded from model training; the docs focus on routing, pricing, and observability instead. Missing for 10: any privacy policy or training-opt-out statement, zero-data-retention terms, provider-level data-use guarantees.", "evidenceIds": [] }, { "productId": "vercel-ai-gateway", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data retention policies, log/data deletion controls, or retention configuration for requests/prompts processed through AI Gateway; observability docs mention logging/exporting requests but nothing about retention duration or deletion mechanisms.", "evidenceIds": [] }, { "productId": "vercel-ai-gateway", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of any telemetry opt-out or privacy/data-collection controls; the gateway's observability features are about logging usage for the account owner, not about disabling tracking. Missing for 10: any documented telemetry opt-out setting, privacy policy on data retention/training use, or way to disable request logging.", "evidenceIds": [] }, { "productId": "vercel-ai-gateway", "storyId": "provision-via-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Docs describe setting budgets, API keys, and monitoring spend through the Vercel dashboard/CLI, and a general Vercel OpenAPI spec exists, but there is no evidence of a documented admin API specifically for programmatically provisioning AI Gateway resources, keys, or budgets. Missing for 10: explicit admin/API endpoints for creating gateways, generating keys, or setting budgets programmatically, and any docs or examples showing this workflow.", "evidenceIds": [ "vercel-ai-gateway-docs-16", "vercel-ai-gateway-docs-12", "vercel-ai-gateway-probe-3" ] }, { "productId": "vercel-ai-gateway", "storyId": "rate-limit-smoothing", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "The gateway supports failover/retry across providers, per-provider timeouts for fast failover, and ordering/sorting of providers (docs-3, docs-9, docs-10, docs-13, docs-20, docs-29), which can mitigate a single provider's rate limits by shifting traffic elsewhere. However, there's no documented mechanism for spreading load across multiple API keys for the same provider, nor any explicit request queuing/throttling feature to smooth bursts against rate limits. Missing for 10: multi-key load balancing, explicit request queuing/throttling controls, and any rate-limit-specific documentation or independent verification.", "evidenceIds": [ "vercel-ai-gateway-docs-3", "vercel-ai-gateway-docs-9", "vercel-ai-gateway-docs-10", "vercel-ai-gateway-docs-13", "vercel-ai-gateway-docs-29" ] }, { "productId": "vercel-ai-gateway", "storyId": "request-logging", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs describe a Logs page with per-request detail, search by request ID, filtering by model/provider/status, and export as CSV/JSON, plus a Requests section with summaries by project/API key and a Usage section with metrics — covering latency, tokens, cost, and routing detail. Missing for 10: independent/hands-on confirmation that latency and cost are explicitly visible per logged request (only vendor docs cited), and no third-party corroboration of the observability UI.", "evidenceIds": [ "vercel-ai-gateway-docs-12", "vercel-ai-gateway-docs-23", "vercel-ai-gateway-docs-30", "vercel-ai-gateway-docs-5" ] }, { "productId": "vercel-ai-gateway", "storyId": "response-caching", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs explicitly describe a `caching: 'auto'` option in providerOptions.gateway that lets the gateway apply provider-appropriate caching automatically, directly supporting cost/latency reduction on repeated requests, and community pricing discussion confirms cache-read pricing exists for at least one model. However, details on cache hit-rate guarantees, TTL, scope, or independent benchmarks of latency/cost savings are absent. Missing for 10: independent hands-on benchmarks showing latency/cost reduction, documentation on cache TTL/invalidation/scope, confirmation caching works uniformly across all providers rather than being provider-dependent.", "evidenceIds": [ "vercel-ai-gateway-docs-11", "vercel-ai-gateway-docs-21", "vercel-ai-gateway-comm-2" ] }, { "productId": "vercel-ai-gateway", "storyId": "retry-policies", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly describe automatic failover/retry to other providers on failure (docs-3), configurable provider ordering/fallback with `order`, `only`, `sort` (docs-9, docs-29), per-provider timeouts to trigger fast failover (docs-10), and even automatic retry with system credentials when BYOK fails (docs-13). This directly covers automatic retry policies for transient provider errors. Missing for 10: no independent/hands-on confirmation of retry behavior in production, and no explicit documentation of configurable retry counts/backoff intervals distinct from provider ordering.", "evidenceIds": [ "vercel-ai-gateway-docs-3", "vercel-ai-gateway-docs-9", "vercel-ai-gateway-docs-10", "vercel-ai-gateway-docs-13", "vercel-ai-gateway-docs-29" ] }, { "productId": "vercel-ai-gateway", "storyId": "streaming-passthrough", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The gateway advertises full OpenAI/Anthropic SDK compatibility and AI SDK integration (docs-7, docs-18, docs-28, docs-31), and since streaming is a standard capability of those client libraries, it is implicitly supported when pointing existing clients at the gateway's base URL. However, no evidence explicitly documents streaming/token-by-token responses, SSE support, or shows this working across all providers. Missing for 10: explicit docs on streaming endpoints/parameters, hands-on confirmation of token-by-token streaming across multiple providers, and any independent verification.", "evidenceIds": [ "vercel-ai-gateway-docs-7", "vercel-ai-gateway-docs-18", "vercel-ai-gateway-docs-28", "vercel-ai-gateway-docs-31" ] }, { "productId": "vercel-ai-gateway", "storyId": "structured-outputs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes unified APIs, OpenAI-compatibility, provider routing, embeddings, and observability, but nowhere mentions structured JSON-schema/response_format output support or how it is normalized across different providers. Missing for 10: any documentation of JSON-schema/structured-output support, cross-provider schema normalization, or examples using response_format/json_schema.", "evidenceIds": [] }, { "productId": "vercel-ai-gateway", "storyId": "tool-call-fidelity", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Vercel AI Gateway supports a unified API/schema (OpenAI-compatible and AI SDK) that lets developers call tools/functions across many providers with consistent code, including an explicit ToolLoopAgent example with tool calling, provider routing/fallback, and no-rewrite compatibility with existing OpenAI/Anthropic clients. missing for 10: independent hands-on verification of cross-provider tool-call schema consistency (e.g., confirming tool-calling behaves identically across Anthropic/OpenAI/other models) and any documented edge cases or limitations in function-calling parity across providers.", "evidenceIds": [ "vercel-ai-gateway-docs-2", "vercel-ai-gateway-docs-7", "vercel-ai-gateway-docs-18", "vercel-ai-gateway-docs-26", "vercel-ai-gateway-docs-28", "vercel-ai-gateway-probe-4" ] }, { "productId": "vercel-ai-gateway", "storyId": "tracing-integrations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The gateway provides a Logs UI with filtering and CSV/JSON export of request logs, which supports manual export of logs, but there's no evidence of native integration with external observability stacks (e.g., OpenTelemetry export, webhook/streaming to Datadog/Grafana, or trace export in standard formats like OTLP). missing for 10: OpenTelemetry/OTLP trace export, native integrations with third-party observability platforms, automated/streaming log export (vs manual CSV/JSON download), independent confirmation of export usability.", "evidenceIds": [ "vercel-ai-gateway-docs-23", "vercel-ai-gateway-docs-12", "vercel-ai-gateway-docs-30" ] }, { "productId": "vercel-ai-gateway", "storyId": "unified-api-many-providers", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Extensive first-party docs describe a single unified API/key across hundreds of providers, OpenAI-compatible client support, and provider routing/fallback controls, and a live probe confirms the OpenAI-compatible /v1/models endpoint works unauthenticated. There is a minor community-reported caveat about BYOK requests being blocked when Vercel balance hits $0, which slightly limits the 'bring your own key' flow but doesn't contradict the core unified-API story. Missing for 10: independent hands-on developer confirmation of multi-provider switching in production beyond docs and one probe.", "evidenceIds": [ "vercel-ai-gateway-docs-1", "vercel-ai-gateway-docs-2", "vercel-ai-gateway-docs-7", "vercel-ai-gateway-docs-18", "vercel-ai-gateway-docs-28", "vercel-ai-gateway-probe-4", "vercel-ai-gateway-comm-1" ] }, { "productId": "vercel-ai-gateway", "storyId": "virtual-keys", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs show a single Gateway-managed API key (or OIDC token) replaces direct provider keys, with per-API-key usage tracking and budget caps that imply teams/apps can have distinct managed keys, and BYOK/system-credential fallback keeps raw provider secrets out of app code. Missing for 10: explicit dashboard walkthrough for creating/rotating multiple named keys per team or app, and documentation of per-key role/permission scoping.", "evidenceIds": [ "vercel-ai-gateway-docs-1", "vercel-ai-gateway-docs-12", "vercel-ai-gateway-docs-16", "vercel-ai-gateway-docs-13", "vercel-ai-gateway-docs-14", "vercel-ai-gateway-docs-27" ] } ]