[ { "productId": "baseten", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Baseten hosts a verified llms.txt (HTTP 200, indexing docs) and an agent-setup page directing agents to install skills/MCP servers to search the docs, confirmed live via a working MCP endpoint. missing for 10: independent third-party confirmation of an agent successfully using llms.txt end-to-end beyond Baseten's own probes.", "evidenceIds": [ "baseten-probe-1", "baseten-probe-2", "baseten-docs-5", "baseten-probe-4", "baseten-probe-rt-3" ] }, { "productId": "baseten", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Baseten is an ML model deployment/inference infrastructure platform (model serving, training, fine-tuning) rather than an analytics/BI product that ingests a user's own data and surfaces AI-generated insights inside a dashboard. None of the evidence describes Baseten analyzing customer data and returning insights/suggestions — this axis is a category error for an inference/serving platform.", "evidenceIds": [] }, { "productId": "baseten", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Baseten's evidence covers async inference (deferred single requests via webhook/polling), model deployment, and training, but there is no evidence of scheduling, triggers, or autonomous multi-step automations running in the background — async inference is single-request deferral, not an automation/workflow engine.", "evidenceIds": [] }, { "productId": "baseten", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Baseten's evidence shows MCP servers and a Baseten 'skill' that let external coding agents (Claude Code, Codex, etc.) manage a Baseten workspace — this is the reverse of a built-in in-product assistant; nothing in the evidence describes a first-party AI assistant living inside the Baseten UI/dashboard that a user can delegate platform tasks to.", "evidenceIds": [ "baseten-docs-5", "baseten-docs-22", "baseten-docs-34", "baseten-probe-4", "baseten-probe-rt-3" ] }, { "productId": "baseten", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Baseten provides a CLI (truss push, Baseten CLI/Switch), config.yaml-driven deployment, OpenAI-compatible REST APIs with async/webhook/polling support, and API-key-based auth/rate limiting, all of which support scripted, non-interactive CI/automation workflows. missing for 10: explicit CI/CD pipeline examples (e.g. GitHub Actions integration), independent hands-on confirmation of headless CLI use in automation, and documented non-interactive auth flow for CI secrets.", "evidenceIds": [ "baseten-gh-2", "baseten-gh-6", "baseten-docs-9", "baseten-docs-27", "baseten-docs-30", "baseten-docs-37", "baseten-docs-38", "baseten-docs-18" ] }, { "productId": "baseten", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Baseten's evidence only shows it exposing its own MCP server so external coding agents (Claude Code, Codex, Pi) can call Baseten's workspace tools — the reverse relationship. There is no evidence that Baseten itself can consume/plug in third-party MCP servers to gain access to their tools during inference or agent workflows.", "evidenceIds": [ "baseten-docs-5", "baseten-docs-22", "baseten-docs-34", "baseten-probe-rt-3", "baseten-probe-4" ] }, { "productId": "baseten", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Baseten documents and hosts an official MCP server (docs.baseten.co/agent-setup, docs.baseten.co/mcp) that lets coding agents manage the Baseten workspace and search docs, and a live probe confirms the endpoint responds to MCP initialize with valid JSON-RPC capabilities. missing for 10: independent/community hands-on validation of the MCP server beyond the vendor's own probe, and more detail on the full tool surface exposed.", "evidenceIds": [ "baseten-docs-5", "baseten-probe-4", "baseten-probe-rt-3" ] }, { "productId": "baseten", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Baseten ships an official skill and MCP server so a coding agent can 'manage your Baseten workspace and search these docs' via natural language, and a probe confirms a live, reachable MCP endpoint responding to JSON-RPC calls — this is genuine natural-language operability of the product through an agent intermediary. However the docs give only a high-level description of what workspace actions are exposed, with no detailed command catalog or independent hands-on verification of breadth/reliability. Missing for 10: a documented list of specific natural-language-operable actions (deploy, scale, monitor, etc.), and independent/hands-on evidence confirming the MCP tools work reliably end-to-end.", "evidenceIds": [ "baseten-docs-5", "baseten-probe-4", "baseten-probe-rt-3" ] }, { "productId": "baseten", "storyId": "agentic-official-cli", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Baseten ships official CLIs: Truss (`uvx truss push` to build/deploy models from a config.yaml, per baseten-gh-2/6) and Baseten Switch (a CLI referenced at /reference/cli/baseten-switch for routing coding-agent traffic, baseten-docs-34), plus one-command checkpoint deployment (baseten-docs-29). missing for 10: independent/hands-on verification of CLI behavior, and a full CLI reference/command list beyond scattered doc mentions.", "evidenceIds": [ "baseten-gh-2", "baseten-gh-6", "baseten-docs-29", "baseten-docs-34", "baseten-docs-22" ] }, { "productId": "baseten", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Baseten exposes an OpenAI/Anthropic-compatible inference API (docs-21, docs-30), a documented CLI/Truss deployment workflow, training APIs (docs-32, docs-36), and usage/rate-limit endpoints, all backed by extensive public docs and a live, probe-verified /v1/models endpoint and MCP server. Missing for 10: a discoverable OpenAPI/swagger spec (probe-3 shows 404s across candidate paths) and independent third-party API usage reports beyond community anecdotes.", "evidenceIds": [ "baseten-docs-21", "baseten-docs-30", "baseten-docs-9", "baseten-docs-32", "baseten-docs-10", "baseten-probe-rt-1", "baseten-probe-3", "baseten-gh-2" ] }, { "productId": "baseten", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Baseten documents API keys, usage monitoring by API key, and workspace budgets, but no evidence describes scoped/least-privilege credential issuance (e.g., role-based permissions, restricted-scope keys, or per-agent credential minting). Monitoring usage by key is not the same as issuing scoped credentials.", "evidenceIds": [ "baseten-docs-10", "baseten-docs-23", "baseten-docs-38", "baseten-docs-24" ] }, { "productId": "baseten", "storyId": "agentic-sdks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Baseten ships the official open-source Truss SDK/CLI for packaging and deploying models (github.com/basetenlabs/truss) and also exposes OpenAI-compatible and Anthropic Messages-compatible endpoints so developers can use those official client SDKs (and LangChain) directly against Baseten (baseten-docs-21, baseten-docs-33, baseten-gh-1/2/5/6). This gives AI-native users multiple first-party and compatible SDK paths for building agentic workflows. Missing for 10: a dedicated first-party 'Baseten Python client' distinct from Truss, and independent/hands-on corroboration of SDK reliability beyond docs and repo README.", "evidenceIds": [ "baseten-gh-1", "baseten-gh-2", "baseten-gh-5", "baseten-gh-6", "baseten-docs-21", "baseten-docs-33", "baseten-docs-30" ] }, { "productId": "baseten", "storyId": "agentic-webhooks", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs mention that async inference results can be delivered via webhook callback, showing Baseten supports webhook-based notification for job completion, but there is no dedicated webhook subscription system (event types, signing, retry, subscription management) described anywhere in the evidence. Missing for 10: a general event-subscription/webhook API or docs page (event catalog, signature verification, delivery guarantees), coverage beyond async-inference completion notifications, and any independent confirmation of webhook reliability.", "evidenceIds": [ "baseten-docs-9", "baseten-docs-27", "baseten-docs-35" ] }, { "productId": "baseten", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows extensive prose documentation (structured outputs, function calling, streaming, pricing) but no interactive API reference or runnable-example playground; a direct probe for an OpenAPI/Swagger spec (which typically powers interactive references) returned 404 on all candidate paths.", "evidenceIds": [ "baseten-probe-3", "baseten-docs-6", "baseten-docs-7", "baseten-docs-8" ] }, { "productId": "baseten", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe for a machine-readable API spec (openapi.json, swagger.json, and related paths) returned 404 on all candidates, and no docs page claims to publish an OpenAPI/Swagger spec — only that the API is OpenAI/Anthropic-compatible in shape, which is not the same as Baseten publishing its own downloadable spec.", "evidenceIds": [ "baseten-probe-3", "baseten-docs-21", "baseten-docs-30" ] }, { "productId": "baseten", "storyId": "api-sandbox", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs mention that deployments 'support stable environments for development, staging, and production' (baseten-docs-20) and that Truss models 'behave the same in development and production' (baseten-gh-1), implying separate non-prod environments exist. However there is no detailed documentation of how sandbox/staging is isolated from production data, no guidance on seeding test data, and no independent confirmation of this workflow. Missing for 10: explicit sandbox/staging setup guide, data-isolation guarantees, and hands-on/community corroboration of safe non-prod testing.", "evidenceIds": [ "baseten-docs-20", "baseten-gh-1" ] }, { "productId": "baseten", "storyId": "api-versioning-policy", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Baseten documents a deprecation policy for Model APIs, instructing users to migrate to dedicated deployments when weights are deprecated (baseten-docs-12), showing some commitment to managing API lifecycle. However, there is no evidence of explicit API versioning (e.g., version numbers, changelogs) and probes for an OpenAPI/swagger spec returned 404s, suggesting no formal versioned API contract is published. Missing for 10: explicit API version scheme, published changelog, machine-readable API spec, and independent confirmation of deprecation timelines.", "evidenceIds": [ "baseten-docs-12", "baseten-probe-3" ] }, { "productId": "baseten", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Baseten's async inference is explicitly described as suited to 'batch work, long documents' where a request ID is returned and completion happens later via webhook or polling, which supports processing many items without holding open connections. However, there is no documented batch/bulk API endpoint for submitting many items in a single call, nor bulk management operations (e.g., bulk model/deployment updates) across the platform. Missing for 10: explicit multi-item batch submission API, bulk resource management (models/deployments/keys) in one operation, and independent evidence of throughput at scale for bulk workloads.", "evidenceIds": [ "baseten-docs-9", "baseten-docs-27", "baseten-docs-35" ] }, { "productId": "baseten", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Baseten's docs describe async inference completing via webhook or polling, but this is a fixed completion-notification mechanism, not a user-defined rules engine for triggering arbitrary actions on events (e.g., alerts, autoscaling policies, custom conditional workflows). No evidence of any rule-definition or event-trigger automation feature exists in the pack.", "evidenceIds": [] }, { "productId": "baseten", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Baseten's docs describe async inference (webhook/polling), training jobs, and deployments, but nothing in the evidence pack mentions a scheduler, cron-like trigger, or recurring/automated job orchestration feature. Since Baseten already runs jobs (training, inference) a scheduling capability is a fair ask, but no evidence shows it exists.", "evidenceIds": [] }, { "productId": "baseten", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Baseten is an ML model deployment and training platform, not an automation/workflow builder that produces user-authored 'automations' needing version review and rollback; the evidence pack only discusses model deployments, training jobs, and inference APIs, none of which map to this axis.", "evidenceIds": [] }, { "productId": "baseten", "storyId": "availability-sla", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Baseten documents Pro/Enterprise tiers, active-active multi-cloud deployments, and a public status page, but no evidence pack item states an actual SLA (uptime percentage, response-time guarantee, or contractual commitment) for any paid or enterprise tier.", "evidenceIds": [ "baseten-docs-24", "baseten-docs-15", "baseten-probe-rt-2" ] }, { "productId": "baseten", "storyId": "batch-discounted-jobs", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Baseten documents asynchronous inference (request ID returned quickly, completion via webhook/polling) explicitly for batch work and long documents, so the submission mechanism is real. However, the only documented pricing discount is for KV-cached input tokens, not for async/batch requests specifically — there is no evidence of a documented discount rate for async vs real-time inference pricing. missing for 10: a documented async/batch discount rate or pricing tier distinct from real-time pricing, first-party pricing page showing the discount percentage.", "evidenceIds": [ "baseten-docs-9", "baseten-docs-27", "baseten-docs-35", "baseten-docs-11", "baseten-docs-10" ] }, { "productId": "baseten", "storyId": "coding-agent-integrations", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Baseten has dedicated first-party docs for connecting coding agents (Claude Code, Codex CLI, Pi) via Baseten Switch, plus a documented MCP server/skill for agent workspace management, and OpenAI/Anthropic-compatible endpoints enabling framework integrations like LangChain. This is corroborated by a live probe confirming the MCP endpoint responds correctly. Missing for 10: independent hands-on validation of the coding-agent connection flow beyond docs/probes.", "evidenceIds": [ "baseten-docs-4", "baseten-docs-5", "baseten-docs-16", "baseten-docs-22", "baseten-docs-34", "baseten-docs-33", "baseten-probe-4", "baseten-probe-rt-3" ] }, { "productId": "baseten", "storyId": "day-zero-model-availability", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence describes Baseten's general capabilities (Model APIs, Truss deployment, training) but contains no claims or evidence about how quickly newly released open-weight models are added to the hosted Model APIs catalog after their public release. Missing for 10: any documentation or track record on model-catalog update cadence, day-1 support claims, or examples of specific new model releases being onboarded quickly.", "evidenceIds": [] }, { "productId": "baseten", "storyId": "dedicated-endpoints-autoscale", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Baseten clearly supports deploying models on dedicated GPUs/dedicated infrastructure (baseten-docs-2, baseten-docs-17, baseten-docs-30) with production-stable environments (baseten-docs-20) and resilient multi-cluster deployment (baseten-docs-15), which is distinct from the shared Model APIs pool (baseten-docs-1). However, no evidence explicitly describes autoscaling behavior for these dedicated deployments (scale-to-zero, concurrency-based scaling, etc.), so the isolation-plus-autoscaling combination is only partially substantiated. Missing for 10: explicit autoscaling documentation for dedicated deployments, and independent/hands-on confirmation of traffic isolation from the shared pool.", "evidenceIds": [ "baseten-docs-2", "baseten-docs-17", "baseten-docs-30", "baseten-docs-20", "baseten-docs-15", "baseten-docs-1" ] }, { "productId": "baseten", "storyId": "documented-rate-limits", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Baseten has a dedicated pricing-and-limits doc covering request/token rate limits, rate-limit response headers, tier upgrades (Basic→Pro/Enterprise via email verification), workspace budgets, and usage monitoring by API key/model. However, it lacks a detailed table of exact numeric limits per tier and doesn't cover rate limits for dedicated-deployment (non-Model-APIs) inference. missing for 10: explicit numeric limit tables per tier, independent/community corroboration of documented limits, coverage of dedicated deployment rate limits (not just Model APIs).", "evidenceIds": [ "baseten-docs-10", "baseten-docs-23", "baseten-docs-24", "baseten-docs-37", "baseten-docs-38" ] }, { "productId": "baseten", "storyId": "drop-in-provider-switch", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Baseten documents OpenAI Chat Completions API compatibility for Model APIs, explicitly enabling drop-in use with familiar client SDKs (e.g., LangChain's ChatOpenAI via base_url swap), and even documents a Switch tool that can route requests between Baseten and other providers like Anthropic/OpenAI mid-workflow. This directly supports agent-driven provider switching without provider-specific code changes. Missing for 10: independent/hands-on confirmation of seamless mid-workflow switching beyond vendor docs, and no explicit statement of full drop-in parity with OpenAI's complete API surface.", "evidenceIds": [ "baseten-docs-21", "baseten-docs-30", "baseten-docs-33", "baseten-docs-16", "baseten-docs-34" ] }, { "productId": "baseten", "storyId": "embeddings-endpoint", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Baseten's general-purpose model deployment (Truss, custom model hosting, transformers/diffusers/vLLM support) can host embedding or reranking models on dedicated infrastructure, and a community report confirms real-world 'Baseten.co embedding workloads' being run on the platform. However, there is no first-party documentation of a dedicated embeddings or rerank API/product page, no mention of specific embedding model catalog entries, or built-in reranking features analogous to competitors' turnkey embedding endpoints. missing for 10: dedicated embeddings/rerank API docs, named embedding/rerank models in the catalog, first-party guidance on building RAG retrieval pipelines end-to-end.", "evidenceIds": [ "baseten-comm-3", "baseten-docs-2", "baseten-docs-17", "baseten-gh-4", "baseten-gh-5" ] }, { "productId": "baseten", "storyId": "fine-tune-and-serve", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Baseten documents Training Jobs/Loops for fine-tuning supported base models (Axolotl/TRL/custom loops), automatic checkpoint syncing, and one-CLI-command deployment of any synced checkpoint to production inference on the same platform, matching the story end-to-end. missing for 10: independent/hands-on third-party confirmation of the fine-tune-to-serve workflow (community evidence only covers unrelated Chains/embedding topics) and explicit list of which base models are 'supported' for fine-tuning.", "evidenceIds": [ "baseten-docs-3", "baseten-docs-13", "baseten-docs-14", "baseten-docs-28", "baseten-docs-29", "baseten-docs-32", "baseten-docs-36" ] }, { "productId": "baseten", "storyId": "low-latency-serving", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Baseten's docs and GitHub materials show explicit architecture for latency-sensitive, high-throughput serving: token-by-token streaming (baseten-docs-8, baseten-docs-26), dedicated GPU deployment with TensorRT-LLM/vLLM/SGLang optimized builds via Truss (baseten-gh-2, baseten-gh-4, baseten-gh-6), automatic KV-cache discounting for repeated prompt tokens (baseten-docs-11), and active-active multi-cluster routing to avoid capacity-related slowdowns (baseten-docs-15). Async inference is also offered for non-latency-critical batch work, showing the platform differentiates paths for latency vs throughput use cases (baseten-docs-9, baseten-docs-27).\nmissing for 10: published quantitative TTFT/throughput benchmarks or independent third-party latency comparisons; one community note (baseten-comm-3) suggests client-side bottlenecks can arise in high-throughput client code, an important but non-contradicting caveat.", "evidenceIds": [ "baseten-docs-8", "baseten-docs-26", "baseten-docs-11", "baseten-docs-15", "baseten-gh-2", "baseten-gh-4", "baseten-gh-6", "baseten-docs-9", "baseten-docs-27", "baseten-comm-3" ] }, { "productId": "baseten", "storyId": "measured-speed-claims", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers Baseten's product features (Model APIs, Truss, training, coding-agent integrations) but contains no published benchmark numbers, tokens-per-second figures, latency claims, or load-testing guides; the only performance-adjacent comment is a third-party observation about client-side GIL bottlenecks, which is unrelated to Baseten's own speed claims.", "evidenceIds": [] }, { "productId": "baseten", "storyId": "model-deprecation-policy", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Baseten has a dedicated deprecation docs page instructing users to migrate to a dedicated deployment with deprecated model weights, showing some documented deprecation handling exists, but the evidence pack shows no explicit advance-notice period, timeline, or notification process. Missing for 10: concrete advance-notice window/SLA, notification channel details, and independent confirmation that notice is actually given before removal.", "evidenceIds": [ "baseten-docs-12" ] }, { "productId": "baseten", "storyId": "models-endpoint-discovery", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A live probe confirms the OpenAI-style GET /v1/models endpoint exists at https://inference.baseten.co/v1/models and returns proper JSON auth-error responses when unauthenticated, consistent with Baseten's documented OpenAI-compatible Chat Completions API (baseten-docs-21, baseten-docs-30). This lets an agent enumerate the model catalog programmatically once authenticated. Missing for 10: explicit first-party documentation of the /v1/models endpoint's response schema/fields and an authenticated hands-on example showing the returned catalog contents.", "evidenceIds": [ "baseten-probe-rt-1", "baseten-docs-21", "baseten-docs-30" ] }, { "productId": "baseten", "storyId": "multimodal-endpoints", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Baseten's Truss framework explicitly supports 'diffusers' (used for image-generation models) and any open-source framework, and the platform lets you deploy 'open-source, fine-tuned, or custom' models on dedicated GPUs, implying vision/audio/image models can be hosted alongside text. However, the hosted 'Model APIs' product is explicitly scoped to 'supported language models' (baseten-docs-1), and no evidence names specific vision, audio, or image-generation models or endpoints being offered as first-class hosted APIs — multimodal support relies on self-deploying via Truss rather than a turnkey catalog like text chat models get.\n\nmissing for 10: explicit vision/audio/image-generation model catalog or API examples, first-party documentation naming specific multimodal models supported out-of-the-box.", "evidenceIds": [ "baseten-docs-1", "baseten-docs-17", "baseten-gh-4", "baseten-gh-5", "baseten-docs-19" ] }, { "productId": "baseten", "storyId": "open-weight-catalog-breadth", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Baseten's docs confirm a serverless 'Model APIs' offering for calling supported language models without deploying them, with OpenAI/Anthropic-compatible endpoints, implying a hosted catalog exists — but no evidence enumerates specific open-weight families like Llama, Qwen, DeepSeek, or GPT-OSS, and the live models endpoint requires an API key to inspect. missing for 10: explicit list of supported model families, breadth/currency of catalog, independent confirmation of specific models available on shared serverless endpoints.", "evidenceIds": [ "baseten-docs-1", "baseten-docs-21", "baseten-docs-30", "baseten-probe-rt-1" ] }, { "productId": "baseten", "storyId": "openai-compatible-endpoint", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Baseten explicitly documents an OpenAI Chat Completions-compatible endpoint for its Model APIs, and shows real-world usage where only base_url and API key need to change (e.g. LangChain's ChatOpenAI pointed at Baseten). A live probe confirms an OpenAI-style /v1/models endpoint responding at inference.baseten.co. Missing for 10: no independent third-party hands-on report explicitly confirming a drop-in swap with the official OpenAI SDK client itself (only LangChain example and docs claims).", "evidenceIds": [ "baseten-docs-21", "baseten-docs-30", "baseten-docs-33", "baseten-probe-rt-1" ] }, { "productId": "baseten", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Baseten's core workflows — deploying models (Truss/config.yaml/CLI), running inference (OpenAI/Anthropic-compatible endpoints, streaming, async, structured outputs, function calling), and training (Loops API calls, Training Jobs CLI deploy) — are all API/CLI-driven, and usage/budget can be queried by API key, suggesting strong API-UI parity for the platform's main functions. However, there is no explicit documentation asserting full UI/API parity, no discoverable OpenAPI/reference spec (probe found only 404s), and no evidence that workspace/account-management UI features (team, billing settings) are exposed via API. Missing for 10: an explicit parity statement or full API reference, confirmation of workspace/admin functions being API-accessible, and independent verification of parity beyond inference/training/deploy workflows.", "evidenceIds": [ "baseten-gh-2", "baseten-docs-28", "baseten-docs-29", "baseten-docs-36", "baseten-docs-9", "baseten-docs-6", "baseten-docs-7", "baseten-docs-38", "baseten-probe-3" ] }, { "productId": "baseten", "storyId": "openness-full-export", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Baseten's tooling leans open: models are packaged via the open-source Truss format (config.yaml, no proprietary container lock-in), and training checkpoints are synced to external storage as the job runs, so a user's model weights and training artifacts aren't trapped in a proprietary format. However there is no explicit documentation of exporting workspace data, logs, usage/billing history, or deployment configs in bulk, and no dedicated 'account export' or data-portability feature. missing for 10: explicit full-account/data export tooling, documentation of exporting logs/usage/config metadata, and any user testimonial of migrating away with all data intact.", "evidenceIds": [ "baseten-gh-1", "baseten-gh-2", "baseten-docs-28", "baseten-docs-29", "baseten-docs-14" ] }, { "productId": "baseten", "storyId": "openness-open-license", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Baseten's core platform (inference service, dashboard, orchestration/MCM) is proprietary SaaS with no evidence of an open license; only the Truss packaging library is open-source on GitHub, which is a peripheral client tool rather than the product itself. missing for 10: no open license for the actual Baseten platform/service source, no repository for the core inference or orchestration engine, only a tangential SDK is open.", "evidenceIds": [ "baseten-gh-1", "baseten-gh-2", "baseten-gh-3", "baseten-gh-4", "baseten-gh-5", "baseten-gh-6" ] }, { "productId": "baseten", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Baseten is a managed cloud inference/training platform (SaaS), not open-source software designed for self-hosting the core product; there is no evidence of a self-hostable core, and the category itself is a hosted service, making this axis inapplicable.", "evidenceIds": [] }, { "productId": "baseten", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions region selection, data residency options, or multi-region storage controls for user data; deployments are described as active-active across clusters/clouds for reliability (baseten-docs-15), not as a user-selectable residency control. Missing for 10: any mention of region/residency selection, data storage location controls, or compliance-driven data locality options.", "evidenceIds": [ "baseten-docs-15" ] }, { "productId": "baseten", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack content addresses data usage for training, opt-out controls, or any privacy/data-retention policy commitments; the axis is applicable (Baseten processes customer data/prompts and could plausibly offer such guarantees) but nothing in the docs, GitHub, or community evidence confirms it.", "evidenceIds": [] }, { "productId": "baseten", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers deployment, training, inference APIs, pricing, and coding-agent integration, but no documentation addresses data retention policies, deletion of inputs/outputs, or user control over stored data/logs. This is a fair axis for an AI inference/hosting platform (buyers routinely ask about data handling), so it applies, but there is no evidence it is delivered.", "evidenceIds": [] }, { "productId": "baseten", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry/usage-tracking opt-out settings for Baseten's platform, CLI, or SDK; the evidence only covers inference features, training, and pricing/usage monitoring for billing purposes, not privacy telemetry controls.", "evidenceIds": [] }, { "productId": "baseten", "storyId": "prompt-caching-support", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs explicitly describe automatic prompt caching: cached input tokens are served from KV cache at a discounted rate with no request flags needed, directly addressing latency/cost reduction on repeated context. missing for 10: no quantified latency/cost benchmarks, no independent/community corroboration of caching behavior in practice.", "evidenceIds": [ "baseten-docs-11", "baseten-docs-10" ] }, { "productId": "baseten", "storyId": "serve-custom-weights", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Baseten documents deploying custom/fine-tuned/open-source models via Truss (config.yaml + weights, supports transformers/diffusers/vLLM/etc.), and its Training product syncs LoRA/checkpoint artifacts that can be deployed as production endpoints with one CLI command, directly covering custom weights and LoRA adapter serving. Missing for 10: explicit LoRA-adapter-specific hot-swapping/multi-adapter serving docs and independent hands-on confirmation beyond vendor docs/GitHub README.", "evidenceIds": [ "baseten-docs-2", "baseten-docs-17", "baseten-gh-1", "baseten-gh-2", "baseten-gh-4", "baseten-docs-28", "baseten-docs-29", "baseten-docs-13" ] }, { "productId": "baseten", "storyId": "spend-controls-budgets", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Baseten docs mention setting a workspace budget and monitoring usage by API key/model, plus rate limits (request/token) that cap throughput, which supports pricing-limits awareness (baseten-docs-10, -23, -37, -38). However, there's no evidence of automated spending caps that hard-stop billing or proactive budget alerts/notifications — the workspace budget appears to be a settable figure without documented enforcement or alerting mechanics. Missing for 10: explicit alerting/notification system, enforcement behavior when budget is exceeded, and independent confirmation that budgets actually cap spend rather than just track it.", "evidenceIds": [ "baseten-docs-10", "baseten-docs-23", "baseten-docs-37", "baseten-docs-38" ] }, { "productId": "baseten", "storyId": "status-page-uptime", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "A live probe confirms Baseten operates a public status page at status.baseten.co that returns HTTP 200 and shows current operational status, which is the core reliability-status signal a founder would check before committing production traffic. However, the evidence pack does not confirm the page includes a visible incident history/archive log, only that it renders current status.\n\nmissing for 10: explicit confirmation that the status page displays past incident history/timeline, not just live status.", "evidenceIds": [ "baseten-probe-rt-2" ] }, { "productId": "baseten", "storyId": "streaming-responses", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Baseten's docs explicitly describe SSE token-by-token streaming ('Return model output token by token as it is generated' / 'Streaming sends tokens as they are generated over server-sent events') as a first-class inference mode alongside async, with OpenAI/Anthropic-compatible client SDK support for easy integration. Missing for 10: no independent/hands-on benchmark or community confirmation of streaming latency/responsiveness in practice.", "evidenceIds": [ "baseten-docs-8", "baseten-docs-26", "baseten-docs-21" ] }, { "productId": "baseten", "storyId": "structured-outputs-json-schema", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Baseten has dedicated first-party docs on structured outputs enforcing JSON schemas for reliable data extraction, plus separate function/tool calling support and OpenAI-compatible client integration (e.g., LangChain with_structured_output) confirming reliable parsing workflows. Missing for 10: independent/hands-on validation of schema-enforcement reliability and no mention of grammar-based (e.g., regex/CFG) constraints beyond JSON schema.", "evidenceIds": [ "baseten-docs-6", "baseten-docs-7", "baseten-docs-25", "baseten-docs-33", "baseten-docs-21" ] }, { "productId": "baseten", "storyId": "tool-calling-fidelity", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Baseten docs confirm function/tool-calling support (\"lets a model choose a tool and produce its arguments from a user request\") and structured outputs for reliable JSON generation, but the pack never mentions parallel tool calls, multi-step agent loops, or any reliability testing on open models. Missing for 10: explicit parallel tool-call support docs, multi-step agent loop examples, and independent/hands-on evidence of reliability across open models.", "evidenceIds": [ "baseten-docs-7", "baseten-docs-25", "baseten-docs-6", "baseten-docs-33" ] }, { "productId": "baseten", "storyId": "transparent-token-pricing", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs confirm Model APIs are billed per token and that usage/rate limits can be queried without contacting sales, but no evidence shows an actual public price list with per-model per-token rates; the models endpoint even requires an API key to enumerate the catalog. missing for 10: a public pricing/rate-card page listing token prices per model, independent confirmation that pricing is visible without account creation or sales contact.", "evidenceIds": [ "baseten-docs-10", "baseten-docs-24", "baseten-probe-rt-1" ] }, { "productId": "cerebras", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Cerebras hosts a live llms.txt index (verified via probe returning HTTP 200 with a structured documentation index) and individual docs pages are also served in markdown form with explicit pointers back to llms.txt, exactly matching the agent-oriented docs pattern. This is directly confirmed by hands-on probes, not just vendor claims. Missing for 10: independent/community confirmation that an agent has actually consumed llms.txt successfully in practice.", "evidenceIds": [ "cerebras-probe-1", "cerebras-probe-2", "cerebras-docs-1" ] }, { "productId": "cerebras", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Cerebras Inference is a raw LLM inference API/platform (chat completions, tool calling, streaming, etc.) used by developers to build other applications; it is not itself a product with user data stores or dashboards that surface 'AI-generated insights from my data.' This story targets an end-user analytics/data product, which is a different category than an inference backend.", "evidenceIds": [] }, { "productId": "cerebras", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Cerebras is an inference API/hardware provider, not an agent-orchestration platform, but its Batch API lets requests be processed asynchronously in the background without immediate results, which is a thin building block for autonomous background automations. There is no evidence of scheduling, triggers, workflow orchestration, or persistent autonomous agents — missing for 10: scheduled/triggered automation framework, agent orchestration/state management, independent evidence of autonomous background runs beyond simple async batch calls.", "evidenceIds": [ "cerebras-docs-8" ] }, { "productId": "cerebras", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "cerebras", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Cerebras Inference is a pure REST API with official Python/Node SDKs, OpenAI-compatible endpoints, and a Batch API for asynchronous request processing—all of which are inherently headless and scriptable for CI/automation pipelines (cerebras-docs-1, cerebras-docs-3, cerebras-docs-8, cerebras-docs-17, cerebras-gh-2). Community reports confirm real-world automated/agentic usage (coding agents, voice assistants) via API keys without needing the console UI (cerebras-comm-6, cerebras-comm-16), though some hit rate-limit friction in automated integrations (cerebras-comm-11). Missing for 10: explicit CI/CD examples (e.g., GitHub Actions), dedicated CLI tool documentation, and no first-party guidance on running in headless/CI environments specifically.", "evidenceIds": [ "cerebras-docs-1", "cerebras-docs-3", "cerebras-docs-8", "cerebras-docs-17", "cerebras-gh-2", "cerebras-comm-6", "cerebras-comm-16", "cerebras-comm-11" ] }, { "productId": "cerebras", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "cerebras", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "cerebras", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "cerebras", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence only shows Python/Node SDKs and a web playground/quickstart; there is no mention of an official Cerebras CLI tool anywhere in the docs, GitHub repos, or community discussion.", "evidenceIds": [ "cerebras-docs-17", "cerebras-gh-2", "cerebras-docs-2" ] }, { "productId": "cerebras", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Cerebras ships a well-documented public REST API (OpenAI-compatible), official SDKs (Python/Node), quickstart docs, streaming, structured outputs, tool calling, batch API, and a live API endpoint confirmed by probe (HTTP 403 auth-gated but functional/JSON). Community evidence confirms real-world usage via SDKs and integrations (Cursor, claude-code-router, llm-cerebras plugin). Missing for 10: a discoverable OpenAPI/swagger spec (probe found 404s on all candidate paths), which would round out formal API documentation.", "evidenceIds": [ "cerebras-docs-1", "cerebras-docs-3", "cerebras-docs-4", "cerebras-docs-5", "cerebras-docs-6", "cerebras-docs-7", "cerebras-docs-8", "cerebras-gh-2", "cerebras-probe-rt-1", "cerebras-probe-1", "cerebras-comm-12", "cerebras-comm-16", "cerebras-probe-3" ] }, { "productId": "cerebras", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of scoped, least-privilege API key/credential issuance (e.g., role-based keys, permission scopes, or per-agent restricted tokens) — docs only mention basic API key usage for authentication, not fine-grained credential scoping. Missing for 10: any mention of scoped/permissioned API keys, role-based access control, or credential restriction features for agents.", "evidenceIds": [ "cerebras-docs-1", "cerebras-docs-3", "cerebras-probe-rt-1" ] }, { "productId": "cerebras", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Cerebras ships official Python and Node/TypeScript SDKs (pip install cerebras_cloud_sdk, cerebras-cloud-sdk-node on GitHub) plus OpenAI-compatible client support, with docs covering streaming, tool calling, structured outputs, and batch APIs—clearly agentic-workflow-friendly. Community evidence confirms real-world SDK/agent integration (coding agents, Cursor support) though with some rough edges like rate-limit friction. Missing for 10: independent quality assessment of SDK docs/API reference completeness and broader language SDK coverage beyond Python/Node.", "evidenceIds": [ "cerebras-docs-17", "cerebras-gh-2", "cerebras-gh-1", "cerebras-docs-3", "cerebras-docs-4", "cerebras-docs-7", "cerebras-docs-5", "cerebras-docs-6", "cerebras-comm-12", "cerebras-comm-6" ] }, { "productId": "cerebras", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack mentions webhooks, event subscriptions, or callback-based notifications; Cerebras Inference appears to be a synchronous/streaming inference API with no webhook mechanism documented.", "evidenceIds": [] }, { "productId": "cerebras", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Cerebras docs offer a quickstart with code snippets and a no-key Cloud Console playground for testing prompts, plus rich per-capability doc pages (streaming, tool use, structured outputs) with example code. However, there's no evidence of a true interactive API reference (e.g., Swagger/OpenAPI 'try it' explorer) — a probe explicitly found no openapi.json/swagger spec at expected paths, and no citation shows runnable code execution directly embedded in the reference docs. missing for 10: an OpenAPI/Swagger-style interactive reference, confirmation that code examples in docs are directly runnable/editable in-browser, independent user confirmation of using such a feature.", "evidenceIds": [ "cerebras-docs-1", "cerebras-docs-2", "cerebras-docs-3", "cerebras-probe-3", "cerebras-probe-1" ] }, { "productId": "cerebras", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Cerebras is OpenAI-API-compatible and documents endpoints extensively, but a direct probe for machine-readable spec files (openapi.json, swagger.json, etc.) returned 404 on all candidate paths, and no evidence pack item links to a downloadable OpenAPI/Swagger spec.", "evidenceIds": [ "cerebras-probe-3" ] }, { "productId": "cerebras", "storyId": "api-sandbox", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Cerebras offers a no-key Cloud Console playground and $5 free credits to 'prototype prompts, agents, and real-time apps before you spend a dollar,' which lets a user experiment without hitting a paid/production billing tier, but there is no dedicated 'sandbox' API mode, test keys, or explicit separation from production data/environment documented. missing for 10: an explicit sandbox/test-mode endpoint or key type, documentation guaranteeing isolation from production data, and independent confirmation that free-tier usage never touches the same infra as production workloads.", "evidenceIds": [ "cerebras-docs-2", "cerebras-docs-13", "cerebras-docs-25" ] }, { "productId": "cerebras", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of API versioning scheme or a documented deprecation policy; docs cover OpenAI-compatible endpoints, streaming, tool use, etc., but nothing on version lifecycle or deprecation timelines. The openapi.json probe even returned 404s, and no changelog or deprecation notice pages are cited.", "evidenceIds": [ "cerebras-probe-3", "cerebras-docs-3", "cerebras-docs-4" ] }, { "productId": "cerebras", "storyId": "automation-bulk-operations", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Cerebras offers a documented Batch API for processing groups of requests asynchronously, directly supporting bulk operations across many items, and rate-limit/caching docs describe handling high-volume token throughput. missing for 10: no hands-on/community evidence validating batch API at scale, and no details on batch size limits or job management UX.", "evidenceIds": [ "cerebras-docs-8", "cerebras-docs-12", "cerebras-docs-19", "cerebras-docs-22" ] }, { "productId": "cerebras", "storyId": "automation-rules-engine", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Cerebras Inference is a raw inference API/compute provider (fast LLM inference, OpenAI-compatible endpoint, tool calling, batch processing); it has no concept of user-defined trigger rules or event-driven automation—that's a workflow/automation platform axis, not an inference API axis.", "evidenceIds": [] }, { "productId": "cerebras", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Cerebras Inference is a raw inference API/hardware platform (chat completions, batch, streaming, tool calls) — it provides no job scheduler, cron, or workflow orchestration layer for recurring automated tasks. Scheduling recurring jobs is a workflow/orchestration concern that belongs to a client application built on top of the API, not to the inference service itself.", "evidenceIds": [] }, { "productId": "cerebras", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Cerebras Inference is an inference API/hardware service, not an automation/workflow builder; there is no concept of versioning, reviewing, or rolling back 'automations' in this product category. This story applies to workflow/agent-builder tools, not a raw inference API provider.", "evidenceIds": [] }, { "productId": "cerebras", "storyId": "availability-sla", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows a public status page and dedicated/reserved-capacity endpoints for enterprise customers, but nowhere in docs or pricing pages is an explicit uptime SLA percentage, credit policy, or contractual availability guarantee stated for paid or enterprise tiers. Community threads even highlight unpredictable rate-limiting and onboarding issues rather than confirming a formal SLA.", "evidenceIds": [ "cerebras-docs-9", "cerebras-docs-24", "cerebras-probe-rt-2", "cerebras-comm-11" ] }, { "productId": "cerebras", "storyId": "batch-discounted-jobs", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Cerebras documents a Batch API for asynchronous, non-immediate processing of grouped requests (cerebras-docs-8), satisfying the async batch-submission part of the story. However, no evidence pack item documents a specific discounted price for batch jobs versus real-time pricing — the pricing pages only mention free credits, $10 self-serve tier, and rate-limit multipliers (cerebras-docs-13, cerebras-docs-14, cerebras-docs-16), not a batch discount. missing for 10: documented batch discount percentage or rate card, independent confirmation of batch pricing savings.", "evidenceIds": [ "cerebras-docs-8", "cerebras-docs-13", "cerebras-docs-14", "cerebras-docs-16" ] }, { "productId": "cerebras", "storyId": "coding-agent-integrations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Cerebras publishes first-party OpenAI-compatibility docs and SDKs (cerebras-docs-3, cerebras-docs-4, cerebras-gh-1/2) that let any OpenAI-compatible coding agent or framework plug in by swapping API key/base URL, and community reports confirm 'official support' for tools like Cursor (cerebras-comm-12). However there is no dedicated first-party guide for specific agent frameworks (e.g., LangChain, Cursor, Claude Code) beyond generic OpenAI-compat instructions, and one report shows friction integrating with claude-code-router (422 wrong_api_format, cerebras-comm-11). Missing for 10: explicit named integration guides/tutorials for popular coding agents or agent frameworks, and confirmation that such integrations work smoothly end-to-end.", "evidenceIds": [ "cerebras-docs-3", "cerebras-docs-4", "cerebras-gh-1", "cerebras-gh-2", "cerebras-comm-12", "cerebras-comm-11" ] }, { "productId": "cerebras", "storyId": "day-zero-model-availability", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Community comments suggest Cerebras adds new open-weight models fairly often (e.g. hosting Qwen 3.8 27B, DFlash2 draft model) and docs show a model catalog exists, but there's no concrete evidence of turnaround time from a model's public release to availability on Cerebras, and one comment notes a newly released model (Qwen 3.8) wasn't yet available via a partner (OpenRouter), implying some lag. missing for 10: documented release-to-availability timelines, first-party announcements tying model launches to Cerebras availability, and independent confirmation of consistent fast onboarding of new open-weight models.", "evidenceIds": [ "cerebras-comm-1", "cerebras-comm-4", "cerebras-comm-6", "cerebras-docs-18", "cerebras-docs-21" ] }, { "productId": "cerebras", "storyId": "dedicated-endpoints-autoscale", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Cerebras documents dedicated endpoints as private, reserved capacity isolated from the shared serverless pool, including support for custom fine-tuned models — directly matching the isolation requirement. However, there is no evidence of autoscaling on dedicated capacity; docs describe reserved/provisioned instances without any scaling mechanism mentioned. Missing for 10: explicit autoscaling capability, capacity/quota controls, and independent confirmation of dedicated-endpoint behavior in production.", "evidenceIds": [ "cerebras-docs-9", "cerebras-docs-24", "cerebras-docs-10" ] }, { "productId": "cerebras", "storyId": "documented-rate-limits", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Cerebras has a dedicated rate-limits doc explaining TPM/RPM mechanics and how cache hit rate affects effective throughput (cerebras-docs-12/19/22), and the pricing page states tiers scale limits (e.g., 10x higher on the $10 self-serve tier vs free) (cerebras-docs-14/20/26). Community reports confirm rate limits are real and enforced in practice (cerebras-comm-11), corroborating the docs. However, the evidence never shows concrete numeric RPM/TPM values per tier or a full scaling table across all tiers (free, self-serve, dedicated), so a developer can't fully predict exact limits before hitting them. Missing for 10: explicit numeric rate-limit tables per tier, dedicated-endpoint tier limits, and independent confirmation that documented numbers match real-world enforcement.", "evidenceIds": [ "cerebras-docs-12", "cerebras-docs-19", "cerebras-docs-22", "cerebras-docs-14", "cerebras-docs-20", "cerebras-docs-26", "cerebras-comm-11" ] }, { "productId": "cerebras", "storyId": "drop-in-provider-switch", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Cerebras explicitly documents OpenAI-compatible chat completions requiring only base URL/API key/model swap ('two code changes'), and supports streaming, tool calling, image_url content, and structured outputs matching OpenAI's API shape (cerebras-docs-3, cerebras-docs-4, cerebras-docs-15, cerebras-docs-23). However, real-world integration reports show friction: a user hit a 'wrong_api_format' 422 error integrating with claude-code-router (cerebras-comm-11), and Cursor integration was described as 'patchy' before official support was added (cerebras-comm-12), suggesting the compatibility layer isn't always frictionless in practice. Missing for 10: independent verification of drop-in compatibility across multiple agent frameworks without errors, and no OpenAPI spec is publicly served (cerebras-probe-3) to confirm exact schema parity.", "evidenceIds": [ "cerebras-docs-3", "cerebras-docs-4", "cerebras-docs-15", "cerebras-docs-23", "cerebras-comm-11", "cerebras-comm-12" ] }, { "productId": "cerebras", "storyId": "embeddings-endpoint", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of embeddings or reranking models/endpoints anywhere in the docs, SDKs, or model catalog — Cerebras Inference documentation focuses solely on chat/completions, tool use, structured outputs, and streaming for LLMs. No mention of an embeddings API or reranking capability.", "evidenceIds": [] }, { "productId": "cerebras", "storyId": "fine-tune-and-serve", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Cerebras dedicated endpoints explicitly support deploying custom fine-tuned models alongside standard variants on reserved capacity (cerebras-docs-9, cerebras-docs-10, cerebras-docs-24), covering the 'serve' half of the story. However, there is no evidence that Cerebras itself provides a fine-tuning service/API — the docs imply fine-tuning happens elsewhere and the resulting model is uploaded/deployed to a dedicated endpoint, not that the platform trains it. missing for 10: an actual fine-tuning API/pipeline on Cerebras, documentation of supported base models for tuning, and any hands-on confirmation of the full fine-tune-then-serve workflow.", "evidenceIds": [ "cerebras-docs-9", "cerebras-docs-10", "cerebras-docs-24" ] }, { "productId": "cerebras", "storyId": "low-latency-serving", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Cerebras' whole value proposition centers on speed: dedicated wafer-scale inference, TTFT-optimized SDK warming (cerebras-gh-1), streaming API (cerebras-docs-5), dedicated non-shared capacity for consistent latency (cerebras-docs-24), and cache-hit optimizations for throughput (cerebras-docs-12/19/22). Independent hands-on community reports strongly corroborate extreme throughput/low-latency (300+ tok/s coding model, beating 8x H100 clusters, 'needlessly fast' code gen) across multiple HN threads (cerebras-comm-1,6,7,10,13,14,15). Missing for 10: no first-party published TTFT/tok-s benchmark numbers in this pack, and some community reports note rate-limit throttling and onboarding friction that slightly tempers the sustained-throughput story (cerebras-comm-11,4).", "evidenceIds": [ "cerebras-gh-1", "cerebras-docs-5", "cerebras-docs-24", "cerebras-docs-12", "cerebras-docs-19", "cerebras-docs-22", "cerebras-comm-1", "cerebras-comm-6", "cerebras-comm-7", "cerebras-comm-10", "cerebras-comm-13", "cerebras-comm-14", "cerebras-comm-15", "cerebras-comm-11" ] }, { "productId": "cerebras", "storyId": "measured-speed-claims", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "There is no first-party benchmark page or load-testing guide in the evidence pack, but community reports repeatedly cite concrete tok/s figures (e.g., 'break 300 tok/s', comparisons showing Cerebras far outpacing 100 tok/s H100 clusters) and the SDK docs mention a TTFT-reduction mechanism, giving some quantitative backing for speed claims. Missing for 10: an official published benchmark/whitepaper with tokens-per-second numbers, a load-testing guide, or independent third-party benchmark reports (e.g., Artificial Analysis) directly cited in the pack.", "evidenceIds": [ "cerebras-comm-6", "cerebras-comm-7", "cerebras-comm-10", "cerebras-comm-13", "cerebras-gh-1" ] }, { "productId": "cerebras", "storyId": "model-deprecation-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item describes a deprecation policy, model retirement timeline, or advance-notice process for hosted models; documentation covers model selection but not lifecycle/deprecation.", "evidenceIds": [] }, { "productId": "cerebras", "storyId": "models-endpoint-discovery", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "A live probe confirms `GET https://api.cerebras.ai/v1/models` is a real, JSON-speaking, OpenAI-style endpoint (403 unauthenticated, not 404), and docs repeatedly assert OpenAI API compatibility and a public models catalog page (cerebras-docs-3, -4, -21). However, no first-party doc page explicitly documents the /v1/models endpoint schema/response, nor is there an OpenAPI spec (all openapi.json paths 404). missing for 10: explicit documented endpoint reference/response schema for /v1/models, no OpenAPI spec confirmation, no independent hands-on report of enumerating the catalog via this endpoint.", "evidenceIds": [ "cerebras-probe-rt-1", "cerebras-docs-3", "cerebras-docs-4", "cerebras-docs-21", "cerebras-probe-3" ] }, { "productId": "cerebras", "storyId": "multimodal-endpoints", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs show only a narrow vision capability (base64 image_url input support for chat completions) but no evidence of dedicated vision, audio, or image-generation models being served on the platform, nor documentation of separate multimodal model endpoints. missing for 10: audio input/output model support, image-generation model support, explicit vision-model catalog entries, and any hands-on/community confirmation of using multimodal (non-text) capabilities.", "evidenceIds": [ "cerebras-docs-15", "cerebras-docs-21", "cerebras-docs-18" ] }, { "productId": "cerebras", "storyId": "open-weight-catalog-breadth", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Cerebras docs confirm a public model catalog with guidance to pick the right model and a models/overview page for browsing all available public endpoints, and community reports confirm live usage of Llama 3.1 70B and Qwen models on shared endpoints. However, no evidence explicitly names DeepSeek or GPT-OSS in the catalog, and one community note flags a context-size limitation (128k) for at least one hosted model, suggesting the catalog's breadth/parity with 'peers' isn't fully documented. missing for 10: explicit confirmation of DeepSeek and GPT-OSS availability, and a full enumerated model list showing parity across all cited families.", "evidenceIds": [ "cerebras-docs-11", "cerebras-docs-18", "cerebras-docs-21", "cerebras-comm-1", "cerebras-comm-4", "cerebras-comm-7" ] }, { "productId": "cerebras", "storyId": "openai-compatible-endpoint", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Official docs explicitly state existing OpenAI SDK apps can switch to Cerebras by changing only the API key, base URL, and model ID, and marketing reiterates 'just two code changes'; a live probe confirms the API serves an OpenAI-style /v1/models endpoint. Community reports (e.g. Cursor/OpenRouter integrations) corroborate real-world drop-in usage. Missing for 10: independent hands-on confirmation of a literal SDK base_url swap with zero other code changes.", "evidenceIds": [ "cerebras-docs-3", "cerebras-docs-4", "cerebras-docs-23", "cerebras-probe-rt-1", "cerebras-comm-12" ] }, { "productId": "cerebras", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Cerebras's API surface is extensive and well-documented (chat completions, streaming, tool calling, structured outputs, batch API, dedicated endpoints, model catalog) and the console playground is explicitly described as just a convenience UI on top of the same API, suggesting strong API/UI parity. However there's no explicit statement or independent verification that every console feature (e.g., dedicated-endpoint provisioning, billing/account management, fine-tuned model deployment) is fully API-driven rather than requiring console/sales interaction, and community reports mention waitlists and manual onboarding steps outside the API. Missing for 10: explicit parity documentation, evidence that account/billing/deployment actions are API-accessible, and independent confirmation of full UI-to-API feature parity.", "evidenceIds": [ "cerebras-docs-2", "cerebras-docs-3", "cerebras-docs-4", "cerebras-docs-9", "cerebras-docs-10", "cerebras-docs-24", "cerebras-comm-8", "cerebras-comm-17" ] }, { "productId": "cerebras", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence describes any data-export mechanism (usage logs, fine-tuned model weights, account data) in open formats; the closest analog—OpenAI API compatibility—only covers code portability for switching inference providers, not actual data export/exit tooling. Missing for 10: any documented data export feature, format, or exit/offboarding process.", "evidenceIds": [] }, { "productId": "cerebras", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Cerebras Inference is a closed, hosted API service; the evidence pack shows only client SDKs (Node/Python) on GitHub, not the source of the inference service or model weights under an open license. There is no evidence the core product's source code is available for review.", "evidenceIds": [] }, { "productId": "cerebras", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Cerebras Inference is a cloud API/hardware service built around proprietary wafer-scale chips; there is no evidence of a downloadable/self-hostable core product, and self-hosting is not a plausible axis for this hardware-dependent inference cloud service.", "evidenceIds": [] }, { "productId": "cerebras", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions data residency, regional storage options, or any control over where data is processed/stored; only dedicated endpoint (capacity isolation, not geography) and general API docs are present.", "evidenceIds": [] }, { "productId": "cerebras", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item addresses a data-privacy/training-opt-out policy, data retention terms, or zero-retention agreement for Cerebras Inference API usage; nothing confirms user data is excluded from model training.", "evidenceIds": [] }, { "productId": "cerebras", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item discusses data retention policies, deletion controls, or privacy/data-handling settings for Cerebras Inference; documentation focuses on API usage, performance, and pricing only.", "evidenceIds": [] }, { "productId": "cerebras", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry, usage tracking, or opt-out privacy controls for Cerebras Inference; the docs focus on API usage, pricing, and performance, not data collection settings.", "evidenceIds": [] }, { "productId": "cerebras", "storyId": "prompt-caching-support", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm Cerebras supports prompt/prefix caching for rate-limit purposes—cached tokens don't count toward uncached TPM limits, letting engineers push more total throughput—implying reduced cost/latency impact on repeated context. However, there's no explicit documentation on cache TTL, hit-rate mechanics, pricing discount for cached tokens, or independent benchmarks showing actual latency/cost reduction from caching in practice. missing for 10: documented cache pricing/discount, cache TTL/eviction details, independent latency benchmarks demonstrating caching benefit.", "evidenceIds": [ "cerebras-docs-12", "cerebras-docs-19", "cerebras-docs-22" ] }, { "productId": "cerebras", "storyId": "serve-custom-weights", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Cerebras' dedicated endpoint docs state customers can 'deploy your custom fine-tuned models alongside standard model variants' on reserved capacity, which implies some path to serve custom fine-tuned weights — but this is only mentioned for the enterprise 'dedicated endpoint' tier, not the standard self-serve API, and there is no mention of LoRA adapter support, upload workflow, or self-serve model registration. Missing for 10: LoRA adapter upload/serving, self-serve (non-dedicated) custom weight upload process, independent/hands-on confirmation that custom fine-tuned models can actually be deployed.", "evidenceIds": [ "cerebras-docs-9", "cerebras-docs-10", "cerebras-docs-24" ] }, { "productId": "cerebras", "storyId": "spend-controls-budgets", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of spending caps, budget alerts, or usage-limit controls in Cerebras docs; only rate-limit tiers and free credit amounts are mentioned, not billing caps or alerts a founder could set to bound spend.", "evidenceIds": [ "cerebras-docs-12", "cerebras-docs-13", "cerebras-docs-14" ] }, { "productId": "cerebras", "storyId": "status-page-uptime", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "A live public status page (status.cerebras.ai) was confirmed via probe, returning HTTP 200 and showing an 'operational' status, which supports founders checking service health before committing production traffic. However, there's no evidence in the pack of a visible incident history log or historical uptime records on that page, and community threads note some real-world reliability hiccups (rate-limit bursts, onboarding scaling issues) without connecting them to the status page. Missing for 10: documented incident history/timeline on the status page, uptime SLA data, and independent confirmation that past incidents are publicly logged.", "evidenceIds": [ "cerebras-probe-rt-2", "cerebras-comm-5", "cerebras-comm-11" ] }, { "productId": "cerebras", "storyId": "streaming-responses", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Official docs explicitly describe streaming responses that send messages back in chunks and display them incrementally as the model generates them, and the SDK/API are OpenAI-compatible so standard SSE streaming semantics apply; community reports independently corroborate extremely fast token generation experienced in real-time apps. missing for 10: explicit SSE protocol details/code sample and independent hands-on confirmation specifically of streaming (vs just raw speed).", "evidenceIds": [ "cerebras-docs-5", "cerebras-docs-3", "cerebras-docs-4", "cerebras-comm-13", "cerebras-comm-7" ] }, { "productId": "cerebras", "storyId": "structured-outputs-json-schema", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Cerebras docs explicitly describe a Structured Outputs feature that constrains model responses to a JSON schema for reliable parsing, alongside OpenAI-API compatibility that typically carries this through standard SDKs. Missing for 10: independent/hands-on confirmation of schema-enforcement reliability, details on grammar-based constraints beyond JSON schema, and coverage of edge cases (nested schemas, strict mode) in evidence.", "evidenceIds": [ "cerebras-docs-6", "cerebras-docs-3", "cerebras-docs-4" ] }, { "productId": "cerebras", "storyId": "tool-calling-fidelity", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Cerebras documents a tool-calling/function-calling capability (cerebras-docs-7) as part of its OpenAI-compatible API, and community reports mention using it for coding agents (cerebras-comm-6, cerebras-comm-12) suggesting real agentic integrations exist. However, the docs pack contains no detail on parallel or multi-step tool-call handling, and one user reports API format errors when integrating with an agent router (cerebras-comm-11), hinting at possible friction in tool-use compatibility. missing for 10: explicit documentation of parallel tool calls, multi-step tool-call chaining, and independent benchmarks confirming reliability of tool-calling in long agent loops.", "evidenceIds": [ "cerebras-docs-7", "cerebras-comm-6", "cerebras-comm-12", "cerebras-comm-11" ] }, { "productId": "cerebras", "storyId": "transparent-token-pricing", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Cerebras publishes a public pricing page with self-serve signup, free credits, and tiered self-serve pricing ($10 minimum, 10x rate limits), showing pricing information is not gated behind a sales call for the basic tier (cerebras-docs-13,14,20,25,26). However, none of the evidence shows an actual published per-token $/M-token rate for each hosted model, and one pricing-related doc references access via 'partner APIs' plus a community comment explicitly wonders whether trying the cloud is possible without contacting sales, suggesting the full price list isn't clearly self-evident. Missing for 10: an explicit per-model per-token price table, confirmation that all hosted models (not just self-serve tiers) have listed rates, and independent corroboration that no sales contact is needed to see model-level pricing.", "evidenceIds": [ "cerebras-docs-13", "cerebras-docs-14", "cerebras-docs-20", "cerebras-docs-25", "cerebras-docs-16", "cerebras-comm-17" ] }, { "productId": "deepinfra", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Direct probes show llms.txt, docs.md, and openapi.json all return 404 on deepinfra.com, and no evidence pack item claims an agent-oriented docs format exists; the only machine-readable endpoint found is the models list, not documentation.", "evidenceIds": [ "deepinfra-probe-1", "deepinfra-probe-2", "deepinfra-probe-3" ] }, { "productId": "deepinfra", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "DeepInfra is a raw inference API/cloud platform (LLMs, vision, embeddings, image/video generation) for developers to build on, not an end-user product that stores 'your data' and surfaces AI-generated insights/suggestions within its own UI. This story applies to data-holding SaaS products, not to an infrastructure/API provider.", "evidenceIds": [ "deepinfra-docs-11", "deepinfra-docs-19" ] }, { "productId": "deepinfra", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "deepinfra", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "deepinfra", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "DeepInfra is a REST/OpenAI-compatible API (base_url + api_key model, keyless model enumeration) that is inherently scriptable and stateless, making it straightforward to call from CI/automation pipelines without any UI dependency. missing for 10: no explicit CI/CD examples (GitHub Actions, cron jobs), no official CLI, and no independent hands-on report of someone running it headlessly in a pipeline.", "evidenceIds": [ "deepinfra-docs-15", "deepinfra-docs-25", "deepinfra-probe-rt-1", "deepinfra-docs-12" ] }, { "productId": "deepinfra", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "deepinfra", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "DeepInfra is an inference API/platform (not an agent), so an official MCP server is a fair axis to expect, but no evidence in the pack mentions MCP support at all—only OpenAI-compatible REST API docs.", "evidenceIds": [] }, { "productId": "deepinfra", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "deepinfra", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an official DeepInfra CLI tool; docs focus on REST/OpenAI-SDK compatibility, and probes for docs/openapi/llms.txt turned up nothing about a CLI. DeepInfra is an inference API/cloud platform, so an official CLI is a plausible and fair capability to ask about, but none is evidenced.", "evidenceIds": [ "deepinfra-docs-1", "deepinfra-probe-1", "deepinfra-probe-2", "deepinfra-probe-3" ] }, { "productId": "deepinfra", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "DeepInfra publishes extensive documentation for a fully OpenAI-compatible REST API (chat completions, embeddings, vision, image/video, speech, service tiers, reasoning_effort, fail_fast, etc.), confirmed live via a keyless probe that enumerated 189 models through the documented endpoint. Missing for 10: no discoverable OpenAPI/swagger spec or llms.txt for machine-readable agent consumption (all returned 404), so an AI agent must rely on human-oriented docs rather than a formal machine spec.", "evidenceIds": [ "deepinfra-docs-1", "deepinfra-docs-2", "deepinfra-docs-15", "deepinfra-docs-25", "deepinfra-docs-26", "deepinfra-docs-27", "deepinfra-docs-28", "deepinfra-probe-rt-1", "deepinfra-probe-1", "deepinfra-probe-3" ] }, { "productId": "deepinfra", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of scoped or least-privilege API key/credential management (e.g., per-agent tokens, permission scoping, role-based keys) — only a single generic API token model is implied via base_url/api_key usage. Missing for 10: any docs on creating multiple scoped keys, permission/role restrictions, credential expiration, or per-agent token issuance.", "evidenceIds": [ "deepinfra-docs-15", "deepinfra-docs-30" ] }, { "productId": "deepinfra", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "DeepInfra explicitly documents OpenAI SDK compatibility as a drop-in replacement (just change base_url, api_key, model) with no code migration required, and a live probe confirms the OpenAI-compatible endpoint is functional and enumerable. This directly satisfies building against an official/well-known SDK ecosystem rather than a bespoke library. missing for 10: no dedicated first-party DeepInfra-branded SDK packages mentioned (relies on OpenAI SDK compatibility), and no independent third-party developer corroboration beyond docs and endpoint probes.", "evidenceIds": [ "deepinfra-docs-1", "deepinfra-docs-14", "deepinfra-docs-15", "deepinfra-docs-25", "deepinfra-docs-30", "deepinfra-probe-rt-1" ] }, { "productId": "deepinfra", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any webhook subscription or event-notification mechanism anywhere in the docs; DeepInfra is a request/response inference API with no mention of async event callbacks.", "evidenceIds": [] }, { "productId": "deepinfra", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The docs pack shows static code snippets (e.g., OpenAI SDK base_url examples) but no evidence of an interactive, runnable API console or playground; probes for openapi.json/swagger.json all returned 404, and no llms.txt/docs.md machine-readable spec exists, indicating no interactive reference is exposed.", "evidenceIds": [ "deepinfra-probe-1", "deepinfra-probe-2", "deepinfra-probe-3", "deepinfra-docs-15" ] }, { "productId": "deepinfra", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Direct probes for llms.txt, docs.md, and common OpenAPI/swagger spec paths all returned 404, indicating no machine-readable API spec is published; the docs only describe OpenAI SDK compatibility, not a downloadable OpenAPI/JSON schema.", "evidenceIds": [ "deepinfra-probe-1", "deepinfra-probe-2", "deepinfra-probe-3" ] }, { "productId": "deepinfra", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes DeepInfra as a pay-per-token inference API with zero data retention, private dedicated deployments, and GPU clusters, but there is no mention of a distinct sandbox/test environment, test API keys, or any mechanism to separate test traffic from production data — instead it just uses live API keys against production endpoints. missing for 10: sandbox/test environment or mode, staging API keys, documented separation between test and production data flows.", "evidenceIds": [ "deepinfra-docs-9", "deepinfra-docs-23", "deepinfra-docs-12" ] }, { "productId": "deepinfra", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows OpenAI-compatible endpoints and a model catalog but contains no documentation of API versioning scheme, changelog, or deprecation/sunset policy for models or endpoints; probes even show missing openapi/spec files (404s), reinforcing the absence of such formal API-lifecycle documentation.", "evidenceIds": [ "deepinfra-probe-2", "deepinfra-probe-3", "deepinfra-docs-15" ] }, { "productId": "deepinfra", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The docs mention a 'flex' pricing tier suited to asynchronous, non-production workloads like data enrichment and model evaluations, but there is no documented batch API, bulk upload endpoint, or mechanism to submit and process many items in a single call — only single-request chat/completions semantics are described.", "evidenceIds": [ "deepinfra-docs-5", "deepinfra-docs-27" ] }, { "productId": "deepinfra", "storyId": "automation-rules-engine", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "DeepInfra is an inference API/hosting platform, not an automation/workflow product; it provides no event-trigger or rules-engine capability, and this axis is a category error for its product type.", "evidenceIds": [] }, { "productId": "deepinfra", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "DeepInfra is an inference API/GPU hosting platform, not an orchestration or workflow-scheduling product; scheduling recurring jobs is outside its product category (wrong axis) rather than a missing feature.", "evidenceIds": [] }, { "productId": "deepinfra", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "DeepInfra is an inference API/GPU cloud provider, not an automation/workflow builder; there is no concept of user-created automations to version, review, or roll back — this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "deepinfra", "storyId": "availability-sla", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack includes pricing, features, and a public status page, but nowhere states an uptime/availability SLA percentage or enterprise contractual guarantee — no mention of SLA terms for paid or enterprise tiers.", "evidenceIds": [ "deepinfra-probe-rt-2", "deepinfra-docs-31", "deepinfra-docs-24" ] }, { "productId": "deepinfra", "storyId": "batch-discounted-jobs", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "DeepInfra documents a 'flex' service_tier that gives a 20% discount versus standard pricing and is explicitly recommended for 'asynchronous workloads' and lower-priority tasks, which partially matches the story's intent of discounted async processing. However, this is a request-level QoS flag on the standard synchronous chat API, not a true batch-job system (no file upload, job submission ID, or polling/webhook completion pattern like OpenAI's Batch API). Missing for 10: a documented batch-job submission endpoint (upload/queue a set of prompts, get a job ID, poll or receive callback), explicit SLA/turnaround time for batch jobs, and independent confirmation of the discount and workflow in practice.", "evidenceIds": [ "deepinfra-docs-5", "deepinfra-docs-16", "deepinfra-docs-27" ] }, { "productId": "deepinfra", "storyId": "coding-agent-integrations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "DeepInfra documents a drop-in OpenAI-compatible endpoint (base_url swap, same SDK, model catalog) which is the mechanism most coding agents/agent frameworks rely on to add a custom provider, and a live probe confirms the OpenAI-style /models endpoint works keylessly. However, the evidence pack contains no first-party guide naming or walking through integration with any specific coding agent or agent framework (e.g., LangChain, LlamaIndex, Cursor, Continue) — only generic 'point your OpenAI SDK here' instructions. missing for 10: named integration guides for specific coding agents/agent frameworks, third-party/community confirmation of successful agent hookups.", "evidenceIds": [ "deepinfra-docs-1", "deepinfra-docs-2", "deepinfra-docs-15", "deepinfra-docs-30", "deepinfra-probe-rt-1" ] }, { "productId": "deepinfra", "storyId": "day-zero-model-availability", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack shows a broad model catalog (189 models) and general inference features, but nothing documents DeepInfra's speed in adding newly released open-weight models after public release — no changelog, launch-day availability claims, or timing data are present.", "evidenceIds": [ "deepinfra-probe-rt-1" ] }, { "productId": "deepinfra", "storyId": "dedicated-endpoints-autoscale", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "DeepInfra docs explicitly offer dedicated instance deployment on A100/H100/H200/B200/B300 GPUs with autoscaling and a private endpoint for data isolation, directly matching the story's request for isolated, dedicated capacity separate from the shared serverless pool. missing for 10: independent/hands-on confirmation of autoscaling behavior under load, SLA/isolation guarantees, and pricing/latency benchmarks for dedicated vs shared tiers.", "evidenceIds": [ "deepinfra-docs-8", "deepinfra-docs-9", "deepinfra-docs-18" ] }, { "productId": "deepinfra", "storyId": "documented-rate-limits", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows DeepInfra documents a fail_fast parameter to get immediate 429s and priority/flex service tiers for speed vs cost tradeoffs, but there is no documentation of actual rate limit numbers, quotas, or how limits scale across usage/spend tiers. Missing for 10: explicit rate-limit tables, per-tier request/token quotas, and any usage-tier scaling documentation.", "evidenceIds": [ "deepinfra-docs-6", "deepinfra-docs-26", "deepinfra-docs-27", "deepinfra-docs-28" ] }, { "productId": "deepinfra", "storyId": "drop-in-provider-switch", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "DeepInfra explicitly documents a drop-in OpenAI-compatible endpoint requiring only base_url/api_key/model changes, and a live probe confirms the OpenAI-compatible models endpoint is reachable and functional, supporting agents switching providers without provider-specific code. Missing for 10: independent third-party (non-vendor) confirmation of seamless mid-workflow agent switching in practice.", "evidenceIds": [ "deepinfra-docs-1", "deepinfra-docs-14", "deepinfra-docs-15", "deepinfra-docs-25", "deepinfra-docs-30", "deepinfra-probe-rt-1" ] }, { "productId": "deepinfra", "storyId": "embeddings-endpoint", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "DeepInfra explicitly offers embeddings and reranking models via an OpenAI-compatible API alongside LLMs, letting developers build retrieval pipelines on one vendor. Missing for 10: independent hands-on validation of embedding/rerank endpoints, benchmark quality comparisons, and detailed API-parameter docs for these specific endpoints.", "evidenceIds": [ "deepinfra-docs-19", "deepinfra-docs-11", "deepinfra-docs-1" ] }, { "productId": "deepinfra", "storyId": "fine-tune-and-serve", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "DeepInfra documents deploying your own fine-tuned weights for serving (docs-8/9/18) and separately offers GPU clusters with SSH for training (docs-10), so the pieces to fine-tune-then-serve exist, but there is no evidence of an integrated managed fine-tuning job/dataset-upload API on a supported base model — customers must train themselves on rented GPUs and then manually deploy the resulting weights as a private endpoint. Missing for 10: a first-party fine-tuning job service (upload dataset, select base model, launch job) that outputs directly into the serving catalog, and any hands-on/community confirmation of this end-to-end flow.", "evidenceIds": [ "deepinfra-docs-8", "deepinfra-docs-9", "deepinfra-docs-10", "deepinfra-docs-18" ] }, { "productId": "deepinfra", "storyId": "low-latency-serving", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "DeepInfra provides explicit priority service tier for faster time-to-first-token and higher throughput during peak demand (docs-4, docs-26), plus fail_fast for latency-sensitive workflows and a flex tier tradeoff, showing deliberate latency/throughput controls. Community evidence corroborates generally low cold-start latency (<10s) for popular models. Missing for 10: independent benchmark data on actual TTFT/throughput numbers and hands-on comparison to competitors.", "evidenceIds": [ "deepinfra-docs-4", "deepinfra-docs-26", "deepinfra-docs-27", "deepinfra-docs-6", "deepinfra-comm-1" ] }, { "productId": "deepinfra", "storyId": "measured-speed-claims", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes qualitative service tiers (priority/flex) that claim to affect speed, and one anecdotal HN comment about cold-start latency, but there are no published tokens-per-second figures, latency benchmarks, or load-testing guides substantiating DeepInfra's speed claims.", "evidenceIds": [ "deepinfra-docs-4", "deepinfra-docs-26", "deepinfra-docs-27", "deepinfra-comm-1" ] }, { "productId": "deepinfra", "storyId": "model-deprecation-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions a deprecation policy, sunset notice period, or advance notice process for removing hosted models; the docs cover API usage, pricing, and infrastructure but nothing about model lifecycle/removal policy.", "evidenceIds": [] }, { "productId": "deepinfra", "storyId": "models-endpoint-discovery", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A live probe confirms GET https://api.deepinfra.com/v1/openai/models returns HTTP 200 with a machine-readable list of 189 models, and DeepInfra's docs establish that its API is a documented OpenAI-compatible drop-in (same base_url, same client libraries), which conventionally includes the /v1/models listing endpoint. Missing for 10: a dedicated docs page explicitly describing the /v1/models endpoint schema/parameters rather than relying on general OpenAI-compatibility claims.", "evidenceIds": [ "deepinfra-probe-rt-1", "deepinfra-docs-14", "deepinfra-docs-15", "deepinfra-docs-2" ] }, { "productId": "deepinfra", "storyId": "multimodal-endpoints", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly list Vision & OCR, Speech (Whisper/TTS), and Image & Video generation (FLUX, Stable Diffusion, text-to-video) as first-class model categories alongside chat, all accessible via the same OpenAI-compatible API/platform. Missing for 10: independent hands-on confirmation of calling these non-text endpoints and more detail on request/response formats for each modality.", "evidenceIds": [ "deepinfra-docs-11", "deepinfra-docs-13", "deepinfra-docs-19", "deepinfra-docs-20", "deepinfra-docs-21", "deepinfra-docs-22" ] }, { "productId": "deepinfra", "storyId": "open-weight-catalog-breadth", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "DeepInfra's docs confirm an OpenAI-compatible shared serverless endpoint where you 'set model to a model from our catalog,' and a live probe confirms 189 models are enumerable via the API without cold-start dedicated deployment. However, the evidence pack never explicitly names or confirms coverage of Llama, Qwen, DeepSeek, or GPT-OSS specifically as catalog entries. Missing for 10: explicit documentation or listing naming these specific model families as available, and independent commentary on catalog breadth/freshness beyond the raw count.", "evidenceIds": [ "deepinfra-docs-15", "deepinfra-docs-11", "deepinfra-probe-rt-1" ] }, { "productId": "deepinfra", "storyId": "openai-compatible-endpoint", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Documentation explicitly and repeatedly states developers only need to change base_url and api_key to point an existing OpenAI SDK at DeepInfra, with a live probe confirming the OpenAI-compatible endpoint is functional and reachable. Missing for 10: independent third-party hands-on confirmation of a full SDK migration beyond docs and a keyless probe.", "evidenceIds": [ "deepinfra-docs-1", "deepinfra-docs-15", "deepinfra-docs-30", "deepinfra-probe-rt-1" ] }, { "productId": "deepinfra", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "DeepInfra's docs show broad API coverage matching most UI-facing capabilities: chat/completions, embeddings, vision/OCR, speech, image/video generation, private model deployment, and GPU clusters are all exposed via API endpoints, and the models endpoint is even queryable keylessly. However, there's no explicit statement of full UI/API parity, no OpenAPI/swagger spec was found (probes returned 404s), and no evidence covers account/billing/dashboard-management actions being API-accessible. missing for 10: explicit parity documentation, a discoverable OpenAPI schema, and evidence that account/billing/UI-console actions are also API-controllable.", "evidenceIds": [ "deepinfra-docs-2", "deepinfra-docs-8", "deepinfra-docs-9", "deepinfra-docs-10", "deepinfra-docs-19", "deepinfra-docs-20", "deepinfra-docs-21", "deepinfra-docs-22", "deepinfra-probe-1", "deepinfra-probe-2", "deepinfra-probe-3", "deepinfra-probe-rt-1" ] }, { "productId": "deepinfra", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "DeepInfra's evidence pack focuses entirely on its OpenAI-compatible inference API, model catalog, and infrastructure offerings; there is no mention of any user data export feature, open-format data portability, or account data download capability. Given the zero-retention policy, DeepInfra does not appear to store persistent user data, but no evidence at all addresses export tooling for whatever account/usage data does exist.", "evidenceIds": [ "deepinfra-docs-23" ] }, { "productId": "deepinfra", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "deepinfra", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "DeepInfra is a hosted inference cloud service (running models on its own GPU infrastructure); there is no evidence of, nor any product category expectation for, self-hosting the core DeepInfra platform itself. GPU cluster rental (docs-10) lets users run their own workloads on rented hardware, but that is not self-hosting DeepInfra's own product.", "evidenceIds": [] }, { "productId": "deepinfra", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of region selection or data residency controls; only zero-retention/SOC2/ISO27001 privacy claims are mentioned, with no mention of choosing storage region or geographic data residency options.", "evidenceIds": [] }, { "productId": "deepinfra", "storyId": "privacy-no-training", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "DeepInfra explicitly states a zero data retention policy covering inputs, outputs, and user data, backed by SOC 2 and ISO 27001 certifications, directly addressing the no-training-data-use concern. Missing for 10: independent/third-party audit confirmation or explicit contractual opt-out language beyond the marketing statement.", "evidenceIds": [ "deepinfra-docs-23" ] }, { "productId": "deepinfra", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "DeepInfra states a 'zero retention policy' for inputs/outputs/user data and cites SOC 2 and ISO 27001 certification, implying strong default privacy posture, but there is no documented user-facing mechanism to configure retention periods, request deletion of stored data, or verify enforcement of the zero-retention claim. missing for 10: explicit data-deletion/retention-control API or settings, independent audit/verification of the zero-retention claim, and documentation of what data (if any) is logged/retained by default.", "evidenceIds": [ "deepinfra-docs-23" ] }, { "productId": "deepinfra", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes a zero-retention data policy and SOC2/ISO27001 certification, but this addresses data retention rather than a specific telemetry/usage-tracking opt-out control; no docs mention any setting, header, or account option to disable telemetry or usage analytics.", "evidenceIds": [] }, { "productId": "deepinfra", "storyId": "prompt-caching-support", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of prompt caching, prefix caching, or any context-reuse feature that reduces latency/cost on repeated prompts; it only covers OpenAI-compatible API basics, priority/flex service tiers, and dedicated deployments, none of which address caching of repeated context.", "evidenceIds": [] }, { "productId": "deepinfra", "storyId": "serve-custom-weights", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "DeepInfra documents deploying custom fine-tuned LLM weights on dedicated GPU instances (A100/H100/H200/B200/B300) with autoscaling and private endpoints, directly addressing the 'serve my own custom model weights' part of the story. However, there is no explicit mention of LoRA adapter upload/serving specifically, nor details on the upload workflow, supported formats, or independent/hands-on confirmation that this works as claimed. Missing for 10: LoRA adapter support specifics, upload/serving workflow documentation, independent verification of custom weight deployment.", "evidenceIds": [ "deepinfra-docs-8", "deepinfra-docs-9", "deepinfra-docs-18" ] }, { "productId": "deepinfra", "storyId": "spend-controls-budgets", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of spending caps, budget alerts, usage limits, or billing controls anywhere in DeepInfra's docs or pages; only pay-per-token pricing and service-tier options (priority/flex) are described. This is a fair question for a usage-based API platform, so absence of evidence means 'none' rather than 'na'.", "evidenceIds": [ "deepinfra-docs-12", "deepinfra-docs-24", "deepinfra-docs-31" ] }, { "productId": "deepinfra", "storyId": "status-page-uptime", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "A probe confirms DeepInfra runs a public status page (status.deepinfra.com) that returns HTTP 200 and displays operational status, which a founder could check before committing production traffic. However, the evidence does not show an incident-history log, uptime SLA, or past-incident detail on that page, so the story is only partially confirmed. Missing for 10: evidence of incident history/timeline, uptime percentage stats, or independent commentary on past outages.", "evidenceIds": [ "deepinfra-probe-rt-2" ] }, { "productId": "deepinfra", "storyId": "streaming-responses", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "DeepInfra positions itself as a drop-in OpenAI-compatible API (same SDK, same code), which implies standard OpenAI SDK features like `stream=True` token-by-token SSE would work unchanged, but no evidence pack item explicitly documents or demonstrates streaming responses, SSE format, or token-by-token behavior. Missing for 10: explicit streaming/SSE documentation or examples, benchmarks on streaming latency, and independent confirmation that stream=True works as expected.", "evidenceIds": [ "deepinfra-docs-1", "deepinfra-docs-15", "deepinfra-docs-30" ] }, { "productId": "deepinfra", "storyId": "structured-outputs-json-schema", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack details DeepInfra's OpenAI-compatible API, pricing tiers, and model catalog, but contains no mention of JSON schema enforcement, structured output modes, or grammar-constrained decoding. Missing for 10: any documentation of `response_format`/json_schema support, grammar-based constrained generation, or examples showing reliable JSON parsing guarantees.", "evidenceIds": [] }, { "productId": "deepinfra", "storyId": "tool-calling-fidelity", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack only shows generic OpenAI-compatible API claims (base URL swap, chat completions) and unrelated features (priority tiers, GPU deployment, image/video models) but never mentions function/tool calling, parallel tool calls, or multi-step agentic tool use, nor any benchmark/community report on tool-calling fidelity. Missing for 10: explicit documentation of a `tools`/`function_call` parameter, evidence of parallel tool call support, and any hands-on or benchmark confirmation that tool calling works reliably in agent loops.", "evidenceIds": [] }, { "productId": "deepinfra", "storyId": "transparent-token-pricing", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs repeatedly state pay-as-you-go, per-token pricing with no contracts or sales required (docs-12, docs-16, docs-24, docs-31), strongly implying self-serve access to prices, and the model catalog is enumerable via a keyless API (deepinfra-probe-rt-1). However, no evidence pack item shows an actual public price list or per-model $/token table a founder could browse. missing for 10: explicit citation of a pricing page or API field listing per-token prices for each hosted model, independent confirmation that prices are visible without account/sales contact.", "evidenceIds": [ "deepinfra-docs-12", "deepinfra-docs-16", "deepinfra-docs-24", "deepinfra-docs-31", "deepinfra-probe-rt-1" ] }, { "productId": "fireworks-ai", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Direct probe evidence confirms llms.txt is live at https://docs.fireworks.ai/llms.txt (HTTP 200) with a documentation index, and per-page .md endpoints (e.g. introduction.md) explicitly point agents back to the llms.txt index, making the docs agent-consumable by design. Missing for 10: no independent third-party report of an agent successfully using this to complete a task end-to-end.", "evidenceIds": [ "fireworks-ai-probe-1", "fireworks-ai-probe-2" ] }, { "productId": "fireworks-ai", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Fireworks AI is an inference/fine-tuning infrastructure platform (APIs, model hosting, deployments) rather than an end-user product that holds 'your data' and surfaces AI-generated insights/suggestions within a UI. This story targets data-analytics/SaaS-style products, not a model-serving API platform, so the axis is a category mismatch.", "evidenceIds": [] }, { "productId": "fireworks-ai", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "fireworks-ai", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Fireworks' dashboard offers a guided flow where a user 'describes the task, reviews the plan and cost, approves the run' for fine-tuning — a narrow assistant-like feature — but there's no evidence of a general-purpose in-product AI assistant that can be delegated broader tasks across the platform. Missing for 10: evidence of a persistent conversational/agentic assistant embedded in the console, scope beyond fine-tuning setup, and independent corroboration of its capabilities.", "evidenceIds": [ "fireworks-ai-docs-31" ] }, { "productId": "fireworks-ai", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Fireworks is API-first (OpenAI-compatible REST endpoint) and ships a CLI (firectl) for scripted deployment/fine-tuning plus async batch-inference for high-volume automated jobs, all of which are naturally usable headlessly in CI pipelines. A live probe confirms the API endpoint is reachable and speaks JSON, requiring just an API key for auth (standard for CI use). Missing for 10: explicit CI/CD pipeline examples (e.g., GitHub Actions integration) and documented non-interactive auth/service-account flows for automated environments.", "evidenceIds": [ "fireworks-ai-docs-17", "fireworks-ai-docs-4", "fireworks-ai-docs-20", "fireworks-ai-docs-23", "fireworks-ai-docs-33", "fireworks-ai-probe-rt-1" ] }, { "productId": "fireworks-ai", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "fireworks-ai", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Fireworks AI is an inference/hosting platform with API compatibility, tool-calling, and fine-tuning features, but no evidence anywhere in the pack of an official MCP server for connecting agents. Absence of evidence for this applicable capability means 'none'.", "evidenceIds": [] }, { "productId": "fireworks-ai", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The only evidence of natural-language operation is a single marketing line about a 'guided path' where you 'describe the task' to kick off fine-tuning — the rest of the product (inference API, deployments, benchmarking, CLI) is operated via code/API/CLI, not NL commands. Missing for 10: documentation of an NL-driven interface for core inference/deployment tasks, any chat-based control plane, or independent corroboration that the 'describe the task' feature works as an agentic NL interface.", "evidenceIds": [ "fireworks-ai-docs-31" ] }, { "productId": "fireworks-ai", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence shows an official CLI (`firectl`) used for deployment commands like creating LoRA deployments, confirming Fireworks ships a CLI. However, the evidence pack lacks any dedicated documentation, install guide, or broader command reference showing its scope for AI-native/agentic workflows beyond one example command. missing for 10: install/setup docs, full command reference, independent hands-on usage confirming agentic/automation use cases.", "evidenceIds": [ "fireworks-ai-docs-23" ] }, { "productId": "fireworks-ai", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Fireworks exposes an OpenAI-compatible REST API (chat completions, tool calling, structured outputs, embeddings, batch inference) documented extensively, with a live public endpoint confirmed by probe (api.fireworks.ai returning proper JSON auth errors) and a public status page. missing for 10: no publicly hosted OpenAPI/swagger spec was found (404s on standard paths), slightly reducing machine-readability of the API contract.", "evidenceIds": [ "fireworks-ai-docs-1", "fireworks-ai-docs-2", "fireworks-ai-docs-19", "fireworks-ai-docs-36", "fireworks-ai-probe-rt-1", "fireworks-ai-probe-rt-2", "fireworks-ai-probe-3" ] }, { "productId": "fireworks-ai", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack of scoped/least-privilege API key management (e.g., role-based keys, permission scoping, per-agent credential issuance) — only general auth requirements are mentioned (401 without a key). Missing for 10: docs on creating scoped/restricted API keys, role-based access control, per-agent credential issuance, and any permission-granularity settings.", "evidenceIds": [ "fireworks-ai-probe-rt-1" ] }, { "productId": "fireworks-ai", "storyId": "agentic-sdks", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Fireworks documents an OpenAI-compatible API and CLI (firectl) that let developers reuse existing OpenAI SDKs and tooling, but no evidence in the pack names a dedicated first-party Fireworks Python/JS SDK, its GitHub repo, or client library documentation. missing for 10: explicit official Fireworks SDK docs/repo, language coverage (Python/JS/Go), and independent developer confirmation of SDK usage.", "evidenceIds": [ "fireworks-ai-docs-1", "fireworks-ai-docs-15", "fireworks-ai-docs-17", "fireworks-ai-docs-36" ] }, { "productId": "fireworks-ai", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack of a webhook subscription mechanism or event notification system for Fireworks AI; the docs focus on inference, fine-tuning, and deployment APIs with no mention of webhooks or event-driven callbacks.", "evidenceIds": [] }, { "productId": "fireworks-ai", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows extensive text docs and a reference to an api-reference path, but no evidence of an interactive, runnable-example API reference (e.g., embedded code sandbox, live API console); in fact the OpenAPI/swagger probe returned 404 on all candidate paths, indicating no discoverable interactive spec. Missing for 10: an interactive API explorer, runnable code snippets, or a live OpenAPI/Swagger UI.", "evidenceIds": [ "fireworks-ai-probe-3", "fireworks-ai-probe-rt-1", "fireworks-ai-probe-1" ] }, { "productId": "fireworks-ai", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Direct probes for OpenAPI/swagger spec files at all standard paths returned 404, and no docs page or evidence pack entry links to a downloadable machine-readable API spec; the API is described as 'OpenAI-compatible' but no explicit OpenAPI/Swagger artifact is provided.", "evidenceIds": [ "fireworks-ai-probe-3" ] }, { "productId": "fireworks-ai", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a sandbox/test environment, staging API keys, or any mechanism to test without touching production data/billing; documentation focuses on production inference, fine-tuning, and deployment features only.", "evidenceIds": [] }, { "productId": "fireworks-ai", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of a versioned API scheme or documented deprecation policy; docs mention OpenAI-compatible API and drop-in replacement but nothing about version numbers, changelogs, or sunset/deprecation timelines. The OpenAPI spec probe even 404'd on all candidate paths, suggesting limited API-versioning documentation. Missing for 10: explicit API versioning scheme, published deprecation/sunset policy, changelog of breaking changes.", "evidenceIds": [ "fireworks-ai-probe-3", "fireworks-ai-docs-17", "fireworks-ai-docs-36" ] }, { "productId": "fireworks-ai", "storyId": "automation-bulk-operations", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Fireworks explicitly supports batch inference for processing large volumes of requests asynchronously at discounted rates, which directly enables bulk operations across many items (e.g., bulk generation, classification, embeddings). This is documented as a first-party feature with clear pricing/mechanics, though evidence lacks hands-on validation of batch job semantics (job status, size limits, error handling) or independent corroboration. missing for 10: hands-on/independent verification of batch job workflow, documented size/rate limits, and error-handling behavior for large batch jobs.", "evidenceIds": [ "fireworks-ai-docs-4", "fireworks-ai-docs-20" ] }, { "productId": "fireworks-ai", "storyId": "automation-rules-engine", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Fireworks AI is an inference/hosting/fine-tuning platform, not an automation/rules-engine product; there's no concept of event-triggered rules in its evidence, and this axis is a category mismatch rather than a missing feature.", "evidenceIds": [] }, { "productId": "fireworks-ai", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Fireworks AI is an inference/fine-tuning/model-hosting platform, not a workflow/job orchestration or scheduling product; scheduling recurring jobs or workflows is outside its product category and belongs to orchestration tools built on top of it.", "evidenceIds": [] }, { "productId": "fireworks-ai", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Fireworks AI is an inference/fine-tuning hosting platform; versioning, reviewing, and rolling back 'automations' (workflow/agent automations) is not a category it addresses—it's a wrong axis for this product type, not a missing feature.", "evidenceIds": [] }, { "productId": "fireworks-ai", "storyId": "availability-sla", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of a stated uptime/availability SLA (e.g., 99.9% commitment) for paid or enterprise tiers — only a public status page and an optional 'priority' service tier for better reliability during peak periods, which is not a contractual SLA.", "evidenceIds": [ "fireworks-ai-docs-6", "fireworks-ai-docs-18", "fireworks-ai-probe-rt-2" ] }, { "productId": "fireworks-ai", "storyId": "batch-discounted-jobs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Fireworks documents a dedicated Batch Inference API that processes requests asynchronously at a documented 50% discount off serverless per-token pricing, directly matching the story. Missing for 10: independent/hands-on confirmation of actual batch job submission and discount realization, and no SLA/turnaround details beyond the discount claim.", "evidenceIds": [ "fireworks-ai-docs-4", "fireworks-ai-docs-20" ] }, { "productId": "fireworks-ai", "storyId": "coding-agent-integrations", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Fireworks documents an OpenAI-compatible API, tool/function calling, and structured outputs, which implicitly supports plugging into agent frameworks that use the OpenAI SDK, but there is no first-party guide explicitly targeting coding agents or agent frameworks (e.g., LangChain, AutoGen, Cursor, Continue) in the evidence. missing for 10: dedicated integration guides for named coding agents/agent frameworks, tutorials showing agent setup with Fireworks endpoints, and community corroboration of such integrations.", "evidenceIds": [ "fireworks-ai-docs-2", "fireworks-ai-docs-11", "fireworks-ai-docs-15", "fireworks-ai-docs-36" ] }, { "productId": "fireworks-ai", "storyId": "day-zero-model-availability", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Marketing copy claims 'instant access to the most popular OSS models' and 'run the latest open models with a single line of code' plus '100+ supported models', implying rapid onboarding of new open-weight releases, but there is no concrete evidence (e.g., specific model, release-to-availability timeline, changelog) demonstrating actual speed of adding new models after public release. missing for 10: concrete turnaround-time examples/announcements for specific new open-weight model releases, independent confirmation of day-0/near-day-0 availability, and any changelog or blog evidence of catalog update cadence.", "evidenceIds": [ "fireworks-ai-docs-12", "fireworks-ai-docs-16", "fireworks-ai-docs-28" ] }, { "productId": "fireworks-ai", "storyId": "dedicated-endpoints-autoscale", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Fireworks explicitly documents on-demand deployments giving dedicated GPUs isolated from shared serverless traffic, with autoscaling configuration including scale-to-zero and minimum replica settings. Missing for 10: independent hands-on validation of autoscaling behavior under load and explicit SLA/isolation guarantees beyond docs claims (one community comment concerns fine-tuning cost, not dedicated-capacity autoscaling).", "evidenceIds": [ "fireworks-ai-docs-21", "fireworks-ai-docs-38", "fireworks-ai-docs-14", "fireworks-ai-docs-5", "fireworks-ai-docs-30" ] }, { "productId": "fireworks-ai", "storyId": "documented-rate-limits", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows references to service tiers (priority, fast) and notes on-demand deployments have 'no hard rate limits', but there is no documented table or page specifying actual rate-limit numbers (RPM/TPM) or how they scale across usage tiers for serverless usage. Developers have no concrete documented limits to plan against before hitting them in production.", "evidenceIds": [ "fireworks-ai-docs-21", "fireworks-ai-docs-18", "fireworks-ai-docs-6" ] }, { "productId": "fireworks-ai", "storyId": "drop-in-provider-switch", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Fireworks explicitly documents itself as a drop-in replacement using the OpenAI-compatible chat-completions API (docs-1,11,15,27,36), and a live probe confirms the OpenAI-style /v1/models endpoint is functioning at api.fireworks.ai (probe-rt-1), supporting seamless mid-workflow provider swaps without code changes. Missing for 10: independent hands-on agent-switching test (e.g. LangChain/agent framework confirming no code changes needed) and a public OpenAPI spec (probe-3 shows 404s for openapi.json).", "evidenceIds": [ "fireworks-ai-docs-1", "fireworks-ai-docs-11", "fireworks-ai-docs-15", "fireworks-ai-docs-27", "fireworks-ai-docs-36", "fireworks-ai-probe-rt-1" ] }, { "productId": "fireworks-ai", "storyId": "embeddings-endpoint", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Fireworks docs explicitly state support for 'embeddings & reranking in search & context retrieval' and list embeddings among its 100+ supported model types, all via the same OpenAI-compatible API used for other inference, letting a developer avoid a second vendor for retrieval pipelines. Missing for 10: no dedicated embeddings/rerank API reference or usage example, no independent/hands-on confirmation of rerank model quality or throughput.", "evidenceIds": [ "fireworks-ai-docs-37", "fireworks-ai-docs-28", "fireworks-ai-docs-36" ] }, { "productId": "fireworks-ai", "storyId": "fine-tune-and-serve", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Fireworks documents a complete fine-tune-and-serve loop on one platform: SFT/RFT training up to 1T+ params, same API/data format as inference, and LoRA deployment via live-merge or multi-LoRA with a single firectl command, producing a model that serves identically to a natively trained one. Missing for 10: independent/hands-on confirmation of end-to-end fine-tune→serve quality and reliability beyond vendor docs (the only community evidence found addresses cost, not functionality).", "evidenceIds": [ "fireworks-ai-docs-10", "fireworks-ai-docs-26", "fireworks-ai-docs-8", "fireworks-ai-docs-23", "fireworks-ai-docs-24", "fireworks-ai-docs-32", "fireworks-ai-docs-39", "fireworks-ai-docs-1" ] }, { "productId": "fireworks-ai", "storyId": "low-latency-serving", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Fireworks explicitly documents Fast variants for latency-sensitive workloads, dedicated on-demand GPUs for predictable low-latency/high-throughput, sticky session-affinity routing to boost cache hit rate, a priority service tier, and an open-source benchmarking tool to measure/optimize deployment performance. missing for 10: independent third-party latency/throughput benchmarks corroborating the claims, and no direct rebuttal of speed claims in community evidence (only pricing complaints, which are off-topic).", "evidenceIds": [ "fireworks-ai-docs-34", "fireworks-ai-docs-14", "fireworks-ai-docs-21", "fireworks-ai-docs-13", "fireworks-ai-docs-25", "fireworks-ai-docs-6", "fireworks-ai-docs-9", "fireworks-ai-docs-17" ] }, { "productId": "fireworks-ai", "storyId": "measured-speed-claims", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Fireworks documents an open-source benchmarking tool that customers can use to measure their own deployment's throughput/latency, and it markets 'Fast' variants and on-demand deployments with claims of 'lower latency, higher throughput' — but the evidence pack contains no actual published tokens-per-second numbers, latency benchmarks, or third-party load-testing results substantiating these speed claims. Missing for 10: concrete published TPS/latency figures, independent benchmark comparisons, or a load-testing guide with real numbers rather than just a tool pointer.", "evidenceIds": [ "fireworks-ai-docs-9", "fireworks-ai-docs-22", "fireworks-ai-docs-34", "fireworks-ai-docs-14", "fireworks-ai-docs-21" ] }, { "productId": "fireworks-ai", "storyId": "model-deprecation-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack references a deprecation policy, model retirement notice period, or sunset process for hosted models; docs cover inference, fine-tuning, deployment, and pricing but nothing about model lifecycle/retirement communication.", "evidenceIds": [] }, { "productId": "fireworks-ai", "storyId": "models-endpoint-discovery", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "A live probe confirms `GET https://api.fireworks.ai/inference/v1/models` is a real, JSON-speaking, OpenAI-style endpoint (401 without a key, meaning it works with one), consistent with Fireworks' documented OpenAI-compatible API surface. Missing for 10: an explicit first-party docs page specifically describing the /v1/models listing endpoint and its response schema, and an authenticated hands-on confirmation showing the actual model list output.", "evidenceIds": [ "fireworks-ai-probe-rt-1", "fireworks-ai-docs-36", "fireworks-ai-docs-17" ] }, { "productId": "fireworks-ai", "storyId": "multimodal-endpoints", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly list 100+ supported models across text, vision, audio, image, and embeddings, with vision models specifically noted for analyzing images/documents, all on the same OpenAI-compatible API. Missing for 10: no independent/hands-on corroboration of image-generation or audio model usage, and no concrete API examples for non-text modalities beyond the feature list.", "evidenceIds": [ "fireworks-ai-docs-28", "fireworks-ai-docs-29", "fireworks-ai-docs-37", "fireworks-ai-docs-36" ] }, { "productId": "fireworks-ai", "storyId": "open-weight-catalog-breadth", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs confirm serverless access to 100+ open-source models across modalities via OpenAI-compatible APIs, with 'instant access to the most popular OSS models' and single-line-of-code deployment; the catalog explicitly spans text/vision/audio/image/embeddings. Specific families like Llama/Qwen/DeepSeek/GPT-OSS aren't individually enumerated in this evidence pack, and the model catalog itself wasn't independently enumerable (API requires a key). Missing for 10: explicit per-family model list confirmation, independent enumeration of catalog contents.", "evidenceIds": [ "fireworks-ai-docs-12", "fireworks-ai-docs-16", "fireworks-ai-docs-28", "fireworks-ai-docs-36", "fireworks-ai-probe-rt-1" ] }, { "productId": "fireworks-ai", "storyId": "openai-compatible-endpoint", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly state Fireworks is a drop-in replacement for OpenAI (same API), points client at api.fireworks.ai with OpenAI-compatible endpoints, and a live probe confirms the OpenAI-style /v1/models endpoint is reachable and speaks JSON. Missing for 10: no explicit hands-on developer account showing a real OpenAI SDK code snippet with only base_url/api_key changed being run successfully.", "evidenceIds": [ "fireworks-ai-docs-1", "fireworks-ai-docs-15", "fireworks-ai-docs-17", "fireworks-ai-docs-36", "fireworks-ai-probe-rt-1" ] }, { "productId": "fireworks-ai", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "low", "rationale": "Fireworks documents API/CLI (firectl) access to essentially every major capability — inference, fine-tuning, LoRA deployment, model upload, autoscaling, batch inference, benchmarking — implying an API-first architecture where the UI is largely a thin layer over these APIs. However, there is no explicit statement or audit confirming full UI/API feature parity, and no evidence addressing whether any UI-only conveniences (e.g., dashboard analytics, billing, team management) lack API equivalents. Missing for 10: an explicit parity statement or audit, evidence covering account/billing/UI-only features, and independent confirmation that no UI feature is API-inaccessible.", "evidenceIds": [ "fireworks-ai-docs-7", "fireworks-ai-docs-23", "fireworks-ai-docs-8", "fireworks-ai-docs-10", "fireworks-ai-docs-5", "fireworks-ai-docs-4", "fireworks-ai-docs-22", "fireworks-ai-docs-17" ] }, { "productId": "fireworks-ai", "storyId": "openness-full-export", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Fireworks documents open, OpenAI-compatible APIs and identical SFT data formats, plus the ability to upload custom/fine-tuned models from Hugging Face, S3, or Azure Blob Storage, which implies some data/model portability rather than lock-in. However there is no explicit documentation of a data export tool, account data download, or guidance for migrating fine-tuning datasets or deployed models back out of the platform. Missing for 10: explicit data-export/download feature docs, confirmation that user-uploaded training data or fine-tuned models can be freely exported (not just uploaded), and any account/data-deletion or portability policy.", "evidenceIds": [ "fireworks-ai-docs-1", "fireworks-ai-docs-7", "fireworks-ai-docs-33", "fireworks-ai-docs-27" ] }, { "productId": "fireworks-ai", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Fireworks AI is a proprietary inference/hosting platform; there is no evidence of an open-license source-code release for the core product (only an 'open-source benchmarking tool' side utility is mentioned, not the platform itself). No repository, license file, or source availability is documented.", "evidenceIds": [ "fireworks-ai-docs-9", "fireworks-ai-docs-22" ] }, { "productId": "fireworks-ai", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Fireworks AI is a hosted inference/training cloud service; there is no evidence of a self-hostable core product (e.g., open-sourced platform binary/container for on-prem deployment). Self-hosting is not a plausible axis for this managed SaaS/API offering, so this is a category mismatch rather than an unmet capability.", "evidenceIds": [] }, { "productId": "fireworks-ai", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack mentions data residency, regional storage options, or geographic controls for where data is stored/processed on Fireworks AI; the docs cover inference, fine-tuning, deployment, and pricing but not region selection.", "evidenceIds": [] }, { "productId": "fireworks-ai", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item addresses a data-training opt-out, privacy policy, or data-retention/no-train guarantee for inputs sent to Fireworks AI's inference or fine-tuning APIs; all citations concern performance, deployment, and pricing features. missing for 10: explicit privacy policy or terms stating user data is not used for model training, an opt-out/opt-in control, and any independent confirmation of this practice.", "evidenceIds": [] }, { "productId": "fireworks-ai", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack discusses data retention policies, deletion controls, or privacy/compliance mechanisms for user data or fine-tuning datasets; the docs cover inference, fine-tuning, deployment, and pricing but never data retention/deletion. missing for 10: documented data retention policy, user-facing deletion/erasure controls, data handling/compliance certifications (SOC2/GDPR), retention configuration options.", "evidenceIds": [] }, { "productId": "fireworks-ai", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items address telemetry, usage tracking, opt-out controls, or privacy settings for Fireworks AI; the docs cover inference, fine-tuning, and deployment features but never mention telemetry/data-collection opt-out. missing for 10: any documentation of telemetry collection, a privacy/opt-out setting, or usage-tracking disclosure.", "evidenceIds": [] }, { "productId": "fireworks-ai", "storyId": "prompt-caching-support", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Fireworks documents an explicit prompt-caching mechanism: an optional x-session-affinity sticky-routing key that pins repeated requests to the same replica specifically to 'maximize prompt-cache hit rate,' directly addressing latency/cost benefits for repeated context on serverless deployments. However, details are thin — no documentation of cache TTL/eviction policy, no quantified latency/cost savings numbers, and no independent benchmarks or hands-on confirmation of cache hit rates. missing for 10: quantified latency/cost savings from cache hits, cache eviction/TTL policy details, independent or hands-on corroboration of the caching behavior.", "evidenceIds": [ "fireworks-ai-docs-13", "fireworks-ai-docs-25" ] }, { "productId": "fireworks-ai", "storyId": "serve-custom-weights", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Fireworks docs explicitly cover uploading custom models from Hugging Face/S3/Azure Blob, plus deploying LoRA adapters via live-merge or multi-LoRA with a single firectl command. This directly matches the ml-engineer story of uploading and serving custom weights/adapters. Missing for 10: independent hands-on verification of the upload/serve workflow beyond vendor docs.", "evidenceIds": [ "fireworks-ai-docs-7", "fireworks-ai-docs-8", "fireworks-ai-docs-23", "fireworks-ai-docs-24", "fireworks-ai-docs-32", "fireworks-ai-docs-33", "fireworks-ai-docs-39" ] }, { "productId": "fireworks-ai", "storyId": "spend-controls-budgets", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack of any spending cap, budget alert, or usage limit configuration feature; only cost-related mentions are pricing structures (serverless, batch discounts) not budget controls. A community report even highlights an unexpectedly high bill with no indication of caps to prevent it, reinforcing the absence of this capability.", "evidenceIds": [ "fireworks-ai-comm-1" ] }, { "productId": "fireworks-ai", "storyId": "status-page-uptime", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "A public status page at status.fireworks.ai is confirmed live and shows current operational status, satisfying the core ask of checking uptime before committing production traffic. However, the evidence pack does not confirm the page includes a historical incident log or past-outage records. missing for 10: explicit confirmation of incident history/timeline on the status page, independent user reports referencing past outages logged there.", "evidenceIds": [ "fireworks-ai-probe-rt-2" ] }, { "productId": "fireworks-ai", "storyId": "streaming-responses", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack repeatedly describes Fireworks as an OpenAI-compatible inference API but never explicitly documents SSE/token-streaming behavior (e.g., a `stream: true` parameter or chunked response format) for chat/completions endpoints; docs-17 and docs-36 only allude to 'sending tokens' and API compatibility without confirming streaming responses. Missing for 10: explicit streaming API docs, SSE example/code snippet, or hands-on confirmation of token-by-token delivery.", "evidenceIds": [ "fireworks-ai-docs-17", "fireworks-ai-docs-36" ] }, { "productId": "fireworks-ai", "storyId": "structured-outputs-json-schema", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Fireworks docs explicitly describe a Structured Outputs feature to 'force model output to conform to a JSON schema' and ensure responses 'conform to your specified format, making them easy to parse,' directly matching the story. Missing for 10: independent/hands-on corroboration of reliability across models and no mention of grammar-based constraints beyond JSON schema.", "evidenceIds": [ "fireworks-ai-docs-3", "fireworks-ai-docs-19" ] }, { "productId": "fireworks-ai", "storyId": "tool-calling-fidelity", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Fireworks documents tool/function calling as a supported feature (docs-2) and structured JSON-schema outputs (docs-19), which underpin agent tool-use loops, but there is no documentation or evidence specifically addressing parallel tool calls, multi-step tool-use reliability, or fidelity benchmarks against OpenAI-style tool calling on open models. missing for 10: explicit parallel/multi-step tool-calling documentation, reliability/fidelity benchmarks, independent hands-on validation of agent-loop tool use.", "evidenceIds": [ "fireworks-ai-docs-2", "fireworks-ai-docs-19" ] }, { "productId": "fireworks-ai", "storyId": "transparent-token-pricing", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs describe a self-serve, pay-per-token model ('point your client at api.fireworks.ai... pay only for what you use', batch inference at '50% off Serverless per-token prices') implying pricing is accessible without sales contact, and a community user cites a specific billed dollar amount for usage, suggesting transparent metering. However, no evidence pack item directly cites or shows Fireworks' public pricing page listing per-model per-token rates. Missing for 10: a direct citation to the pricing page enumerating per-token rates for each hosted model, and confirmation that all 100+ models have listed public prices rather than requiring contact for some tiers.", "evidenceIds": [ "fireworks-ai-docs-17", "fireworks-ai-docs-4", "fireworks-ai-docs-20", "fireworks-ai-comm-1" ] }, { "productId": "groq", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Direct probe evidence confirms llms.txt is live (HTTP 200) at console.groq.com/llms.txt, and Groq also serves per-page .md docs (docs/overview.md) explicitly designed for agent consumption, indicating agent-oriented docs beyond just llms.txt. Missing for 10: no independent/community confirmation of an agent actually using llms.txt in practice.", "evidenceIds": [ "groq-probe-1", "groq-probe-2" ] }, { "productId": "groq", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Groq is a raw LLM inference API/platform (chat completions, tool use, batch processing) — it has no first-party product surface that holds 'my data' and surfaces insights/suggestions from it; that would be a downstream application built on Groq, not Groq itself.", "evidenceIds": [] }, { "productId": "groq", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Groq's evidence covers only inference primitives (chat completions, tool use, MCP tool integration, batch processing, structured outputs) — none of which describe a scheduling/trigger system for autonomous background automations; batch jobs are asynchronous request batches, not persistent background agents. Missing for 10: any documented scheduler, cron/trigger mechanism, persistent background agent runtime, or workflow orchestration feature that lets a user 'set up' an automation to run unattended.", "evidenceIds": [ "groq-docs-29", "groq-docs-5", "groq-docs-6" ] }, { "productId": "groq", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "groq", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Groq is a pure API/inference service accessed via OpenAI-compatible REST endpoints and SDKs (API key + base_url swap), with batch processing for large-scale asynchronous automation and confirmed live, key-gated JSON endpoints — all of which are inherently scriptable/headless with no GUI dependency, ideal for CI pipelines. Missing for 10: explicit CI/CD examples (e.g., GitHub Actions) or docs specifically framing headless/automation use cases beyond batch API references.", "evidenceIds": [ "groq-docs-2", "groq-docs-7", "groq-docs-29", "groq-gh-1", "groq-probe-rt-1", "groq-docs-10" ] }, { "productId": "groq", "storyId": "agentic-mcp-client", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Groq's Responses/Chat API supports remote MCP: you provide an MCP server URL and Groq handles tool orchestration without implementing tool logic yourself, documented explicitly in first-party docs and confirmed via a probe of the dedicated MCP docs page. Missing for 10: independent hands-on user reports specifically validating MCP tool use (community evidence covers speed/reliability but not MCP), and no details on multi-server or auth edge cases.", "evidenceIds": [ "groq-docs-6", "groq-docs-17", "groq-docs-28", "groq-probe-4" ] }, { "productId": "groq", "storyId": "agentic-mcp-server", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "There is a probe hit indicating an official MCP-server doc page exists (console.groq.com/docs/mcp), but the evidence pack contains no actual content, setup instructions, or independent confirmation of this server — the detailed MCP docs quoted (groq-docs-6/17/28) describe Groq's API acting as an MCP *client* consuming remote MCP servers, not Groq serving as an MCP server for other agents to connect to. Missing for 10: quoted documentation of the official MCP server's endpoint/connection instructions, evidence of an agent successfully connecting to Groq via that server, and independent/community confirmation it works as advertised.", "evidenceIds": [ "groq-probe-4", "groq-docs-6", "groq-docs-28" ] }, { "productId": "groq", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Groq's Chat Completions/Responses API is inherently natural-language driven, and its tool-use and remote-MCP features let a model take autonomous actions from NL instructions, which supports agentic operation on top of the API. However, this is evidence of building NL-driven agents *with* Groq's models, not of an NL command interface for operating the Groq platform/product itself (e.g., managing keys, billing, deployments via chat). Missing for 10: an NL/chat interface for controlling the Groq platform itself, independent verification of agentic tool-use reliability in production.", "evidenceIds": [ "groq-docs-14", "groq-docs-5", "groq-docs-27", "groq-docs-6", "groq-docs-28" ] }, { "productId": "groq", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of an official Groq CLI tool; documentation covers SDKs (Python/TypeScript), REST API, and MCP integration but no CLI is mentioned anywhere in the evidence pack.", "evidenceIds": [] }, { "productId": "groq", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Groq ships an extensively documented, OpenAI-compatible REST API (chat completions, streaming, tool use, structured outputs, batch, service tiers, LoRA, Responses API) with a live confirmed endpoint (401 on missing key, not 404) and a public status page. Missing for 10: a discoverable formal OpenAPI/Swagger spec (all candidate paths 404).", "evidenceIds": [ "groq-docs-1", "groq-docs-2", "groq-docs-25", "groq-docs-14", "groq-docs-27", "groq-docs-16", "groq-docs-23", "groq-probe-1", "groq-probe-2", "groq-probe-3", "groq-probe-rt-1", "groq-probe-rt-2" ] }, { "productId": "groq", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers billing spend limits and general API key auth (single api_key parameter) but shows no support for scoped/least-privilege credentials such as role-based keys, granular permission scopes, or per-agent restricted tokens; only one flat API key model is documented.", "evidenceIds": [ "groq-docs-2", "groq-docs-25", "groq-docs-12", "groq-docs-24" ] }, { "productId": "groq", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Groq maintains an official TypeScript SDK (groq-typescript) with concrete usage examples (audio transcription, file uploads) and extensive first-party docs covering chat completions, streaming, tool use, structured outputs, and OpenAI-compatible client usage, all of which are core to building against official SDKs. Missing for 10: explicit evidence of an official Python SDK repo (only OpenAI-compatible base_url swap is shown for Python) and independent/community validation specifically of SDK reliability rather than general API performance.", "evidenceIds": [ "groq-gh-1", "groq-gh-2", "groq-gh-3", "groq-docs-2", "groq-docs-25", "groq-docs-10", "groq-docs-5", "groq-docs-16" ] }, { "productId": "groq", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack of a webhooks/event-subscription feature for Groq's API—only synchronous/streaming inference, batch, tool-use/MCP, and admin/billing docs are covered.", "evidenceIds": [] }, { "productId": "groq", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Groq's docs (console.groq.com/docs) are extensive and code-example-rich (curl/Python/JS snippets for chat, tool-use, structured outputs, batch, MCP, etc.), functioning as a de facto API reference, but there's no evidence of an interactive, in-browser 'runnable' playground or embedded code execution — probes show no discoverable OpenAPI/swagger spec (404s) and no mention of a try-it-now console feature. Missing for 10: an actual interactive/executable API reference (e.g., Swagger UI, embedded runnable code sandbox), confirmation of an OpenAPI spec, and independent user confirmation of using such an interactive reference.", "evidenceIds": [ "groq-docs-1", "groq-docs-2", "groq-docs-10", "groq-probe-2", "groq-probe-3" ] }, { "productId": "groq", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Groq's API is OpenAI-compatible and well documented, but there is no evidence of a downloadable OpenAPI/Swagger spec; a direct probe of common spec paths (openapi.json, swagger.json, etc.) all returned 404, and no docs page links to a machine-readable spec.", "evidenceIds": [ "groq-probe-3" ] }, { "productId": "groq", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "groq", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item documents API versioning conventions or a formal deprecation policy for Groq's endpoints or models; docs cover OpenAI-compatibility, features, and pricing but not lifecycle/versioning guarantees.", "evidenceIds": [] }, { "productId": "groq", "storyId": "automation-bulk-operations", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Groq's Batch API explicitly supports submitting thousands of API requests asynchronously at 50% lower cost with a 24h-7day processing window, directly satisfying bulk operations across many items, and higher-throughput Flex processing further supports large-scale workloads. Missing for 10: independent hands-on verification of batch job completion/reliability at scale and details on per-batch item limits or error handling.", "evidenceIds": [ "groq-docs-7", "groq-docs-29", "groq-docs-9", "groq-docs-30" ] }, { "productId": "groq", "storyId": "automation-rules-engine", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Groq is an LLM inference API/platform (fast inference, tool-use, structured outputs, batch processing) — it has no workflow/automation engine for defining rules that trigger actions on events; this axis belongs to automation platforms, not an inference provider.", "evidenceIds": [] }, { "productId": "groq", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Groq is an inference API/hardware provider (LLM completions, batch, tool-use, MCP) with no scheduling/orchestration layer for recurring jobs or workflows; this is a category error for the product type, not a missing feature.", "evidenceIds": [] }, { "productId": "groq", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Groq is an inference API/hardware provider, not an automation-builder platform; there is no concept of building 'automations' to version, review, or roll back within its product scope.", "evidenceIds": [] }, { "productId": "groq", "storyId": "availability-sla", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows service tiers, flex processing, and a public status page, but nowhere is there a stated uptime/response-time SLA commitment for paid or enterprise customers; enterprise tiers are only referenced via 'call us for quotes' community comments with no SLA terms disclosed.", "evidenceIds": [ "groq-docs-8", "groq-docs-31", "groq-probe-rt-2", "groq-comm-16" ] }, { "productId": "groq", "storyId": "batch-discounted-jobs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Groq's official docs explicitly describe an asynchronous batch API delivering 50% lower cost than standard on-demand pricing, with a 24-hour to 7-day processing window and no impact on standard rate limits (groq-docs-7, groq-docs-29). This directly matches the story's requirement of documented discounted async batch inference. Missing for 10: independent/hands-on confirmation of the batch discount actually being applied in practice, and more detail on job submission/monitoring workflow beyond the docs summary.", "evidenceIds": [ "groq-docs-7", "groq-docs-29" ] }, { "productId": "groq", "storyId": "coding-agent-integrations", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Groq's docs clearly document OpenAI-compatible base_url swapping (groq-docs-2/25), tool-use/function-calling (groq-docs-5/27), a Responses API that is 'fully compatible with OpenAI's Responses API' (groq-docs-23), and first-party remote MCP server support where 'Groq handles all orchestration' (groq-docs-6/17/28, groq-probe-4) — all of which make it straightforward to plug into agent frameworks that speak the OpenAI protocol or MCP. However, there are no named first-party guides for specific coding-agent/agent-framework integrations (e.g., LangChain, Cursor, AutoGPT) beyond the generic OpenAI-compat pointer, and community reports flag real-world reliability quirks (groq-comm-12/13/14) that add friction beyond the docs' 'simple to integrate' framing. Missing for 10: named integration guides for specific agent frameworks/coding agents, and independent hands-on confirmation of smooth agent-framework plug-in.", "evidenceIds": [ "groq-docs-2", "groq-docs-25", "groq-docs-5", "groq-docs-27", "groq-docs-6", "groq-docs-17", "groq-docs-28", "groq-docs-23", "groq-probe-4", "groq-comm-12", "groq-comm-14" ] }, { "productId": "groq", "storyId": "day-zero-model-availability", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Groq's docs list several open-weight models (e.g. GPT-OSS 120B, Llama, Mixtral) and community reports mention using Kimi K2 'fast' shortly after its release, suggesting some open-weight models are added promptly, but there is no explicit documentation of a release-to-availability SLA or a systematic track record of rapid onboarding. Missing for 10: explicit timeline commitments or a changelog showing new open-weight model releases added within days, and independent verification beyond a single anecdote.", "evidenceIds": [ "groq-docs-3", "groq-comm-15", "groq-docs-11" ] }, { "productId": "groq", "storyId": "dedicated-endpoints-autoscale", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Groq's service tiers (on-demand, flex, auto) and batch processing, all within the shared serverless API, but no documentation of dedicated GPU capacity or customer-isolated infrastructure with autoscaling; one community remark about 'call us for quotes' for newer models hints at an enterprise track but gives no detail on dedicated capacity or autoscaling.", "evidenceIds": [ "groq-docs-8", "groq-docs-9", "groq-docs-19", "groq-comm-16" ] }, { "productId": "groq", "storyId": "documented-rate-limits", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Groq's docs describe multiple service tiers (on-demand, flex, auto) and note Flex Processing gives '10x higher rate limits' for paid customers, plus batch processing having 'no impact to your standard rate limits' and spend-limit alerts — showing tiered scaling exists and is documented at a high level. However, no evidence pack citation shows a dedicated rate-limits page with concrete numeric request/token-per-minute limits per tier (free vs paid vs flex), so a developer can't get exact production-planning numbers from what's cited. Missing for 10: a documented numeric rate-limit table per usage tier, and independent confirmation that these documented limits match real-world behavior.", "evidenceIds": [ "groq-docs-8", "groq-docs-9", "groq-docs-18", "groq-docs-19", "groq-docs-29", "groq-docs-30", "groq-docs-31", "groq-docs-24" ] }, { "productId": "groq", "storyId": "drop-in-provider-switch", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Groq's docs explicitly instruct developers to reuse OpenAI client libraries by just swapping api_key and base_url to https://api.groq.com/openai/v1, and this is corroborated by a live probe showing the OpenAI-style /v1/models endpoint responding in the expected JSON error format. Chat completions, streaming, tool-use, and structured outputs are all documented against this same OpenAI-compatible surface, supporting drop-in provider switching mid-workflow. Missing for 10: independent/hands-on confirmation from a real agent framework that switching providers requires zero code changes, and clarity on any OpenAI parameters/features that are NOT supported (full parity claim not verified end-to-end).", "evidenceIds": [ "groq-docs-2", "groq-docs-25", "groq-docs-1", "groq-probe-rt-1", "groq-docs-14" ] }, { "productId": "groq", "storyId": "embeddings-endpoint", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an embeddings endpoint or reranking model/API in Groq's documentation; the evidence pack covers chat completions, tool use, transcription, LoRA, batch, and service tiers but never mentions embeddings or rerank capabilities.", "evidenceIds": [] }, { "productId": "groq", "storyId": "fine-tune-and-serve", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Groq's docs show it supports serving pre-made LoRA adapters for inference (groq-docs-11, groq-docs-20, groq-docs-32), but there is no evidence of any fine-tuning/training capability on the platform itself — adapters must be trained elsewhere and uploaded. No mention of a fine-tuning API, training jobs, or dataset ingestion for model customization.", "evidenceIds": [ "groq-docs-11", "groq-docs-20", "groq-docs-32" ] }, { "productId": "groq", "storyId": "low-latency-serving", "verdict": "disputed", "quality": 6, "confidence": "medium", "rationale": "Early hands-on community reports strongly corroborate Groq's speed claims (491 T/s and ~450 t/s on Mixtral, deterministic low-latency architecture, streaming support docs), and Groq documents service tiers/flex-processing explicitly for latency/throughput tuning. However, more recent independent reports directly contradict this, describing Groq's hosted model performance as 'really poor for the token price' and 'always worse than traditional hosts,' plus reliability complaints ('so many random errors'), which concretely undercuts the fast/high-throughput positioning for current models. Missing for 10: recent (not 2024-era) throughput benchmarks, first-party TTFT numbers, and resolution of the conflicting recent user reports on performance-for-price.", "evidenceIds": [ "groq-comm-1", "groq-comm-2", "groq-comm-3", "groq-comm-4", "groq-docs-8", "groq-docs-18", "groq-docs-10", "groq-comm-12", "groq-comm-13", "groq-comm-14" ] }, { "productId": "groq", "storyId": "measured-speed-claims", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "There is no first-party published benchmark or load-testing page in the evidence (docs cover service tiers, flex processing, batch, streaming, but no concrete TPS/latency numbers or methodology). Independent hands-on community reports do supply concrete numbers (491 T/s and 450+ TPS for Mixtral, deterministic low-latency architecture claims) supporting the speed claim, but later community threads directly contradict this, calling performance 'really poor for the token price' and reliability 'worse than traditional hosts' with frequent errors. Missing for 10: an official Groq benchmark/load-testing page with methodology, and resolution of the conflicting recent community performance complaints.", "evidenceIds": [ "groq-comm-1", "groq-comm-2", "groq-comm-3", "groq-comm-4", "groq-comm-13", "groq-comm-14", "groq-docs-8", "groq-docs-9" ] }, { "productId": "groq", "storyId": "model-deprecation-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item documents a deprecation policy, sunset timeline, or advance-notice commitment for hosted models being removed; community comments even reference abrupt shifts (e.g., new models moved to 'call us for quotes') without formal notice process. This is a fair question for a model-hosting API but no supporting documentation exists.", "evidenceIds": [ "groq-comm-16" ] }, { "productId": "groq", "storyId": "models-endpoint-discovery", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Groq exposes an OpenAI-compatible /v1/models endpoint (confirmed live via probe returning JSON error structure, not 404), and docs establish OpenAI SDK/client compatibility with base_url https://api.groq.com/openai/v1, which per OpenAI spec includes the models list endpoint; this is documented and independently verified via a real GET request. missing for 10: no explicit Groq-authored documentation page specifically describing the /v1/models endpoint response schema, and no successful authenticated enumeration example shown (only the 401 unauthenticated probe).", "evidenceIds": [ "groq-probe-rt-1", "groq-docs-2", "groq-docs-25", "groq-docs-1" ] }, { "productId": "groq", "storyId": "multimodal-endpoints", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Groq's docs confirm audio transcription (Whisper models via groq-gh-2) and vision input support through the Responses API (text and image inputs, groq-docs-13), extending beyond pure text chat. However, there is no evidence of image-generation models or text-to-speech/audio-generation capability on the platform. missing for 10: image-generation model support, audio-generation/TTS support, independent hands-on confirmation of multimodal calls.", "evidenceIds": [ "groq-gh-2", "groq-docs-13", "groq-docs-23" ] }, { "productId": "groq", "storyId": "open-weight-catalog-breadth", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Groq's docs confirm OpenAI-compatible serverless endpoints hosting multiple model families, explicitly naming GPT-OSS 120B and (via community reports) Llama and Mixtral models, but the evidence pack never mentions Qwen or DeepSeek being hosted, so the 'broad catalog' claim is only partially substantiated. missing for 10: explicit documentation of Qwen and DeepSeek model availability, and a public model list/catalog page confirming breadth beyond GPT-OSS/Llama/Mixtral.", "evidenceIds": [ "groq-docs-3", "groq-docs-25", "groq-comm-1", "groq-comm-2", "groq-probe-rt-1" ] }, { "productId": "groq", "storyId": "openai-compatible-endpoint", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Groq's docs explicitly instruct developers to point OpenAI SDK clients at Groq by only changing api_key and base_url to https://api.groq.com/openai/v1, and a live probe confirms the OpenAI-style /v1/models endpoint is functional. This is corroborated by consistent documentation across chat completions, streaming, tool use, and structured outputs all using OpenAI SDK conventions. Missing for 10: independent third-party confirmation of a drop-in swap working flawlessly in production (community evidence focuses on speed/reliability rather than compatibility specifics).", "evidenceIds": [ "groq-docs-2", "groq-docs-25", "groq-probe-rt-1", "groq-docs-1" ] }, { "productId": "groq", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Groq's API surface covers essentially all the model-facing capabilities shown in its console/docs — chat completions, streaming, tool use, structured outputs, batch processing, service tiers, LoRA inference, and MCP remote tool use are all documented as API features (groq-docs-2,5,6,7,8,11,16,28). However, account/billing controls like spend limits (groq-docs-12/24) appear to be console-only with no documented API equivalent, and the probe found no public OpenAPI spec (all candidate paths 404, groq-probe-3), making it unclear whether the full API surface is discoverable/self-describing the way the UI is. Missing for 10: API-based management of billing/spend limits, a published OpenAPI/schema confirming full UI-API parity, and independent confirmation that every console feature (e.g., playground-only settings) has an API equivalent.", "evidenceIds": [ "groq-docs-2", "groq-docs-5", "groq-docs-6", "groq-docs-7", "groq-docs-11", "groq-docs-16", "groq-docs-28", "groq-docs-12", "groq-probe-3" ] }, { "productId": "groq", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Groq's docs cover API usage, models, tool-use, batch processing and billing, but nothing addresses exporting account data, conversation history, or batch outputs in open/portable formats, nor any account-closure data dump. Even though Groq's Responses API is 'stateful,' there is no documented export or data-portability mechanism.", "evidenceIds": [] }, { "productId": "groq", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "groq", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Groq is a hosted inference API/cloud service built on proprietary LPU hardware; there is no evidence (or plausibility) of a self-hostable core product, as the value proposition is entirely tied to Groq's own datacenter hardware. Self-hosting is a category error for this product type, not a missing feature.", "evidenceIds": [ "groq-comm-8", "groq-docs-1" ] }, { "productId": "groq", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, region selection, or geographic storage controls anywhere in Groq's docs, GitHub, or community discussion.", "evidenceIds": [] }, { "productId": "groq", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item addresses data usage/training policies, opt-out controls, or privacy commitments regarding whether user data is used to train Groq's models.", "evidenceIds": [] }, { "productId": "groq", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack items address data retention policies, deletion controls, or user-facing privacy/data management settings for Groq's API; the docs focus on inference features, tooling, pricing, and rate limits with nothing on retention/deletion.", "evidenceIds": [] }, { "productId": "groq", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item addresses telemetry, usage tracking, opt-out settings, or privacy controls for Groq's API or console; the docs cover inference features, pricing, and infra but nothing about data collection preferences.", "evidenceIds": [] }, { "productId": "groq", "storyId": "prompt-caching-support", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item mentions prompt/prefix caching, context caching, or any discounted repeated-context pricing on Groq's API. Docs cover streaming, batch processing, service tiers, and flex processing, but none describe caching repeated prompt prefixes to cut latency or cost, so this applicable axis is unaddressed.", "evidenceIds": [] }, { "productId": "groq", "storyId": "serve-custom-weights", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Groq's docs explicitly describe LoRA adapter upload and serving ('Upload your existing LoRA adapters to run specialized inference' — groq-docs-11/20/32), directly matching part of the story. However, evidence only covers LoRA adapters on top of existing base models, not uploading fully custom model weights, and there's no independent/hands-on confirmation of this LoRA workflow actually working in practice. Missing for 10: evidence of custom full-weight model upload/serving, and independent corroboration that LoRA upload works reliably in production.", "evidenceIds": [ "groq-docs-11", "groq-docs-20", "groq-docs-32" ] }, { "productId": "groq", "storyId": "spend-controls-budgets", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Groq's billing docs explicitly state users can set automated spending limits and receive budget alerts, directly matching the story. Missing for 10: no independent/hands-on corroboration of the spend-limit UI actually working, and no detail on granularity (per-project vs account-wide) or alert channels.", "evidenceIds": [ "groq-docs-12", "groq-docs-24" ] }, { "productId": "groq", "storyId": "status-page-uptime", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "A probe confirms a live public status page at groqstatus.com that returns HTTP 200 and shows 'operational' status, indicating Groq does operate a status page a founder could check before committing production traffic. However, the evidence doesn't confirm the page includes detailed incident history, uptime history, or granular component-level status, and there's no first-party documentation describing the status page's features. missing for 10: confirmation of incident history/log on the status page, documented SLA or uptime commitments, and independent corroboration beyond a single automated probe.", "evidenceIds": [ "groq-probe-rt-2" ] }, { "productId": "groq", "storyId": "streaming-responses", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly document setting stream=True for token-by-token delta streaming via the OpenAI-compatible API, aligned with real-time display use cases, and independent community testimonials corroborate very high token throughput consistent with responsive streaming UX. Missing for 10: explicit confirmation of SSE transport format (docs describe iterator/deltas but don't spell out 'SSE' verbatim) and no independent hands-on benchmark specifically of streaming latency/SSE behavior.", "evidenceIds": [ "groq-docs-10", "groq-docs-15", "groq-docs-22", "groq-docs-26", "groq-comm-1", "groq-comm-2", "groq-comm-6" ] }, { "productId": "groq", "storyId": "structured-outputs-json-schema", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Groq's docs explicitly describe Structured Outputs with JSON Schema and a strict:true mode that uses constrained decoding to guarantee schema-conforming output, plus tool-use docs confirming JSON schema-based function calling. This is first-party documented and consistent with OpenAI-compatible API design. Missing for 10: independent/hands-on verification that strict mode reliably parses in production and no community corroboration of structured-output reliability specifically.", "evidenceIds": [ "groq-docs-4", "groq-docs-16", "groq-docs-21", "groq-docs-27" ] }, { "productId": "groq", "storyId": "tool-calling-fidelity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Groq's docs clearly document OpenAI-compatible function/tool calling (JSON-schema tool definitions, strict-mode structured outputs, and now Remote MCP where 'Groq handles all orchestration') which supports agentic tool use, but none of the evidence explicitly confirms parallel tool calls or multi-step tool-use loops working reliably. Community reports also raise reliability concerns ('so many random errors, so many silly quirks') that create some doubt about robustness for agent loops, though not a concrete tool-calling failure. missing for 10: explicit documentation/examples of parallel tool calling, multi-step agentic loop reliability testing, and independent hands-on verification that tool calls remain faithful across steps.", "evidenceIds": [ "groq-docs-5", "groq-docs-27", "groq-docs-4", "groq-docs-6", "groq-docs-28", "groq-comm-12", "groq-comm-13" ] }, { "productId": "groq", "storyId": "transparent-token-pricing", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Groq's docs discuss billing mechanics (spend limits, flex-processing pricing parity, batch discount) implying a public pricing model, but no evidence pack item shows an actual public per-token price sheet for every hosted model, and a community report explicitly states Groq switched to 'call us for quotes' for newer models, contradicting the claim of fully public pricing without sales contact. Missing for 10: an actual public pricing page listing per-token rates for each model, and confirmation that newer/premium models are not gated behind sales quotes.", "evidenceIds": [ "groq-docs-12", "groq-docs-24", "groq-docs-30", "groq-comm-16", "groq-comm-7" ] }, { "productId": "morph", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Morph confirms llms.txt is live and documented (HTTP 200) with a full documentation index, and also exposes .md versions of docs pages explicitly for agent consumption, directly matching the story's ask. Missing for 10: no independent/community confirmation of agents actually consuming llms.txt in practice.", "evidenceIds": [ "morph-probe-1", "morph-probe-2", "morph-docs-1" ] }, { "productId": "morph", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Morph is developer infrastructure (fast-apply code editing, model routing, classifiers) accessed via API/SDK/MCP, not a product with a UI surfacing AI-generated insights from a user's own data; this story targets data-analytics/BI-style products and is a category mismatch for Morph.", "evidenceIds": [] }, { "productId": "morph", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Morph is presented as an LLM inference/API platform (fast apply, compact, batch completions, canary/reflex model switching) rather than a background automation or agent-orchestration product; nothing in the evidence describes setting up autonomous, self-running background automations or scheduled agentic tasks. The closest features (batch processing, canary traffic routing) are developer-triggered infra operations, not autonomous background automations.", "evidenceIds": [ "morph-docs-13", "morph-docs-3", "morph-docs-4" ] }, { "productId": "morph", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Morph is a developer-facing API/infrastructure platform (fast-apply, chat models, MCP tools) meant to be embedded into other products or coding agents — it is not itself an end-user product with a built-in assistant UI to delegate tasks to. This axis is a category error for an API/infra offering, not a missing feature.", "evidenceIds": [] }, { "productId": "morph", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Morph is delivered as an OpenAI/Anthropic-compatible HTTP API (morph-docs-1, morph-docs-16) plus a batch mode explicitly designed for running 'thousands of chat completions offline' (morph-docs-13), both of which are naturally scriptable/headless and usable from CI pipelines. However, there is no explicit CI/CD documentation, GitHub Actions example, or automation-pipeline guide in the evidence pack. Missing for 10: explicit CI/automation guide or example, dedicated headless-mode docs beyond generic API access, independent confirmation of CI usage.", "evidenceIds": [ "morph-docs-1", "morph-docs-13", "morph-docs-16", "morph-probe-1" ] }, { "productId": "morph", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Morph is an LLM API/tool-serving platform (fast-apply, reflexes, model routing) that itself exposes an MCP server for other agents (e.g., Claude Code) to consume — it is not an agent host that ingests external MCP servers as tool sources. The story's client-side capability (plugging MCP servers into Morph so it can use their tools) is a category mismatch for this product type; all MCP evidence describes Morph being served TO clients, not consuming servers itself.", "evidenceIds": [ "morph-docs-5", "morph-docs-9", "morph-probe-4" ] }, { "productId": "morph", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Morph publishes an official MCP server (guides/mcp.md, quickstart 'Install the Morph MCP so Claude Code can read it') that lets external agents like Claude Code connect to its fast file-editing/search tools, confirmed live via probe [morph-probe-4]. missing for 10: independent/hands-on confirmation that the MCP server works as documented (community evidence only covers the apply model, not MCP connectivity), and no detail on MCP tool list/config beyond the guide.", "evidenceIds": [ "morph-docs-9", "morph-docs-5", "morph-docs-10", "morph-probe-4" ] }, { "productId": "morph", "storyId": "agentic-nl-commands", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Morph is a backend model-serving/API and MCP-tool provider consumed programmatically by coding agents (Claude Code, Cursor, etc.); it is not itself a user-facing surface where an end user types natural-language commands. The natural-language interface lives in the calling agent, not in Morph, so this axis is a category mismatch for this kind of infrastructure product.", "evidenceIds": [] }, { "productId": "morph", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Morph is API/SDK-focused (OpenAI-compatible endpoints, MCP integration, Fast Apply, etc.) but no evidence pack item mentions an official Morph CLI tool; community only references third-party CLIs (llm.datasette.io) connecting to Morph's API, not a first-party CLI shipped by Morph.", "evidenceIds": [] }, { "productId": "morph", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Morph documents a public OpenAI-compatible API at api.morphllm.com/v1 plus a native Anthropic Messages endpoint, with SDK docs covering chat, Fast Apply, Compact, Batch, caching, and dedicated endpoints, and this is corroborated by third-party usage (HN commenters, github demos) actually driving it programmatically. Missing for 10: no discoverable OpenAPI/swagger spec (404s on all probed paths) and mixed community reports on output reliability reduce polish.", "evidenceIds": [ "morph-docs-1", "morph-docs-11", "morph-docs-12", "morph-docs-13", "morph-docs-16", "morph-probe-1", "morph-probe-3", "morph-gh-1", "morph-comm-1", "morph-comm-5" ] }, { "productId": "morph", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows only a single API key covering all models/tools with no mention of scoped, least-privilege, or role-based credential issuance for agents; no docs on permission scoping, sub-keys, or credential restriction. missing for 10: scoped/limited-permission API key creation, per-agent credential issuance, role-based access control docs.", "evidenceIds": [ "morph-docs-1" ] }, { "productId": "morph", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Morph documents OpenAI-SDK and Anthropic Messages-API compatibility (point any OpenAI SDK at api.morphllm.com, native /v1/messages support), and community evidence confirms real developers wiring it up via OpenAI-compatible clients (e.g. llm.datasette.io CLI). However there's no evidence of a first-party Morph-branded SDK package, and an OpenAPI spec probe returned 404s across all candidate paths, suggesting no formal machine-readable API spec is published. missing for 10: dedicated first-party Morph SDK package, discoverable OpenAPI/swagger spec, broader language-SDK coverage beyond OpenAI/Anthropic compatibility.", "evidenceIds": [ "morph-docs-1", "morph-docs-7", "morph-docs-16", "morph-comm-5", "morph-probe-3" ] }, { "productId": "morph", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions webhooks or any event-subscription mechanism; Morph's documented surface is API endpoints, MCP integration, and model tooling, with no webhook capability described.", "evidenceIds": [] }, { "productId": "morph", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows standard markdown docs (llms.txt, introduction.md) but no interactive API reference or runnable-example explorer — probes for an OpenAPI/Swagger spec returned 404 on every candidate path, and no docs mention a live API console. A community mention of a separate 'playground' relates to testing the apply model, not an API reference tool, and doesn't establish this capability.", "evidenceIds": [ "morph-probe-3", "morph-probe-1", "morph-probe-2" ] }, { "productId": "morph", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Morph is API-based (OpenAI-compatible endpoint), so an OpenAPI spec is a fair expectation, but a direct probe found all candidate OpenAPI/swagger spec paths returned 404, and no evidence pack item shows a downloadable machine-readable spec.", "evidenceIds": [ "morph-probe-3" ] }, { "productId": "morph", "storyId": "api-sandbox", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Morph is an API/model-serving platform for code editing, embeddings, and agent tooling — it has no concept of a 'sandbox vs production data' environment for testing; this axis is a category error for its product type.", "evidenceIds": [] }, { "productId": "morph", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of API versioning scheme (e.g., v1/v2) or a documented deprecation policy anywhere in the docs; the /v1 path is just the base URL, not a versioning/deprecation commitment. OpenAPI spec probes also 404, further indicating no formal versioned API contract is published.", "evidenceIds": [ "morph-probe-3", "morph-docs-1" ] }, { "productId": "morph", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Morph offers batch offline completions (morph-docs-13) and fast bulk file-editing/apply capabilities (morph-docs-11, morph-gh-1), which support some bulk/automation-depth use cases, but there is no documented feature for bulk operations across many arbitrary items (e.g., batch file renames, mass repo-wide refactors, multi-item dataset operations) beyond chat completions and code apply. missing for 10: explicit bulk/batch operation APIs beyond chat completions, evidence of bulk operations across many discrete items (files, records, tickets) rather than just token throughput, independent verification of bulk reliability at scale.", "evidenceIds": [ "morph-docs-13", "morph-docs-11", "morph-gh-1" ] }, { "productId": "morph", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Morph's 'Reflexes' are classifiers that detect events (jailbreak, frustration, loop) and can gate automatic actions like canary traffic-splitting or full migration between models, which is a narrow form of event-triggered automation. However there's no documented general-purpose rule engine letting users define arbitrary custom rules/actions beyond this migration-gating use case. Missing for 10: a general user-facing rules/automation builder, documentation of broader trigger-action mappings beyond model migration, and independent confirmation of custom Reflex-to-action wiring.", "evidenceIds": [ "morph-docs-3", "morph-docs-4", "morph-docs-6" ] }, { "productId": "morph", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Morph is an API/model platform for code editing, apply, search, and agent-provider migration—no evidence of any scheduler, cron-like trigger, or recurring workflow/job execution feature in the docs or community posts.", "evidenceIds": [] }, { "productId": "morph", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Morph is an API/tooling platform for fast code editing, model routing, and reflex classifiers — there's no evidence of any versioning, review, or rollback mechanism for 'automations' (e.g., no workflow history, diff review UI for automation configs, or revert capability). Nothing in the evidence pack addresses this axis at all.", "evidenceIds": [] }, { "productId": "morph", "storyId": "availability-sla", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions an SLA, uptime commitment, or reliability guarantee for paid/enterprise tiers — dedicated endpoints and enterprise privacy terms are mentioned but no stated availability percentage or SLA terms are documented.", "evidenceIds": [ "morph-docs-15", "morph-comm-6" ] }, { "productId": "morph", "storyId": "batch-discounted-jobs", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Morph documents a batch component explicitly for offline, asynchronous chat completions at 'half price' (morph-docs-13), directly matching the story's async-job-with-discount claim. However, there's no detail on job submission/status API mechanics, SLA/turnaround time, or independent confirmation of the discount in practice. missing for 10: detailed batch API docs (submission, polling, output retrieval), documented turnaround SLAs, independent/community verification of discount pricing.", "evidenceIds": [ "morph-docs-13" ] }, { "productId": "morph", "storyId": "coding-agent-integrations", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Morph provides first-party documented guides for integrating with coding agents/frameworks: OpenAI-compatible endpoint (morph-docs-1), native Anthropic Messages API for Claude Code (morph-docs-7), MCP server setup for Claude Code (morph-docs-5, morph-docs-9, morph-probe-4), and Codex integration via edit_file (morph-docs-8). Community evidence (goose author, HN discussion) corroborates real-world use in agent frameworks. Missing for 10: independent verification of MCP/agent integration reliability beyond docs and mixed community feedback on apply accuracy.", "evidenceIds": [ "morph-docs-1", "morph-docs-5", "morph-docs-7", "morph-docs-8", "morph-docs-9", "morph-probe-4", "morph-comm-1" ] }, { "productId": "morph", "storyId": "day-zero-model-availability", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence shows Morph supports a curated set of open-weight chat models (e.g., Kimi K3) and offers OpenAI/Anthropic-compatible access, but there is no documentation or report about how quickly new open-weight model releases are added to the platform. A community comment even questions why only an older model was listed on OpenRouter, suggesting possible lag, but this is not a concrete contradiction of a specific claim. Missing for 10: any release-cadence documentation, changelog, or independent confirmation of rapid onboarding of newly released open-weight models.", "evidenceIds": [ "morph-docs-3", "morph-docs-16", "morph-comm-8" ] }, { "productId": "morph", "storyId": "dedicated-endpoints-autoscale", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Morph docs mention 'dedicated endpoints' where you reserve model capacity and plan and Morph provisions/operates it, which implies isolated capacity, but there is no mention of autoscaling, traffic isolation guarantees, or GPU-level specifics. missing for 10: explicit autoscaling behavior, confirmation of traffic isolation from shared serverless pool, GPU capacity details, independent corroboration of dedicated endpoint performance.", "evidenceIds": [ "morph-docs-15" ] }, { "productId": "morph", "storyId": "documented-rate-limits", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence mentions rate limits exist (morph-docs-16: 'same open source chat models, token billing, and rate limits') but never documents actual limit numbers, tiers, or how they scale with usage — no pricing/tier table or rate-limit reference is shown. Missing for 10: an actual rate-limit table or tier breakdown, numeric thresholds, and documentation on how limits change across plans.", "evidenceIds": [ "morph-docs-16", "morph-docs-15" ] }, { "productId": "morph", "storyId": "drop-in-provider-switch", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Morph explicitly documents an OpenAI-compatible endpoint (and Anthropic Messages compatibility) plus first-class provider-switching workflows—canary traffic splits and full migration from OpenAI/Anthropic to another model—showing the standard API enables mid-workflow provider switches without code changes. Community evidence (morph-comm-5) independently confirms the OpenAI-compatible endpoint works with third-party tools. missing for 10: independent hands-on verification of the canary/migration switching feature itself (only vendor docs), and no evidence of switching away from Morph to another provider.", "evidenceIds": [ "morph-docs-1", "morph-docs-3", "morph-docs-4", "morph-docs-16", "morph-probe-1", "morph-comm-5" ] }, { "productId": "morph", "storyId": "embeddings-endpoint", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Morph's evidence focuses on chat completions, Fast Apply code editing, compaction, batch processing, caching, and dedicated endpoints — none of the docs mention embeddings generation or reranking capabilities for retrieval pipelines.", "evidenceIds": [] }, { "productId": "morph", "storyId": "fine-tune-and-serve", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Morph offers 'Reflexes' — small custom classifiers that train in ~30s on your data and are then served/gated in production (morph-docs-6, morph-docs-3, morph-docs-4) — plus dedicated endpoints to reserve capacity for a chosen model (morph-docs-15). This is a narrow, specialized form of 'train on your data and serve it' rather than fine-tuning a general-purpose base chat/completion model. Missing for 10: explicit fine-tuning API/pipeline for full base language models (not just small classifiers), documentation on custom dataset upload/training for chat models, and evidence that a fine-tuned base model (vs. a Reflex classifier) can be served alongside the standard endpoints.", "evidenceIds": [ "morph-docs-6", "morph-docs-3", "morph-docs-4", "morph-docs-15" ] }, { "productId": "morph", "storyId": "low-latency-serving", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Morph publishes concrete high-throughput numbers (10,500+ tok/s Fast Apply, 33,000 tok/s Compact, ~90ms classifiers, prefix caching with no surcharge) aimed squarely at low-latency/high-throughput serving, and independent community use (goose author) corroborates a real speed boost in practice. Missing for 10: explicit time-to-first-token benchmarks and independent third-party throughput verification beyond vendor docs and one anecdotal HN comment.", "evidenceIds": [ "morph-docs-8", "morph-docs-11", "morph-docs-12", "morph-docs-2", "morph-docs-14", "morph-comm-1" ] }, { "productId": "morph", "storyId": "measured-speed-claims", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Morph publishes specific throughput numbers (10,500+ tok/s Fast Apply, 33,000 tok/s Compact, 90ms classifiers, 30s Reflex training) across its docs and even one independent community post corroborates a speed/iteration boost, but there is no independent benchmark suite, third-party load-testing report, or reproducible methodology, and one community report describes inaccurate output rather than confirming throughput. missing for 10: independent benchmark/load-testing reports, published methodology or test conditions behind the tok/s figures, and resolution of the community accuracy complaint.", "evidenceIds": [ "morph-docs-8", "morph-docs-11", "morph-docs-12", "morph-docs-2", "morph-docs-6", "morph-gh-1", "morph-comm-1", "morph-comm-2" ] }, { "productId": "morph", "storyId": "model-deprecation-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any documented deprecation policy or advance-notice commitment for hosted models; docs cover model capabilities, migration, and endpoints but nothing about model lifecycle/removal notice.", "evidenceIds": [] }, { "productId": "morph", "storyId": "models-endpoint-discovery", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Morph documents an OpenAI-compatible base URL (/v1) and multiple endpoints (/v1/messages, batch, caching, dedicated endpoints) but no evidence pack item mentions a GET /v1/models catalog endpoint, and the OpenAPI schema probe returned 404s across all candidate paths, meaning no documented machine-readable catalog listing was found.", "evidenceIds": [ "morph-docs-1", "morph-probe-3" ] }, { "productId": "morph", "storyId": "multimodal-endpoints", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "All evidence describes Morph as a text/code-focused LLM API platform (chat completions, Fast Apply code editing, Reflexes classifiers, batch/caching) with no mention of vision, audio, or image-generation model endpoints. This axis applies to an LLM API platform, but no evidence supports multimodal capability.", "evidenceIds": [] }, { "productId": "morph", "storyId": "open-weight-catalog-breadth", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Morph's docs confirm an OpenAI-compatible endpoint serving 'open-weight chat models' broadly, with prefix caching enabled for 'every open source model' and specific mention of migrating between providers like Kimi K3, but no evidence explicitly lists Llama, Qwen, DeepSeek, GPT-OSS or enumerates a broad multi-family catalog. Missing for 10: an explicit model catalog/list naming specific open-weight families (Llama, Qwen, DeepSeek, GPT-OSS), and independent confirmation of catalog breadth beyond Kimi K3.", "evidenceIds": [ "morph-docs-1", "morph-docs-3", "morph-docs-14", "morph-docs-16" ] }, { "productId": "morph", "storyId": "openai-compatible-endpoint", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly state to point any OpenAI SDK at https://api.morphllm.com/v1 with a single API key, corroborated by a probe confirming the llms.txt describes it as OpenAI-compatible, and community evidence confirms it was actually wired up with an OpenAI-compatible client tool. missing for 10: independent hands-on confirmation of a truly drop-in swap with zero code changes beyond base URL/key, and no official OpenAPI spec found to verify full compatibility surface.", "evidenceIds": [ "morph-docs-1", "morph-probe-1", "morph-comm-5" ] }, { "productId": "morph", "storyId": "openness-api-parity", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Morph is fundamentally API-first — nearly all documented functionality (chat completions, Fast Apply, Compact, Reflexes, batch, dedicated endpoints, MCP) is exposed via the API/SDK per morph-docs-1 through morph-docs-16, and a playground/dashboard UI is only mentioned in passing (morph-comm-2). There's no explicit documentation enumerating dashboard-only features (billing, endpoint provisioning UI, playground testing) and confirming each has an API equivalent, so full parity isn't demonstrated. Missing for 10: explicit UI/dashboard feature list mapped to API endpoints, confirmation that account/billing/endpoint-management actions available in the dashboard are also API-accessible, and independent verification of parity.", "evidenceIds": [ "morph-docs-1", "morph-docs-15", "morph-comm-2", "morph-probe-1" ] }, { "productId": "morph", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of a data export feature or open-format export of user data/history; Morph is an API/tooling platform for code editing and model access, but nothing addresses exporting stored user data for portability or account exit.", "evidenceIds": [] }, { "productId": "morph", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Morph is a hosted API/SaaS product; no evidence of an open-license source repository for the core product (morph-demos is just example code, not the product source). No license or open-source claim appears anywhere in the evidence pack.", "evidenceIds": [] }, { "productId": "morph", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Morph is presented entirely as a hosted API/SaaS product (api.morphllm.com endpoints, dedicated endpoints as reserved capacity, prefix caching, batch processing) with no mention of on-premises deployment, downloadable server binaries, Docker images, or open-source release of the core service. Evidence pack contains no self-hosting instructions or claims.", "evidenceIds": [ "morph-docs-1", "morph-docs-15", "morph-probe-1", "morph-probe-3" ] }, { "productId": "morph", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, regional storage options, or geographic controls; the closest is a comment about retention/training policy tiers (morph-comm-6), which is about usage not location. Missing for 10: any documentation of region selection, data residency guarantees, or geo-specific hosting options.", "evidenceIds": [ "morph-comm-6" ] }, { "productId": "morph", "storyId": "privacy-no-training", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "A community-sourced excerpt of Morph's privacy policy (morph-comm-6) shows Enterprise Tier users can opt out of having code/data used for training, satisfying the story, but Free Tier data is explicitly used to train models by default — a significant caveat requiring a paid tier to actually prevent training use. No first-party Morph docs in the evidence pack directly address data retention/training opt-out. Missing for 10: first-party documentation of the privacy/data-use policy and opt-out mechanism, independent confirmation that Enterprise-tier exclusion is honored in practice.", "evidenceIds": [ "morph-comm-6" ] }, { "productId": "morph", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows a general privacy-policy statement about free vs enterprise data usage (morph-comm-6), with a community member explicitly asking for clarification — there is no documented user-facing control for data retention settings or deletion requests. Missing for 10: any docs on data retention windows, user-initiated deletion/export tools, or enterprise data handling guarantees beyond the raw policy snippet.", "evidenceIds": [ "morph-comm-6" ] }, { "productId": "morph", "storyId": "privacy-telemetry-optout", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Community evidence indicates a tiered privacy policy where free-tier code may be used for training while Enterprise-tier data is not used beyond immediate processing, implying paying/enterprise users can effectively opt out of data usage — but there is no explicit telemetry/usage-tracking opt-out control or documentation for individual users. missing for 10: explicit telemetry opt-out toggle/setting, first-party privacy documentation confirming opt-out mechanics, independent confirmation beyond a single HN comment.", "evidenceIds": [ "morph-comm-6" ] }, { "productId": "morph", "storyId": "prompt-caching-support", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Morph docs explicitly state prefix caching is enabled by default for all open-source models with no configuration and no cache-write surcharge, directly addressing latency/cost reduction on repeated context. Missing for 10: independent benchmark or community corroboration of actual latency/cost savings from caching specifically (community evidence covers Fast Apply speed, not caching).", "evidenceIds": [ "morph-docs-14" ] }, { "productId": "morph", "storyId": "serve-custom-weights", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Morph's docs describe serving open-weight chat models, dedicated capacity reservations, and training small in-house 'Reflex' classifiers, but nothing about ml-engineers uploading their own custom model weights or LoRA adapters for serving. missing for 10: any weight/adapter upload API, LoRA fine-tuning support, or documentation of bringing custom checkpoints to Morph's serving layer.", "evidenceIds": [ "morph-docs-6", "morph-docs-15", "morph-docs-1" ] }, { "productId": "morph", "storyId": "spend-controls-budgets", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions spending caps, budget alerts, usage limits, or billing controls of any kind; docs focus on models, tools, and pricing mechanics like batch/cache but not cap-setting features. This axis applies to any paid API/cloud product but no capability is documented, so it's 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "morph", "storyId": "status-page-uptime", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a public status page or incident history for Morph anywhere in the pack; only docs, demos, and community discussion threads are present. missing for 10: a public status page URL, uptime/incident history, SLA commitments.", "evidenceIds": [] }, { "productId": "morph", "storyId": "streaming-responses", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Morph advertises full OpenAI SDK/API compatibility at api.morphllm.com/v1 (morph-docs-1), which implies standard streaming support, but no evidence explicitly documents SSE/token-by-token streaming, a `stream` parameter, or any latency/responsiveness benchmarks for streamed output. missing for 10: explicit docs or examples showing `stream=true`/SSE usage, first-party or community confirmation of token-by-token streaming behavior, and any measured time-to-first-token metrics.", "evidenceIds": [ "morph-docs-1", "morph-probe-1" ] }, { "productId": "morph", "storyId": "structured-outputs-json-schema", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Morph's docs cover OpenAI-compatible endpoints, Fast Apply, compaction, batch, caching, and MCP integration, but no evidence describes JSON schema/grammar-constrained structured output enforcement for reliable parsing. Missing for 10: any mention of response_format/json_schema support, grammar constraints, or validation guarantees on model outputs.", "evidenceIds": [] }, { "productId": "morph", "storyId": "tool-calling-fidelity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Morph documents OpenAI-compatible and Anthropic Messages-compatible endpoints and specialized tools like edit_file/Fast Apply meant to plug into agent loops (Claude Code, Codex) with high speed and claimed 98% accuracy, which implies decent tool-call fidelity for open models. However there's no explicit documentation of parallel or multi-step tool-call handling, and a hands-on community report describes the apply/tool behavior as 'completely broken' with unrequested edits, undercutting full confidence in reliability. Missing for 10: explicit parallel tool-call support docs, multi-step agent-loop stability guarantees, and independent verification resolving the reported apply failures.", "evidenceIds": [ "morph-docs-7", "morph-docs-8", "morph-docs-11", "morph-docs-9", "morph-comm-2", "morph-comm-7" ] }, { "productId": "morph", "storyId": "transparent-token-pricing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item shows a public pricing page or per-token price list; docs mention batch pricing at 'half price' and mention of a 'plan' for dedicated endpoints but no explicit public per-token rates are cited, and no pricing page was probed or found.", "evidenceIds": [] }, { "productId": "together-ai", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Together AI publishes a working llms.txt (HTTP 200) with a documentation index, per-page .md docs, and even a live MCP server for docs lookup, all independently confirmed via probes plus first-party docs on agent-skills. missing for 10: no independent third-party report of an agent actually consuming llms.txt successfully in practice.", "evidenceIds": [ "together-ai-probe-1", "together-ai-probe-2", "together-ai-probe-rt-3", "together-ai-docs-12", "together-ai-docs-30" ] }, { "productId": "together-ai", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Together AI is an inference/training infrastructure API platform; it does not host 'your data' inside a product surface with dashboards or analytics from which it generates insights and suggestions — that role belongs to end-user applications built on top of it, not to Together AI itself.", "evidenceIds": [] }, { "productId": "together-ai", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "together-ai", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "together-ai", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Together AI is an API/SDK-first inference platform with OpenAI-compatible REST endpoints, official server-side SDKs (TypeScript/Python), batch inference, streaming via SSE, and structured/function-calling outputs — all natively usable in headless CI/automation pipelines without any UI dependency. Live probes confirm the API and status endpoints are reachable programmatically. missing for 10: no explicit CI/CD example or GitHub Actions template in evidence, no documented CLI exit-code/scripting conventions, and no independent case study of a CI pipeline using Together AI headlessly.", "evidenceIds": [ "together-ai-docs-1", "together-ai-docs-2", "together-ai-docs-3", "together-ai-gh-1", "together-ai-gh-2", "together-ai-probe-rt-1", "together-ai-docs-21" ] }, { "productId": "together-ai", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence only shows Together AI publishing its own MCP server (docs lookup) for coding agents to consume — i.e., Together acting as an MCP *server*, not as a client that lets users plug external MCP servers into it. There's no documentation or probe showing Together's platform/API can connect to and use tools from user-supplied MCP servers, despite having function/tool-calling APIs. missing for 10: any evidence of MCP-client support (configuring/connecting external MCP servers within Together's product), integration of MCP tool results into its agentic function-calling flow.", "evidenceIds": [ "together-ai-docs-12", "together-ai-docs-19", "together-ai-docs-20", "together-ai-docs-30", "together-ai-probe-4", "together-ai-probe-rt-3", "together-ai-docs-9", "together-ai-docs-16" ] }, { "productId": "together-ai", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Together AI documents an official Docs MCP server that gives agents live access to documentation, and a live probe confirms the MCP endpoint at docs.together.ai/mcp responds correctly to an initialize call with proper protocol capabilities. This is corroborated by both first-party docs and independent hands-on verification. Missing for 10: broader coverage beyond docs lookup (e.g., an MCP server exposing inference/model-management actions rather than just documentation retrieval), and no community discussion confirming real-world agent integration experiences.", "evidenceIds": [ "together-ai-docs-12", "together-ai-docs-19", "together-ai-docs-20", "together-ai-docs-30", "together-ai-probe-4", "together-ai-probe-rt-3" ] }, { "productId": "together-ai", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Together AI is an inference/training API platform operated via REST API, SDKs, CLI, and console — the evidence shows structured commands (API calls, CLI syntax like 'tg beta endpoints deploy...') rather than any natural-language command interface for operating the platform itself. The agent-skills/MCP docs only help external coding agents write correct Together AI code, not let a user operate Together AI's own product surface via natural language.", "evidenceIds": [] }, { "productId": "together-ai", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs mention an official CLI (the 'tg' command, e.g. 'tg beta endpoints deploy...') and explicitly state fine-tuning jobs can be launched 'through the API/SDK, or with the CLI', confirming a real CLI exists for agentic/dev workflows. However, evidence lacks a dedicated CLI reference page, install instructions, full command coverage, or independent hands-on corroboration. Missing for 10: dedicated CLI documentation/installation guide, comprehensive command reference, independent user verification of CLI usage.", "evidenceIds": [ "together-ai-docs-25", "together-ai-docs-17" ] }, { "productId": "together-ai", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Together AI exposes an OpenAI-compatible REST API with official SDKs (TypeScript/JS), documented endpoints for chat, batch, fine-tuning, dedicated deployment, and a live, publicly reachable models endpoint confirmed by probe; docs are extensive and public. Missing for 10: a discoverable formal OpenAPI/swagger spec (probe found all candidate paths 404).", "evidenceIds": [ "together-ai-docs-1", "together-ai-docs-2", "together-ai-gh-2", "together-ai-probe-rt-1", "together-ai-probe-3" ] }, { "productId": "together-ai", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of scoped or least-privilege API key issuance (e.g., per-project keys, role/permission scoping, or restricted-scope tokens for agents); docs only mention a single API key used for authentication, with no mention of scoping controls.", "evidenceIds": [ "together-ai-docs-1" ] }, { "productId": "together-ai", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Together AI provides an official TypeScript/JavaScript SDK (github repo) and OpenAI-compatible API/SDK usage, with docs covering chat, structured outputs, function calling, streaming, fine-tuning, and dedicated deployment all launchable via API/SDK. This directly supports AI-native, agentic development against official SDKs, including agent loop patterns and tool calling. Missing for 10: independent hands-on developer reviews specifically praising SDK quality/completeness, and no official Python SDK repo cited in evidence pack.", "evidenceIds": [ "together-ai-gh-1", "together-ai-gh-2", "together-ai-docs-1", "together-ai-docs-8", "together-ai-docs-9", "together-ai-docs-17", "together-ai-docs-7" ] }, { "productId": "together-ai", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of webhook subscription support anywhere in the docs, SDKs, or probes; Together AI offers SSE streaming and a docs MCP server, but nothing about webhook event subscriptions for async notifications (e.g., fine-tune job completion, batch job status).", "evidenceIds": [] }, { "productId": "together-ai", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows extensive prose documentation (guides, overview pages) but no reference to an interactive API reference or runnable/try-it-now code examples; a direct probe for OpenAPI/Swagger specs at standard paths returned 404 for all candidates, indicating no interactive spec-driven reference was found. No community or hands-on evidence confirms a runnable API console exists.", "evidenceIds": [ "together-ai-probe-3", "together-ai-probe-1", "together-ai-probe-2" ] }, { "productId": "together-ai", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Active probing for standard OpenAPI/swagger spec locations (openapi.json, swagger.json, etc.) all returned 404, and no docs page in the pack references a downloadable OpenAPI/machine-readable spec for the Together API.", "evidenceIds": [ "together-ai-probe-3" ] }, { "productId": "together-ai", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of a dedicated sandbox/staging environment; the only related mention is a test account with trial credit on the same production API, not an isolated sandbox for testing without touching production data.", "evidenceIds": [ "together-ai-comm-3" ] }, { "productId": "together-ai", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows a documented, OpenAI-compatible REST API (with a /v1/ endpoint) but contains no mention of a versioning scheme or a documented deprecation policy for API changes. Absence of evidence for this applicable capability means it cannot be credited as delivered.", "evidenceIds": [] }, { "productId": "together-ai", "storyId": "automation-bulk-operations", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Together AI offers an asynchronous Batch API explicitly designed for bulk workloads across many items at once, at up to 50% lower cost than synchronous calls, alongside dedicated endpoints and fine-tuning pipelines that can process large datasets end-to-end. missing for 10: no documented batch size/throughput limits, no independent/hands-on report validating batch job behavior at scale, and no detail on input format flexibility for large item counts.", "evidenceIds": [ "together-ai-docs-3", "together-ai-docs-23", "together-ai-docs-28" ] }, { "productId": "together-ai", "storyId": "automation-rules-engine", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Together AI is an inference/training/hosting API platform, not a workflow-automation or event-rule system; there is no evidence of a rules engine, triggers, or event-driven actions being a category feature. This story targets automation-platform capabilities that fall outside Together AI's product scope (model inference, fine-tuning, dedicated endpoints), so the axis does not apply.", "evidenceIds": [] }, { "productId": "together-ai", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Together AI documents async batch workloads, fine-tuning jobs, and dedicated endpoints, but nowhere in the evidence is there a scheduler, cron-like trigger, or recurring-job/workflow orchestration mechanism; jobs appear to be triggered manually via API/CLI/console rather than on a recurring schedule.", "evidenceIds": [ "together-ai-docs-3", "together-ai-docs-17", "together-ai-docs-28" ] }, { "productId": "together-ai", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Together AI is an inference/training API platform, not an automation/workflow builder; there is no concept of user-created 'automations' to version, review, or roll back in this evidence pack.", "evidenceIds": [] }, { "productId": "together-ai", "storyId": "availability-sla", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows a public status page and various product docs, but nothing states an actual uptime/availability SLA percentage or enterprise contractual guarantee. Absence of evidence for this applicable capability yields none.", "evidenceIds": [ "together-ai-probe-rt-2" ] }, { "productId": "together-ai", "storyId": "batch-discounted-jobs", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Together AI has documented batch inference at up to 50% lower cost than real-time serverless pricing, directly matching the story's requirement for asynchronous jobs at a documented discount [together-ai-docs-3]. missing for 10: independent/hands-on corroboration of actual batch job submission and discount realization beyond the docs page, and more detail on submission workflow/API specifics.", "evidenceIds": [ "together-ai-docs-3" ] }, { "productId": "together-ai", "storyId": "coding-agent-integrations", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Together AI documents first-party agent-skills and an MCP server specifically for coding agent integration (docs-agent-skills, docs-12/19/20/24/29/30), confirmed live via probe (probe-4, probe-rt-3), plus OpenAI-compatible base URL swap for framework compatibility (docs-1) and function/tool-calling docs for agentic workflows (docs-8/9). Missing for 10: independent hands-on reports specifically validating the agent-skills/MCP integration with popular frameworks like LangChain or Claude Code.", "evidenceIds": [ "together-ai-docs-1", "together-ai-docs-12", "together-ai-docs-19", "together-ai-docs-20", "together-ai-docs-30", "together-ai-docs-8", "together-ai-docs-9", "together-ai-probe-4", "together-ai-probe-rt-3" ] }, { "productId": "together-ai", "storyId": "day-zero-model-availability", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack shows Together AI hosts 100+ open-source models and supports fine-tuning/dedicated deployment, but there is no evidence—first-party or independent—about the speed or process of adding newly released open-weight models to the catalog. missing for 10: any documentation or community report on time-to-availability for new model releases, changelog/release-notes evidence, or comparison of catalog freshness versus competitors.", "evidenceIds": [] }, { "productId": "together-ai", "storyId": "dedicated-endpoints-autoscale", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly describe Dedicated Model Inference (DMI) for serving models on reserved/isolated hardware distinct from the shared serverless pool, with no hard rate limits, and a dedicated scaling doc showing configurable autoscaling via replica limits, plus deployment of fine-tuned or custom models via CLI. This directly matches the story's requirements of dedicated GPU capacity, autoscaling, and isolation from shared serverless traffic. Missing for 10: independent/hands-on verification of autoscaling behavior in production and concrete GPU-type/SLA details beyond docs.", "evidenceIds": [ "together-ai-docs-4", "together-ai-docs-10", "together-ai-docs-18", "together-ai-docs-25", "together-ai-docs-31" ] }, { "productId": "together-ai", "storyId": "documented-rate-limits", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Together AI documents a rate-limit model explicitly (docs-14): dynamic per-organization, per-model limits based on live model capacity and recent successful usage, and notes dedicated endpoints have no hard rate limits (docs-31). This gives developers some visibility into how limits work before hitting them, but there's no documented tiered structure (e.g., spend-based tiers with specific numeric thresholds) showing how limits scale as usage/spend grows over time. Missing for 10: explicit usage-tier table or thresholds, numeric rate-limit values per tier, guidance on how to request/track tier upgrades.", "evidenceIds": [ "together-ai-docs-14", "together-ai-docs-31" ] }, { "productId": "together-ai", "storyId": "drop-in-provider-switch", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly state OpenAI-compatibility requires only changing API key and base URL, meaning agents/tools written against the standard chat-completions API can point at Together with no provider-specific code changes; the live /v1/models endpoint confirms OpenAI-style REST shape. This directly supports switching to/away from Together mid-workflow with minimal friction. Missing for 10: independent third-party confirmation of drop-in swap behavior (e.g., a framework's changelog showing zero-code-change provider swap) and explicit mention of streaming/tool-call parity with OpenAI's spec beyond docs.", "evidenceIds": [ "together-ai-docs-1", "together-ai-docs-26", "together-ai-probe-rt-1" ] }, { "productId": "together-ai", "storyId": "embeddings-endpoint", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm a unified API that includes embeddings alongside chat/image/audio (together-ai-docs-21), supporting retrieval pipelines without a separate embeddings vendor. However, no evidence in the pack documents a dedicated rerank endpoint or reranking models, so the reranking half of the story is unsupported. Missing for 10: dedicated rerank API/model documentation, embeddings model list/dimensions, and independent confirmation of embeddings quality/usage in RAG pipelines.", "evidenceIds": [ "together-ai-docs-21" ] }, { "productId": "together-ai", "storyId": "fine-tune-and-serve", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs clearly describe fine-tuning (LoRA or full) on supported base models via console/API/CLI, with Together AI handling the full lifecycle including data upload, training, hosting, and inference on a dedicated endpoint, and dedicated endpoints explicitly support deploying fine-tuned models. This is a first-party documented end-to-end workflow with concrete CLI examples. Missing for 10: independent/hands-on confirmation from a third party that a fine-tune-then-serve workflow works as described.", "evidenceIds": [ "together-ai-docs-5", "together-ai-docs-11", "together-ai-docs-17", "together-ai-docs-18", "together-ai-docs-23", "together-ai-docs-28" ] }, { "productId": "together-ai", "storyId": "low-latency-serving", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Together AI's docs claim strong speed characteristics — 'no provisioning latency' for serverless, DMI offering 'better performance' and 'no hard rate limits', dynamic rate limits based on live capacity, streaming SSE support, and automatic prompt caching to speed repeated prefills — all directly supporting fast TTFT and high throughput [together-ai-docs-2][together-ai-docs-31][together-ai-docs-14][together-ai-gh-1][together-ai-docs-13]. However, an independent hands-on comparison explicitly disputes this, stating Together AI was 'at the bottom of the pile' for large-scale deployment speed/pricing versus competitors like Cerebras, DeepInfra, and Hyperbolic [together-ai-comm-5], while another community post gives a positive but vaguer 'good speed' impression [together-ai-comm-4] — a genuine mixed/contradicted signal rather than mere skepticism. Missing for 10: independent benchmarks (TTFT/tokens-per-second numbers) confirming or refuting the vendor's latency claims, and resolution of the conflicting community reports.", "evidenceIds": [ "together-ai-docs-2", "together-ai-docs-31", "together-ai-docs-14", "together-ai-docs-13", "together-ai-gh-1", "together-ai-comm-4", "together-ai-comm-5" ] }, { "productId": "together-ai", "storyId": "measured-speed-claims", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains only vague qualitative claims ('no provisioning latency', 'Better performance' for dedicated endpoints) and subjective community remarks ('good speed') but no published tokens-per-second figures, latency benchmarks, or load-testing methodology/guides that would let an ml-engineer verify speed claims.", "evidenceIds": [ "together-ai-docs-2", "together-ai-docs-31", "together-ai-comm-4" ] }, { "productId": "together-ai", "storyId": "model-deprecation-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions a deprecation policy, sunset schedule, or advance-notice commitment for retiring hosted models; docs cover serverless/dedicated inference, fine-tuning, and agent tooling but nothing about model lifecycle/deprecation notices.", "evidenceIds": [] }, { "productId": "together-ai", "storyId": "models-endpoint-discovery", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Together AI exposes an OpenAI-compatible /v1/models endpoint confirmed live via probe (returns 401 without key, meaning it exists and speaks JSON, per together-ai-probe-rt-1), and docs confirm OpenAI-compatibility for base URL/API key swaps (together-ai-docs-1) enabling programmatic catalog enumeration. missing for 10: a dedicated docs page explicitly documenting the /v1/models endpoint schema/response format, and independent hands-on confirmation of a successful authenticated call listing the full catalog.", "evidenceIds": [ "together-ai-probe-rt-1", "together-ai-docs-1" ] }, { "productId": "together-ai", "storyId": "multimodal-endpoints", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Together AI's docs explicitly state you can 'Call chat, image, audio, embedding, and more through one API' on the serverless platform, indicating multimodal model support beyond text chat on the same platform [together-ai-docs-21]. This is corroborated by the broader serverless/dedicated inference docs describing a unified API/base URL approach [together-ai-docs-1, together-ai-docs-2]. Missing for 10: no independent hands-on report or example code specifically calling vision/audio/image-generation endpoints, and no detailed model list confirming breadth of non-text modalities.", "evidenceIds": [ "together-ai-docs-21", "together-ai-docs-2", "together-ai-docs-1" ] }, { "productId": "together-ai", "storyId": "open-weight-catalog-breadth", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs confirm serverless endpoints serving '100+ open-source models' with per-token pricing, no provisioning latency, and OpenAI-compatible API for easy model switching, and community evidence corroborates real-world use of Llama-family models on the platform. However, the evidence pack never explicitly enumerates Qwen, DeepSeek, or GPT-OSS by name, so full breadth across all cited model families isn't directly confirmed. Missing for 10: explicit documentation naming Qwen/DeepSeek/GPT-OSS as catalog entries, and independent verification of catalog breadth beyond the generic '100+ models' claim.", "evidenceIds": [ "together-ai-docs-2", "together-ai-docs-1", "together-ai-docs-26", "together-ai-comm-2", "together-ai-probe-rt-1" ] }, { "productId": "together-ai", "storyId": "openai-compatible-endpoint", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly state you can point an existing client at Together by changing only the API key and base URL, and the API is OpenAI-compatible with a live /v1/models endpoint confirmed via probe. missing for 10: independent developer testimonial specifically confirming drop-in OpenAI SDK compatibility (community evidence covers pricing/UX but not this specific mechanic), and no explicit code snippet shown in evidence.", "evidenceIds": [ "together-ai-docs-1", "together-ai-probe-rt-1", "together-ai-docs-26" ] }, { "productId": "together-ai", "storyId": "openness-api-parity", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs show broad API/CLI/SDK coverage mirroring UI actions—fine-tuning can be launched 'from the console, through the API/SDK, or with the CLI', dedicated endpoints can be deployed via CLI (tg beta endpoints deploy), and inference, batch, and cluster provisioning are all API-accessible—suggesting strong UI/API parity for core workflows. However, there's no explicit documentation enumerating full parity (e.g., billing, account/org management, usage dashboards) or confirming every UI-only feature has an API equivalent, and no OpenAPI spec was found (probe shows all openapi.json paths 404), making completeness hard to verify. Missing for 10: an explicit parity statement or API coverage for account/billing/admin functions, and a published OpenAPI spec confirming full API surface.", "evidenceIds": [ "together-ai-docs-17", "together-ai-docs-25", "together-ai-docs-23", "together-ai-docs-6", "together-ai-docs-3", "together-ai-probe-3" ] }, { "productId": "together-ai", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers inference, fine-tuning, dedicated endpoints, and agent tooling, but contains no mention of a data export feature, downloadable account data, or a documented way to retrieve fine-tuning datasets/model weights in open formats and leave the platform. This is a fair question for a hosted AI platform that stores prompts, fine-tuning data, and trained models, so absence of evidence means 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "together-ai", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers Together AI's hosted inference, fine-tuning, and dedicated endpoint features, and its client SDKs (e.g., together-typescript) are open on GitHub, but there is no evidence that Together AI's core platform/product source is released under an open license—only that it serves open-source models to customers.", "evidenceIds": [] }, { "productId": "together-ai", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Together AI is a hosted inference/fine-tuning cloud service, not open-source software distributed for self-hosting; nothing in the evidence describes deploying the core Together platform on one's own infrastructure. Self-hosting is a category error for this SaaS-style product.", "evidenceIds": [] }, { "productId": "together-ai", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions data residency, regional storage options, or geographic control over where data/models are hosted; only privacy policy criticism unrelated to residency is present. Missing for 10: any documentation of region selection, data residency guarantees, or geo-specific hosting options.", "evidenceIds": [] }, { "productId": "together-ai", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No first-party documentation describes an opt-out, data-retention control, or 'no training on your data' policy; the only relevant evidence is a community report describing Together AI's privacy policy as broad/vague, allowing data use 'for other purposes,' which points toward the opposite of a training-opt-out guarantee.", "evidenceIds": [ "together-ai-comm-1", "together-ai-comm-2" ] }, { "productId": "together-ai", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item documents user-facing data retention or deletion controls (e.g., API/dashboard settings to delete stored prompts or data). The only relevant community evidence (together-ai-comm-1) actually raises concerns that Together AI's privacy policy allows broad data collection with vague retention/usage language, which is the opposite of demonstrating retention/deletion controls.", "evidenceIds": [ "together-ai-comm-1" ] }, { "productId": "together-ai", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No documentation or feature reference describes a telemetry/usage-tracking opt-out mechanism; the only related evidence is a community critique noting Together AI's privacy policy broadly collects prompts, IPs, and contact info with vague 'other purposes' language and no opt-out mentioned, reinforcing the absence of this capability.", "evidenceIds": [ "together-ai-comm-1", "together-ai-comm-2" ] }, { "productId": "together-ai", "storyId": "prompt-caching-support", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Together AI documents automatic prompt/prefix caching for serverless chat models, explicitly stating cached input tokens are billed at a steep discount with no toggle needed, directly reducing cost on repeated context. missing for 10: independent/hands-on benchmark confirming latency reduction, and clarity on which models/latency gains apply beyond the cost-discount framing.", "evidenceIds": [ "together-ai-docs-13", "together-ai-docs-22", "together-ai-docs-27" ] }, { "productId": "together-ai", "storyId": "serve-custom-weights", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Together AI's docs clearly support fine-tuning (LoRA or full) and deploying the resulting fine-tuned models to dedicated inference endpoints, covering the LoRA-adapter serving use case end-to-end (data upload, training, hosting, inference). However, there's no evidence of uploading externally-trained/custom weights not produced via Together's own fine-tuning pipeline (e.g., bringing your own checkpoint from another framework).\n\nmissing for 10: evidence of uploading externally-trained custom weights/checkpoints (not produced by Together's fine-tuning), and independent/hands-on confirmation of this workflow.", "evidenceIds": [ "together-ai-docs-11", "together-ai-docs-18", "together-ai-docs-23", "together-ai-docs-28", "together-ai-docs-17" ] }, { "productId": "together-ai", "storyId": "spend-controls-budgets", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of spending caps, budget alerts, or usage-based cutoffs; docs mention rate limits (dynamic, based on capacity/usage) and pricing structures but nothing about setting a spend ceiling or receiving alerts to prevent runaway bills.", "evidenceIds": [] }, { "productId": "together-ai", "storyId": "status-page-uptime", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "A live public status page at status.together.ai returns HTTP 200 and shows service status ('operational'), confirming a real status dashboard exists that a founder could check before committing production traffic. Missing for 10: no evidence of detailed incident history/postmortems or uptime SLA history visible in the pack, so depth of incident history can't be fully confirmed.", "evidenceIds": [ "together-ai-probe-rt-2" ] }, { "productId": "together-ai", "storyId": "streaming-responses", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Together AI's SDKs explicitly document SSE-based streaming support ('We provide support for streaming responses using Server Sent Events (SSE)'), and the API is OpenAI-compatible, implying standard streaming chat completion endpoints. Missing for 10: no independent hands-on latency benchmark or explicit token-by-token example snippet in the evidence pack.", "evidenceIds": [ "together-ai-gh-1", "together-ai-gh-2", "together-ai-docs-1" ] }, { "productId": "together-ai", "storyId": "structured-outputs-json-schema", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Together AI's docs explicitly describe structured outputs where supported models return JSON conforming to a supplied schema, avoiding retries/fragile parsing, alongside function/tool calling and multi-step agentic tool call chaining. This directly matches the story's request for JSON-schema-enforced structured outputs; missing for 10: no independent/hands-on validation of schema-conformance reliability or grammar-based constraints beyond JSON schema.", "evidenceIds": [ "together-ai-docs-7", "together-ai-docs-8", "together-ai-docs-9", "together-ai-docs-16" ] }, { "productId": "together-ai", "storyId": "tool-calling-fidelity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Together AI documents native function/tool calling with explicit support for multi-step (chained) and multi-turn agentic tool use, positioned as the foundation for agent loops (together-ai-docs-8,9,15,16). However, there is no independent/hands-on evidence confirming reliability of parallel tool calls or that it avoids breakage across open models in practice — only first-party docs. missing for 10: independent benchmarks or community reports on parallel tool-call accuracy, explicit parallel tool-calling documentation, and hands-on verification that agent loops don't break on open models.", "evidenceIds": [ "together-ai-docs-8", "together-ai-docs-9", "together-ai-docs-15", "together-ai-docs-16" ] }, { "productId": "together-ai", "storyId": "transparent-token-pricing", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs advertise \"per-token pricing\" for 100+ serverless models and community evidence confirms self-serve signup with visible per-token cost (e.g. \"$0.0001/1K tokens\", $25 trial credit) with no sales contact required. However, no evidence pack item directly shows a public pricing page enumerating rates for every hosted model, and the live models API requires an API key to enumerate the catalog. Missing for 10: a documented public pricing page/table covering all models, independent confirmation that all model prices (not just examples) are visible pre-signup.", "evidenceIds": [ "together-ai-docs-2", "together-ai-comm-3", "together-ai-comm-4", "together-ai-probe-rt-1" ] } ]