[ { "productId": "bland", "storyId": "agent-consumes-mcp-tools", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Bland ships an MCP *server* so coding agents can control the Bland account (docs-11/20/27, probe-3/probe-rt-2), which is the opposite role from what the story asks — the voice agent itself acting as an MCP *client* that plugs in external MCP servers as tool sources mid-call. Tool/function calling is documented only via custom HTTP API integrations (bland-docs-4, bland-docs-25), with no mention of the agent consuming MCP servers as a toolset source during calls.", "evidenceIds": [ "bland-docs-11", "bland-docs-20", "bland-docs-27", "bland-probe-3", "bland-probe-rt-2", "bland-docs-4", "bland-docs-25" ] }, { "productId": "bland", "storyId": "agent-provisions-voice-agent", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Bland's docs show all three provisioning steps available programmatically: pathway/agent creation via API (bland-docs-1, bland-docs-18), phone number acquisition/porting/Twilio and SIP attachment (bland-docs-3, bland-docs-16, bland-docs-17), and call placement via API or batch calls (bland-docs-6, bland-docs-23). Both the CLI ('make calls, build and test pathways, configure phone numbers' — bland-docs-12) and the MCP server ('place and inspect calls, build and validate pathways, manage agents' — bland-docs-11/20/27) explicitly cover the full create-agent/attach-number/place-call lifecycle without the dashboard, and runtime probes confirm both the CLI and hosted MCP endpoint are live and functional (bland-probe-rt-1, bland-probe-rt-2). Missing for 10: a single consolidated end-to-end tutorial/example walking through create→attach→call in one flow, and independent (non-vendor) confirmation of the full pipeline working end-to-end.", "evidenceIds": [ "bland-docs-1", "bland-docs-3", "bland-docs-6", "bland-docs-11", "bland-docs-12", "bland-docs-16", "bland-docs-17", "bland-docs-20", "bland-probe-rt-1", "bland-probe-rt-2" ] }, { "productId": "bland", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Bland has a confirmed live llms.txt at docs.bland.ai/llms.txt (HTTP 200) providing agent-oriented documentation, plus an official MCP server and CLI explicitly designed for AI coding agents to interact with the platform via natural language. missing for 10: independent third-party confirmation that agents actually consume llms.txt successfully in practice, and broader agent-oriented docs beyond the single llms.txt file.", "evidenceIds": [ "bland-probe-1", "bland-docs-11", "bland-docs-20", "bland-probe-3", "bland-probe-rt-2" ] }, { "productId": "bland", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Bland offers LLM-judge Evals to grade call quality and an MCP integration that can 'query analytics' on your account data, which are AI-generated evaluative outputs derived from your call data, but there's no dedicated insights/suggestions dashboard or proactive recommendation feature described. Missing for 10: a native analytics/insights UI, evidence of proactive suggestions surfaced to users, and independent confirmation of these AI-generated insights in practice.", "evidenceIds": [ "bland-docs-7", "bland-docs-22", "bland-docs-9", "bland-docs-11", "bland-docs-27" ] }, { "productId": "bland", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bland supports batch calls, webhooks, pathways, and scheduled/triggered call campaigns that run without manual intervention, which constitute a form of autonomous background automation for voice workflows. However, this is scoped to phone-call automation only, not general-purpose background task/agent scheduling. missing for 10: evidence of a generic scheduler/cron-like trigger system, independent hands-on validation of unattended background runs, and confirmation of failure handling/monitoring for long-running autonomous automations.", "evidenceIds": [ "bland-docs-6", "bland-docs-5", "bland-docs-18", "bland-docs-30", "bland-docs-23", "bland-docs-25" ] }, { "productId": "bland", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Bland's core product is a built-in AI voice agent that users delegate tasks to (placing/answering calls, executing pathways, calling APIs, remembering context) rather than a separate feature bolted on — e.g., 'Dispatch AI phone calls to call customers, leads, and to streamline operations' and pathway/tool/memory docs show rich task delegation to the built-in agent. Missing for 10: independent/hands-on evidence of real-world task delegation outcomes beyond vendor docs.", "evidenceIds": [ "bland-docs-23", "bland-docs-18", "bland-docs-25", "bland-docs-9", "bland-docs-30" ] }, { "productId": "bland", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Bland offers a full programmatic/REST API for pathways, calls, batch calls, webhooks, and evals, plus an official CLI (verified runtime installable keylessly via npx) for terminal-based automation, enabling headless/CI usage. missing for 10: no explicit CI/CD pipeline example (e.g., GitHub Actions) or independent case study confirming CI usage beyond docs and CLI probe.", "evidenceIds": [ "bland-docs-1", "bland-docs-6", "bland-docs-12", "bland-probe-4", "bland-probe-rt-1", "bland-docs-7" ] }, { "productId": "bland", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Bland's evidence only documents it exposing an outbound MCP server so that external AI coding agents can call Bland's own tools (docs-11, docs-20, docs-27, probe-rt-2) — the reverse direction of this story. There is no evidence that Bland itself can consume/plug in third-party MCP servers as a client; its tool integration story is limited to custom HTTP API endpoints and webhooks (bland-docs-4, bland-docs-25).", "evidenceIds": [ "bland-docs-11", "bland-docs-20", "bland-docs-27", "bland-docs-4", "bland-docs-25", "bland-probe-rt-2" ] }, { "productId": "bland", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Bland ships an official MCP server (docs and runtime probe confirm it's live and gated by API key) that lets AI coding agents place/inspect calls, build pathways, manage agents, query analytics, run evals, and search docs — directly fulfilling the story. Missing for 10: independent third-party hands-on review of the MCP server beyond vendor docs/probe.", "evidenceIds": [ "bland-docs-11", "bland-docs-20", "bland-docs-27", "bland-probe-3", "bland-probe-rt-2" ] }, { "productId": "bland", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Bland offers an official MCP server that lets AI agents operate the entire account (calls, pathways, agents, analytics, evals) via natural language, plus an official CLI, both confirmed live via runtime probes. missing for 10: independent/hands-on third-party review of the MCP/CLI natural-language experience beyond vendor docs and probes.", "evidenceIds": [ "bland-docs-11", "bland-docs-20", "bland-docs-27", "bland-probe-3", "bland-probe-rt-2", "bland-docs-12", "bland-probe-4", "bland-probe-rt-1" ] }, { "productId": "bland", "storyId": "agentic-official-cli", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Bland ships an official CLI (bland-cli) documented to manage the entire account from the terminal, and a runtime probe confirms it installs and runs via npx keylessly. missing for 10: independent third-party review/usage reports of the CLI beyond the official docs and one probe run.", "evidenceIds": [ "bland-docs-12", "bland-probe-4", "bland-probe-rt-1" ] }, { "productId": "bland", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Bland documents a comprehensive public API (pathways, calls, tools, webhooks, evals, batch calls) plus SDKs, CLI, and an official MCP server, and runtime probes confirm the CLI installs and the hosted MCP endpoint is live and gated as documented, showing agentic programmatic control. Missing for 10: a formally published OpenAPI/Swagger spec (probe found 404s at standard OpenAPI paths), so machine-readable spec discoverability is unconfirmed.", "evidenceIds": [ "bland-docs-1", "bland-docs-4", "bland-docs-6", "bland-docs-11", "bland-docs-12", "bland-probe-3", "bland-probe-4", "bland-probe-rt-1", "bland-probe-rt-2", "bland-probe-2" ] }, { "productId": "bland", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows generic API-key auth and JWT-based webhook verification (bland-docs-15), but nothing about issuing scoped or least-privilege credentials specific to an agent's permissions (e.g., role-based API keys, scoped tokens limiting call/pathway/account access). Missing for 10: documentation of scoped API key creation, permission levels, or per-agent credential restriction.", "evidenceIds": [ "bland-docs-15" ] }, { "productId": "bland", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bland documents a Web Agent SDK for embedding voice agents (React/Vanilla JS/Node), a CLI, and a REST API used throughout tutorials, giving AI-native developers concrete building blocks; runtime probes confirm the CLI installs and runs. However, no dedicated 'official SDK' page for server-side languages (Python/Node backend SDK) is evidenced, and openapi/swagger spec endpoints all 404, suggesting the API reference isn't machine-consumable in a standard SDK-generation format. Missing for 10: a clearly documented multi-language backend SDK (Python/Node) beyond the browser widget SDK, and a working OpenAPI spec for auto-generating clients.", "evidenceIds": [ "bland-docs-13", "bland-docs-12", "bland-probe-4", "bland-probe-rt-1", "bland-probe-2", "bland-docs-1" ] }, { "productId": "bland", "storyId": "agentic-webhooks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bland documents post-call webhooks (automatic HTTP notifications sent after a call completes) and pathway-level webhook execution at specific conversation points, showing genuine event-driven webhook support tied to call lifecycle. However, evidence only covers call-related events (completion, in-call triggers) — missing for 10: a general-purpose event subscription/webhook management API covering other account events (e.g., evals, batch campaign status, pathway publishes), and any independent confirmation of webhook reliability/configuration options.", "evidenceIds": [ "bland-docs-5", "bland-docs-30" ] }, { "productId": "bland", "storyId": "ai-assisted-agent-authoring", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Bland's MCP server lets external AI coding agents 'build and validate pathways ... through natural language' (bland-docs-11/20/27), and the testbed/evals tools support iterating on and grading prompts, which loosely supports AI-assisted authoring. However there is no evidence of a built-in, first-party generative feature where Bland's own platform AI drafts a full pathway/flow/test-cases from a plain-language description inside the product itself — the closest capability requires an external AI agent connecting via MCP. Missing for 10: a native 'describe your agent, we generate the pathway/prompts/tests' feature, in-product prompt-improvement AI, and independent/hands-on confirmation of AI-generated flows.", "evidenceIds": [ "bland-docs-11", "bland-docs-20", "bland-docs-27", "bland-docs-8", "bland-docs-28", "bland-docs-7", "bland-docs-22" ] }, { "productId": "bland", "storyId": "ai-call-qa-scoring", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Bland's Evals feature explicitly lets users define LLM-judge agents that grade calls on custom dimensions (quality, resolution, etc.), and the Testbed lets you replay and analyze specific call nodes to spot failures — directly matching automated call review/scoring. However, this requires the user to configure eval criteria rather than being a fully out-of-the-box automatic analysis, and there's no evidence of a pre-built default 'failure flagging' report. Missing for 10: evidence of fully automatic, no-setup call scoring/dashboard, and independent/hands-on validation of eval accuracy.", "evidenceIds": [ "bland-docs-7", "bland-docs-22", "bland-docs-8", "bland-docs-28" ] }, { "productId": "bland", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Docs reference an API reference for pathways but no evidence of an interactive playground with runnable examples; a probe for OpenAPI/Swagger specs at standard paths returned 404s, suggesting no such interactive reference exists.", "evidenceIds": [ "bland-docs-1", "bland-probe-2" ] }, { "productId": "bland", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe found no OpenAPI/Swagger spec at any expected location (all 404s), and no evidence pack item shows a downloadable machine-readable API spec despite extensive API documentation.", "evidenceIds": [ "bland-probe-2" ] }, { "productId": "bland", "storyId": "api-sandbox", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bland provides draft/staging pathway editing where live production calls are unaffected, plus a testbed for isolated node testing, canary deployments, and staged version adoption — all functioning as sandbox-like mechanisms distinct from production. However, there's no explicit 'sandbox environment' or dedicated test account/data isolation concept described, and testing still appears to involve real calls/production infrastructure rather than a fully isolated non-production environment. missing for 10: a dedicated sandbox/test-mode account distinct from production billing and phone infrastructure, explicit documentation of synthetic/non-production test data, and independent confirmation that testbed/draft testing never touches real production call data or costs.", "evidenceIds": [ "bland-docs-2", "bland-docs-19", "bland-docs-8", "bland-docs-28", "bland-docs-29", "bland-docs-21" ] }, { "productId": "bland", "storyId": "api-versioning-policy", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Bland documents infrastructure versioning concepts (staged/canary rollouts, choosing when to adopt a new release, draft vs. production pathway versions) but there is no evidence of a documented API versioning scheme (e.g., v1/v2 endpoints) or an explicit deprecation policy for its APIs, and the OpenAPI spec probe returned 404s. missing for 10: documented API version numbering/endpoints, an explicit deprecation/sunset policy for APIs, published OpenAPI spec, and independent confirmation of versioning practices.", "evidenceIds": [ "bland-docs-14", "bland-docs-29", "bland-docs-31", "bland-probe-2" ] }, { "productId": "bland", "storyId": "automation-bulk-operations", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Batch calls let users upload a CSV of recipients to initiate high-volume call campaigns, directly enabling bulk operations across many items, and the CLI/MCP server extend programmatic/bulk management of pathways, agents, and calls. Missing for 10: independent/hands-on verification of batch call performance at scale, and documentation of bulk operations beyond calls (e.g., bulk pathway or number management).", "evidenceIds": [ "bland-docs-6", "bland-docs-11", "bland-docs-12", "bland-probe-rt-1" ] }, { "productId": "bland", "storyId": "automation-rules-engine", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Bland's pathways let users define conditional logic that executes webhooks/API calls at specific conversation nodes, and post-call webhooks automatically fire HTTP notifications when a call-completion event occurs — this is a documented rules-trigger-action-on-event mechanism. missing for 10: independent/hands-on verification of the webhook triggering in production, and evidence of event types beyond call-based ones (e.g., generic account-level event automation).", "evidenceIds": [ "bland-docs-5", "bland-docs-30", "bland-docs-25", "bland-docs-18" ] }, { "productId": "bland", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Bland's docs cover batch calls, pathways, webhooks, and API integrations, but nothing describes scheduling recurring jobs or workflows (e.g., cron-like triggers for calls or campaigns) — batch calls are one-off CSV uploads, not recurring schedules.", "evidenceIds": [ "bland-docs-6", "bland-docs-23" ] }, { "productId": "bland", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bland pathways support draft/staging/production versioning, A/B testing new releases against live production, and canary/staged rollout with adoption control (bland-docs-2, bland-docs-14, bland-docs-19, bland-docs-21, bland-docs-29, bland-docs-31), plus a testbed and evals for reviewing behavior before shipping (bland-docs-7, bland-docs-8, bland-docs-22, bland-docs-28). However, there is no explicit documentation of an automated 'rollback' mechanism to revert a live pathway/release to a prior version — only forward-adoption and canary controls are described. Missing for 10: explicit rollback/revert-to-previous-version capability, version history/diff view, and independent confirmation of rollback in practice.", "evidenceIds": [ "bland-docs-2", "bland-docs-14", "bland-docs-19", "bland-docs-21", "bland-docs-29", "bland-docs-31", "bland-docs-7", "bland-docs-8", "bland-docs-22", "bland-docs-28" ] }, { "productId": "bland", "storyId": "barge-in-interruptions", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers pathways, webhooks, batch calls, evals, MCP/CLI integrations, and infrastructure features, but contains no mention of barge-in, interruption handling, stopping TTS mid-utterance, or turn recovery logic — a core voice-agent capability that would be a fair and expected axis for this product type. missing for 10: any documentation or claim about detecting user interruptions, halting agent speech, updating context after a barge-in, and resuming/recovering the conversational turn.", "evidenceIds": [] }, { "productId": "bland", "storyId": "batch-outbound-campaigns", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Bland's docs explicitly describe batch calls for uploading CSV recipient lists to run high-volume outbound campaigns, backed by scheduling/throughput-related infrastructure like SIP integration, phone number management, and analytics/evals to monitor campaign performance. Missing for 10: explicit documentation of scheduling controls (e.g., call windows/timing) and rate-limiting/throughput knobs specifically, plus independent hands-on verification of batch campaign behavior at scale.", "evidenceIds": [ "bland-docs-6", "bland-docs-23", "bland-docs-17", "bland-docs-16" ] }, { "productId": "bland", "storyId": "build-phone-agent-fast", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs cover the whole first-call workflow: creating a pathway/prompt (bland-docs-18, bland-docs-26, bland-docs-30), voice cloning/selection (bland-docs-10), phone number setup via own Twilio or new inbound numbers (bland-docs-3, bland-docs-24), and dispatching outbound calls (bland-docs-23), plus a CLI and MCP server confirmed live at runtime (bland-probe-rt-1, bland-probe-rt-2) that let a developer configure and place calls quickly. Missing for 10: no independent hands-on account of a developer actually completing a first call within an hour, and one community comment notes cost concerns rather than time-to-first-call, so onboarding speed is only documented, not externally verified.", "evidenceIds": [ "bland-docs-18", "bland-docs-26", "bland-docs-30", "bland-docs-10", "bland-docs-3", "bland-docs-24", "bland-docs-23", "bland-docs-4", "bland-probe-rt-1", "bland-probe-rt-2" ] }, { "productId": "bland", "storyId": "call-analytics-dashboard", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence confirms Bland has an analytics layer (MCP server can 'query analytics') and quality-grading tools like Evals (LLM judges scoring call dimensions) and a testbed for reviewing call interactions, implying some built-in metrics exist. However, there is no direct evidence of an actual dashboard UI showing success rates, call durations, outcomes, or sentiment trends over time — analytics access shown is via MCP/API query rather than a visual dashboard. Missing for 10: screenshots or docs of a native analytics dashboard, explicit mention of success-rate/duration/sentiment metrics, and independent confirmation the dashboard requires no custom building.", "evidenceIds": [ "bland-docs-20", "bland-docs-22", "bland-docs-28", "bland-probe-3" ] }, { "productId": "bland", "storyId": "call-transfer-escalation", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers pathways, webhooks, SIP, MCP, CLI, and other Bland features, but contains no mention of call transfer (warm or blind) or handing off a live call to a human agent with context. This is a standard telephony capability that could plausibly be documented, so absence of evidence yields 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "bland", "storyId": "concurrency-scaling-limits", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs mention batch calling for high-volume campaigns and enterprise infrastructure controls (canary releases, staged rollout), implying some capacity for scale, but there is no documented concurrency limit, rate ceiling, or auto-scaling guarantee, and no evidence that capacity increases don't require contacting sales/support. Missing for 10: explicit concurrent-call limits, auto-scaling documentation, and evidence that scaling doesn't require manual requests to Bland.", "evidenceIds": [ "bland-docs-6", "bland-docs-29", "bland-docs-31" ] }, { "productId": "bland", "storyId": "consent-recording-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of call recording consent/disclosure features, per-call recording toggles, or configurable data retention policies anywhere in Bland's docs; while this is a fair compliance axis for a voice AI platform, nothing in the pack substantiates it.", "evidenceIds": [] }, { "productId": "bland", "storyId": "custom-voice-cloning", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Bland documents voice cloning itself (a ~10-second clean sample sets the quality ceiling) which supports the 'clone a custom brand voice' half of the story, but no evidence describes a documented consent process, verification, or authorization requirement for cloning someone's voice. missing for 10: documented consent/verification workflow for voice cloning, legal/compliance guidance on brand-voice rights, independent confirmation of the cloning feature's fidelity.", "evidenceIds": [ "bland-docs-10" ] }, { "productId": "bland", "storyId": "documented-latency-numbers", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items mention latency numbers, benchmarks, or tuning guidance for end-to-end voice response time; documentation covers pathways, SIP, MCP, CLI, and enterprise features but nothing about speed/latency metrics or optimization guidance.", "evidenceIds": [] }, { "productId": "bland", "storyId": "dtmf-ivr-voicemail", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers pathways, SIP, webhooks, evals, and MCP/CLI tooling, but nowhere mentions DTMF keypress sending, IVR menu navigation, or voicemail detection/leaving — capabilities that are plausible for a voice telephony platform but are simply undocumented here.", "evidenceIds": [] }, { "productId": "bland", "storyId": "dynamic-call-personalization", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Evidence shows personalization mechanisms exist—Memory for per-caller context (bland-docs-9), batch calls that likely carry per-recipient data (bland-docs-6), and pathway webhooks/API integrations that could fetch live data (bland-docs-4, bland-docs-30)—but there is no explicit documentation of a 'dynamic variables' injection API or call-time variable substitution mechanism. missing for 10: explicit dynamic-variable/request_data injection documentation, examples of per-call variable interpolation into prompts, independent confirmation of personalization working in practice.", "evidenceIds": [ "bland-docs-9", "bland-docs-6", "bland-docs-4", "bland-docs-30" ] }, { "productId": "bland", "storyId": "hipaa-regulated-workloads", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no mention of HIPAA, BAA, SOC 2 certification, or data-residency options anywhere in the docs or probes; only a generic tagline calling Bland an 'enterprise' platform for 'regulated' workflows without specifics. Missing for 10: HIPAA/BAA documentation, SOC 2 report or certification evidence, data-residency/region controls, and any compliance attestations.", "evidenceIds": [ "bland-probe-1" ] }, { "productId": "bland", "storyId": "knowledge-base-grounding", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Bland is a voice AI/phone agent platform; evidence covers pathways, memory, tools, MCP, CLI, etc., but nowhere mentions a document knowledge base or RAG capability for grounding agent responses in uploaded content. Since a voice/conversational agent platform could plausibly ship this (many competitors do), absence of evidence makes this 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "bland", "storyId": "live-call-monitoring", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Bland provides post-call webhooks, evals with LLM judges, and an MCP integration that can 'query analytics' — giving some after-the-fact quality/analytics visibility — but there is no documented live-call monitoring dashboard, real-time alerting, or error-rate-spike notification system in the evidence pack. Missing for 10: live/real-time call monitoring UI, configurable alert thresholds, error-rate spike detection/paging, independent confirmation these exist in production.", "evidenceIds": [ "bland-docs-5", "bland-docs-7", "bland-docs-22", "bland-docs-27", "bland-docs-28" ] }, { "productId": "bland", "storyId": "mid-call-tool-calling", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bland's pathway/tools docs confirm agents can call external APIs and execute webhooks mid-conversation (bland-docs-4, bland-docs-25, bland-docs-30), which supports live function calling during a call. However, no evidence describes mechanisms for avoiding dead air (e.g., filler speech, streaming partial responses) while waiting on API results. missing for 10: explicit documentation of latency-masking/filler-speech behavior during API calls, and independent/hands-on confirmation of smooth conversational flow.", "evidenceIds": [ "bland-docs-4", "bland-docs-25", "bland-docs-30", "bland-docs-18" ] }, { "productId": "bland", "storyId": "multilingual-conversations", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no mention of multi-language support, language detection, or mid-call language switching anywhere in Bland's docs; only pathways, TTS voice cloning, and infrastructure features are documented. Missing for 10: any documentation of multilingual conversation support, automatic language detection, or dynamic language switching mid-call.", "evidenceIds": [] }, { "productId": "bland", "storyId": "noise-robustness", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions noise suppression, audio filtering, or handling of noisy real-world call environments; documentation covers pathways, integrations, MCP, CLI, SIP, and enterprise release features but nothing about audio quality/noise handling.", "evidenceIds": [] }, { "productId": "bland", "storyId": "openness-api-parity", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Bland's docs show extensive programmatic control mirroring UI features: pathway creation/versioning, testbed, evals, batch calls, webhooks, SIP/phone number management, and a CLI/MCP server that explicitly lets users 'manage your entire Bland account from the terminal' and perform the same actions (calls, pathways, agents, analytics) as the UI. This breadth strongly supports API parity, though there's no explicit first-party statement guaranteeing 100% feature parity and no discoverable OpenAPI spec (all candidate URLs 404), so full parity isn't independently confirmed. Missing for 10: an explicit parity guarantee/documentation and a machine-readable OpenAPI spec, plus independent hands-on confirmation that every UI action has an API equivalent.", "evidenceIds": [ "bland-docs-1", "bland-docs-12", "bland-docs-11", "bland-docs-20", "bland-docs-8", "bland-docs-6", "bland-docs-7", "bland-probe-4", "bland-probe-rt-1", "bland-probe-2" ] }, { "productId": "bland", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of a data export feature or open-format export of account data (pathways, call logs, memories, etc.); the docs cover CLI, MCP, and SDK integrations but nothing about exporting user data for portability/exit.", "evidenceIds": [] }, { "productId": "bland", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Bland is a closed, commercial SaaS voice AI platform; there is no evidence of any open-source license or public source code repository. A community comment explicitly asks for an open-source alternative, implying Bland itself is not open source. This is an applicable axis (a product could publish open-source components) but no evidence supports it.", "evidenceIds": [ "bland-comm-1" ] }, { "productId": "bland", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Bland is documented as a hosted enterprise voice AI platform (call dispatch, pathways, SIP, MCP, CLI) with no mention of an on-prem/self-hosted deployment option; 'enterprise' release controls (docs-14, docs-29, docs-31) only govern version adoption timing on Bland's own infrastructure, not customer self-hosting. A community post explicitly looks for an open-source self-hostable alternative because Bland itself doesn't offer this.", "evidenceIds": [ "bland-docs-14", "bland-docs-29", "bland-docs-31", "bland-comm-1" ] }, { "productId": "bland", "storyId": "phone-number-provisioning", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Bland's docs explicitly document creating/managing inbound phone numbers, outbound call dispatch via API, bringing your own Twilio numbers, and SIP for both inbound/outbound, all programmatically accessible, plus a CLI/MCP that manage phone numbers and calls end-to-end. Missing for 10: no explicit REST API reference/OpenAPI spec confirmed (probe shows openapi.json 404s) and no independent hands-on developer report of a full provision+call round trip.", "evidenceIds": [ "bland-docs-23", "bland-docs-24", "bland-docs-3", "bland-docs-16", "bland-docs-17", "bland-docs-12", "bland-probe-2" ] }, { "productId": "bland", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions data residency, regional hosting options, or geographic storage controls for Bland; enterprise/infra docs discuss release versioning and SIP/canary deployments but not region selection. Missing for 10: any mention of data residency options, region-specific hosting, or compliance-driven storage location controls.", "evidenceIds": [] }, { "productId": "bland", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data-training opt-out or AI-training privacy policies for Bland; nothing about training-data usage or opt-out controls is mentioned across any of the docs or probes.", "evidenceIds": [] }, { "productId": "bland", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items address data retention policies, data deletion controls, or privacy/compliance settings for call recordings, transcripts, or memory data. This axis clearly applies to an enterprise voice AI platform handling call data, but nothing in the evidence documents retention periods, deletion APIs, or GDPR/CCPA-style data controls. Missing for 10: retention policy docs, data deletion API/endpoint, compliance certifications, memory/data purge mechanism.", "evidenceIds": [] }, { "productId": "bland", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items mention telemetry, analytics opt-out, or usage-tracking controls for Bland's own platform; this is a fair question for an enterprise SaaS product but is simply unaddressed in the evidence.", "evidenceIds": [] }, { "productId": "bland", "storyId": "realtime-transcription-quality", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Bland is a phone-call AI platform where transcription accuracy is clearly relevant, but the evidence pack contains no mention of STT provider selection, language model choice for transcription, or key-term/vocabulary boosting features. missing for 10: STT provider selection, transcription accuracy documentation, custom key terms/vocabulary support, language model configuration for transcription.", "evidenceIds": [] }, { "productId": "bland", "storyId": "recordings-transcripts-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item explicitly documents an API or endpoint for retrieving full call recordings or transcripts programmatically; references to call logs (testbed) and post-call webhooks hint at underlying data but never confirm a recording/transcript retrieval capability.", "evidenceIds": [] }, { "productId": "bland", "storyId": "self-host-oss-runtime", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Bland is presented throughout as a hosted enterprise SaaS platform (managed infrastructure, release adoption controls, canary deployments on Bland's own containers) with no mention of open-source code or a self-hostable runtime; a community post even shows a user asking for an open-source alternative because Bland itself isn't one.", "evidenceIds": [ "bland-docs-14", "bland-docs-29", "bland-comm-1" ] }, { "productId": "bland", "storyId": "semantic-turn-detection", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no mention of turn-taking, end-of-turn detection, VAD, or interruption handling mechanisms of any kind — nothing addresses how Bland decides when a speaker has finished talking. This is a fair and applicable axis for a voice AI platform, but no capability is documented; missing for 10: any mention of end-of-turn/turn-taking model, VAD configuration, or handling of slow speakers/pauses.", "evidenceIds": [] }, { "productId": "bland", "storyId": "simulated-agent-testing", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Bland provides explicit pre-production testing tools: Evals (LLM-judge grading of call quality), the Testbed (replay/edit/re-run node interactions on historical or test chats), staging promotion and draft-vs-production pathway separation, and canary/A/B rollout testing against real calls before full deployment. Together these let a developer simulate conversations and grade agent behavior before real phone calls go live.\n\nmissing for 10: no independent/hands-on evidence of eval accuracy or testbed usage from outside vendor docs, and no explicit description of a pure text-based conversation simulator separate from testbed/staging.", "evidenceIds": [ "bland-docs-7", "bland-docs-8", "bland-docs-22", "bland-docs-28", "bland-docs-2", "bland-docs-19", "bland-docs-21", "bland-docs-29" ] }, { "productId": "bland", "storyId": "sip-trunk-byoc", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Bland's docs explicitly document inbound/outbound SIP trunking to connect a customer's own carrier or PBX, plus number porting, and separately support connecting an existing Twilio account/numbers. Missing for 10: no mention of Telnyx import specifically and no independent/hands-on validation of SIP setup success.", "evidenceIds": [ "bland-docs-17", "bland-docs-16", "bland-docs-3" ] }, { "productId": "bland", "storyId": "structured-outcome-extraction", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bland documents post-call webhooks that automatically deliver call data to a developer's server after each call, and evals let you grade/classify calls (dispositions) via LLM judges — both align with the API/webhook delivery and outcome-tagging parts of the story. However, there's no explicit documentation of structured entity extraction (e.g., named fields like names, dates, custom entities) as a distinct capability, nor a described webhook payload schema. Missing for 10: explicit entity-extraction feature docs, sample webhook payload showing structured outcome/entity/disposition fields, and independent confirmation of the data delivered.", "evidenceIds": [ "bland-docs-5", "bland-docs-22", "bland-docs-30" ] }, { "productId": "bland", "storyId": "transparent-per-minute-pricing", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item shows published per-minute or usage pricing, a pricing page, or any cost calculator; the only pricing-adjacent mention is a community complaint that Bland 'gets expensive' with no figures. Missing for 10: published price list, per-minute rate documentation, cost calculator or estimator tool.", "evidenceIds": [ "bland-comm-1" ] }, { "productId": "bland", "storyId": "visual-flow-builder", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Bland's 'Conversational Pathways' feature is explicitly a node-based flow builder where founders give instructions at specific conversation points, test individual node interactions in a pathway editor/testbed, and publish drafts separately from production—matching branching/states/handoffs without code (bland-docs-18, 26, 30, 28, 19). Missing for 10: explicit confirmation of a drag-and-drop visual canvas UI (docs describe 'nodes' and 'pathway editor' but no screenshot/UI walkthrough) and independent hands-on corroboration beyond vendor docs.", "evidenceIds": [ "bland-docs-18", "bland-docs-26", "bland-docs-30", "bland-docs-28", "bland-docs-19", "bland-docs-1" ] }, { "productId": "bland", "storyId": "voice-library-choice", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs show voice cloning support (a 10-second sample sets voice quality) but there is no evidence of a broad pre-built voice library to choose from, nor any mention of plugging in multiple third-party TTS providers. Missing for 10: documented voice library/catalog, multi-provider TTS integration, and any comparison of voice options.", "evidenceIds": [ "bland-docs-10" ] }, { "productId": "bolna", "storyId": "agent-consumes-mcp-tools", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows Bolna exposing its OWN MCP server so external clients (Claude, Cursor, Windsurf) can manage a Bolna account (bolna-docs-17, bolna-probe-3) — the reverse direction of the story, which asks whether a Bolna voice agent can consume external MCP servers as tool sources mid-call. No documentation shows Bolna agents acting as an MCP client or importing toolsets from third-party MCP servers during a call. missing for 10: any evidence of Bolna voice agents connecting to external MCP servers as tool sources, mid-call toolset loading via MCP.", "evidenceIds": [ "bolna-docs-17", "bolna-probe-3" ] }, { "productId": "bolna", "storyId": "agent-provisions-voice-agent", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Bolna offers all three surfaces needed for agent-native provisioning: a documented API quickstart to create an agent, attach a number, and place a call (bolna-docs-14, bolna-docs-15), a native Go CLI to manage agents, calls, and phone numbers from the terminal (bolna-docs-16, bolna-probe-4), and an official MCP server to connect Claude/Cursor/Windsurf clients (bolna-docs-17, bolna-probe-3), all without requiring dashboard interaction. missing for 10: independent/hands-on confirmation that MCP or CLI can complete the full create-attach-call flow end-to-end (only vendor docs), and no OpenAPI spec was resolvable (bolna-probe-2) to verify API completeness.", "evidenceIds": [ "bolna-docs-14", "bolna-docs-15", "bolna-docs-16", "bolna-docs-17", "bolna-probe-3", "bolna-probe-4", "bolna-probe-2" ] }, { "productId": "bolna", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A probe confirms llms.txt is live and returns 200 with structured agent-oriented doc links (bolna-probe-1), and Bolna also documents an official MCP server for connecting AI clients like Claude/Cursor to its docs/account (bolna-docs-17, bolna-probe-3), directly enabling agent-native consumption of its documentation. Missing for 10: no independent third-party confirmation that agents successfully use llms.txt in practice.", "evidenceIds": [ "bolna-probe-1", "bolna-docs-17", "bolna-probe-3" ] }, { "productId": "bolna", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Bolna offers agent-level call_summary_enabled to auto-generate AI summaries and an extraction-categories API to pull structured insights out of call data, both of which are AI-generated insights derived from user data (calls). However, there's no evidence of broader cross-call analytics, trend insights, or proactive suggestions surfaced in a dashboard beyond per-call outputs. Missing for 10: aggregated/dashboard-level insights across multiple calls, proactive recommendation surfacing, and independent confirmation of quality of generated summaries/extractions.", "evidenceIds": [ "bolna-docs-1", "bolna-docs-2" ] }, { "productId": "bolna", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bolna's workflows feature lets users build multi-step outreach sequences (voice calls, WhatsApp, API calls, waits, retries) driven entirely through the API, which supports background automation, and the CLI can manage batches of calls autonomously. However there's no documented evidence of scheduling/triggers, monitoring/alerting on autonomous runs, or a dashboard for managing long-running background jobs beyond the API-driven workflow primitives. missing for 10: scheduling/trigger mechanisms, autonomous-run monitoring/alerting, independent hands-on confirmation of unattended long-running workflows.", "evidenceIds": [ "bolna-docs-12", "bolna-docs-16", "bolna-probe-4" ] }, { "productId": "bolna", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "bolna", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bolna offers a CLI (single Go binary) for managing agents, calls, phone numbers, and batches from the terminal, plus a REST API for scripted call creation and transcript retrieval — both usable in CI/automation pipelines without a GUI. However, there's no explicit CI/pipeline example, headless-mode documentation, or exit-code/automation-specific guidance. missing for 10: explicit CI integration examples/guides, documented non-interactive/headless flags or exit codes, third-party confirmation of CI usage.", "evidenceIds": [ "bolna-docs-16", "bolna-probe-4", "bolna-docs-14", "bolna-docs-15" ] }, { "productId": "bolna", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Bolna ships an MCP *server* so external clients like Claude, Cursor, or Windsurf can control a Bolna account (bolna-docs-17, bolna-probe-3) — this is the reverse direction from the story, which asks whether Bolna's own agents can plug in external MCP servers to gain their tools. No evidence describes Bolna voice agents consuming or calling external MCP servers.", "evidenceIds": [ "bolna-docs-17", "bolna-probe-3" ] }, { "productId": "bolna", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Bolna documents an official MCP server that lets Claude, Cursor, Windsurf, or any MCP client connect to a Bolna account, confirmed by both docs and an independent probe hitting the live page. Missing for 10: independent hands-on validation or community confirmation of the MCP server working in practice beyond the docs/probe.", "evidenceIds": [ "bolna-docs-17", "bolna-probe-3" ] }, { "productId": "bolna", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Bolna ships an official MCP server that lets AI-native tools like Claude, Cursor, or Windsurf connect to a Bolna account and operate it (manage agents, calls, etc.) via natural-language commands, confirmed both in docs and via a live probe. Missing for 10: independent/hands-on verification that the MCP integration works as described, and no additional natural-language control surfaces (e.g., chat-based dashboard) beyond MCP.", "evidenceIds": [ "bolna-docs-17", "bolna-probe-3" ] }, { "productId": "bolna", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Bolna documents an official CLI (single Go binary) letting users manage agents, calls, phone numbers, and batches from the terminal, corroborated by a live docs probe confirming the page exists. missing for 10: no independent/hands-on community usage reports or version/release history confirming real-world adoption beyond first-party docs.", "evidenceIds": [ "bolna-docs-16", "bolna-probe-4" ] }, { "productId": "bolna", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Bolna provides a documented public API with quickstart, authentication, agent/call/extraction endpoints, and workflow automation entirely through the API, plus a CLI and MCP server built on top of it. Missing for 10: a discoverable OpenAPI/Swagger spec (probe found all standard OpenAPI paths 404) and independent third-party corroboration of API robustness.", "evidenceIds": [ "bolna-docs-14", "bolna-docs-15", "bolna-docs-12", "bolna-docs-16", "bolna-probe-2", "bolna-probe-4" ] }, { "productId": "bolna", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Bolna documents API key creation but there is no evidence of scoped or least-privilege credential controls (e.g., role-based permissions, scoped tokens, restricted access levels) — only a single account-wide key generation flow is described.", "evidenceIds": [ "bolna-docs-15" ] }, { "productId": "bolna", "storyId": "agentic-sdks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Bolna documents an official Web Call SDK (bolna-docs-18) under a 'developer-resources/sdks' path, plus a CLI and MCP server, giving AI-native builders some official tooling to build against, but there is no evidence of broader language SDKs (e.g., Python, Node, Go client libraries) beyond the browser-based Web Call SDK. missing for 10: multi-language server-side SDKs, independent/hands-on verification of SDK usability, breadth beyond the single web-call SDK.", "evidenceIds": [ "bolna-docs-18", "bolna-docs-16", "bolna-probe-4", "bolna-docs-17" ] }, { "productId": "bolna", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions webhooks or event subscription for Bolna; the docs cover agents, calls, CLI, MCP, SIP, pricing, and security but nothing about outbound event notifications. Missing for 10: any webhook registration/config API, event payload documentation, or third-party confirmation of webhook delivery.", "evidenceIds": [] }, { "productId": "bolna", "storyId": "ai-assisted-agent-authoring", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence shows Bolna's own platform AI generating or improving prompts, flows, or test cases from a description; the closest feature (build-with-ai/mcp) is an MCP server that lets external AI clients like Claude or Cursor manage the account via API, not the platform itself authoring agents. Templates can be cloned, but that's static copying, not AI-generated authoring.", "evidenceIds": [ "bolna-docs-17", "bolna-docs-5" ] }, { "productId": "bolna", "storyId": "ai-call-qa-scoring", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Bolna offers post-call analysis primitives — configurable extraction-categories (bolna-docs-1) and toggleable call summaries (bolna-docs-2) — which could be used to build custom scoring/flagging, but there's no documented automated QA/quality-scoring, failure-flagging, or resolution-analysis feature out of the box. Missing for 10: dedicated call-scoring/QA dashboard, automatic failure detection, resolution-analysis reporting, and independent evidence these extraction fields are used for review purposes.", "evidenceIds": [ "bolna-docs-1", "bolna-docs-2" ] }, { "productId": "bolna", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Bolna has an API reference doc listing basics like API key creation, but there is no evidence of an interactive reference with runnable/try-it examples, and a probe for a machine-readable OpenAPI spec (which typically powers such interactive docs) returned 404 on all checked paths.", "evidenceIds": [ "bolna-docs-15", "bolna-probe-2" ] }, { "productId": "bolna", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe for OpenAPI/Swagger spec files at all standard locations returned 404s, and no evidence pack item shows a downloadable machine-readable API spec despite having an API reference page.", "evidenceIds": [ "bolna-probe-2", "bolna-docs-15" ] }, { "productId": "bolna", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions a sandbox environment, test mode, staging account, or separation from production data for Bolna; the closest concepts (self-hosting, BYOK, data residency) are about deployment/privacy, not a distinct sandbox/test mode. missing for 10: any mention of a sandbox/test environment, test API keys, or safe testing workflow that avoids production data.", "evidenceIds": [] }, { "productId": "bolna", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "There is a changelog page referenced and an API reference, but no evidence of API versioning scheme or a documented deprecation policy anywhere in the pack; OpenAPI spec probes even returned 404s. Missing for 10: versioned API endpoints/headers, explicit deprecation policy documentation, migration/versioning changelog entries.", "evidenceIds": [ "bolna-probe-2", "bolna-docs-15" ] }, { "productId": "bolna", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "The CLI explicitly manages 'batches' of calls alongside agents and phone numbers (bolna-docs-16), and the platform claims to power 'thousands of inbound and outbound calls every minute' (bolna-docs-22), implying built-in bulk calling capability. However, there is no documentation of a specific bulk API endpoint, CSV/list upload, or bulk agent/category management workflow. Missing for 10: explicit bulk-operation API docs (e.g., batch call creation, bulk agent updates), a documented interface for uploading many items at once, and independent confirmation of bulk throughput.", "evidenceIds": [ "bolna-docs-16", "bolna-docs-22", "bolna-docs-12" ] }, { "productId": "bolna", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Bolna's workflow feature lets users chain voice calls, WhatsApp messages, API calls, waits and retries via the API, which is a form of automation, but the docs never describe defining conditional 'if event X happens then do Y' rules or an event/trigger system — it's presented as sequential outreach flows rather than rule-based event triggers. Missing for 10: explicit event-trigger/rule definition syntax, conditional branching logic, and any webhook-or-event-driven automation examples beyond the sequential workflow description.", "evidenceIds": [ "bolna-docs-12" ] }, { "productId": "bolna", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Bolna's workflow docs describe multi-step outreach sequences with waits and retries (bolna-docs-12), but there is no evidence of a scheduler for recurring/cron-based jobs or workflows that re-trigger automatically over time. missing for 10: recurring/scheduled trigger mechanism, cron-like job definitions, evidence of periodic re-execution.", "evidenceIds": [ "bolna-docs-12" ] }, { "productId": "bolna", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Bolna is a voice AI agent platform (agent creation, calls, workflows, telephony) but the evidence pack shows no version control, review workflow, or rollback mechanism for automations/agents — no mention of agent versioning, change history, approval flows, or revert capability.", "evidenceIds": [] }, { "productId": "bolna", "storyId": "barge-in-interruptions", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions barge-in, interruption handling, speech-stop, or turn recovery mechanics; the docs cover extraction categories, summaries, providers, CLI, MCP, pricing, etc., but nothing about turn-taking or interruption behavior. missing for 10: any documentation of barge-in/interruption detection, mid-speech stop mechanics, context update on interruption, turn recovery logic.", "evidenceIds": [] }, { "productId": "bolna", "storyId": "batch-outbound-campaigns", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The CLI explicitly manages 'batches' of calls alongside agents/phone numbers, and marketing copy claims support for 'thousands of inbound and outbound calls every minute,' plus a workflows API for multi-step outreach with waits/retries — together implying batch campaign and throughput capability. However, there's no explicit documentation of scheduling controls (e.g., time windows, campaign start/stop) or throughput/rate-limiting configuration for outbound campaigns specifically. Missing for 10: explicit scheduling/calendar controls for campaigns, documented throughput/rate-limit configuration knobs, and any hands-on/independent evidence of batch campaign execution.", "evidenceIds": [ "bolna-docs-16", "bolna-docs-22", "bolna-docs-12", "bolna-probe-4" ] }, { "productId": "bolna", "storyId": "build-phone-agent-fast", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs directly support the story: templates can be cloned and customized, build/deploy claimed under 5 minutes, voice/accent selection, phone number and telephony setup (BYOT/SIP), and a quickstart walking through key verification, agent creation, placing a call, and fetching a transcript in ~5 minutes. This covers prompt (agent config/templates), voice (voice/provider selection), and phone number (telephony/SIP) end-to-end for a first live call within an hour. Missing for 10: independent/hands-on confirmation of the 'under 5 minutes' or 'first call in an hour' claims beyond vendor docs, and no third-party success report.", "evidenceIds": [ "bolna-docs-4", "bolna-docs-5", "bolna-docs-8", "bolna-docs-13", "bolna-docs-14", "bolna-docs-6" ] }, { "productId": "bolna", "storyId": "call-analytics-dashboard", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows API-level features like call summaries (call_summary_enabled) and extraction categories, but there is no mention of a dashboard UI presenting success rates, durrations, outcomes, or sentiment analytics — missing for 10: any dashboard/UI screenshot or docs describing aggregated call analytics views.", "evidenceIds": [ "bolna-docs-1", "bolna-docs-2" ] }, { "productId": "bolna", "storyId": "call-transfer-escalation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack describes call transfer functionality (warm or blind) to a human agent, or context handoff during a live call; the closest items relate to model switching, summaries, and telephony/SIP trunking but not transfer escalation.", "evidenceIds": [] }, { "productId": "bolna", "storyId": "concurrency-scaling-limits", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Bolna's marketing page claims it can 'power thousands of inbound and outbound calls every minute' and docs mention self-hosting via Docker/Kubernetes for 'scalable enterprise deployments,' but there are no documented concurrency limits, rate-limit tables, or self-serve capacity-scaling mechanisms (e.g., no API rate-limit docs, no autoscaling guide, no numeric SLA). missing for 10: documented numeric concurrency/rate limits, self-serve scale-up process, autoscaling architecture details, independent verification of claimed call volume.", "evidenceIds": [ "bolna-docs-22", "bolna-docs-9" ] }, { "productId": "bolna", "storyId": "consent-recording-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows security/data-residency features and call-summary toggles, but nothing about call-recording consent, per-call recording enable/disable controls, disclosure/announcement features, or configurable retention periods for recordings/transcripts. No mention of recording consent mechanisms or retention settings anywhere in the docs pack.", "evidenceIds": [] }, { "productId": "bolna", "storyId": "custom-voice-cloning", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows Bolna lets founders choose among third-party voices/accents/styles (bolna-docs-8) and integrates providers like ElevenLabs (bolna-docs-6), but there is no mention of a voice-cloning feature for a custom brand voice, nor any documented consent process for cloning. Missing for 10: voice cloning capability, consent/verification workflow documentation, any policy on brand voice usage.", "evidenceIds": [ "bolna-docs-8", "bolna-docs-6" ] }, { "productId": "bolna", "storyId": "documented-latency-numbers", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item provides documented latency numbers, benchmarks, or tuning guidance for end-to-end voice latency; only vague marketing claims like 'build in under 5 minutes' or 'power thousands of calls' appear, with no actual latency figures or turn-taking tuning documentation.", "evidenceIds": [ "bolna-docs-4", "bolna-docs-22" ] }, { "productId": "bolna", "storyId": "dtmf-ivr-voicemail", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions DTMF keypress sending, IVR menu navigation, or voicemail detection/leaving; only unrelated telephony features like SIP trunking, call summaries, and workflows are documented.", "evidenceIds": [] }, { "productId": "bolna", "storyId": "dynamic-call-personalization", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers agent creation, extraction categories, call summaries, workflows, and providers, but nothing documents passing per-caller dynamic variables or context payloads at call time to personalize a conversation.", "evidenceIds": [] }, { "productId": "bolna", "storyId": "hipaa-regulated-workloads", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Bolna documents Indian data residency and self-hosting via Docker/Kubernetes for private enterprise deployments, but there is no mention of HIPAA/BAA support or SOC 2 certification anywhere in the evidence pack. missing for 10: HIPAA/BAA attestation, SOC 2 report or certification, broader multi-region data-residency options beyond India.", "evidenceIds": [ "bolna-docs-20", "bolna-docs-9" ] }, { "productId": "bolna", "storyId": "knowledge-base-grounding", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Bolna is a voice AI agent platform (telephony, TTS/STT, LLM providers, workflows) but the evidence pack contains no mention of a knowledge base, document ingestion, or RAG capability for grounding agent responses in custom content. This axis clearly applies to conversational agent-building platforms, so absence of evidence yields 'none'.", "evidenceIds": [] }, { "productId": "bolna", "storyId": "live-call-monitoring", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers call summaries, extraction categories, transcripts, CLI, MCP, and SIP/telephony features, but nothing describes live-call monitoring dashboards, real-time observability, or alerting on agent misbehavior or error-rate spikes. No mention of metrics, alert thresholds, webhooks for anomaly detection, or a monitoring console for platform engineers. Missing for 10: real-time monitoring dashboard, alerting/notification system for errors or misbehavior, error-rate tracking, on-call/integration with observability tools.", "evidenceIds": [] }, { "productId": "bolna", "storyId": "mid-call-tool-calling", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers extraction categories, call summaries, workflows, and custom LLM providers, but nothing describes an agent invoking external APIs or custom functions mid-call and voicing the result seamlessly. Missing for 10: documentation of a function-calling/tool-use API, examples of mid-call API invocation, and any mention of latency-masking/filler techniques to avoid dead air.", "evidenceIds": [] }, { "productId": "bolna", "storyId": "multilingual-conversations", "verdict": "partial", "quality": 6, "confidence": "low", "rationale": "Docs claim the agent \"detects and switches mid-call on its own\" and marketing mentions \"multilingual intelligence\" for thousands of calls, directly supporting the story, but there's no detail on supported languages, configuration options, or independent/hands-on verification of mid-call switching reliability. Missing for 10: language list/configuration docs, hands-on or third-party confirmation of switching accuracy, and edge-case handling details.", "evidenceIds": [ "bolna-docs-3", "bolna-docs-22" ] }, { "productId": "bolna", "storyId": "noise-robustness", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions noise suppression, background noise handling, or audio filtering settings for calls; the docs cover extraction categories, summaries, telephony, SDKs, and pricing but not audio quality/noise robustness features. Missing for 10: any mention of noise suppression/audio filtering configuration, VAD tuning for noisy environments, or documented handling of real-world call noise.", "evidenceIds": [] }, { "productId": "bolna", "storyId": "openness-api-parity", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs show the platform is API-first — agent creation, extraction categories, call summaries, workflows, and telephony are all described as configurable via the API, and a CLI and MCP server both wrap that same API, implying broad parity. However, there is no explicit documentation stating full UI/API feature parity, and probes found no discoverable OpenAPI/swagger spec (404s), leaving some doubt about completeness of coverage for every UI action.\nMissing for 10: explicit UI-API parity statement, published OpenAPI/swagger spec, independent confirmation that all UI actions have API equivalents.", "evidenceIds": [ "bolna-docs-1", "bolna-docs-2", "bolna-docs-12", "bolna-docs-16", "bolna-docs-17", "bolna-probe-2", "bolna-probe-4" ] }, { "productId": "bolna", "storyId": "openness-full-export", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Bolna's API lets you fetch call transcripts (bolna-docs-14) and the platform can be self-hosted via Docker/Kubernetes (bolna-docs-9), with an open-source core framework (bolna-gh-1), which together provide some data portability, but there is no documented bulk 'export all data' feature or guarantee of open, standard export formats covering call recordings, agent configs, analytics, etc. Missing for 10: an explicit full-data export tool/endpoint, documented open format specification for exported data, and independent confirmation that a user can fully migrate away with all their data.", "evidenceIds": [ "bolna-docs-14", "bolna-docs-9", "bolna-gh-1" ] }, { "productId": "bolna", "storyId": "openness-open-license", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "The GitHub repo description explicitly calls Bolna 'the end-to-end open source production ready framework,' supporting that its source is publicly available under an open-license framing, but no specific license (e.g., MIT/Apache) is named or verified, and no independent corroboration exists. Missing for 10: explicit license identification (e.g., LICENSE file contents), confirmation the entire product (not just the framework) is open-sourced, and independent verification.", "evidenceIds": [ "bolna-gh-1" ] }, { "productId": "bolna", "storyId": "openness-self-host", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Bolna is explicitly open source (GitHub repo) and documentation provides dedicated on-premise instructions for self-hosting via Docker or Kubernetes for enterprise deployments, directly satisfying the self-host story. Missing for 10: independent/hands-on confirmation of a successful self-hosted deployment and more detail on feature parity between self-hosted and cloud versions.", "evidenceIds": [ "bolna-gh-1", "bolna-docs-9" ] }, { "productId": "bolna", "storyId": "phone-number-provisioning", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bolna's API/CLI clearly support placing outbound calls and receiving inbound calls (quickstart shows placing a call and fetching transcript via API, CLI manages calls and phone numbers, marketing claims thousands of inbound/outbound calls per minute), and BYOT lets you attach SIP trunks/numbers. However there's no explicit documented endpoint for provisioning/purchasing new phone numbers through the API itself — only CLI mention of managing 'phone numbers' and bringing your own trunk. Missing for 10: explicit phone-number provisioning API endpoint/documentation, independent confirmation of end-to-end inbound+outbound flow.", "evidenceIds": [ "bolna-docs-14", "bolna-docs-16", "bolna-docs-13", "bolna-docs-22", "bolna-probe-4" ] }, { "productId": "bolna", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs explicitly mention Indian data residency as an available option, and self-hosting via Docker/Kubernetes lets enterprises control where data lives entirely. However, there's no broader multi-region selection (e.g., EU, US) documented beyond India, and no detail on how residency is configured or enforced. Missing for 10: documented multi-region choice beyond India, configuration/UI for selecting region, and independent confirmation of residency enforcement.", "evidenceIds": [ "bolna-docs-20", "bolna-docs-9" ] }, { "productId": "bolna", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item addresses a training-data opt-out or policy on AI model training use; security docs cover data residency and IP whitelisting only, not training-data usage controls. missing for 10: any explicit statement on AI training data usage/opt-out policy, data processing agreement terms, or contractual guarantee against training use.", "evidenceIds": [] }, { "productId": "bolna", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of data retention controls, deletion APIs, or user-facing privacy/data-lifecycle settings; only tangential mentions of security (IP whitelisting, India data residency) which do not address retention/deletion. Self-hosting option exists but that's deployment control, not a retention/deletion feature.", "evidenceIds": [ "bolna-docs-20", "bolna-docs-21", "bolna-docs-9" ] }, { "productId": "bolna", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item addresses telemetry, analytics opt-out, or usage-tracking controls for the platform itself; existing security/data-residency docs don't cover this. Missing for 10: any documentation of telemetry collection or an opt-out/disable mechanism, privacy policy statement on usage tracking, self-host confirmation of no phone-home behavior.", "evidenceIds": [] }, { "productId": "bolna", "storyId": "realtime-transcription-quality", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm STT provider choice (Deepgram, Azure, ElevenLabs) and full LLM control including custom OpenAI-compatible endpoints, plus transcript retrieval via API, giving developers real provider/model flexibility. However there is no evidence of key-term/vocabulary boosting or accuracy tuning controls, nor any real-time transcription accuracy benchmarks. Missing for 10: key-term/keyword-boost configuration, accuracy/latency benchmarks for real-time transcription, and independent validation of transcription quality.", "evidenceIds": [ "bolna-docs-6", "bolna-docs-11", "bolna-docs-14" ] }, { "productId": "bolna", "storyId": "recordings-transcripts-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm fetching call transcripts via API (quickstart mentions 'fetch the transcript') and call summaries can be toggled, but there is no explicit documentation of a dedicated recordings-retrieval endpoint or bulk/programmatic export for all calls. missing for 10: explicit call-recording download/API endpoint, confirmation that recordings are retained and accessible for every call, and independent verification of transcript completeness/format.", "evidenceIds": [ "bolna-docs-14", "bolna-docs-2", "bolna-docs-1" ] }, { "productId": "bolna", "storyId": "self-host-oss-runtime", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Bolna is explicitly an open-source framework on GitHub, and docs include dedicated enterprise instructions for self-hosting on Docker or Kubernetes for private/enterprise deployments. Missing for 10: independent hands-on reports confirming successful self-host deployment and more detail on infrastructure requirements/limitations of the self-hosted version vs SaaS.", "evidenceIds": [ "bolna-gh-1", "bolna-docs-9" ] }, { "productId": "bolna", "storyId": "semantic-turn-detection", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence describes model-based end-of-turn/turn-taking detection distinct from VAD silence timeouts; evidence only covers mid-call model switching, extraction categories, and unrelated features. Missing for 10: any mention of endpointing model, semantic turn detection, or configurable end-of-turn logic beyond silence thresholds.", "evidenceIds": [] }, { "productId": "bolna", "storyId": "simulated-agent-testing", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any simulated conversation testing, evals, or pre-production testing framework for agents; the evidence pack only covers deployment, telephony, providers, pricing, and SDKs. Missing for 10: any mention of test/eval tooling, simulated call testing, or conversation quality scoring before live calls.", "evidenceIds": [] }, { "productId": "bolna", "storyId": "sip-trunk-byoc", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Bolna explicitly documents 'Bring Your Own Telephony (BYOT)' allowing any standards-compliant SIP trunk to connect to the platform, plus IP whitelisting requirements for the connection, directly addressing carrier/PBX independence from bundled telephony. missing for 10: explicit step-by-step Twilio/Telnyx number import walkthrough and independent/hands-on verification of SIP trunk setup working end-to-end.", "evidenceIds": [ "bolna-docs-13", "bolna-docs-21" ] }, { "productId": "bolna", "storyId": "structured-outcome-extraction", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Bolna provides extraction-categories API to attach custom entity/data extraction to agents and a call_summary_enabled toggle for summaries, plus transcript fetch via API — covering entities, summaries, and outcomes. However, there's no explicit documentation of a unified structured 'disposition' field, no clear webhook delivery mechanism for post-call structured payloads (only API fetch is shown), and no example of the combined extraction+summary+disposition JSON schema. missing for 10: webhook-based delivery of extraction results, documented disposition/outcome field, end-to-end example combining all three data types in one payload.", "evidenceIds": [ "bolna-docs-1", "bolna-docs-2", "bolna-docs-14" ] }, { "productId": "bolna", "storyId": "transparent-per-minute-pricing", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "There is a dedicated call-pricing doc page and mentions that BYOK/connected providers aren't charged (bolna-docs-19, bolna-docs-10), indicating some usage-based pricing structure exists, but no evidence pack item actually quotes a per-minute rate, tiered pricing table, or a cost calculator a founder could use to estimate cost per call. Missing for 10: explicit per-minute/usage rate figures, a pricing calculator or worked cost example, and independent confirmation that published rates are accurate.", "evidenceIds": [ "bolna-docs-19", "bolna-docs-10" ] }, { "productId": "bolna", "storyId": "visual-flow-builder", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Bolna is built around APIs, CLI, and MCP server for defining agents and multi-step outreach sequences (bolna-docs-12, bolna-docs-16, bolna-docs-17), but there is no mention of a visual/no-code builder with branching, states, or handoffs; workflows are described as 'driven entirely through the Bolna API.' Missing for 10: any visual flow-builder UI, drag-and-drop branching/state design, no-code handoff configuration.", "evidenceIds": [ "bolna-docs-12", "bolna-docs-16", "bolna-docs-17", "bolna-gh-1" ] }, { "productId": "bolna", "storyId": "voice-library-choice", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm multiple TTS providers (Deepgram, Azure, ElevenLabs, and more) plus voice/accent/style customization, satisfying the 'plug in providers' part of the story. However, there's no evidence of a broad pre-built voice library or catalog to browse/select from, only provider integration. Missing for 10: evidence of a large built-in voice library/catalog, provider-count breadth beyond three named examples, independent/hands-on confirmation of voice quality or selection UX.", "evidenceIds": [ "bolna-docs-6", "bolna-docs-8" ] }, { "productId": "deepgram", "storyId": "agent-consumes-mcp-tools", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence confirms mid-call function calling exists for the Voice Agent (deepgram-docs-2) but only documents an MCP *server* built into the CLI that lets coding tools access Deepgram APIs (deepgram-docs-13, deepgram-probe-4) — this is the reverse direction, not the Voice Agent acting as an MCP client to pull in external toolsets mid-call. No evidence shows the Voice Agent itself can connect to MCP servers as a tool source.", "evidenceIds": [ "deepgram-docs-2", "deepgram-docs-13", "deepgram-probe-4" ] }, { "productId": "deepgram", "storyId": "agent-provisions-voice-agent", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Deepgram documents agent creation via reusable agent configs (UUID-based), telephony connectivity for inbound/outbound calls, and a CLI with a built-in MCP server giving AI coding tools API access — the building blocks for programmatic provisioning exist. However, the evidence shows phone-call handling is done via a Twilio bridge (server code required) rather than a native Deepgram 'attach number/place call' API, and no doc shows the CLI or MCP server actually exposing agent-create + number-attach + call-place as a single end-to-end flow. Missing for 10: explicit CLI/MCP commands for provisioning a phone number and placing a call, and confirmation that this full workflow avoids external glue code/dashboard steps.", "evidenceIds": [ "deepgram-docs-4", "deepgram-docs-5", "deepgram-docs-13", "deepgram-docs-18", "deepgram-probe-4", "deepgram-probe-5" ] }, { "productId": "deepgram", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Direct probe confirms llms.txt exists at developers.deepgram.com/llms.txt (HTTP 200) with explicit AI-agent instructions, and docs pages support .md suffix for clean markdown retrieval, both verified by live probes. missing for 10: no independent third-party confirmation of an agent actually consuming this in practice.", "evidenceIds": [ "deepgram-probe-1", "deepgram-probe-2" ] }, { "productId": "deepgram", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Deepgram Voice Agent's API capabilities (function calling, multi-agent orchestration, telephony, LLM providers, prompting) but contains no mention of an in-product analytics dashboard, conversation insights, or AI-generated suggestions derived from a user's own data. While such a feature (e.g., call analytics/insights) is plausible for a voice AI platform, no evidence shows Deepgram surfaces this.", "evidenceIds": [] }, { "productId": "deepgram", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "deepgram", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "deepgram", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "The Voice Agent is a WebSocket API (no GUI dependency) and Deepgram ships a terminal-based `dg` CLI for scripting Deepgram operations (transcribe, synthesize, manage account) which is inherently automatable/CI-friendly, and temporary tokens support secure automated auth. However, there is no explicit documentation of running the Voice Agent itself headlessly in CI pipelines, no CI/CD examples, and the CLI's primary use-cases described are transcription/synthesis rather than orchestrating full voice-agent sessions programmatically. Missing for 10: dedicated CI/headless automation guide for Voice Agent sessions, example CI pipeline configs, and confirmation the CLI can drive the Agent API end-to-end rather than just STT/TTS.", "evidenceIds": [ "deepgram-docs-1", "deepgram-docs-12", "deepgram-docs-14", "deepgram-probe-5" ] }, { "productId": "deepgram", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Deepgram Voice Agent supports function calling for custom tool use, but there is no evidence the Voice Agent can act as an MCP client to consume external MCP servers' tools. The only MCP-related evidence is a built-in MCP *server* in the `dg` CLI that lets coding tools access Deepgram's APIs — the opposite direction of the story.", "evidenceIds": [ "deepgram-docs-2", "deepgram-docs-13", "deepgram-probe-4" ] }, { "productId": "deepgram", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Deepgram ships an official MCP server (bundled in the dg CLI) that lets AI coding tools/agents connect directly to Deepgram's APIs, confirmed both in docs and via a dedicated agentic-tools page. This directly satisfies the story of connecting an agent to Deepgram via an official MCP server. Missing for 10: independent/hands-on verification that the MCP server works reliably in practice, and more detail on which Voice Agent capabilities specifically are exposed through it.", "evidenceIds": [ "deepgram-docs-13", "deepgram-probe-4", "deepgram-docs-12" ] }, { "productId": "deepgram", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Deepgram Voice Agent is fundamentally a conversational system: users interact via natural spoken language, and the agent supports function calling to perform tasks, live prompting to shape behavior, and mid-call message injection—all driven by natural-language conversation rather than rigid commands. This is core, well-documented functionality (docs-2, docs-7, docs-11) directly matching the story's intent for an ai-native/agentic persona. Missing for 10: independent/hands-on validation of natural-language command accuracy and no evidence of complex multi-turn command chaining reliability from third-party sources.", "evidenceIds": [ "deepgram-docs-2", "deepgram-docs-7", "deepgram-docs-11", "deepgram-docs-3" ] }, { "productId": "deepgram", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Deepgram ships an official 'dg' CLI for transcribing files, streaming audio, synthesizing speech, and managing accounts from the terminal, and it even embeds an MCP server for AI coding tools, directly supporting agentic/AI-native workflows. Missing for 10: independent/hands-on corroboration beyond first-party docs and more detail on CLI coverage of Voice Agent-specific features.", "evidenceIds": [ "deepgram-docs-12", "deepgram-docs-13", "deepgram-probe-5", "deepgram-probe-4" ] }, { "productId": "deepgram", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Deepgram publishes a documented public API (WebSocket-based Voice Agent API plus OpenAPI spec confirmed live), extensive docs covering endpoints, function calling, LLM providers, telephony, and auth, and even a CLI/MCP server for programmatic/agentic access, making it clearly drivable by an AI-native user. Missing for 10: independent third-party corroboration of API usage beyond vendor docs.", "evidenceIds": [ "deepgram-docs-1", "deepgram-probe-3", "deepgram-docs-12", "deepgram-docs-13", "deepgram-probe-5", "deepgram-docs-14" ] }, { "productId": "deepgram", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Deepgram documents temporary, short-lived tokens as an alternative to traditional API keys for secure, limited-duration access (deepgram-docs-14), which partially satisfies a least-privilege credential story, but there is no evidence of granular scopes/permissions (e.g., restricting a key to specific endpoints or agent capabilities) or role-based access control for agents. Missing for 10: documented scope/permission granularity, per-agent key restriction, and evidence of enforcement/verification of least-privilege in practice.", "evidenceIds": [ "deepgram-docs-14" ] }, { "productId": "deepgram", "storyId": "agentic-sdks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Evidence shows an official 'Browser Agent SDK' with composable packages, plus an OpenAPI spec, CLI, and MCP server for developer tooling, indicating some official SDK/build tooling exists. However, evidence does not mention broader server-side/language SDKs (Python, Node, Go, etc.) commonly expected for building agentic applications, nor independent corroboration of SDK quality. Missing for 10: multi-language SDK documentation, independent/hands-on developer reports, broader agentic build examples beyond browser.", "evidenceIds": [ "deepgram-docs-6", "deepgram-probe-3", "deepgram-probe-5", "deepgram-probe-4" ] }, { "productId": "deepgram", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Deepgram Voice Agent operates via a persistent WebSocket connection for real-time audio streaming, not webhook-based event subscription; no documentation mentions webhooks for event notifications (e.g., call completion, transcript ready, errors). Absence of evidence for an applicable capability yields 'none'.", "evidenceIds": [] }, { "productId": "deepgram", "storyId": "ai-assisted-agent-authoring", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers building agents manually (prompting, function calling, reusable configs, MCP server for coding tools) but there is no mention of the platform's own AI generating or improving prompts, flows, or test cases from a description; the MCP server exposes Deepgram APIs to external coding assistants rather than being an AI author within the platform itself.", "evidenceIds": [ "deepgram-docs-7", "deepgram-docs-4", "deepgram-docs-13", "deepgram-probe-4" ] }, { "productId": "deepgram", "storyId": "ai-call-qa-scoring", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers Deepgram Voice Agent's real-time conversation infrastructure (STT/TTS, function calling, telephony, prompting, A/B testing of voices/prompts) but contains no mention of post-call AI review, automated quality scoring, failure flagging, or resolution analysis of completed calls.", "evidenceIds": [] }, { "productId": "deepgram", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows an OpenAPI spec exists and docs pages are available as clean Markdown, but there is no evidence of an interactive API reference with runnable/try-it-now examples (e.g., embedded playground, live code execution, or Swagger/Postman-style explorer). Missing for 10: interactive explorer UI, runnable/executable code snippets, live request/response testing.", "evidenceIds": [ "deepgram-probe-3", "deepgram-probe-1", "deepgram-probe-2" ] }, { "productId": "deepgram", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A probe confirms Deepgram publishes a machine-readable OpenAPI spec at openapi.json containing the 'openapi' key, directly satisfying the story. Missing for 10: no first-party docs page explicitly announcing/describing the OpenAPI spec's coverage or versioning.", "evidenceIds": [ "deepgram-probe-3" ] }, { "productId": "deepgram", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions a sandbox environment, test/dev API keys, or a way to test without touching production data or usage; only production endpoints, EU endpoint, and opt-out flags are documented. Missing for 10: sandbox/test-mode environment, documentation of non-production keys or test credits, any guidance on isolating test traffic from production data.", "evidenceIds": [] }, { "productId": "deepgram", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows extensive Voice Agent API documentation, an OpenAPI spec, CLI, and MCP server, but contains no mention of API versioning scheme or a documented deprecation policy for the Voice Agent API.", "evidenceIds": [] }, { "productId": "deepgram", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "deepgram", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Deepgram Voice Agent supports function calling that lets the agent trigger actions during a live conversation, and features like InjectAgentMessage and multi-agent handoff suggest some event-driven behavior, but this is developer-defined logic (via function calling code) rather than a declarative rules engine for automatically triggering actions on arbitrary events. missing for 10: a documented rules/trigger engine (if-this-then-that style), event subscription system, or automation workflow builder distinct from manual function-calling code.", "evidenceIds": [ "deepgram-docs-2", "deepgram-docs-3", "deepgram-docs-11" ] }, { "productId": "deepgram", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "deepgram", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Deepgram's Reusable Agent Configurations let you persist a named agent config and reference it by UUID, and support A/B testing two configs in parallel, which is adjacent to versioning, but there is no documented version history, diff/review workflow, or explicit rollback mechanism. missing for 10: version history tracking, review/approval workflow, explicit rollback capability, changelog or diffing between config versions.", "evidenceIds": [ "deepgram-docs-4", "deepgram-docs-17" ] }, { "productId": "deepgram", "storyId": "barge-in-interruptions", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs explicitly claim barge-in support where users can talk over the agent and it stops instantly, plus mid-call context tools like InjectAgentMessage for updating conversation state. However, evidence lacks detail on how turn recovery/context updating works technically after interruption, and there's no independent/hands-on corroboration of graceful recovery in practice. missing for 10: technical details on context-state recovery post-interruption, independent verification of barge-in reliability, edge-case handling (e.g., rapid interruptions, latency of stop).", "evidenceIds": [ "deepgram-docs-8", "deepgram-docs-11", "deepgram-docs-1" ] }, { "productId": "deepgram", "storyId": "batch-outbound-campaigns", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Deepgram's docs confirm outbound telephony connectivity (docs-5, docs-18) and per-call features like barge-in and prompting, but there is no mention of campaign-level batch dialing, call scheduling, or throughput/rate controls for outbound campaigns — this is a real-time conversation API, not a dialer/campaign manager.", "evidenceIds": [ "deepgram-docs-5", "deepgram-docs-18" ] }, { "productId": "deepgram", "storyId": "build-phone-agent-fast", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Deepgram provides all core building blocks — prompting, voice/LLM selection, telephony via Twilio bridging, and a single WebSocket API — that together could plausibly get a developer to a first live call quickly. However, the telephony path requires standing up your own server to bridge Twilio audio to the Voice Agent WebSocket, which is nontrivial integration work rather than a turnkey 'phone number in one hour' flow, and there's no first-party quickstart or time-to-first-call benchmark cited. missing for 10: an end-to-end quickstart/tutorial demonstrating full setup within an hour, evidence of a managed/no-code telephony number provisioning path, and independent hands-on confirmation of setup speed.", "evidenceIds": [ "deepgram-docs-1", "deepgram-docs-5", "deepgram-docs-7", "deepgram-docs-9", "deepgram-docs-18", "deepgram-docs-8" ] }, { "productId": "deepgram", "storyId": "call-analytics-dashboard", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Deepgram is a developer API/infrastructure product; evidence shows mentions of A/B testing metrics (conversion, CSAT, containment rate) as data points but no evidence of an actual built-in dashboard for call analytics, success rates, durations, sentiment, or outcomes that founders can view without building their own.", "evidenceIds": [ "deepgram-docs-17" ] }, { "productId": "deepgram", "storyId": "call-transfer-escalation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers telephony connectivity, function calling, multi-agent handoff between AI agents, and mid-call message injection, but nothing documents a warm/blind transfer to a human agent (e.g., SIP REFER, call transfer function, or context handoff to a live operator).", "evidenceIds": [] }, { "productId": "deepgram", "storyId": "concurrency-scaling-limits", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions concurrency limits, rate limits, per-account call caps, scaling guidance, or capacity request processes for the Voice Agent API; the docs cover architecture, features, and integrations but nothing about scale/concurrency documentation.", "evidenceIds": [] }, { "productId": "deepgram", "storyId": "consent-recording-controls", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Deepgram documents general data-privacy controls (per-request opt-out from retention via mip_opt_out, EU-region processing endpoint) that touch on data retention/residency, but there is no explicit documentation of per-call recording toggles, consent/disclosure workflows, or configurable retention windows tied to call recordings specifically. Missing for 10: explicit per-call recording enable/disable controls, documented retention period configuration for stored call audio/transcripts, and consent/disclosure feature support.", "evidenceIds": [ "deepgram-docs-15", "deepgram-docs-16" ] }, { "productId": "deepgram", "storyId": "custom-voice-cloning", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a voice cloning feature or consent process; docs cover STT/TTS model selection, multilingual voices, and agent configuration but never mention custom voice cloning or consent workflows.", "evidenceIds": [] }, { "productId": "deepgram", "storyId": "documented-latency-numbers", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers architecture, features (barge-in, telephony, function calling, multi-agent), and CLI/SDK tooling, but contains no documented latency numbers (e.g., ms end-to-end) or tuning guidance for reducing turn-taking latency. missing for 10: quantified latency benchmarks, latency tuning/configuration guidance, any performance dashboard or whitepaper on speed claims.", "evidenceIds": [ "deepgram-docs-1", "deepgram-docs-8", "deepgram-docs-18" ] }, { "productId": "deepgram", "storyId": "dtmf-ivr-voicemail", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence covers telephony connectivity, function calling, barge-in, and multi-agent orchestration, but nothing documents DTMF keypress sending, IVR menu navigation, or voicemail detection/leaving capabilities.", "evidenceIds": [] }, { "productId": "deepgram", "storyId": "dynamic-call-personalization", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Deepgram supports prompting, reusable agent configs (passed at call time via UUID), function calling, and mid-call message injection, which together allow injecting per-call context/variables into agent behavior; A/B testing of configs also implies runtime parameterization. However there is no explicit documented mechanism for templated dynamic variables (e.g., {{caller_name}}) or a dedicated per-caller context API akin to other platforms. missing for 10: explicit dynamic variable templating syntax, dedicated per-caller metadata injection API, hands-on example of personalizing a call with caller-specific data.", "evidenceIds": [ "deepgram-docs-4", "deepgram-docs-7", "deepgram-docs-11", "deepgram-docs-17", "deepgram-docs-2" ] }, { "productId": "deepgram", "storyId": "hipaa-regulated-workloads", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Evidence shows EU data-residency endpoint and an opt-out from data retention/training, indicating some compliance/trust infrastructure, but there is no mention of HIPAA/BAA support or SOC 2 certification anywhere in the pack. missing for 10: HIPAA/BAA documentation, SOC 2 attestation evidence, broader regional residency options beyond EU.", "evidenceIds": [ "deepgram-docs-15", "deepgram-docs-16" ] }, { "productId": "deepgram", "storyId": "knowledge-base-grounding", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers architecture, function calling, multi-agent handoff, telephony, prompting, and LLM providers, but nowhere documents a built-in knowledge base or RAG feature for grounding the agent on user documents. Function calling ([deepgram-docs-2]) could theoretically be wired to an external retrieval system, but that is not the same as a built-in KB/RAG capability, and no such feature is described.", "evidenceIds": [ "deepgram-docs-2", "deepgram-docs-3", "deepgram-docs-7" ] }, { "productId": "deepgram", "storyId": "live-call-monitoring", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of live monitoring dashboards, error-rate alerting, or observability/analytics tooling for production calls; docs cover agent architecture, function calling, telephony, and configuration but nothing about monitoring or alerting on agent misbehavior.", "evidenceIds": [] }, { "productId": "deepgram", "storyId": "mid-call-tool-calling", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm function calling is a supported feature for performing tasks mid-conversation (deepgram-docs-2), and the agent architecture is a single low-latency WebSocket for audio in/out (deepgram-docs-1) with barge-in and inject-message features (deepgram-docs-8, deepgram-docs-11) suggesting attention to latency/dead-air. However there's no explicit documentation or example describing how function-call latency is masked (e.g., filler speech, streaming partial results) while awaiting an API response. Missing for 10: concrete guidance/examples on avoiding dead air during function execution, sample code showing async function calls with the agent speaking a holding phrase, and independent/hands-on verification of smooth latency handling.", "evidenceIds": [ "deepgram-docs-2", "deepgram-docs-1", "deepgram-docs-8", "deepgram-docs-11" ] }, { "productId": "deepgram", "storyId": "multilingual-conversations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Deepgram's docs confirm multilingual voice agent support, noting that STT and TTS model choices must be configured for the target language (deepgram-docs-10), but the evidence pack does not explicitly document automatic language detection or dynamic switching mid-call — only static multilingual configuration is described. Missing for 10: explicit documentation of automatic language detection, mid-call language switching mechanics, and independent/hands-on confirmation of this behavior in production.", "evidenceIds": [ "deepgram-docs-10" ] }, { "productId": "deepgram", "storyId": "noise-robustness", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions noise suppression, background noise filtering, or audio denoising features for the Voice Agent; coverage focuses on barge-in, telephony, multi-agent orchestration, and LLM/TTS configuration instead.", "evidenceIds": [] }, { "productId": "deepgram", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Deepgram is fundamentally API-first — the Voice Agent is a WebSocket API with full configuration (prompting, function calling, multi-agent, telephony, reusable configs) exposed programmatically, and even the CLI/MCP server let AI-native users manage things from the terminal or via agentic tools rather than a GUI. However, evidence doesn't show a full-featured UI to compare against (e.g., a dashboard/console) nor confirm that every console feature (like A/B testing dashboards, analytics views) has a documented API equivalent. missing for 10: explicit mapping of console/UI-only features (analytics dashboards, A/B test result views) to API endpoints, and confirmation no UI-exclusive functionality exists.", "evidenceIds": [ "deepgram-docs-1", "deepgram-docs-4", "deepgram-docs-12", "deepgram-docs-13", "deepgram-docs-17", "deepgram-probe-3", "deepgram-probe-4", "deepgram-probe-5" ] }, { "productId": "deepgram", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence describes any data export, account data portability, or open-format export/deletion workflow letting a user take their data and leave; docs cover retention opt-out and EU processing but not export tooling. missing for 10: data export mechanism, open-format export documentation, account closure/data portability guarantees.", "evidenceIds": [] }, { "productId": "deepgram", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Deepgram Voice Agent is a proprietary cloud API/SDK product; no evidence anywhere in the pack points to an open-source license or public source repository for the core Voice Agent service. Only API docs, CLI, and SDK usage are documented, none of which imply open-licensed source availability.", "evidenceIds": [] }, { "productId": "deepgram", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Deepgram Voice Agent is a cloud API/SaaS product; all evidence points to hosted WebSocket endpoints, EU regional endpoints, and cloud-based configuration — there is no mention of a self-hostable or on-premises deployment package for the core voice agent model/inference stack. missing for 10: any self-hosting/on-prem deployment option, downloadable model weights or container, docs on running the core service outside Deepgram's cloud.", "evidenceIds": [ "deepgram-docs-1", "deepgram-docs-16" ] }, { "productId": "deepgram", "storyId": "phone-number-provisioning", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm Deepgram Voice Agent supports inbound/outbound calls and telephony integration (e.g., via Twilio bridging audio to a single WebSocket), but there is no evidence Deepgram itself provisions or manages phone numbers—developers must bring their own Twilio account/number and build a bridging server. Missing for 10: native phone-number provisioning API, first-party telephony number management, and evidence of outbound call initiation directly through Deepgram's API without a third-party carrier.", "evidenceIds": [ "deepgram-docs-5", "deepgram-docs-18", "deepgram-docs-8" ] }, { "productId": "deepgram", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Deepgram documents an EU-specific endpoint (api.eu.deepgram.com) for customers requiring EU data processing, and a per-request opt-out for model improvement data retention, showing some regional/data-handling control. However, this is limited to EU vs. default (US) with no evidence of broader region selection (e.g., APAC, other jurisdictions) or a general residency configuration option across the platform. Missing for 10: documented list of all available regions, self-serve region selection in dashboard/API config, and independent confirmation of residency guarantees beyond docs.", "evidenceIds": [ "deepgram-docs-16", "deepgram-docs-15" ] }, { "productId": "deepgram", "storyId": "privacy-no-training", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Deepgram documents a per-request opt-out (mip_opt_out=true) from its Model Improvement Program, with opted-out requests not retained, directly addressing training-data use. However, the evidence only shows a per-request flag rather than an account-wide default-off setting, and there's no independent verification that opt-out requests are truly excluded from training. Missing for 10: account/org-level opt-out default, third-party audit or independent confirmation of non-training use, and clarity on default behavior when the flag is omitted.", "evidenceIds": [ "deepgram-docs-15" ] }, { "productId": "deepgram", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Deepgram documents a per-request opt-out (mip_opt_out=true) that prevents retention, and references data-privacy compliance including an EU-specific endpoint, giving users some retention control. However, there is no documented self-service deletion mechanism for already-retained data, no stated retention periods, and no dashboard/API for managing or purging stored voice data. Missing for 10: explicit data deletion API/console, documented retention duration policy, and confirmation of deletion for non-opted-out data.", "evidenceIds": [ "deepgram-docs-15", "deepgram-docs-16" ] }, { "productId": "deepgram", "storyId": "privacy-telemetry-optout", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Deepgram documents an opt-out mechanism (mip_opt_out=true) for model improvement/data retention on a per-request basis, which is a form of telemetry/usage-data opt-out, but this is narrowly scoped to training data retention rather than general telemetry/usage tracking (e.g., analytics, product usage metrics). Missing for 10: documentation of a broader telemetry/usage-tracking opt-out setting, confirmation this covers all usage data beyond model-improvement retention, and independent verification that opting out has no side effects.", "evidenceIds": [ "deepgram-docs-15" ] }, { "productId": "deepgram", "storyId": "realtime-transcription-quality", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm real-time transcription via the unified WebSocket API, selectable LLM providers (docs-9), and multilingual STT/TTS model choices (docs-10), giving developers meaningful control over STT and LLM models. However, there is no evidence of 'key terms' or keyword-boosting controls specific to the Voice Agent's transcription pipeline. Missing for 10: explicit key-term/keyword-boost configuration support, independent benchmarking of transcription accuracy.", "evidenceIds": [ "deepgram-docs-1", "deepgram-docs-9", "deepgram-docs-10" ] }, { "productId": "deepgram", "storyId": "recordings-transcripts-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Voice Agent architecture, function calling, telephony bridging, LLM providers, and data privacy opt-outs, but nowhere describes an API or mechanism to retrieve full call recordings or persisted transcripts after a call ends. No endpoint, storage feature, or retrieval workflow is documented for this specific capability.", "evidenceIds": [] }, { "productId": "deepgram", "storyId": "self-host-oss-runtime", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Deepgram Voice Agent is a hosted, cloud-only API accessed via WebSocket; nothing in the evidence indicates open-source runtime code that can be self-hosted on customer infrastructure. All references point to Deepgram-hosted endpoints (api.deepgram.com, api.eu.deepgram.com) and a CLI/SDK for calling the cloud service, not deploying the runtime itself.", "evidenceIds": [ "deepgram-docs-1", "deepgram-docs-16", "deepgram-docs-18" ] }, { "productId": "deepgram", "storyId": "semantic-turn-detection", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence only mentions barge-in (interrupting agent mid-speech) and generic prompting/architecture docs; there is no mention of model-based end-of-turn detection or endpointing that goes beyond VAD/silence timeouts to avoid cutting off slow speakers.", "evidenceIds": [] }, { "productId": "deepgram", "storyId": "simulated-agent-testing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence covers live-call features (function calling, prompting, telephony, A/B testing of live configs) but nothing describes a simulated-conversation or eval framework for pre-production testing of agents before real phone calls. Missing for 10: any documentation of a test/sandbox mode, conversation simulation tool, or eval harness for agents.", "evidenceIds": [ "deepgram-docs-17", "deepgram-docs-4" ] }, { "productId": "deepgram", "storyId": "sip-trunk-byoc", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only documents a generic WebSocket bridge pattern for Twilio (docs-18, docs-8) and a general 'Telephony' feature bullet (docs-5), but there is no mention of SIP trunking, PBX connectivity, or Telnyx number import — the specific mechanisms a platform engineer would need to bring their own carrier.", "evidenceIds": [ "deepgram-docs-5", "deepgram-docs-18", "deepgram-docs-8" ] }, { "productId": "deepgram", "storyId": "structured-outcome-extraction", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers live function calling during a call and various agent configuration/telephony features, but there is no mention of post-call structured data extraction (summaries, entities, dispositions) or webhook delivery of such analytics after a call ends.", "evidenceIds": [] }, { "productId": "deepgram", "storyId": "transparent-per-minute-pricing", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions pricing, per-minute rates, usage-based cost, or any pricing page/calculator for the Voice Agent product; all citations focus on architecture, features, and developer tools.", "evidenceIds": [] }, { "productId": "deepgram", "storyId": "visual-flow-builder", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "All evidence describes a code/API-first architecture (WebSocket connections, JSON Settings messages, system prompts, function calling, CLI/SDK) rather than a visual no-code builder; there is no mention of a drag-and-drop flow designer, branching UI, or state-machine editor. missing for 10: any visual builder UI, no-code branching/state design, or drag-and-drop handoff configuration.", "evidenceIds": [ "deepgram-docs-1", "deepgram-docs-3", "deepgram-docs-4", "deepgram-docs-7", "deepgram-docs-9" ] }, { "productId": "deepgram", "storyId": "voice-library-choice", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence confirms TTS model selection exists (docs-10 references choosing a TTS model) but there is no documentation of a broad voice library or of plugging in multiple third-party TTS providers analogous to the multi-LLM-provider list (docs-9). Missing for 10: evidence of voice catalog/library breadth, evidence of multiple supported TTS providers/vendors, and any provider-switching mechanism for voice output.", "evidenceIds": [ "deepgram-docs-10" ] }, { "productId": "elevenlabs-agents", "storyId": "agent-consumes-mcp-tools", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly describe connecting conversational agents to external MCP servers as a tool source, enabling access to data/processing capabilities via that server mid-call, directly matching the story. Missing for 10: independent hands-on confirmation of mid-call multi-tool retrieval from a connected MCP server (only vendor docs, no third-party validation) and detail on how many tools/toolsets a single MCP connection exposes.", "evidenceIds": [ "elevenlabs-agents-docs-16", "elevenlabs-agents-docs-30", "elevenlabs-agents-docs-46" ] }, { "productId": "elevenlabs-agents", "storyId": "agent-provisions-voice-agent", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs and runtime probes confirm agents can be created and managed via API, a real CLI (@elevenlabs/cli, verified live on npm), and a hosted MCP server (verified live with OAuth), all without the dashboard (docs-1,3,4,27,28,38; probe-rt-1,2). Phone-number attachment is also supported via SIP trunking to route calls to agents (docs-12,36), but there is no explicit documentation or evidence of actually placing/initiating an outbound call end-to-end through the API/CLI/MCP — only routing inbound infrastructure integration is described. Missing for 10: explicit API/CLI/MCP call to place an outbound call, and a concrete end-to-end walkthrough combining create+attach+call in one non-dashboard flow.", "evidenceIds": [ "elevenlabs-agents-docs-1", "elevenlabs-agents-docs-3", "elevenlabs-agents-docs-4", "elevenlabs-agents-docs-12", "elevenlabs-agents-docs-36", "elevenlabs-agents-docs-38", "elevenlabs-agents-probe-rt-1", "elevenlabs-agents-probe-rt-2" ] }, { "productId": "elevenlabs-agents", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "ElevenLabs publishes a working llms.txt at the root (HTTP 200) and structures its docs as machine-readable .md pages with an explicit pointer to a full docs/llms.txt index, exactly matching the ai-native 'point an agent at llms.txt' pattern; this was independently verified via live probes rather than just claimed docs. Missing for 10: no evidence of a hands-on agent actually consuming the file end-to-end to complete a task (only that the endpoint/format is live and documented).", "evidenceIds": [ "elevenlabs-agents-probe-1", "elevenlabs-agents-probe-2" ] }, { "productId": "elevenlabs-agents", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Agent-analysis features (sentiment analysis, success evaluation, structured data extraction, semantic conversation search) generate AI-derived insights from the user's own conversation data, which partially satisfies the story. However, these are evaluation/reporting metrics rather than proactive AI-generated suggestions or recommendations for next actions, and there is no independent/hands-on corroboration of this analytics experience. Missing for 10: evidence of proactive suggestion generation (e.g., recommended actions/optimizations), a dashboard summarization view, and independent verification beyond first-party docs.", "evidenceIds": [ "elevenlabs-agents-docs-20", "elevenlabs-agents-docs-21", "elevenlabs-agents-docs-22", "elevenlabs-agents-docs-23", "elevenlabs-agents-docs-34", "elevenlabs-agents-docs-50" ] }, { "productId": "elevenlabs-agents", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "ElevenLabs Agents supports tool-calling, webhook actions, and multi-step workflows that let a voice agent perform actions (scheduling, API calls, MCP access) during a conversation, and agents can be created/managed via API for programmatic triggering, which give some automation flavor. But all documented action-taking is tied to a live conversational turn (voice/chat) rather than an autonomous background process that runs without a triggering conversation or human interaction, so the 'runs autonomously in the background' framing is only weakly supported. Missing for 10: evidence of scheduled/cron-triggered autonomous runs, evidence of multi-step tasks executing without a live conversation session, and independent confirmation of background automation outside the demoed conversational/tool-call flow.", "evidenceIds": [ "elevenlabs-agents-docs-5", "elevenlabs-agents-docs-8", "elevenlabs-agents-docs-14", "elevenlabs-agents-docs-31", "elevenlabs-agents-docs-16", "elevenlabs-agents-docs-39" ] }, { "productId": "elevenlabs-agents", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "ElevenLabs documents a first-party hosted MCP server letting an assistant like Claude create, configure, and manage agents in the workspace via natural language, with nothing to install locally — a form of task delegation to an AI assistant for managing the product. However this is not a native in-product chat/assistant UI; it requires an external MCP client (e.g., Claude) to act as the assistant, and delegation is scoped to agent-management tasks rather than general product tasks. Missing for 10: a built-in assistant surface inside the ElevenLabs dashboard itself, evidence of broader task delegation beyond agent CRUD, and independent hands-on confirmation of the natural-language workflow.", "evidenceIds": [ "elevenlabs-agents-docs-4", "elevenlabs-agents-docs-28", "elevenlabs-agents-docs-38", "elevenlabs-agents-docs-44", "elevenlabs-agents-probe-rt-2" ] }, { "productId": "elevenlabs-agents", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "ElevenLabs Agents can be managed via a full REST API and an official CLI (verified runtime install/run), enabling scripted/headless creation and configuration of agents outside the dashboard, which supports CI-style automation. However, evidence is docs/probe-based with no CI pipeline examples, no first-class 'run in CI' guidance, and Agent testing is described as pre-deploy validation without confirmation it can be triggered non-interactively/headlessly. Missing for 10: explicit CI/CD integration examples, headless testing invocation docs, and independent hands-on confirmation of automated pipelines.", "evidenceIds": [ "elevenlabs-agents-docs-1", "elevenlabs-agents-docs-39", "elevenlabs-agents-docs-24", "elevenlabs-agents-probe-rt-1" ] }, { "productId": "elevenlabs-agents", "storyId": "agentic-mcp-client", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly describe connecting conversational agents to external MCP servers so agents can access data sources and tools via MCP, directly matching the story, and this is corroborated by dedicated documentation pages (mcp.md) rather than just passing mention. Missing for 10: independent/hands-on verification of an actual third-party MCP server being plugged in and tool-calls succeeding in production (only docs and a runtime probe on the separate hosted-server feature are present).", "evidenceIds": [ "elevenlabs-agents-docs-16", "elevenlabs-agents-docs-30", "elevenlabs-agents-docs-46" ] }, { "productId": "elevenlabs-agents", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "ElevenLabs explicitly documents and operates a hosted MCP server (api.elevenlabs.io/v1/mcp) that lets external MCP clients like Claude connect to create, configure, and manage agents through natural language, with a runtime probe confirming the live endpoint correctly responds with the MCP OAuth challenge. This is a first-party 'official MCP server' offering, not just client-side MCP consumption, matching the story precisely. Missing for 10: independent/community hands-on confirmation beyond ElevenLabs' own docs and the single runtime probe.", "evidenceIds": [ "elevenlabs-agents-docs-1", "elevenlabs-agents-docs-4", "elevenlabs-agents-docs-38", "elevenlabs-agents-docs-44", "elevenlabs-agents-probe-3", "elevenlabs-agents-probe-rt-2" ] }, { "productId": "elevenlabs-agents", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "ElevenLabs Agents ships a hosted MCP server explicitly designed so assistants like Claude can 'create, configure, and manage the agents in your workspace through natural language, with nothing to install or run locally,' and a runtime probe confirms the endpoint is live and speaks the MCP OAuth flow. This is a direct, first-party natural-language control surface, not just a client-side integration. Missing for 10: independent/community hands-on confirmation of a full natural-language agent-management session (only endpoint liveness was probed, not an actual successful NL task).", "evidenceIds": [ "elevenlabs-agents-docs-4", "elevenlabs-agents-docs-28", "elevenlabs-agents-docs-38", "elevenlabs-agents-docs-44", "elevenlabs-agents-probe-rt-2", "elevenlabs-agents-probe-3" ] }, { "productId": "elevenlabs-agents", "storyId": "agentic-official-cli", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "ElevenLabs documents an official Agents CLI for managing agents, and runtime probing confirms `npx @elevenlabs/cli` installs and runs keylessly from npm, corroborating the docs claim with hands-on evidence. Missing for 10: deeper documentation of full CLI command surface/subcommands beyond basic install confirmation.", "evidenceIds": [ "elevenlabs-agents-docs-1", "elevenlabs-agents-docs-3", "elevenlabs-agents-docs-27", "elevenlabs-agents-probe-rt-1" ] }, { "productId": "elevenlabs-agents", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "ElevenLabs Agents ships a documented public API alongside a dashboard, CLI, and hosted MCP server for programmatic agent creation/management, with quickstart docs explicitly noting 'creating an agent via the API' and full customization (tools, voice, analysis, testing) accessible programmatically. Runtime probes confirm the CLI installs keylessly and the hosted MCP endpoint is live and functional. Missing for 10: no independent third-party API reference/SDK usage examples or community corroboration beyond first-party docs.", "evidenceIds": [ "elevenlabs-agents-docs-1", "elevenlabs-agents-docs-39", "elevenlabs-agents-docs-8", "elevenlabs-agents-probe-rt-1", "elevenlabs-agents-probe-rt-2" ] }, { "productId": "elevenlabs-agents", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers agent-side authentication (protecting inbound access to a deployed agent) and OAuth for the hosted MCP server, but there is no documentation of issuing scoped or least-privilege API keys/tokens that limit what an agent itself can do via the ElevenLabs API. Missing for 10: any docs on API key permission scopes, workspace role-based tokens, or restricted-credential issuance for agents.", "evidenceIds": [ "elevenlabs-agents-docs-10", "elevenlabs-agents-docs-43", "elevenlabs-agents-probe-rt-2" ] }, { "productId": "elevenlabs-agents", "storyId": "agentic-sdks", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs confirm programmatic access via the ElevenLabs API and an official CLI (`@elevenlabs/cli`, verified runtime) for building agents, but no evidence pack item explicitly documents a first-party language SDK (Python/Node/etc.) for Agents beyond the generic API reference. Missing for 10: explicit official SDK docs/repos, SDK code examples, and independent confirmation of SDK feature parity with the API.", "evidenceIds": [ "elevenlabs-agents-docs-1", "elevenlabs-agents-docs-39", "elevenlabs-agents-probe-rt-1" ] }, { "productId": "elevenlabs-agents", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes 'webhook tools' that let an agent make outbound calls to external endpoints during a conversation (docs-14, docs-29, docs-31), which is the opposite of subscribing to platform-emitted events via webhooks. There is no documented mechanism for a developer to register a webhook URL to receive event notifications (e.g., conversation-ended, call-completed) from ElevenLabs Agents.", "evidenceIds": [ "elevenlabs-agents-docs-14", "elevenlabs-agents-docs-29", "elevenlabs-agents-docs-31" ] }, { "productId": "elevenlabs-agents", "storyId": "ai-assisted-agent-authoring", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "ElevenLabs documents that an external AI assistant (e.g., Claude) can create, configure, and manage agents through natural language via the hosted MCP server, and agent-testing docs describe turning real conversations into test cases for regression testing — both are AI-assisted authoring flows. However, this relies on a third-party AI client (Claude) rather than a built-in, platform-native 'describe your agent and we generate it' copilot for prompts/flows within the dashboard itself. Missing for 10: evidence of a first-party generative wizard that authors prompts/flows purely from a natural-language description inside the ElevenLabs UI, and evidence of AI-generated (not just conversation-derived) test-case authoring.", "evidenceIds": [ "elevenlabs-agents-docs-4", "elevenlabs-agents-docs-38", "elevenlabs-agents-docs-25", "elevenlabs-agents-docs-24", "elevenlabs-agents-probe-rt-2" ] }, { "productId": "elevenlabs-agents", "storyId": "ai-call-qa-scoring", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs explicitly describe automated post-call analysis: 'Success evaluation' scores conversation quality/goal achievement, 'Sentiment analysis' flags user sentiment, and 'Data collection' extracts structured resolution details — directly matching the story's scoring/flagging/resolution-analysis ask. Missing for 10: no independent/hands-on validation of analysis accuracy, and no explicit 'failure flagging' terminology beyond success-criteria evaluation.", "evidenceIds": [ "elevenlabs-agents-docs-20", "elevenlabs-agents-docs-21", "elevenlabs-agents-docs-22", "elevenlabs-agents-docs-34", "elevenlabs-agents-docs-56" ] }, { "productId": "elevenlabs-agents", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains extensive markdown documentation for ElevenLabs Agents (quickstart, customization, tools, etc.) but nothing describes an interactive API reference page with runnable/'try it' examples — no mention of a Swagger/OpenAPI explorer, live code sandbox, or embedded runnable snippets. Since ElevenLabs Agents exposes an API, this axis is applicable, but no evidence supports the specific capability.", "evidenceIds": [] }, { "productId": "elevenlabs-agents", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents the API, CLI, dashboard, and hosted MCP server for ElevenLabs Agents, but nowhere mentions a downloadable OpenAPI/Swagger spec or machine-readable schema for the API. This axis clearly applies to an API-driven product, but no evidence shows it being delivered.", "evidenceIds": [] }, { "productId": "elevenlabs-agents", "storyId": "api-sandbox", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs describe an 'agent testing' feature to verify conversational responses, tool usage, and multi-turn outcomes 'before you deploy', implying a pre-production test workflow, and tool execution can run in a 'sandboxed environment' on ElevenLabs' infrastructure. However there is no explicit documentation of a separate sandbox/staging environment or test data isolation from production conversation/analytics data. Missing for 10: explicit sandbox/staging environment separate from production data, documentation of test-vs-prod data isolation, independent confirmation that test runs don't affect production analytics or logs.", "evidenceIds": [ "elevenlabs-agents-docs-24", "elevenlabs-agents-docs-51", "elevenlabs-agents-docs-15", "elevenlabs-agents-docs-45" ] }, { "productId": "elevenlabs-agents", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions API versioning scheme, version numbers, or any documented deprecation policy for ElevenLabs Agents' API/CLI/MCP interfaces. missing for 10: versioned API endpoints or version headers, changelog/release notes, explicit deprecation policy or sunset timelines, migration guides for breaking changes.", "evidenceIds": [] }, { "productId": "elevenlabs-agents", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence documents API, CLI, and MCP-based management of agents, but nothing describes bulk/batch operations (e.g., batch-create, batch-update, or batch-delete across many agents or conversations at once). This axis is fair to ask given the product exposes an API and CLI, but no evidence confirms bulk-operation support.", "evidenceIds": [] }, { "productId": "elevenlabs-agents", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "ElevenLabs Agents supports tool-calling and webhook actions that are triggered automatically based on conversation events (e.g., 'trigger authenticated actions based on the conversation, like scheduling meetings or initiating order returns'), plus multi-step visual workflows and MCP-based external actions. This covers event-triggered automation within a conversation but the evidence doesn't show general rule-based triggers on arbitrary system events (e.g., scheduled/external triggers outside a live conversation) — missing for 10: documentation of standalone event/rule engine independent of a live conversation, examples of conditional branching logic beyond tool calls, and independent verification of automation reliability.", "evidenceIds": [ "elevenlabs-agents-docs-8", "elevenlabs-agents-docs-14", "elevenlabs-agents-docs-31", "elevenlabs-agents-docs-5", "elevenlabs-agents-docs-16" ] }, { "productId": "elevenlabs-agents", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers multi-step workflow building, tool calling, MCP integration, and analysis, but no documentation mentions scheduling recurring jobs, cron-like triggers, or automated recurring workflow execution. This is a fair capability to expect from an automation-focused agent platform, but no evidence supports it.", "evidenceIds": [] }, { "productId": "elevenlabs-agents", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The docs describe building, testing, and analyzing agents (agent-testing, agent-analysis) but there is no mention of version history, change review workflows, or rollback/revert capability for agent configurations. Missing for 10: version history UI, rollback/restore of prior agent configs, change-review/approval workflow, audit trail of edits.", "evidenceIds": [] }, { "productId": "elevenlabs-agents", "storyId": "barge-in-interruptions", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs explicitly document configuring turn-taking, interruptions, and timeout settings, and conversation-flow settings that 'determine how your assistant handles periods of user silence, interruptions during speech, and turn-taking behavior,' confirming barge-in handling is a first-class configurable feature. However, evidence lacks detail on how context is updated after an interruption or how turn recovery specifically behaves, and there's no hands-on/independent validation of graceful recovery in practice. Missing for 10: technical detail on context truncation/update after interruption, independent or hands-on testing confirming graceful recovery, and latency benchmarks for barge-in specifically.", "evidenceIds": [ "elevenlabs-agents-docs-33", "elevenlabs-agents-docs-42", "elevenlabs-agents-docs-19" ] }, { "productId": "elevenlabs-agents", "storyId": "batch-outbound-campaigns", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents phone-number/SIP trunking integration for routing calls to agents, but contains no mention of batch outbound calling, campaign scheduling, or throughput/rate controls for outbound call volume. Since telephony campaign management is a fair capability to expect from a voice-agent telephony product, absence of evidence here is 'none' rather than 'na'.", "evidenceIds": [ "elevenlabs-agents-docs-12", "elevenlabs-agents-docs-36" ] }, { "productId": "elevenlabs-agents", "storyId": "build-phone-agent-fast", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs show a clear quickstart path — create an agent via dashboard/API, pick from 5k+ voices, and connect phone numbers via SIP trunking to route calls to the agent without changing existing phone infrastructure — which together cover the prompt+voice+phone number pieces of the story. However, there's no explicit end-to-end 'first call in under an hour' walkthrough or independent timing evidence, and phone-number provisioning specifics (vs. just SIP integration) are thin. Missing for 10: an explicit timed/complete quickstart-to-live-call walkthrough, first-party or independent evidence of the actual time-to-first-call, and clearer native phone-number purchase/setup docs beyond SIP trunking.", "evidenceIds": [ "elevenlabs-agents-docs-1", "elevenlabs-agents-docs-6", "elevenlabs-agents-docs-12", "elevenlabs-agents-docs-36", "elevenlabs-agents-docs-39", "elevenlabs-agents-docs-40" ] }, { "productId": "elevenlabs-agents", "storyId": "call-analytics-dashboard", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm built-in agent-analysis features—success evaluation, sentiment analysis, data extraction, and conversation search—managed via the ElevenAgents dashboard, which covers most of the story's asked-for metrics without custom-building analytics. However, there's no explicit evidence of a dashboard showing aggregated success-rate percentages or call-duration analytics/rollups across calls, only per-conversation criteria and settings. Missing for 10: dashboard screenshots/evidence of aggregate success-rate metrics, call-duration reporting/rollups, and independent corroboration that these appear in a unified analytics view.", "evidenceIds": [ "elevenlabs-agents-docs-20", "elevenlabs-agents-docs-21", "elevenlabs-agents-docs-22", "elevenlabs-agents-docs-23", "elevenlabs-agents-docs-1", "elevenlabs-agents-docs-56" ] }, { "productId": "elevenlabs-agents", "storyId": "call-transfer-escalation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers telephony features like SIP trunking, tools, webhooks, and MCP integrations, but nowhere documents call transfer (warm or blind) to a human agent or passing conversation context during handoff. Missing for 10: any mention of transfer_to_number/transfer_to_agent tool types, warm/blind transfer configuration, or context-passing during escalation.", "evidenceIds": [] }, { "productId": "elevenlabs-agents", "storyId": "concurrency-scaling-limits", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item documents concurrency limits, per-account call caps, autoscaling behavior, or a process for scaling to many simultaneous calls without contacting sales/support; docs cover features like tools, voice customization, SIP trunking, HIPAA, and MCP but nothing on capacity/scale numbers or self-service scaling guarantees.", "evidenceIds": [] }, { "productId": "elevenlabs-agents", "storyId": "consent-recording-controls", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs confirm configurable data-retention settings for conversation transcripts and audio recordings, which addresses the retention half of the story, plus HIPAA/BAA support suggesting compliance posture. However, there is no evidence of per-call recording enable/disable controls or built-in consent/disclosure messaging mechanisms for calls. Missing for 10: per-call recording toggle, consent/disclosure prompt configuration, and any independent verification of retention enforcement.", "evidenceIds": [ "elevenlabs-agents-docs-26", "elevenlabs-agents-docs-52", "elevenlabs-agents-docs-11", "elevenlabs-agents-docs-53" ] }, { "productId": "elevenlabs-agents", "storyId": "custom-voice-cloning", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes selecting from 5k+ pre-built voices and voice customization (speed, multi-voice switching) but contains no mention of a voice-cloning feature for creating a custom brand voice, nor any documented consent process for cloning (e.g., voice verification, spoken consent statement) required by ElevenLabs' broader platform. Missing for 10: any docs on voice cloning/Instant or Professional Voice Cloning for Agents, and any consent-verification workflow.", "evidenceIds": [ "elevenlabs-agents-docs-6", "elevenlabs-agents-docs-17", "elevenlabs-agents-docs-32" ] }, { "productId": "elevenlabs-agents", "storyId": "documented-latency-numbers", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "ElevenLabs markets 'ultra-low latency' (docs-37) and does document conversation-flow tuning knobs like turn-taking, interruption handling and timeouts (docs-33, docs-42, docs-19/49/54), which counts as tuning guidance for latency/turn-taking behavior. However there are no documented end-to-end latency benchmark numbers (e.g., ms figures, percentile breakdowns) anywhere in the evidence pack. Missing for 10: concrete latency benchmark numbers/metrics, methodology for measuring end-to-end latency, and any independent corroboration of the 'ultra-low latency' marketing claim.", "evidenceIds": [ "elevenlabs-agents-docs-37", "elevenlabs-agents-docs-33", "elevenlabs-agents-docs-42", "elevenlabs-agents-docs-19" ] }, { "productId": "elevenlabs-agents", "storyId": "dtmf-ivr-voicemail", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack confirms ElevenLabs Agents supports telephony integration via SIP trunking and phone numbers, so the telephony axis clearly applies, but nothing in the docs mentions DTMF tone sending, IVR menu navigation, or voicemail detection/leaving capabilities.", "evidenceIds": [ "elevenlabs-agents-docs-12", "elevenlabs-agents-docs-36" ] }, { "productId": "elevenlabs-agents", "storyId": "dynamic-call-personalization", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly state 'Use dynamic variables and overrides for per-conversation customization' as a core build feature, directly matching the story of injecting per-caller context at call time. Missing for 10: no independent/hands-on example showing a real dynamic-variable injection call in code, and no detail on variable scoping/limits beyond the single doc mention.", "evidenceIds": [ "elevenlabs-agents-docs-9" ] }, { "productId": "elevenlabs-agents", "storyId": "hipaa-regulated-workloads", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "ElevenLabs Agents documents HIPAA eligibility and BAA availability for eligible customers, directly supporting regulated PHI workloads. However, there is no evidence in the pack of SOC 2 certification or data-residency options, both explicitly named in the story. Missing for 10: SOC 2 attestation/report evidence, data-residency region selection or documentation, independent audit corroboration of HIPAA/BAA claims.", "evidenceIds": [ "elevenlabs-agents-docs-11", "elevenlabs-agents-docs-35", "elevenlabs-agents-docs-53" ] }, { "productId": "elevenlabs-agents", "storyId": "knowledge-base-grounding", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly state agents support uploading documents and enabling RAG for grounded responses, directly matching the story's built-in knowledge base/RAG capability. Missing for 10: independent/hands-on corroboration of RAG quality and details on knowledge base management (chunking, size limits, sources).", "evidenceIds": [ "elevenlabs-agents-docs-7" ] }, { "productId": "elevenlabs-agents", "storyId": "live-call-monitoring", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs show post-call analytics (sentiment analysis, success evaluation, data extraction, conversation search) and a marketing claim to 'monitor' agents, but there is no evidence of live/real-time call monitoring dashboards, error-rate metrics, or alerting/notification mechanisms for misbehaving agents. Missing for 10: real-time monitoring dashboard, error-rate/latency metrics, alerting/notification system, on-call integration (e.g., PagerDuty/Slack alerts).", "evidenceIds": [ "elevenlabs-agents-docs-20", "elevenlabs-agents-docs-22", "elevenlabs-agents-docs-23", "elevenlabs-agents-docs-34", "elevenlabs-agents-docs-37" ] }, { "productId": "elevenlabs-agents", "storyId": "mid-call-tool-calling", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs clearly describe multiple tool-calling mechanisms — webhook tools for external API calls, client-side tools, sandboxed custom JS, and MCP server integration — explicitly for making external function calls mid-conversation to fetch real-time info and trigger authenticated actions. Conversation-flow settings (turn-taking, interruption handling, ultra-low latency) address smooth conversational pacing, though not explicitly framed as avoiding dead air during tool execution. Missing for 10: explicit documentation of filler/hold-speech behavior while awaiting API results, and independent/hands-on verification that tool calls don't introduce audible pauses.", "evidenceIds": [ "elevenlabs-agents-docs-8", "elevenlabs-agents-docs-13", "elevenlabs-agents-docs-14", "elevenlabs-agents-docs-15", "elevenlabs-agents-docs-16", "elevenlabs-agents-docs-29", "elevenlabs-agents-docs-31", "elevenlabs-agents-docs-33", "elevenlabs-agents-docs-37" ] }, { "productId": "elevenlabs-agents", "storyId": "multilingual-conversations", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Docs confirm broad multi-language support (70+ languages, language-specific voice settings) and voice-switching for multi-character use, but there is no explicit documentation of automatic language detection or switching languages mid-call. Missing for 10: explicit language-detection/mid-call switching feature docs, and any hands-on demonstration of a live language switch during a single conversation.", "evidenceIds": [ "elevenlabs-agents-docs-37", "elevenlabs-agents-docs-32", "elevenlabs-agents-docs-17", "elevenlabs-agents-docs-6" ] }, { "productId": "elevenlabs-agents", "storyId": "noise-robustness", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers voice customization, turn-taking/interruption settings, tools, testing, and telephony (SIP) integration, but nowhere mentions noise suppression, background noise handling, or audio filtering for noisy real-world calls. This is a fair capability to expect for a voice-agent platform handling phone calls, but no documentation or feature reference confirms it exists.", "evidenceIds": [ "elevenlabs-agents-docs-33", "elevenlabs-agents-docs-42", "elevenlabs-agents-docs-12" ] }, { "productId": "elevenlabs-agents", "storyId": "openness-api-parity", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs show agents can be created, configured, and managed via the API, CLI, dashboard, or hosted MCP server, and quickstart explicitly says agents can be created 'via the API or the web dashboard,' implying broad parity across surfaces (elevenlabs-agents-docs-1, docs-39, docs-44). However, there's no explicit feature-by-feature confirmation that every dashboard capability (e.g., visual workflow builder, analytics dashboards, testing UI) has a full API equivalent, nor independent verification of parity. Missing for 10: an explicit parity statement or changelog confirming all dashboard features (workflow builder, analytics, testing suite) are fully scriptable via API, and independent/hands-on confirmation of that parity.", "evidenceIds": [ "elevenlabs-agents-docs-1", "elevenlabs-agents-docs-39", "elevenlabs-agents-docs-44", "elevenlabs-agents-docs-5", "elevenlabs-agents-probe-rt-1" ] }, { "productId": "elevenlabs-agents", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence describes a data export feature, open-format data dumps, or account deletion/portability tooling; only conversation retention settings are documented, not export mechanisms. Missing for 10: any documented export API/CLI command, supported open export formats (e.g., JSON/CSV), and confirmation of full data portability/account closure process.", "evidenceIds": [] }, { "productId": "elevenlabs-agents", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "ElevenLabs Agents is a closed, proprietary SaaS product; the CLI is open on npm but no evidence indicates the core Agents platform/backend source is available under an open license. No license file, GitHub repo, or open-source statement is present in the evidence pack.", "evidenceIds": [] }, { "productId": "elevenlabs-agents", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "ElevenLabs Agents is entirely a hosted SaaS product—managed via dashboard, API, CLI, and hosted MCP server, all pointing to ElevenLabs' own infrastructure; there is no evidence of an open-source release, downloadable server binary, or self-hosting option for the core voice-agent runtime. missing for 10: any open-source repo or self-hosted deployment package, docs describing running the core service on one's own infrastructure, independent confirmation of self-hosting.", "evidenceIds": [ "elevenlabs-agents-docs-1", "elevenlabs-agents-docs-4", "elevenlabs-agents-docs-44" ] }, { "productId": "elevenlabs-agents", "storyId": "phone-number-provisioning", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm phone/telephony integration via SIP trunking, letting agents route inbound/outbound calls over existing phone infrastructure, and agents can be created/configured via the API. However, evidence does not explicitly show a documented API endpoint for provisioning new phone numbers or explicitly triggering outbound calls programmatically vs. via SIP trunk setup. Missing for 10: explicit phone-number provisioning API endpoint docs, explicit outbound-call-initiation API example, and independent/hands-on confirmation of inbound+outbound call flows.", "evidenceIds": [ "elevenlabs-agents-docs-12", "elevenlabs-agents-docs-36", "elevenlabs-agents-docs-39" ] }, { "productId": "elevenlabs-agents", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence covers data retention duration settings, HIPAA/BAA eligibility, and infrastructure details, but nothing addresses selectable data region/residency options for storage location. missing for 10: any mention of region selection, data residency controls, or geographic storage options.", "evidenceIds": [ "elevenlabs-agents-docs-26", "elevenlabs-agents-docs-52", "elevenlabs-agents-docs-53" ] }, { "productId": "elevenlabs-agents", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers data retention settings (how long transcripts/audio are stored) and HIPAA/BAA compliance, but nowhere mentions an explicit opt-out or policy regarding use of customer data/conversations for AI model training. missing for 10: explicit training-data opt-out policy or setting, terms-of-service language on model training use, any statement distinguishing enterprise vs free-tier data usage for training.", "evidenceIds": [ "elevenlabs-agents-docs-26", "elevenlabs-agents-docs-52", "elevenlabs-agents-docs-53" ] }, { "productId": "elevenlabs-agents", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs explicitly document configurable retention settings for conversation transcripts and audio recordings, giving users control over how long data is stored. However, evidence does not detail a deletion mechanism (e.g., data subject deletion requests, API/CLI commands to purge stored data) beyond retention duration settings. Missing for 10: explicit deletion/export controls, granular per-conversation deletion API, and independent verification of retention enforcement.", "evidenceIds": [ "elevenlabs-agents-docs-26", "elevenlabs-agents-docs-52" ] }, { "productId": "elevenlabs-agents", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers conversation transcript/audio retention settings (docs-26, docs-52) but nothing about opting out of product telemetry or usage tracking (e.g., CLI analytics, dashboard usage metrics) for the AI-native user. Retention configuration is about customer conversation data, not the vendor's own telemetry collection, so this axis has no supporting evidence.", "evidenceIds": [] }, { "productId": "elevenlabs-agents", "storyId": "realtime-transcription-quality", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Docs confirm agents let developers choose or bring custom LLMs (docs-41) and mention real-time voice interaction (docs-37), but there is no evidence of selectable STT providers, transcription accuracy tuning, or 'key terms'/vocabulary boosting controls anywhere in the pack. Missing for 10: STT provider selection, transcription-specific accuracy settings, key-term/glossary support, and any independent verification of transcription accuracy claims.", "evidenceIds": [ "elevenlabs-agents-docs-41", "elevenlabs-agents-docs-37", "elevenlabs-agents-docs-33" ] }, { "productId": "elevenlabs-agents", "storyId": "recordings-transcripts-api", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs confirm that conversation transcripts and audio recordings are stored with configurable retention, and that conversations can be searched, analyzed, and have structured data extracted (docs-26/52, docs-21/23/56), implying underlying storage of call recordings/transcripts. However, there is no explicit documentation of a specific API endpoint or method for programmatically retrieving full call recordings and transcripts on a per-call basis. Missing for 10: an explicit 'get conversation' or 'get recording/transcript' API reference, example request/response showing programmatic export of audio + transcript per call.", "evidenceIds": [ "elevenlabs-agents-docs-26", "elevenlabs-agents-docs-52", "elevenlabs-agents-docs-21", "elevenlabs-agents-docs-23", "elevenlabs-agents-docs-56" ] }, { "productId": "elevenlabs-agents", "storyId": "self-host-oss-runtime", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "ElevenLabs Agents is entirely a managed/hosted service — the CLI and MCP server are clients/interfaces to ElevenLabs' cloud infrastructure, not open-source runtime code that can be deployed on a platform-engineer's own servers. Nothing in the evidence pack mentions open-sourcing the runtime, self-hosting instructions, or on-prem deployment options; all references (dashboard, API, CLI, hosted MCP, SIP trunking) point to ElevenLabs-operated infrastructure.", "evidenceIds": [ "elevenlabs-agents-docs-1", "elevenlabs-agents-docs-4", "elevenlabs-agents-docs-44", "elevenlabs-agents-probe-rt-1", "elevenlabs-agents-probe-rt-2" ] }, { "productId": "elevenlabs-agents", "storyId": "semantic-turn-detection", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Docs mention configurable turn-taking, interruption handling, and timeout settings (docs-33, docs-42), but nothing in the evidence describes a model-based end-of-turn detector that goes beyond a VAD/silence-timeout mechanism — no mention of semantic or ML-based turn-completion detection to avoid interrupting slow speakers. Missing for 10: any documentation of a dedicated end-of-turn prediction model, benchmarks or comparisons versus simple VAD timeouts, or hands-on evidence that the agent waits appropriately for slow speakers.", "evidenceIds": [ "elevenlabs-agents-docs-33", "elevenlabs-agents-docs-42", "elevenlabs-agents-docs-19" ] }, { "productId": "elevenlabs-agents", "storyId": "simulated-agent-testing", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "ElevenLabs documents an explicit Agent Testing feature that verifies conversational responses, tool usage, and full multi-turn outcomes before deployment, including converting real conversations into simulated test cases, plus success-evaluation criteria for assessing conversation quality/goal achievement — directly matching the pre-deployment simulated testing story. Missing for 10: no independent/hands-on report validating test accuracy or CI-integration details, and no explicit mention of automated eval scoring pipelines beyond docs description.", "evidenceIds": [ "elevenlabs-agents-docs-24", "elevenlabs-agents-docs-25", "elevenlabs-agents-docs-51", "elevenlabs-agents-docs-20", "elevenlabs-agents-docs-34" ] }, { "productId": "elevenlabs-agents", "storyId": "sip-trunk-byoc", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs explicitly describe a SIP trunking integration that lets customers connect their existing phone systems/carrier infrastructure to ElevenLabs agents without changing carriers, directly addressing the core ask of bringing your own PBX/carrier. However, the pack has no explicit mention of importing existing Twilio/Telnyx numbers or detailed carrier onboarding steps, and no independent/hands-on confirmation of a live SIP trunk setup working end-to-end. Missing for 10: explicit Twilio/Telnyx number import instructions, and third-party/hands-on validation of the SIP trunking flow.", "evidenceIds": [ "elevenlabs-agents-docs-12", "elevenlabs-agents-docs-36" ] }, { "productId": "elevenlabs-agents", "storyId": "structured-outcome-extraction", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show structured post-call outputs (data collection with contact/business data extraction, success-evaluation criteria, sentiment analysis) available via 'agent-analysis' features, and the platform is manageable via API. However, the pack never explicitly documents a webhook payload/callback delivering this structured analysis after a call, only webhook *tools* for in-call actions — missing for 10: explicit post-call webhook delivery mechanism/schema for outcomes and dispositions, and confirmation these analysis fields are retrievable via a documented API endpoint.", "evidenceIds": [ "elevenlabs-agents-docs-20", "elevenlabs-agents-docs-21", "elevenlabs-agents-docs-22", "elevenlabs-agents-docs-50", "elevenlabs-agents-docs-56", "elevenlabs-agents-docs-14", "elevenlabs-agents-docs-31" ] }, { "productId": "elevenlabs-agents", "storyId": "transparent-per-minute-pricing", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence item in the pack references pricing, per-minute rates, usage-based billing, or cost calculators for ElevenLabs Agents; all citations are product/docs feature descriptions. Since founders evaluating a voice-agent SaaS could reasonably expect published pricing, the axis applies but is entirely unaddressed in this evidence pack.", "evidenceIds": [] }, { "productId": "elevenlabs-agents", "storyId": "visual-flow-builder", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs explicitly claim a 'visual workflow builder' for multi-step workflows, plus dashboard-based agent creation, conversation-flow settings (turn-taking, interruptions, timeouts), and tools/handoff-like actions (webhooks, client tools) — supporting the no-code, branching, multi-step story. However, evidence never details branching logic, explicit 'states' or agent-to-agent 'handoffs' within the builder, nor independent hands-on confirmation of the visual builder's depth. Missing for 10: concrete detail/screenshots of branching and state logic in the builder, explicit handoff-between-agents feature description, and independent/third-party corroboration of the builder's no-code sufficiency.", "evidenceIds": [ "elevenlabs-agents-docs-5", "elevenlabs-agents-docs-33", "elevenlabs-agents-docs-42", "elevenlabs-agents-docs-39", "elevenlabs-agents-docs-8" ] }, { "productId": "elevenlabs-agents", "storyId": "voice-library-choice", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm a broad in-house voice library (5k+ voices across 31 languages, docs-6) with customization like speed, pronunciation, and multi-voice switching (docs-17/32/48), satisfying the first half of the story. However, there is no evidence of plugging in third-party/alternative TTS providers — the docs only mention bring-your-own LLM (docs-41), not bring-your-own TTS engine. Missing for 10: any documented support for external/third-party TTS provider integration, evidence of provider-agnostic voice pipeline, independent confirmation of using non-ElevenLabs voices.", "evidenceIds": [ "elevenlabs-agents-docs-6", "elevenlabs-agents-docs-17", "elevenlabs-agents-docs-32", "elevenlabs-agents-docs-48", "elevenlabs-agents-docs-41" ] }, { "productId": "livekit-agents", "storyId": "agent-consumes-mcp-tools", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "LiveKit Agents has first-class, documented MCP support (\"first-class support for Model Context Protocol (MCP) servers\", \"Integrate tools provided by MCP servers with one line of code\") via `MCPToolset`, which wraps an MCP server and passes it directly to the agent's tools parameter — enabling mid-call toolset access for voice agents. This is corroborated across both docs and GitHub README. Missing for 10: independent hands-on validation of actual mid-call MCP toolset usage beyond vendor docs.", "evidenceIds": [ "livekit-agents-docs-17", "livekit-agents-docs-23", "livekit-agents-docs-34", "livekit-agents-gh-1", "livekit-agents-gh-3" ] }, { "productId": "livekit-agents", "storyId": "agent-provisions-voice-agent", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "LiveKit provides a real CLI (`lk`), a documented telephony/SIP stack for provisioning inbound/outbound trunks and placing calls, and framework docs explicitly note it's 'built for coding agents like Claude Code, Cursor, and Codex,' suggesting API/CLI-driven workflows are possible without the dashboard. However, the MCP support documented is for the agent's own tool-calling (consuming MCP servers as tools), not an MCP server exposing agent-provisioning/number-attachment/call-placing actions to a coding agent, and no single evidenced workflow shows an agent programmatically creating an agent, attaching a number, and placing a call end-to-end via CLI/API in one flow — the Agent Builder path shown is dashboard/browser-based (no-code), which contradicts the 'without touching dashboard' framing. Missing for 10: an MCP server (or CLI/API recipe) that lets a coding agent itself create an agent, provision/attach a phone number, and place a call in one documented end-to-end sequence.", "evidenceIds": [ "livekit-agents-gh-2", "livekit-agents-docs-26", "livekit-agents-docs-32", "livekit-agents-probe-3", "livekit-agents-probe-rt-1", "livekit-agents-docs-29", "livekit-agents-gh-1", "livekit-agents-docs-34" ] }, { "productId": "livekit-agents", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "LiveKit publishes a working llms.txt (verified live at docs.livekit.io/llms.txt, HTTP 200) and agent-friendly markdown doc endpoints (e.g. /agents/.md), plus explicit docs noting the platform is 'built for coding agents like Claude Code, Cursor, and Codex,' confirming intentional support for AI-native doc consumption. missing for 10: no independent third-party confirmation that external agents successfully consume these endpoints in practice, and no dedicated documentation page explaining the llms.txt/agent-doc strategy itself.", "evidenceIds": [ "livekit-agents-probe-1", "livekit-agents-probe-2", "livekit-agents-docs-29" ] }, { "productId": "livekit-agents", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "livekit-agents", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "LiveKit Agents workers run as persistent 'programmatic participants' deployed to LiveKit Cloud or self-hosted infrastructure, processing realtime streams and telephony calls autonomously in the background without human intervention (docs-8, docs-20, docs-30, docs-26, gh-2), and this is verified hands-on as a real pip-installable OSS framework with a working self-hosted backbone (probe-rt-2, probe-rt-3). Missing for 10: explicit scheduling/cron-trigger mechanisms, independent third-party reports of long-running unattended stability (the one community mention is about observability tooling gaps, not autonomy failure).", "evidenceIds": [ "livekit-agents-docs-8", "livekit-agents-docs-20", "livekit-agents-docs-30", "livekit-agents-docs-26", "livekit-agents-gh-2", "livekit-agents-probe-rt-2", "livekit-agents-probe-rt-3" ] }, { "productId": "livekit-agents", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "livekit-agents", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "LiveKit Agents is a Python/Node.js SDK plus self-hostable server that runs as a headless programmatic participant with no UI dependency; it's pip/npm-installable, has a built-in test framework (pytest/Vitest, agent simulations) suitable for CI, and the server/CLI were verified runtime to work keylessly. missing for 10: no explicit documented CI pipeline example (e.g., GitHub Actions config) or headless-mode confirmation beyond inference from server/testing docs.", "evidenceIds": [ "livekit-agents-docs-8", "livekit-agents-docs-20", "livekit-agents-docs-9", "livekit-agents-docs-10", "livekit-agents-gh-4", "livekit-agents-probe-rt-2", "livekit-agents-probe-rt-3" ] }, { "productId": "livekit-agents", "storyId": "agentic-mcp-client", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs and GitHub explicitly state 'first-class support for Model Context Protocol (MCP) servers' with a documented pattern to wrap an MCP server in an MCPToolset and pass it to the agent's tools parameter, described as one line of code integration. This is first-party documentation without independent hands-on verification of MCP-specific usage. Missing for 10: independent/community confirmation of MCP tool integration working end-to-end, and more detail on multi-server or auth configurations.", "evidenceIds": [ "livekit-agents-docs-17", "livekit-agents-docs-23", "livekit-agents-docs-34", "livekit-agents-gh-1", "livekit-agents-gh-3" ] }, { "productId": "livekit-agents", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "livekit-agents", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "LiveKit ships an 'Agent Builder' that lets users prototype/deploy voice agents in-browser without code, and docs explicitly note the framework is 'built for coding agents like Claude Code, Cursor, and Codex' plus agent-readable llms.txt docs — suggesting some AI-native/no-code operability. However there is no evidence of a native natural-language command interface for operating LiveKit itself (e.g., NL-driven CLI, chatops control plane); the MCP support described is for agents calling external tools, not for controlling LiveKit via NL. Missing for 10: a documented NL-driven interface/CLI for configuring or operating the platform itself, and independent confirmation of the Agent Builder's no-code UX actually being NL-based rather than GUI-based.", "evidenceIds": [ "livekit-agents-docs-29", "livekit-agents-docs-13", "livekit-agents-docs-15", "livekit-agents-probe-1", "livekit-agents-probe-2" ] }, { "productId": "livekit-agents", "storyId": "agentic-official-cli", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "LiveKit ships an official CLI (livekit-cli), documented on GitHub and verified hands-on to install and run (`lk --version` works via brew install), confirming it's a real, functional official CLI supporting the agent workflow. missing for 10: no deep documentation of full CLI command surface for agent-specific workflows within the evidence pack.", "evidenceIds": [ "livekit-agents-probe-3", "livekit-agents-probe-rt-1" ] }, { "productId": "livekit-agents", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "LiveKit Agents ships a well-documented, pip/npm-installable Python and Node.js SDK with extensive public API surface (tools, MCP, turn detection, telephony, testing), plus a llms.txt and hands-on verified installs/runtime confirming the API is real and usable by AI-native developers. Missing for 10: independent third-party API reference/versioning audit and deeper evidence of API stability guarantees beyond docs and probes.", "evidenceIds": [ "livekit-agents-docs-1", "livekit-agents-docs-8", "livekit-agents-docs-20", "livekit-agents-gh-1", "livekit-agents-docs-22", "livekit-agents-probe-1", "livekit-agents-probe-2", "livekit-agents-probe-rt-3" ] }, { "productId": "livekit-agents", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers voice/agent features, tool-calling, MCP, testing, and telephony, but contains no mention of scoped or least-privilege API key/token issuance for agents (e.g., LiveKit's grant-based access tokens or credential scoping). Since LiveKit is a platform with API keys and would plausibly support such scoping, this is an applicable but undocumented axis in the given evidence.", "evidenceIds": [] }, { "productId": "livekit-agents", "storyId": "agentic-sdks", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "LiveKit provides official Python and Node.js SDKs (livekit-agents), verified hands-on to pip-install and import with no key, plus documented plugin ecosystem, tool use, MCP support, testing frameworks, and telephony integration—clearly an official SDK ecosystem for AI-native building. Missing for 10: independent third-party benchmarking of SDK API stability/versioning beyond community latency complaints unrelated to SDK build story.", "evidenceIds": [ "livekit-agents-docs-1", "livekit-agents-docs-12", "livekit-agents-docs-22", "livekit-agents-gh-1", "livekit-agents-probe-rt-3", "livekit-agents-docs-9" ] }, { "productId": "livekit-agents", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack items mention webhooks or an event-subscription mechanism for LiveKit Agents; the docs cover MCP tool integration, telephony, testing, and turn detection, but nothing about webhook-based event notifications.", "evidenceIds": [] }, { "productId": "livekit-agents", "storyId": "ai-assisted-agent-authoring", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows a no-code 'Agent Builder' for browser prototyping and a testing framework, but nothing indicates the platform itself uses AI to generate or improve prompts, flows, or test cases from a natural-language description of the desired agent.", "evidenceIds": [ "livekit-agents-docs-15", "livekit-agents-docs-28", "livekit-agents-docs-9", "livekit-agents-docs-25" ] }, { "productId": "livekit-agents", "storyId": "ai-call-qa-scoring", "verdict": "disputed", "quality": 3, "confidence": "medium", "rationale": "LiveKit ships a dev-time testing framework with 'judges' and agent simulations to evaluate scripted interactions (docs-9/10/25, gh-4), but this is pre-deployment test tooling, not automatic scoring/flagging/resolution-analysis of live production calls. A hands-on community report explicitly says operators running real agents on LiveKit have no built-in way to measure latency or cost per call, let alone automated quality/resolution review, contradicting the idea that the platform reviews calls for you. Missing for 10: production call-level QA scoring, automatic failure flagging on real traffic, resolution/outcome analysis dashboards.", "evidenceIds": [ "livekit-agents-docs-9", "livekit-agents-docs-10", "livekit-agents-gh-4", "livekit-agents-comm-1" ] }, { "productId": "livekit-agents", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows extensive quickstart guides, markdown-based docs (llms.txt), and a no-code 'Agent Builder' prototyping tool, but nothing describes an interactive API reference (e.g., embedded code sandbox, live runnable examples, or Swagger-like explorer) for exploring the SDK/API itself.", "evidenceIds": [] }, { "productId": "livekit-agents", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents LiveKit's SDKs, CLI, docs (llms.txt), and framework capabilities but contains no mention of a downloadable OpenAPI/Swagger spec or other machine-readable API description for LiveKit's server or Agents APIs. Since LiveKit exposes a real API surface (server API, cloud API), this axis applies but is unevidenced.", "evidenceIds": [ "livekit-agents-probe-3", "livekit-agents-probe-1", "livekit-agents-probe-2" ] }, { "productId": "livekit-agents", "storyId": "api-sandbox", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "LiveKit provides a self-hostable dev server (`livekit-server --dev`) that boots locally with placeholder keys, separate from any production deployment, plus a built-in testing framework with behavioral tests and LLM-driven agent simulations that evaluate agent behavior without needing real production data. However, there is no explicitly branded 'sandbox environment' or staging/production data-isolation feature documented — the sandbox-like capability is inferred from dev-mode self-hosting and test simulations rather than a dedicated sandbox product feature. Missing for 10: an explicit sandbox/staging environment offering with documented separation from production data, and independent confirmation that test simulations never touch production data stores.", "evidenceIds": [ "livekit-agents-docs-9", "livekit-agents-docs-10", "livekit-agents-docs-25", "livekit-agents-gh-4", "livekit-agents-probe-rt-2" ] }, { "productId": "livekit-agents", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions API versioning scheme, version compatibility guarantees, or a documented deprecation policy for LiveKit Agents' SDKs or APIs.", "evidenceIds": [] }, { "productId": "livekit-agents", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "livekit-agents", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "LiveKit Agents supports event-driven automatic behaviors — tool-calling on LLM decisions, turn-detector triggering response timing, and adaptive interruption handling triggering barge-in logic — all of which are 'events auto-trigger actions' patterns built into the agent runtime. However, the evidence shows these as built-in framework behaviors and developer-coded event handlers rather than a user-facing declarative rule engine (e.g., no evidence of a 'when X happens do Y' config surface, webhook/rule subscription API, or no-code rule builder). Missing for 10: explicit rule-definition/webhook-trigger API exposed to non-developer users, documentation of a generalized event-subscription system beyond tool calls and turn logic, and independent confirmation of custom rule automation in production use.", "evidenceIds": [ "livekit-agents-docs-6", "livekit-agents-docs-22", "livekit-agents-docs-33", "livekit-agents-docs-5", "livekit-agents-docs-18", "livekit-agents-docs-4", "livekit-agents-docs-19", "livekit-agents-docs-36" ] }, { "productId": "livekit-agents", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers LiveKit Agents' realtime voice/video agent capabilities, tool use, MCP integration, telephony, and testing, but contains no mention of any scheduling, cron-like recurring job, or workflow-automation trigger mechanism. This is a fair capability to ask about for an automation-focused agent framework, but no evidence supports it.", "evidenceIds": [] }, { "productId": "livekit-agents", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "The pricing page explicitly claims 'Instant rollback to a previous agent deployment,' and the testing/simulation framework (behavioral tests, LLM-judge simulations) supports a review step before deployment, but there's no documented versioning system, changelog, or diff/history UI for automations. Missing for 10: explicit version history/diffing of agent configs, a documented review/approval workflow beyond test suites, and independent confirmation that rollback works in practice.", "evidenceIds": [ "livekit-agents-docs-21", "livekit-agents-docs-9", "livekit-agents-docs-10", "livekit-agents-gh-4" ] }, { "productId": "livekit-agents", "storyId": "barge-in-interruptions", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "LiveKit Agents has well-documented, purpose-built features for barge-in: adaptive interruption handling distinguishes intentional interruptions from backchanneling, and the TurnDetector model analyzes acoustic and semantic signals to manage turn-taking recovery. These are first-party docs with specific technical detail on the mechanism (stopping speech, updating context, recovering turn), and the runtime probes confirm the framework is real and installable. Missing for 10: independent/hands-on evidence specifically testing barge-in behavior in production (only community evidence is about latency observability, not interruption handling itself) and no third-party benchmark of interruption accuracy.", "evidenceIds": [ "livekit-agents-docs-4", "livekit-agents-docs-19", "livekit-agents-docs-5", "livekit-agents-docs-18", "livekit-agents-docs-36", "livekit-agents-docs-37", "livekit-agents-probe-rt-3" ] }, { "productId": "livekit-agents", "storyId": "batch-outbound-campaigns", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows LiveKit Agents can place individual outbound calls via SIP outbound trunks (docs-26, docs-32, docs-31), but there is no evidence of any batch campaign management, call scheduling, or throughput/rate-limiting controls for running outbound calling campaigns at scale — these features are simply not documented anywhere in the pack.", "evidenceIds": [ "livekit-agents-docs-26", "livekit-agents-docs-32", "livekit-agents-docs-31", "livekit-agents-gh-2" ] }, { "productId": "livekit-agents", "storyId": "build-phone-agent-fast", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs show a <10‑minute voice-assistant quickstart (prompt+voice) plus a dedicated telephony/SIP stack for inbound/outbound calls (trunks, phone number), and runtime probes confirm the framework and self-hosted server actually install and boot without extra keys. Together these cover prompt, voice, and phone number needed for a first live call within an hour. Missing for 10: a single unified, hands-on-verified tutorial that walks through phone-number provisioning and first live call end-to-end (currently voice-quickstart and telephony docs are separate), and independent (non-vendor) confirmation of the 'within an hour' timeline.", "evidenceIds": [ "livekit-agents-docs-2", "livekit-agents-docs-14", "livekit-agents-docs-26", "livekit-agents-docs-31", "livekit-agents-docs-32", "livekit-agents-gh-2", "livekit-agents-probe-rt-1", "livekit-agents-probe-rt-2", "livekit-agents-probe-rt-3" ] }, { "productId": "livekit-agents", "storyId": "call-analytics-dashboard", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows testing/simulation tools (pytest/Vitest, LLM-driven simulations) but no built-in production analytics dashboard for success rates, durations, outcomes, or sentiment. A community report explicitly confirms this gap — users running LiveKit voice agents said they had 'no way to measure TTFT, latency across STT→LLM→TTS, or even cost per call' and found debugging painful, indicating no such dashboard exists out of the box.", "evidenceIds": [ "livekit-agents-comm-1", "livekit-agents-docs-9", "livekit-agents-docs-10", "livekit-agents-docs-25" ] }, { "productId": "livekit-agents", "storyId": "call-transfer-escalation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "LiveKit Agents documents inbound/outbound telephony (SIP trunks, phone calls) but the evidence pack contains no mention of call transfer (warm or blind) or passing conversational context to a human agent during a live call — this specific escalation capability is never described.", "evidenceIds": [ "livekit-agents-docs-26", "livekit-agents-docs-31", "livekit-agents-docs-32", "livekit-agents-gh-2" ] }, { "productId": "livekit-agents", "storyId": "concurrency-scaling-limits", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs mention running agents on LiveKit Cloud's global infrastructure and self-hosting with an embedded TURN server, implying scale-out capability, but no evidence pack item cites concrete documented concurrency limits, per-instance call caps, or autoscaling guarantees; a community report even flags lack of visibility into per-call cost/latency at scale. missing for 10: published concurrency/capacity numbers, autoscaling documentation, load-testing benchmarks, and confirmation that scaling requires no manual quota requests.", "evidenceIds": [ "livekit-agents-docs-30", "livekit-agents-docs-11", "livekit-agents-comm-1" ] }, { "productId": "livekit-agents", "storyId": "consent-recording-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions call recording, consent disclosure, or data retention controls anywhere in LiveKit Agents docs, GitHub, or community sources; the pack covers telephony, tool use, MCP, testing, and turn detection but nothing about recording/retention compliance features. missing for 10: recording consent/disclosure mechanisms, per-call recording toggles, configurable data retention policies, and any documentation or hands-on evidence of these compliance controls.", "evidenceIds": [] }, { "productId": "livekit-agents", "storyId": "custom-voice-cloning", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "livekit-agents", "storyId": "documented-latency-numbers", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documented end-to-end latency benchmarks or explicit tuning guidance for reducing turn-taking latency — only feature descriptions (turn detector, adaptive interruption handling) without numbers or configuration guidance. A community report explicitly states users could not measure TTFT or STT→LLM→TTS latency, reinforcing that this visibility/documentation is absent.", "evidenceIds": [ "livekit-agents-docs-4", "livekit-agents-docs-5", "livekit-agents-docs-18", "livekit-agents-docs-19", "livekit-agents-comm-1" ] }, { "productId": "livekit-agents", "storyId": "dtmf-ivr-voicemail", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence confirms LiveKit's telephony/SIP stack supports inbound/outbound calls (docs-26, docs-31, docs-32, gh-2), but nothing in the pack mentions DTMF keypress sending, IVR menu navigation, or voicemail detection/leaving capabilities specifically. Missing for 10: any documentation or example of DTMF tone generation, automated IVR menu traversal, or voicemail detection/answering-machine-detection logic.", "evidenceIds": [ "livekit-agents-docs-26", "livekit-agents-docs-31", "livekit-agents-docs-32", "livekit-agents-gh-2" ] }, { "productId": "livekit-agents", "storyId": "dynamic-call-personalization", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers tool use, MCP, telephony, testing, and turn detection, but nothing addresses injecting dynamic variables or per-caller context (e.g., participant/room metadata, job context) into an agent's prompt or session at call start. Missing for 10: any documentation of job/participant metadata APIs, per-call context injection, or dynamic prompt personalization.", "evidenceIds": [] }, { "productId": "livekit-agents", "storyId": "hipaa-regulated-workloads", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of HIPAA/BAA agreements, SOC 2 certification, or data-residency controls anywhere in LiveKit's docs, GitHub repo, or probes; self-hosting and TURN/security items only cover TLS/SSL and self-hosted deployment, not compliance attestations. As a platform serving enterprise/regulated workloads, this axis clearly applies, so lack of evidence yields 'none'.", "evidenceIds": [ "livekit-agents-docs-11", "livekit-agents-docs-27" ] }, { "productId": "livekit-agents", "storyId": "knowledge-base-grounding", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "LiveKit Agents supports calling external APIs/tools that a developer can use to implement RAG (\"Call external APIs or lookup data for RAG\"), and has full LLM tool-use and MCP integration for wiring in retrieval systems, but there is no built-in knowledge base, document ingestion, or vector-store/RAG pipeline shipped by the framework itself — developers must bring their own RAG implementation via the tools API. Missing for 10: built-in document indexing/vector store, out-of-the-box knowledge-base feature, first-party RAG pipeline or example showing document grounding end-to-end.", "evidenceIds": [ "livekit-agents-docs-6", "livekit-agents-docs-22", "livekit-agents-docs-33", "livekit-agents-gh-1", "livekit-agents-docs-17" ] }, { "productId": "livekit-agents", "storyId": "live-call-monitoring", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack items describe production monitoring dashboards, error-rate alerting, or call-quality observability tooling for LiveKit Agents; the only related evidence is a community report explicitly describing the lack of visibility into TTFT, per-call latency, and cost — a gap, not a delivered capability.", "evidenceIds": [ "livekit-agents-comm-1" ] }, { "productId": "livekit-agents", "storyId": "mid-call-tool-calling", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "LiveKit Agents has full documented support for LLM tool/function calling (custom Python/Node functions calling external APIs mid-conversation) plus MCP server tool integration, combined with adaptive interruption handling and turn-detection to avoid awkward dead air while tools execute. Runtime probes confirm the framework is real and installable, corroborating the docs claims.\n\nmissing for 10: independent/hands-on evidence specifically showing tool-call latency handled gracefully in a live conversation (only vendor docs cover this exact combination).", "evidenceIds": [ "livekit-agents-docs-6", "livekit-agents-docs-22", "livekit-agents-docs-33", "livekit-agents-docs-34", "livekit-agents-docs-17", "livekit-agents-docs-4", "livekit-agents-docs-19", "livekit-agents-probe-rt-3" ] }, { "productId": "livekit-agents", "storyId": "multilingual-conversations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack addresses multilingual conversation support, language detection, or dynamic language switching mid-call; the docs cover turn detection, interruption handling, tool use, MCP, and telephony but never mention STT/TTS language selection or switching logic. This is a fair axis for a voice-agent framework (many STT/TTS providers support multi-language), so absence of evidence means 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "livekit-agents", "storyId": "noise-robustness", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers turn detection, interruption handling, telephony, MCP tools, and testing, but contains no mention of noise suppression or audio filtering plugins/features anywhere. Missing for 10: any docs or plugin reference to noise cancellation (e.g. Krisp/BVC), background noise filtering, or audio pre-processing configuration.", "evidenceIds": [] }, { "productId": "livekit-agents", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "LiveKit Agents is API/SDK-first (Python/Node.js) with a full programmatic surface for building agents, tools, telephony, and testing, and there is also a no-code 'Agent Builder' UI for prototyping. However, the evidence doesn't confirm that everything achievable in that browser-based Agent Builder UI (or LiveKit Cloud dashboard features like deployment management, rollback) is equally exposed via API/CLI, so parity between UI and API is not fully demonstrated. missing for 10: explicit evidence that Agent Builder's no-code UI actions (and Cloud dashboard deployment/rollback controls) are all reachable via API/CLI, and confirmation of full CLI/API parity with dashboard features.", "evidenceIds": [ "livekit-agents-docs-13", "livekit-agents-docs-15", "livekit-agents-docs-28", "livekit-agents-docs-21", "livekit-agents-probe-3", "livekit-agents-probe-rt-1", "livekit-agents-docs-8" ] }, { "productId": "livekit-agents", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents LiveKit Agents' open-source, self-hostable nature (Apache-2.0 server, pip/npm installable framework) but contains no explicit mention of data export tooling, open data formats for conversation/session logs, or a documented exit/migration path for user data. Self-hosting mitigates lock-in in principle, but that is not the same as a documented 'export all data' capability.", "evidenceIds": [ "livekit-agents-probe-rt-2", "livekit-agents-probe-rt-3", "livekit-agents-docs-11" ] }, { "productId": "livekit-agents", "storyId": "openness-open-license", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "LiveKit Agents is hosted at github.com/livekit/agents and documented as an open-source framework, with hands-on verification that the packages install and run without any account or key (pip/npm) and that the underlying server is Apache-2.0 OSS. The llms.txt docs also explicitly describe LiveKit as 'an open-source platform.' Missing for 10: an explicit license file/badge citation specifically for the livekit/agents repo (only inferred via probe evidence and sibling repo license).", "evidenceIds": [ "livekit-agents-gh-1", "livekit-agents-probe-1", "livekit-agents-probe-rt-2", "livekit-agents-probe-rt-3", "livekit-agents-probe-3" ] }, { "productId": "livekit-agents", "storyId": "openness-self-host", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "LiveKit provides a documented, Apache-2.0 open-source self-hosting deployment guide including embedded TURN server and SSL setup, and this was hands-on verified: `livekit-server --dev` boots with no account or key, and the open-source livekit-agents framework installs and imports keylessly via pip/npm. This directly demonstrates the core product (server + agents framework) can be self-hosted. Missing for 10: independent third-party production self-hosting case study beyond the probe verification.", "evidenceIds": [ "livekit-agents-docs-11", "livekit-agents-docs-27", "livekit-agents-probe-rt-2", "livekit-agents-probe-rt-3", "livekit-agents-probe-rt-1" ] }, { "productId": "livekit-agents", "storyId": "phone-number-provisioning", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "LiveKit's SIP/telephony stack clearly supports inbound and outbound calling ('LiveKit telephony lets you build AI-powered voice apps that handle inbound and outbound calls', 'Outbound trunks are used to place outgoing calls', 'Enable your voice agent to make or take phone calls') and integrates with the agents runtime via SIP trunks. However, the evidence never shows LiveKit itself provisioning phone numbers through its own API — telephony typically requires configuring an external SIP trunk provider, and no docs here describe a native number-provisioning endpoint. Missing for 10: explicit documentation/evidence of a LiveKit API call that provisions/purchases phone numbers directly (rather than just configuring trunks against externally-acquired numbers), and independent/hands-on confirmation of the inbound+outbound call flow.", "evidenceIds": [ "livekit-agents-docs-26", "livekit-agents-docs-31", "livekit-agents-docs-32", "livekit-agents-gh-2" ] }, { "productId": "livekit-agents", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "LiveKit Agents/server is open-source and self-hostable (embedded TURN server, deployment docs, hands-on verified self-host boot), which lets an operator choose their own hosting region/data location, but there is no explicit documented region-selection or data-residency feature for LiveKit Cloud itself. missing for 10: explicit LiveKit Cloud region-selection/data-residency controls, compliance certifications (e.g. GDPR/SOC2 region guarantees), and any customer-facing UI for choosing storage region.", "evidenceIds": [ "livekit-agents-docs-11", "livekit-agents-docs-27", "livekit-agents-probe-rt-2" ] }, { "productId": "livekit-agents", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data usage for AI model training, opt-out policies, or any privacy commitments regarding training data; the evidence pack is entirely about agent framework features (tools, telephony, testing, MCP support). This is a plausible axis for a platform handling user voice/media data, but absent any documentation on training-data usage or opt-out, it cannot be credited as delivered.", "evidenceIds": [] }, { "productId": "livekit-agents", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of data retention policies, recording/session deletion controls, or configurable retention windows for LiveKit Agents. While self-hosting is documented (implying infrastructure control), no explicit retention/deletion feature or API is evidenced.", "evidenceIds": [] }, { "productId": "livekit-agents", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item discusses telemetry, usage tracking, analytics collection, or an opt-out mechanism for LiveKit Agents; the framework being open-source and self-hostable is not sufficient evidence of an explicit telemetry opt-out control.", "evidenceIds": [] }, { "productId": "livekit-agents", "storyId": "realtime-transcription-quality", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "LiveKit Agents supports swapping STT/LLM providers via its open-source plugin ecosystem and lets you override API key/base URL for any provider (docs-12, docs-24, docs-35), giving real control over transcription and language-model choice. However there is no evidence of support for custom vocabulary/key-term or hotword boosting in STT, and no independent benchmark of transcription accuracy. Missing for 10: key-term/hotword/phrase-hint configuration support, accuracy benchmarks or independent corroboration of transcription quality.", "evidenceIds": [ "livekit-agents-docs-12", "livekit-agents-docs-24", "livekit-agents-docs-35" ] }, { "productId": "livekit-agents", "storyId": "recordings-transcripts-api", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers agent building, tool use, MCP, telephony, and testing but contains no mention of recording APIs, transcript storage, or programmatic retrieval of call recordings/transcripts for platform engineers.", "evidenceIds": [] }, { "productId": "livekit-agents", "storyId": "self-host-oss-runtime", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "LiveKit Agents is open source, pip/npm installable and runs the agent runtime independent of any vendor account, and the underlying WebRTC/SIP server (livekit-server) is Apache-2.0 and self-hostable, confirmed via hands-on probes (booting `livekit-server --dev` locally with no keys, and installing livekit-agents/CLI keylessly) plus official self-hosting deployment docs covering TURN, TLS/domain setup. Missing for 10: no first-party production-scale self-hosting case study or independent report of large-scale self-hosted deployment beyond dev-mode probe.", "evidenceIds": [ "livekit-agents-probe-rt-1", "livekit-agents-probe-rt-2", "livekit-agents-probe-rt-3", "livekit-agents-docs-11", "livekit-agents-docs-27", "livekit-agents-docs-8" ] }, { "productId": "livekit-agents", "storyId": "semantic-turn-detection", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "LiveKit's TurnDetector is a dedicated audio+semantic model (not just VAD silence) that predicts whether a user has finished speaking or has more to say, waiting accordingly, and adaptive interruption handling further distinguishes real barge-ins from backchanneling — directly addressing not talking over slow speakers. This is documented in detail across multiple first-party doc pages plus GitHub feature lists. missing for 10: independent/hands-on benchmark or community validation of the turn-detector's real-world latency/accuracy impact.", "evidenceIds": [ "livekit-agents-docs-5", "livekit-agents-docs-18", "livekit-agents-docs-37", "livekit-agents-docs-4", "livekit-agents-docs-19", "livekit-agents-docs-36" ] }, { "productId": "livekit-agents", "storyId": "simulated-agent-testing", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly describe both behavioral tests (pytest/Vitest) and agent simulations that run end-to-end conversations with an LLM-driven user, evaluating results before deployment, and GitHub notes a 'builtin test framework' with judges — directly matching the testing-before-real-calls story, complemented by separate telephony/SIP support for real calls. Missing for 10: independent/hands-on validation of the eval/simulation framework and explicit documentation tying test simulations to pre-phone-call validation workflows.", "evidenceIds": [ "livekit-agents-docs-9", "livekit-agents-docs-10", "livekit-agents-docs-25", "livekit-agents-docs-38", "livekit-agents-gh-4", "livekit-agents-gh-2", "livekit-agents-docs-26" ] }, { "productId": "livekit-agents", "storyId": "sip-trunk-byoc", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "LiveKit's SIP-based telephony stack lets agents make/receive calls via inbound and outbound SIP trunks, which is the standard mechanism for bringing your own carrier or PBX (docs-26, docs-32, gh-2, docs-31). This is architecture-agnostic SIP, not a bundled/proprietary telephony lock-in, satisfying the platform-engineer's need to connect external trunks. Missing for 10: explicit documented walkthroughs or examples of importing Twilio/Telnyx numbers specifically, and no independent/hands-on confirmation of a real carrier trunk connection succeeding.", "evidenceIds": [ "livekit-agents-docs-26", "livekit-agents-docs-32", "livekit-agents-gh-2", "livekit-agents-docs-31" ] }, { "productId": "livekit-agents", "storyId": "structured-outcome-extraction", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "LiveKit Agents provides tool-calling, MCP integration, and telephony hooks, but nothing in the evidence describes a built-in mechanism for automatically extracting structured call outcomes/entities/dispositions and delivering them via API or webhook after a call ends — that would have to be custom-built by the developer using the tool-calling primitives. Since no such capability is documented, this applicable axis is unmet.", "evidenceIds": [ "livekit-agents-docs-6", "livekit-agents-docs-22", "livekit-agents-docs-26", "livekit-agents-docs-32" ] }, { "productId": "livekit-agents", "storyId": "transparent-per-minute-pricing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The pack references a pricing URL (livekit.io/pricing) only for an unrelated 'rollback' feature and never quotes actual per-minute or usage rates; a community comment even highlights that operators struggled with 'cost per call' visibility, but no evidence shows published pricing figures a founder could use to estimate costs before committing.", "evidenceIds": [ "livekit-agents-docs-21", "livekit-agents-comm-1" ] }, { "productId": "livekit-agents", "storyId": "visual-flow-builder", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "LiveKit mentions a browser-based 'Agent Builder' for no-code prototyping of voice agents, but the evidence never describes visual branching, explicit states, or handoff design — the core framework is fundamentally code-first (Python/Node.js) with tools, MCP, and turn-detection logic. missing for 10: documentation of branching/state UI, handoff modeling, or any screenshots/examples of the Agent Builder's flow-design capabilities beyond a vague no-code prototyping claim.", "evidenceIds": [ "livekit-agents-docs-13", "livekit-agents-docs-15", "livekit-agents-docs-28" ] }, { "productId": "livekit-agents", "storyId": "voice-library-choice", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm a large open-source plugin ecosystem for TTS/STT/LLM providers, plus LiveKit Inference for provider access, and an override mechanism for API key/base URL for providers not natively included, supporting multi-provider TTS flexibility. However, there's no explicit mention of a 'voice library' (e.g., curated voice catalog/selection UI) or enumeration of specific TTS providers/voices, so the 'broad voice library' half of the story is unevidenced. Missing for 10: explicit voice catalog/list of supported TTS providers and voices, evidence of ease of switching between TTS voices, independent confirmation of provider breadth.", "evidenceIds": [ "livekit-agents-docs-12", "livekit-agents-docs-24", "livekit-agents-docs-35" ] }, { "productId": "pipecat", "storyId": "agent-consumes-mcp-tools", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence only shows Pipecat's context-hub *acting as* an MCP server for coding-agent tooling (pipecat-docs-11/38), which is the opposite direction from what the story asks — a voice agent consuming external MCP servers as tool sources mid-call. Function/tool calling is documented via Pipecat Flows (pipecat-docs-6/22/30/39), but no evidence shows the voice agent connecting to or importing toolsets from external MCP servers during a call.", "evidenceIds": [ "pipecat-docs-11", "pipecat-docs-38", "pipecat-docs-6", "pipecat-docs-22" ] }, { "productId": "pipecat", "storyId": "agent-provisions-voice-agent", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Pipecat provides a CLI (`pipecat init`, `pipecat cloud`, `pipecat context-hub`) and code-level telephony hooks (`python bot.py -t twilio -x ngrok...`) that let a coding agent scaffold and run a bot, and the context-hub can register an MCP server for coding-agent tooling, but there is no documented single API/CLI/MCP flow that creates the agent, provisions/attaches a phone number, and places a call — telephony setup still relies on the carrier's (Twilio/Telnyx) own console/API for number purchase and webhook config. missing for 10: a unified provisioning API or CLI command that attaches a phone number and places a call, MCP-driven agent creation (context-hub only builds a docs index, not agent provisioning), evidence of number attachment without touching a carrier dashboard.", "evidenceIds": [ "pipecat-docs-13", "pipecat-docs-23", "pipecat-docs-40", "pipecat-docs-34", "pipecat-docs-37", "pipecat-docs-11", "pipecat-probe-rt-1" ] }, { "productId": "pipecat", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Pipecat provides a working llms.txt endpoint (confirmed HTTP 200 with agent-oriented summary) plus an explicit context-hub CLI feature that registers docs as an MCP server for coding agents and builds a local index, directly enabling agents to consume its docs programmatically. missing for 10: no independent/community confirmation of an agent successfully using llms.txt or context-hub end-to-end.", "evidenceIds": [ "pipecat-probe-1", "pipecat-docs-11", "pipecat-docs-38", "pipecat-probe-rt-1" ] }, { "productId": "pipecat", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "pipecat", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Pipecat supports multi-agent systems where 'specialists hand off, fan out in parallel, and coordinate over a shared bus, locally or distributed' and can 'hand off to subagents for long-running tools and complex tasks,' plus telephony bots that start autonomously when a carrier calls a webhook rather than a user request — suggesting some background/autonomous agent operation. But Pipecat's core model is a live, turn-based conversational pipeline tied to an active voice/video session, not a scheduler or cron-like background automation runner independent of a live interaction. Missing for 10: explicit scheduling/triggering mechanisms for unattended background jobs, evidence of agents running without any live session, and independent confirmation of autonomous multi-agent orchestration in production.", "evidenceIds": [ "pipecat-gh-1", "pipecat-docs-43", "pipecat-docs-13", "pipecat-docs-23" ] }, { "productId": "pipecat", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Pipecat's docs describe a framework/CLI for building voice AI agents, a context-hub that exposes Pipecat's own docs as an MCP server to external coding agents, and dev tooling (runner, evals, Flows) — but none of this is a built-in assistant inside Pipecat's own product surface that a user delegates tasks to. Missing for 10: any first-party 'chat with an assistant' feature in the CLI/dashboard/docs site, evidence of task delegation to an embedded assistant, or independent confirmation of such a feature.", "evidenceIds": [ "pipecat-docs-38", "pipecat-docs-11", "pipecat-probe-rt-1" ] }, { "productId": "pipecat", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Pipecat is a Python framework that is pip-installable with no key required and runs bots as scripts (bot.py) with a CLI supporting init, cloud deploy, and eval subcommands, which supports headless/CI-friendly automation; the Evals system explicitly runs conversations against the real agent programmatically, suitable for CI. However there is no explicit CI/CD example, no documented headless flag or non-interactive mode guidance, and the development runner is framed around local dev/demo UI rather than CI. missing for 10: explicit CI pipeline documentation/example, confirmation that pipecat.runner.run or bot.py can run fully headless without any UI/browser dependency, independent hands-on report of running Pipecat in an automated CI environment.", "evidenceIds": [ "pipecat-docs-8", "pipecat-docs-25", "pipecat-probe-rt-1", "pipecat-probe-rt-2", "pipecat-docs-24", "pipecat-docs-40" ] }, { "productId": "pipecat", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence only shows Pipecat's context-hub CLI *exposing itself* as an MCP server to a coding agent (docs-11/38) — the reverse of what the story asks (Pipecat consuming external MCP servers to gain their tools inside its voice-agent pipelines). No evidence shows Pipecat's Flows/function-calling system connecting to arbitrary MCP servers as a client.", "evidenceIds": [ "pipecat-docs-11", "pipecat-docs-38" ] }, { "productId": "pipecat", "storyId": "agentic-mcp-server", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Pipecat ships an official `pipecat context-hub install` CLI command that registers itself as an MCP server with a coding agent, building a local docs/examples index — confirmed both in docs and via a runtime CLI probe showing the `context-hub` subcommand exists. This is a genuine official MCP server for agent connection, but it's scoped narrowly to indexing Pipecat's own docs/examples for coding assistants rather than exposing Pipecat's runtime voice-agent capabilities to arbitrary MCP clients. Missing for 10: independent/hands-on confirmation that the MCP server actually functions end-to-end (only CLI help text was probed, not live MCP protocol behavior), and broader documentation of what tools/resources the MCP server exposes.", "evidenceIds": [ "pipecat-docs-11", "pipecat-docs-38", "pipecat-probe-rt-1" ] }, { "productId": "pipecat", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Pipecat's entire raison d'être is building voice agents that end users control via natural spoken language — turn detection, interruption/barge-in, and Flows functions that progress conversations based on NL input (pipecat-docs-9, pipecat-docs-19, pipecat-docs-6/22/30) all support NL-driven operation of the *bots* it produces. However, the story is ambiguous about whether it's the bots or the Pipecat tooling itself; Pipecat's own developer-facing surfaces (CLI init/cloud/eval/context-hub) are traditional structured commands, not natural-language commands (pipecat-probe-rt-1, pipecat-docs-34/37). Missing for 10: evidence that the Pipecat framework/CLI itself (not just end-user bots) can be operated by an AI-native user via natural-language instructions rather than code/CLI syntax.", "evidenceIds": [ "pipecat-docs-9", "pipecat-docs-19", "pipecat-docs-6", "pipecat-docs-22", "pipecat-docs-30", "pipecat-probe-rt-1", "pipecat-docs-34" ] }, { "productId": "pipecat", "storyId": "agentic-official-cli", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Pipecat ships an official CLI (pip-installable pipecat-ai[cli], confirmed via runtime probe) supporting init/scaffolding, cloud deploy, eval, and context-hub (MCP registration for coding agents), well documented at docs.pipecat.ai/api-reference/cli/overview. This directly supports AI-native agentic workflows (context-hub install registers as MCP server, writes AGENTS.md/CLAUDE.md for coding agents). missing for 10: independent/third-party review of the CLI's robustness beyond vendor docs and a single runtime probe.", "evidenceIds": [ "pipecat-probe-rt-1", "pipecat-probe-3", "pipecat-docs-11", "pipecat-docs-16", "pipecat-docs-38" ] }, { "productId": "pipecat", "storyId": "agentic-public-api", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Pipecat exposes a well-documented programmatic surface — a Python framework API (frame processors, Flows functions with auto-derived metadata), a runner API, and an official CLI (init/cloud/eval/context-hub) confirmed to work keylessly via runtime probe — giving AI-native users a documented, scriptable way to drive the product. However, probing shows no REST/OpenAPI spec (all openapi.json candidates 404'd), so there's no conventional network-callable public API, only the code-level Python/CLI surface. Missing for 10: a formal REST/OpenAPI-documented HTTP API, and independent hands-on confirmation of driving a full pipeline purely via that public API surface.", "evidenceIds": [ "pipecat-docs-6", "pipecat-docs-7", "pipecat-docs-24", "pipecat-docs-39", "pipecat-probe-2", "pipecat-probe-3", "pipecat-probe-rt-1" ] }, { "productId": "pipecat", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "pipecat", "storyId": "agentic-sdks", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Pipecat is itself an official open-source Python SDK/framework (pip-installable, verified via runtime probe to install and import cleanly), and it also ships a family of official 'Pipecat Client' SDKs for web/mobile integration plus pre-built React components, all documented on the official site. Missing for 10: independent/third-party confirmation of the client SDKs' quality beyond docs (only the core Python framework was hands-on verified).", "evidenceIds": [ "pipecat-docs-10", "pipecat-docs-35", "pipecat-docs-18", "pipecat-docs-42", "pipecat-probe-rt-2", "pipecat-gh-1" ] }, { "productId": "pipecat", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Webhooks appear only as an inbound mechanism (a telephony carrier calling Pipecat's webhook to signal an incoming call), not as an outbound event-subscription system that lets a user register a webhook to receive Pipecat's own events (e.g., call end, transcript ready, errors). No evidence of a webhook subscription API or configurable event notifications for consumers.", "evidenceIds": [ "pipecat-docs-13", "pipecat-docs-23", "pipecat-docs-40" ] }, { "productId": "pipecat", "storyId": "ai-assisted-agent-authoring", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Pipecat has a context-hub that registers as an MCP server so external coding agents (e.g. Claude) can get context about the framework, and an Evals system for testing agent behavior — but these rely on the user writing prompts/flows/tests and connecting third-party AI tools, not Pipecat's own AI generating or improving prompts, flows, or test cases from a description. No evidence of an in-platform AI assistant that authors agent configs.", "evidenceIds": [ "pipecat-docs-11", "pipecat-docs-38", "pipecat-docs-8", "pipecat-docs-25" ] }, { "productId": "pipecat", "storyId": "ai-call-qa-scoring", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Pipecat ships developer-facing dev-time Evals (describe expected conversation, run against real agent) plus metrics/transcripts/audio-recording primitives, but there is no evidence of an AI system that automatically reviews production calls, scores quality, flags failures, or analyzes resolution after the fact — Evals is a pre-deployment testing tool, not post-call analytics.", "evidenceIds": [ "pipecat-docs-8", "pipecat-docs-25", "pipecat-docs-5", "pipecat-docs-4" ] }, { "productId": "pipecat", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Pipecat's docs are static markdown pages (get-started, fundamentals, flows, CLI reference) with no evidence of an interactive API reference or runnable code examples; a direct probe for OpenAPI/Swagger endpoints returned 404 on all candidate paths, indicating no interactive API explorer exists.", "evidenceIds": [ "pipecat-probe-2", "pipecat-probe-3" ] }, { "productId": "pipecat", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe for OpenAPI/swagger specs at all standard locations (openapi.json, swagger.json, etc.) on docs.pipecat.ai returned 404s, and no evidence pack item shows a downloadable machine-readable API spec despite Pipecat having an api-reference docs section and cloud service.", "evidenceIds": [ "pipecat-probe-2" ] }, { "productId": "pipecat", "storyId": "api-sandbox", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Pipecat provides a local development runner (`pipecat.runner.run`) and CLI scaffolding that let developers run and iterate on bots entirely locally, plus a built-in Evals system that runs described conversations against 'your real agent' pipeline for behavior testing — both usable without deploying to Pipecat Cloud or touching a production instance. However, there's no explicit 'sandbox environment' concept with data isolation from production (e.g. no staging vs prod data separation, no test-data seeding, no mention of production data at all since Pipecat doesn't store conversational data centrally). Missing for 10: explicit sandbox/production data isolation, documented data classification for dev vs prod, hands-on confirmation that Evals or local runner never touch live production resources.", "evidenceIds": [ "pipecat-docs-7", "pipecat-docs-24", "pipecat-docs-33", "pipecat-docs-8", "pipecat-docs-25", "pipecat-docs-32" ] }, { "productId": "pipecat", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item discusses API versioning scheme, version numbers surfaced in docs, or any documented deprecation policy/timeline for Pipecat's SDKs or APIs; the only version datum (pip package 1.8.1) shows a version exists but nothing about a deprecation policy is documented.", "evidenceIds": [] }, { "productId": "pipecat", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Pipecat supports 'unlimited concurrency' for running many simultaneous voice sessions and multi-agent fan-out ('fan out in parallel... coordinate over a shared bus'), which could support bulk-style parallel processing of many calls/agents at once. However, there is no explicit bulk-operation tooling (e.g., batch APIs, bulk data import/export, mass-edit of items) documented anywhere in the evidence pack. Missing for 10: explicit bulk/batch operation APIs or workflows, evidence of processing many discrete 'items' (not just concurrent voice sessions) in one operation, and any hands-on confirmation of this at scale.", "evidenceIds": [ "pipecat-docs-12", "pipecat-gh-1" ] }, { "productId": "pipecat", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Pipecat's frame-based pipeline is inherently event-driven — interruption events automatically halt bot output when the user speaks, and Pipecat Flows functions can process data and automatically transition conversation nodes based on function-call events. This gives a real automation layer, but it's scoped to conversation/pipeline events rather than a general-purpose rules engine for arbitrary triggers/actions. Missing for 10: a documented generic rule-definition system (arbitrary trigger→action bindings outside conversation flow), and independent/hands-on evidence of custom event-rule automation beyond the built-in interruption/Flows mechanics.", "evidenceIds": [ "pipecat-docs-2", "pipecat-docs-19", "pipecat-docs-6", "pipecat-docs-22", "pipecat-docs-30", "pipecat-docs-4" ] }, { "productId": "pipecat", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "pipecat", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Pipecat's docs describe building pipelines/Flows in Python code but no evidence of built-in versioning, review workflows, or rollback capabilities for automations (Flows or pipelines) — no changelog, diff, or rollback tooling is mentioned anywhere in the evidence pack.", "evidenceIds": [] }, { "productId": "pipecat", "storyId": "barge-in-interruptions", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Pipecat has explicit, dedicated documentation on interruptions/barge-in describing immediate speech stop, cancellation of in-flight work, and pipeline readiness for new input, plus Smart Turn Detection for natural turn-taking and Krisp-based false-interruption suppression in Pipecat Cloud. Missing for 10: independent hands-on validation of graceful context/turn recovery specifically (community evidence is about debugging/latency tooling, not barge-in correctness) and no explicit documentation of how conversation context is updated/rolled back after an interruption.", "evidenceIds": [ "pipecat-docs-2", "pipecat-docs-19", "pipecat-docs-9", "pipecat-docs-31", "pipecat-docs-36" ] }, { "productId": "pipecat", "storyId": "batch-outbound-campaigns", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Pipecat's telephony docs cover per-call webhook-driven session handling (inbound/outbound single calls via Twilio/Telnyx/etc.) and unlimited concurrency on Pipecat Cloud, but there is no evidence of batch campaign management, call scheduling, pacing/throughput controls, or dialer-list orchestration for outbound campaigns — this is a fair ask for a telephony-capable voice AI framework but no such capability is documented.", "evidenceIds": [ "pipecat-docs-13", "pipecat-docs-23", "pipecat-docs-40", "pipecat-docs-12", "pipecat-docs-28" ] }, { "productId": "pipecat", "storyId": "build-phone-agent-fast", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Pipecat's quickstart gets a browser voice bot running in ~5 minutes with prompt+voice configurable across 200+ providers, and the telephony guide shows the same bot file working with real carriers (Twilio/Telnyx/Plivo) via a one-line command with ngrok tunneling, so all three pieces (prompt, voice, phone number) are documented and connectable to the same codebase. However, the telephony path is a separate 'production' doc requiring carrier account setup/ngrok tunneling not folded into the 5-minute quickstart, and there's no first-party or independent account of someone going from zero to a live phone call within an hour. Missing for 10: an end-to-end timed walkthrough combining phone-number provisioning with the quickstart, and independent hands-on confirmation of the 'first live call within an hour' claim.", "evidenceIds": [ "pipecat-docs-1", "pipecat-docs-17", "pipecat-docs-40", "pipecat-docs-23", "pipecat-docs-13", "pipecat-docs-14", "pipecat-docs-28", "pipecat-probe-rt-1", "pipecat-probe-rt-2" ] }, { "productId": "pipecat", "storyId": "call-analytics-dashboard", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Pipecat exposes low-level building blocks (metrics, transcripts, audio recording, evals) that a developer could use to construct analytics, and a community debugging tool (Whisker) for pipeline tracing — but there is no evidence of a built-in dashboard surfacing success rates, call durations, outcomes, or sentiment without the founder building it themselves.", "evidenceIds": [ "pipecat-docs-5", "pipecat-docs-4", "pipecat-docs-3", "pipecat-comm-4", "pipecat-docs-25" ] }, { "productId": "pipecat", "storyId": "call-transfer-escalation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Pipecat's telephony deployment (webhook/carrier integration for Twilio, Telnyx, etc.), conversation flow transitions via Pipecat Flows, and subagent hand-off for tasks, but nothing addresses call transfer to a human agent (warm or blind) or passing call context/metadata during such a transfer. This is a reasonable telephony axis for a voice AI framework, but no citation demonstrates or documents this capability.", "evidenceIds": [] }, { "productId": "pipecat", "storyId": "concurrency-scaling-limits", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Pipecat Cloud's pricing page asserts 'unlimited concurrency' and offers multiple agent profiles for compute needs, giving a vendor-level answer to scaling, but there is no documented capacity planning guide, per-instance concurrency limits, or benchmarks for self-hosted deployments. Community commentary (a developer porting Pipecat to avoid Python's GIL for 'real concurrency') suggests self-hosted scaling is not frictionless and may require manual work to reach high concurrency. Missing for 10: documented concurrency ceilings/benchmarks for self-hosted deployments, load-testing data, and detail behind the 'unlimited concurrency' claim beyond a pricing-page bullet.", "evidenceIds": [ "pipecat-docs-12", "pipecat-docs-26", "pipecat-comm-6" ] }, { "productId": "pipecat", "storyId": "consent-recording-controls", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Pipecat provides an AudioBufferProcessor that developers can use to capture call recordings, which is a building block a founder could use to implement recording, but there's no documented feature for per-call consent/disclosure toggles or configurable data-retention policies — those would have to be built entirely by the developer. missing for 10: consent/disclosure mechanisms, retention-policy configuration, compliance-specific controls, any documentation tying recording to legal/consent obligations.", "evidenceIds": [ "pipecat-docs-3", "pipecat-docs-21", "pipecat-docs-29" ] }, { "productId": "pipecat", "storyId": "custom-voice-cloning", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Pipecat's evidence covers swapping among 200+ TTS/STT providers and general voice pipeline features, but nothing addresses custom voice cloning or any documented consent process for cloning a brand voice — this is a fair question since Pipecat is a voice-AI framework, but no evidence supports it being delivered.", "evidenceIds": [ "pipecat-docs-14", "pipecat-docs-42" ] }, { "productId": "pipecat", "storyId": "documented-latency-numbers", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Pipecat documents a dedicated STT-latency-tuning guide describing measured P99 latency values per service and a TTFS (Time To Final Segment) metric, plus a built-in metrics system for enabling latency instrumentation — real tuning guidance exists. However, this is scoped to STT stage latency rather than a comprehensive end-to-end voice round-trip latency benchmark, and no actual published numbers/tables or independent corroboration of the claimed P99 figures are shown. Missing for 10: published end-to-end latency benchmark numbers (not just STT), tuning guidance across the full voice pipeline (TTS/LLM/turn-taking), and independent verification of the claimed latency figures.", "evidenceIds": [ "pipecat-docs-15", "pipecat-docs-20", "pipecat-docs-5" ] }, { "productId": "pipecat", "storyId": "dtmf-ivr-voicemail", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Pipecat clearly supports telephony transports (Twilio, Telnyx, Plivo, Exotel, SIP) for building call-handling bots, so this axis applies, but the evidence pack contains no mention of DTMF keypress sending, IVR menu navigation, or voicemail detection/leaving capabilities anywhere in the docs or community sources.", "evidenceIds": [ "pipecat-docs-13", "pipecat-docs-23", "pipecat-docs-28", "pipecat-docs-40" ] }, { "productId": "pipecat", "storyId": "dynamic-call-personalization", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Pipecat Flows functions can process data and transition conversation state, and telephony bots receive caller info via webhook, but nothing documents a mechanism for injecting per-caller dynamic variables/context into the bot's prompt or state at call start for personalization. This is a fair capability to expect from a voice-agent framework, but no citation directly demonstrates it.", "evidenceIds": [ "pipecat-docs-6", "pipecat-docs-13", "pipecat-docs-39" ] }, { "productId": "pipecat", "storyId": "hipaa-regulated-workloads", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack mentions HIPAA/BAA, SOC 2 certification, or data-residency options for Pipecat or Pipecat Cloud; the pack covers pipeline features, telephony, CLI, and dev tooling but nothing on compliance/regulatory posture. missing for 10: HIPAA/BAA documentation, SOC 2 attestation, data-residency configuration options.", "evidenceIds": [] }, { "productId": "pipecat", "storyId": "knowledge-base-grounding", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Pipecat's docs describe voice/multimodal pipeline orchestration, function calling via Flows, telephony, evals, and a context-hub for coding-agent tooling — but nothing about a built-in knowledge base or RAG mechanism for grounding the agent on user documents. The closest hit (context-hub) indexes Pipecat's own docs/API for coding assistants, not user content for the voice agent.", "evidenceIds": [ "pipecat-docs-6", "pipecat-docs-11", "pipecat-docs-38" ] }, { "productId": "pipecat", "storyId": "live-call-monitoring", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Pipecat provides building blocks for observability — built-in metrics (docs-5), turn-event transcripts (docs-4), audio recording (docs-3/21/29), and the community-highlighted Whisker tool that shows a live pipeline graph and frame-level debugger (pipecat-comm-3/4/5) — but these are development/debugging aids, not a production monitoring or alerting system. There is no evidence of dashboards for live production calls, error-rate-spike detection, or alert/notification integrations (e.g., PagerDuty, Slack alerts) for misbehaving agents. Missing for 10: production-grade dashboards for live call monitoring, automated error-rate/anomaly alerting, and integration with on-call/alerting systems.", "evidenceIds": [ "pipecat-docs-5", "pipecat-docs-4", "pipecat-comm-3", "pipecat-comm-4", "pipecat-comm-5" ] }, { "productId": "pipecat", "storyId": "mid-call-tool-calling", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Pipecat Flows functions explicitly support 'processing data by interfacing with external systems and APIs' mid-conversation and auto-derive function schemas from code, while interruption handling and turn detection docs describe how the pipeline manages timing/dead-air around such calls. Missing for 10: no concrete hands-on example/demo showing latency-masking (e.g., filler speech) during a live API call, and no independent benchmark of perceived dead-air during function execution.", "evidenceIds": [ "pipecat-docs-6", "pipecat-docs-22", "pipecat-docs-30", "pipecat-docs-39", "pipecat-docs-2", "pipecat-docs-19", "pipecat-docs-9" ] }, { "productId": "pipecat", "storyId": "multilingual-conversations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence mentions swapping speech/language services across providers (one line of code) but nothing about detecting a spoken language mid-call or dynamically switching STT/TTS/LLM language settings within an active conversation. No docs, community reports, or runtime probes address multilingual or language-switching behavior. missing for 10: language detection mechanism, mid-call language switching mechanism, any docs/demo showing multilingual conversation handling.", "evidenceIds": [ "pipecat-docs-14" ] }, { "productId": "pipecat", "storyId": "noise-robustness", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Pipecat Cloud explicitly ships Krisp-powered noise/background-voice suppression to eliminate false interruptions, and the framework's Smart Turn Detection complements this by recognizing genuine speech cues, keeping the agent coherent on noisy calls. Missing for 10: no first-party docs on self-hosted (non-cloud) noise suppression configuration, and no independent hands-on validation of suppression quality in real noisy environments.", "evidenceIds": [ "pipecat-docs-36", "pipecat-docs-9", "pipecat-docs-31" ] }, { "productId": "pipecat", "storyId": "openness-api-parity", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Pipecat is fundamentally code-first: the CLI covers init/scaffolding, cloud deploy, evals, and context-hub (pipecat-probe-rt-1, pipecat-docs-16/34/37), and the framework itself is the primary interface. But evidence never explicitly confirms that Pipecat Cloud's web dashboard (agent profiles, concurrency settings, monitoring) exposes no exclusive UI-only functionality, nor does it document a full REST/OpenAPI surface (probe shows no discoverable OpenAPI spec, pipecat-probe-2) for programmatic parity with any cloud console. Missing for 10: explicit UI/API parity statement, documented OpenAPI/REST API for Pipecat Cloud console features, and confirmation no dashboard-only settings exist.", "evidenceIds": [ "pipecat-probe-rt-1", "pipecat-docs-16", "pipecat-docs-34", "pipecat-docs-26", "pipecat-probe-2", "pipecat-docs-24" ] }, { "productId": "pipecat", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Pipecat is open-source (BSD-2) and self-hostable, meaning users inherently retain full control of their data (audio, transcripts, logs) rather than it being locked in a vendor's system — recordings via AudioBufferProcessor and transcripts via turn events are captured directly in the developer's own infrastructure. However, there's no explicit documentation of an 'export' feature, standardized open data formats, or migration tooling for leaving a hosted deployment (e.g., Pipecat Cloud). missing for 10: explicit data-export/format documentation, Pipecat Cloud data portability/exit guarantees, independent confirmation of format openness.", "evidenceIds": [ "pipecat-docs-3", "pipecat-docs-4", "pipecat-probe-rt-2", "pipecat-docs-42" ] }, { "productId": "pipecat", "storyId": "openness-open-license", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Pipecat is explicitly documented as an 'Open source Python framework' with a public GitHub repo, and a runtime probe confirms the pip package is BSD-2 licensed and installable/importable without any key, confirming genuine open-source source access. Missing for 10: no explicit citation of a LICENSE file or license text itself, only the probe's characterization.", "evidenceIds": [ "pipecat-docs-42", "pipecat-gh-1", "pipecat-probe-rt-2" ] }, { "productId": "pipecat", "storyId": "openness-self-host", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Pipecat is an open-source Python framework (BSD-2 licensed), pip-installable and runnable with no key/cloud dependency, with a documented dev runner and explicit note that the same bot file is portable to self-hosting setups; local inference (Ollama, Whisper) further confirms self-hosting without vendor lock-in. missing for 10: no dedicated self-hosting/production deployment guide (e.g., Docker/Kubernetes reference architecture) or independent case study of a large-scale self-hosted deployment.", "evidenceIds": [ "pipecat-probe-rt-2", "pipecat-probe-rt-1", "pipecat-docs-32", "pipecat-comm-1", "pipecat-comm-2" ] }, { "productId": "pipecat", "storyId": "phone-number-provisioning", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Docs show Pipecat integrates with telephony carriers (Twilio, Telnyx, Plivo, Exotel, SIP) to receive inbound call webhooks and run bots over PSTN/SIP transports, and CLI examples show running a bot against Twilio (`python bot.py -t twilio -x ...`), covering the 'run calls' half of the story. However, there is no evidence Pipecat itself exposes an API to provision/purchase phone numbers — that is left to the carrier's own console/API, not documented as a Pipecat capability. Missing for 10: native phone-number provisioning API/CLI, outbound call initiation API details, and any first-party number-management workflow.", "evidenceIds": [ "pipecat-docs-13", "pipecat-docs-23", "pipecat-docs-28", "pipecat-docs-40" ] }, { "productId": "pipecat", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Pipecat is an open-source, self-hostable framework (pip-installable, runs locally or on your own infrastructure), which implicitly lets an operator choose where data/compute resides by choosing their own deployment location; however there is no explicit documentation of region/residency selection controls, especially for the hosted Pipecat Cloud offering. Missing for 10: explicit region-selection UI/API for Pipecat Cloud, data-residency compliance documentation (GDPR/SOC2 region guarantees), and confirmation that self-hosting is the only path to residency control.", "evidenceIds": [ "pipecat-probe-rt-2", "pipecat-docs-24", "pipecat-docs-32", "pipecat-docs-26" ] }, { "productId": "pipecat", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no mention of data-training opt-outs, data-usage policies, or privacy commitments regarding customer data and AI model training for either the open-source Pipecat framework or the hosted Pipecat Cloud service. Since Pipecat Cloud is a real hosted product where such a policy could plausibly exist, this axis applies, but no evidence supports it. Missing for 10: any privacy policy, ToS clause, or documentation stating customer data is not used to train models, and no independent corroboration.", "evidenceIds": [] }, { "productId": "pipecat", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Pipecat is a self-hostable open-source framework and also offers a hosted Pipecat Cloud option that processes voice/audio data, so data-retention/deletion controls are a fair question for this product category, but the evidence pack contains no documentation of retention policies, deletion APIs, or data lifecycle controls — only unrelated features like recording/transcription and metrics.", "evidenceIds": [] }, { "productId": "pipecat", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "pipecat", "storyId": "realtime-transcription-quality", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs and community evidence confirm Pipecat lets developers swap STT providers and language models with one line of code (200+ services, local Whisper/Ollama), and it exposes STT-specific latency metrics (TTFS, P99) for tuning accuracy/speed tradeoffs. However, there is no evidence of support for passing custom vocabulary or 'key terms'/boost-word lists to STT providers, and no independent benchmark validating transcription accuracy claims. Missing for 10: explicit key-terms/vocabulary-boost configuration support, and independent/hands-on accuracy validation.", "evidenceIds": [ "pipecat-docs-14", "pipecat-docs-15", "pipecat-docs-20", "pipecat-comm-1", "pipecat-comm-2" ] }, { "productId": "pipecat", "storyId": "recordings-transcripts-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Pipecat provides building blocks — AudioBufferProcessor for capturing audio recordings and turn-event based transcript collection — that a platform engineer could wire into their own storage/retrieval system, but there's no documented built-in centralized recordings/transcripts store or query API for retrieving them programmatically across all calls. Missing for 10: a first-party recordings/transcripts storage service or retrieval API, evidence of per-call metadata/indexing, and any hands-on confirmation of retrieving historical call data at scale.", "evidenceIds": [ "pipecat-docs-3", "pipecat-docs-21", "pipecat-docs-29", "pipecat-docs-4" ] }, { "productId": "pipecat", "storyId": "self-host-oss-runtime", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Pipecat is confirmed as a pip-installable, BSD-2 open-source Python framework that runs keylessly with no vendor lock-in, ships a development runner and CLI, and docs explicitly state bot files are portable across the dev runner, Pipecat Cloud, and 'most production self-hosting setups.' Runtime probes confirm the package installs and imports successfully outside any managed cloud, and community evidence corroborates real-world self-hosted usage (local inference via Ollama/Whisper, custom ports of the framework). Missing for 10: a first-party detailed production self-hosting deployment guide (e.g., Docker/Kubernetes reference architecture) beyond the telephony/local-runner docs.", "evidenceIds": [ "pipecat-probe-rt-2", "pipecat-probe-rt-1", "pipecat-docs-32", "pipecat-docs-42", "pipecat-comm-1", "pipecat-comm-2", "pipecat-gh-1" ] }, { "productId": "pipecat", "storyId": "semantic-turn-detection", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Pipecat documents 'Smart Turn Detection' as an ML model that determines end-of-turn based on intonation and linguistic cues, going beyond simple VAD silence timeouts, directly matching the story (pipecat-docs-9, pipecat-docs-31). Additional STT latency-tuning docs show the framework accounts for provider-specific delays to avoid premature cutoffs (pipecat-docs-15, pipecat-docs-20). Missing for 10: independent/hands-on evidence validating the model's accuracy on slow speakers in production, and no third-party benchmark of false-interruption reduction.", "evidenceIds": [ "pipecat-docs-9", "pipecat-docs-31", "pipecat-docs-15", "pipecat-docs-20" ] }, { "productId": "pipecat", "storyId": "simulated-agent-testing", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Pipecat Evals is a documented, built-in framework feature letting developers describe conversations/expected behavior and run them against the real agent pipeline before deployment, and the runtime probe confirms a working `pipecat eval` CLI subcommand exists alongside `init`/`cloud`. This directly addresses simulated-conversation testing before real phone calls, though evidence doesn't show telephony-specific eval scenarios or third-party validation of eval accuracy. Missing for 10: independent/community corroboration of Evals in practice, and explicit examples of testing telephony-specific call flows via evals.", "evidenceIds": [ "pipecat-docs-8", "pipecat-docs-25", "pipecat-probe-rt-1" ] }, { "productId": "pipecat", "storyId": "sip-trunk-byoc", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs explicitly describe telephony bots being invoked by carrier webhooks including 'Twilio, Telnyx, Plivo, Exotel, or your SIP provider,' and highlight 'Any Transport: WebRTC, SIP, PSTN' plus a working CLI example (`python bot.py -t twilio -x ...ngrok.io`), showing genuine carrier/SIP flexibility rather than lock-in to one bundled provider. Missing for 10: concrete walkthrough of raw SIP trunk/PBX configuration (vs. just carrier webhooks), and independent/hands-on confirmation of a non-Twilio/Telnyx SIP trunk actually working in production.", "evidenceIds": [ "pipecat-docs-13", "pipecat-docs-23", "pipecat-docs-28", "pipecat-docs-40" ] }, { "productId": "pipecat", "storyId": "structured-outcome-extraction", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Pipecat provides building blocks that a developer could use to build this — Flows functions can 'process data by interfacing with external systems and APIs' (function calling) and turn events/transcripts capture user and assistant messages, plus telephony webhooks for call events — but there is no documented turnkey feature for automatic post-call structured extraction (outcomes/entities/dispositions) delivered via a dedicated API or webhook. Missing for 10: a built-in post-call summary/disposition-extraction pipeline, a documented structured-output schema for call outcomes, and an out-of-box webhook delivery mechanism specifically for post-call analytics (only inbound-call webhooks and generic function-calling/transcript primitives are documented).", "evidenceIds": [ "pipecat-docs-6", "pipecat-docs-22", "pipecat-docs-30", "pipecat-docs-39", "pipecat-docs-4", "pipecat-docs-13", "pipecat-docs-23" ] }, { "productId": "pipecat", "storyId": "transparent-per-minute-pricing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack references a Pipecat Cloud pricing page (concurrency, agent profiles) but never shows actual per-minute/usage rates, a pricing table, or a cost calculator that would let a founder estimate cost per call; the core open-source framework itself is free/self-hosted with no billing model at all.", "evidenceIds": [ "pipecat-docs-12", "pipecat-docs-26" ] }, { "productId": "pipecat", "storyId": "visual-flow-builder", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Pipecat Flows lets you define branching conversation states via Python functions/config (docs-6, docs-22, docs-39), but this is a code-first framework with a CLI and Python SDK, not a visual drag-and-drop builder — no evidence pack item mentions any GUI/no-code flow designer. Since competing voice-agent platforms do offer visual builders, this is a fair axis to ask about, and Pipecat's evidence shows the opposite (functions/config authored in code).", "evidenceIds": [ "pipecat-docs-6", "pipecat-docs-22", "pipecat-docs-39", "pipecat-docs-43" ] }, { "productId": "pipecat", "storyId": "voice-library-choice", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs and community evidence confirm Pipecat supports swapping speech services (TTS/STT) across 200+ integrated providers with typically one line of code, plus community confirmation of easy onboarding of any provider and local TTS/STT options (ollama, whisper). Missing for 10: an explicit enumerated 'voice library' list or curated voice catalog UI, and independent benchmark of ease/quality across providers beyond vendor/community anecdotes.", "evidenceIds": [ "pipecat-docs-14", "pipecat-docs-42", "pipecat-comm-1", "pipecat-comm-2", "pipecat-docs-15" ] }, { "productId": "retell", "storyId": "agent-consumes-mcp-tools", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Retell documents connecting single/multi-prompt voice agents to remote MCP servers so the agent can call the server's tools mid-call (retell-docs-6/23/36), directly matching the story, and this is distinct from Retell's own client-side MCP server for building agents. missing for 10: independent third-party corroboration of mid-call MCP tool invocation in production (only first-party docs and a probe of the client-facing MCP server, not the agent-as-MCP-client tool-call path, are available).", "evidenceIds": [ "retell-docs-6", "retell-docs-23", "retell-docs-36", "retell-docs-5", "retell-docs-35" ] }, { "productId": "retell", "storyId": "agent-provisions-voice-agent", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Retell ships an official Node/Python SDK with full voice endpoint coverage, a CLI for managing agents and phone numbers, and an MCP server whose meta-tools (list/get/invoke_api_endpoint) expose the entire API — including agent create/update/publish — to MCP clients like Cursor or Claude Code, and a keyless runtime probe confirms the MCP handshake works end-to-end. However, the flagship quick-start walkthrough is dashboard-centric (create in UI, assign number in 'configuration settings', test/call via dashboard button), and no evidence explicitly shows a CLI/API/MCP call sequence that attaches a number and places a live call without touching the UI. Missing for 10: an explicit end-to-end CLI/MCP example showing number-attach and call-placement commands, and independent confirmation that non-dashboard number provisioning/outbound calling works in practice.", "evidenceIds": [ "retell-docs-2", "retell-docs-3", "retell-docs-4", "retell-docs-22", "retell-docs-47", "retell-probe-rt-1", "retell-probe-rt-2", "retell-docs-32" ] }, { "productId": "retell", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A direct probe confirms an llms.txt file exists at docs.retellai.com/llms.txt returning HTTP 200 with a structured index of the docs, and Retell's docs are also agent-oriented enough to support MCP-based discovery/management of resources. This directly satisfies the story of pointing an agent at llms.txt or agent-oriented docs. Missing for 10: no independent third-party corroboration of llms.txt usage in the wild.", "evidenceIds": [ "retell-probe-1", "retell-docs-4" ] }, { "productId": "retell", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Retell's AI QA feature automatically scores calls on hallucination, knowledge-base accuracy, latency, sentiment, and tool usage to surface quality trends and issues, and custom analytics dashboards let users chart/filter call and chat metrics — both are AI-derived insights from the product's own call data. However, this is scoring/quality analytics rather than proactive generative 'suggestions' (e.g., recommended actions, next-best-response, or coaching tips), and there's no independent/hands-on evidence validating the accuracy or usefulness of these AI-generated insights. Missing for 10: evidence of proactive suggestion/recommendation generation beyond scoring, and third-party corroboration of AI QA insight quality.", "evidenceIds": [ "retell-docs-14", "retell-docs-43", "retell-docs-13", "retell-docs-42", "retell-docs-19" ] }, { "productId": "retell", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Retell agents act autonomously during live calls — function calling lets them independently book appointments, send SMS, transfer calls, and call external APIs without human intervention (retell-docs-5, retell-docs-35), and webhooks push real-time events to downstream systems (retell-docs-12, retell-docs-24). This shows in-call autonomous action-taking, which is a form of background automation once a call is triggered. However, there's no evidence of standalone scheduled/triggered automations running independent of a live voice/chat session (e.g., cron-like outbound campaigns or autonomous multi-step workflows outside conversation context). Missing for 10: evidence of scheduled or event-triggered background automations outside live calls, and independent/hands-on confirmation that autonomous function-calling reliably completes tasks unattended (community reports show some confusion/looping during autonomous task execution, retell-comm-1, retell-comm-3, retell-comm-5).", "evidenceIds": [ "retell-docs-5", "retell-docs-35", "retell-docs-12", "retell-docs-24", "retell-comm-1", "retell-comm-3" ] }, { "productId": "retell", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Retell's dashboard includes 'Conductor', a built-in AI assistant that can draft an entire agent from a plain-English prompt (docs-30), which is a genuine instance of delegating a task to an in-product AI assistant. However this is a single thin mention with no further detail on scope, limits, or other delegable tasks beyond initial agent creation. Missing for 10: documentation of Conductor's full capabilities/limits, evidence of delegating other tasks (not just agent drafting) to a built-in assistant, and independent/hands-on corroboration of Conductor actually working.", "evidenceIds": [ "retell-docs-30" ] }, { "productId": "retell", "storyId": "agentic-headless", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Retell ships official Node.js/Python SDKs with API-key auth and a terminal CLI for managing agents, phone numbers, and resources, both of which are scriptable outside the dashboard, and a runtime probe confirms the CLI installs and runs keylessly via npm — all consistent with headless/CI use. However, there is no explicit CI/automation documentation (e.g., GitHub Actions example, testing-in-pipeline guide) confirming an officially supported headless workflow. Missing for 10: explicit CI/pipeline documentation or example, confirmation of non-interactive auth flow for CI secrets, and independent evidence of real-world CI usage.", "evidenceIds": [ "retell-docs-2", "retell-docs-3", "retell-docs-21", "retell-docs-33", "retell-probe-rt-1", "retell-probe-4" ] }, { "productId": "retell", "storyId": "agentic-mcp-client", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Retell explicitly documents connecting a live voice/chat agent to a remote MCP server so it can invoke that server's tools during calls, directly matching the story of plugging in external MCP servers for tool use. This is a first-party, well-specified capability (not just Retell exposing its own API as MCP) covering both single- and multi-prompt agents. Missing for 10: independent/hands-on verification that a third-party MCP server's tools work reliably mid-call in production.", "evidenceIds": [ "retell-docs-6", "retell-docs-23", "retell-docs-36", "retell-docs-35" ] }, { "productId": "retell", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Retell documents and hosts an official MCP server (mcp.retellai.com) that lets MCP-capable clients like Cursor, Claude Desktop, and Claude Code build and manage Retell voice agents via the Retell API, exposing meta-tools for listing/invoking endpoints. A runtime probe confirms the server completes a full keyless JSON-RPC initialize handshake and exposes its tool set, corroborating the vendor docs. Missing for 10: independent/community usage reports of the MCP server in practice beyond vendor docs and the probe.", "evidenceIds": [ "retell-docs-4", "retell-docs-22", "retell-docs-34", "retell-docs-47", "retell-probe-3", "retell-probe-rt-2" ] }, { "productId": "retell", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Retell supports natural-language operation via Conductor's 'Generate from prompt' feature that drafts an agent from a plain-English description, and via its official MCP server which exposes the full API as agent-callable tools so AI-native clients (Cursor, Claude Desktop, Claude Code) can build/manage agents through natural-language MCP tool calls — confirmed live by a keyless JSON-RPC probe against the hosted MCP endpoint. missing for 10: independent/hands-on evaluation of Conductor's prompt-to-agent quality, and no evidence of NL commands for other everyday operations (e.g., dashboard chat-based control) beyond agent creation/management.", "evidenceIds": [ "retell-docs-30", "retell-docs-4", "retell-docs-22", "retell-docs-34", "retell-docs-47", "retell-probe-3", "retell-probe-rt-2" ] }, { "productId": "retell", "storyId": "agentic-official-cli", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Retell ships an official CLI (`@retell-ai/retell-cli`) documented for managing agents, phone numbers, and knowledge bases from the terminal, and a runtime probe confirms it installs and runs keylessly via npx. Missing for 10: independent/community usage reports of the CLI itself (only docs and one probe run corroborate it).", "evidenceIds": [ "retell-docs-3", "retell-docs-21", "retell-probe-4", "retell-probe-rt-1" ] }, { "productId": "retell", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Retell documents official Node.js/Python SDKs with typed clients and full endpoint coverage, a CLI for managing resources from the terminal, and API references (e.g., clone-voice endpoint) — and a runtime probe confirms the CLI installs/runs keylessly and the hosted MCP server exposes the entire API via meta-tools, proving the API is genuinely agent-drivable. Missing for 10: a publicly discoverable OpenAPI/swagger spec (all standard paths returned 404 in the probe).", "evidenceIds": [ "retell-docs-2", "retell-docs-3", "retell-docs-33", "retell-docs-46", "retell-probe-3", "retell-probe-4", "retell-probe-rt-1", "retell-probe-rt-2", "retell-probe-2" ] }, { "productId": "retell", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents API key auth, SDKs, CLI, and an MCP server, but nowhere describes scoped, role-based, or least-privilege API key/credential issuance (e.g., per-agent or per-permission keys) for Retell. This is a fair axis for an API/voice-agent platform, but no docs or probes show scoped credential support, so it is unproven.", "evidenceIds": [ "retell-docs-2", "retell-docs-33", "retell-probe-rt-2", "retell-docs-4" ] }, { "productId": "retell", "storyId": "agentic-sdks", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Retell documents and ships official typed SDKs for Node.js and Python with API key auth, full endpoint coverage, structured errors, and TypeScript autocomplete, plus an official CLI and MCP server for programmatic/agentic access — all independently confirmed by runtime probes (CLI installs and runs, MCP server completes handshake). Missing for 10: no independent third-party review of SDK quality/DX beyond vendor docs and no public OpenAPI spec discovered.", "evidenceIds": [ "retell-docs-2", "retell-docs-33", "retell-docs-46", "retell-probe-4", "retell-probe-rt-1", "retell-probe-3", "retell-probe-rt-2" ] }, { "productId": "retell", "storyId": "agentic-webhooks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Retell has dedicated first-party documentation for webhooks describing real-time event notifications pushed to your application as events occur, enabling event-driven integrations rather than polling — directly matching the story of subscribing to events via webhooks. Missing for 10: no independent/hands-on corroboration of webhook reliability or event catalog completeness beyond the docs themselves.", "evidenceIds": [ "retell-docs-12", "retell-docs-24" ] }, { "productId": "retell", "storyId": "ai-assisted-agent-authoring", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Retell docs mention a 'Generate from prompt' feature where Conductor drafts an agent from a plain-English description (retell-docs-30), directly supporting AI-assisted agent authoring. However, there's no evidence of AI-driven improvement of existing prompts/flows or automatic test-case generation, and no independent/hands-on confirmation of Conductor's output quality. Missing for 10: documentation on iterative prompt/flow refinement by the platform's AI, automated test-case generation, and community or hands-on verification of Conductor's generated agents.", "evidenceIds": [ "retell-docs-30", "retell-docs-17", "retell-docs-25", "retell-docs-26" ] }, { "productId": "retell", "storyId": "ai-call-qa-scoring", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Retell's AI QA feature automatically scores sampled calls on hallucination, knowledge base accuracy, latency, sentiment, and tool usage, and the Analytics dashboard tracks success rate and other metrics, directly matching the story's core ask of automated call scoring and quality flagging. However, evidence shows AI QA only evaluates a 'sampled set' of calls (not all calls) against user-configured rules, and there's no documented automatic 'resolution analysis' narrative beyond metrics/dashboards, nor independent hands-on validation of AI QA accuracy. missing for 10: evidence of full (non-sampled) coverage, independent/third-party validation of AI QA scoring accuracy, and detail on automated resolution/failure-cause analysis beyond configured metrics.", "evidenceIds": [ "retell-docs-14", "retell-docs-43", "retell-docs-13", "retell-docs-42", "retell-docs-19" ] }, { "productId": "retell", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Retell has API reference doc pages (e.g., api-references/clone-voice.md) and SDKs/CLI/MCP tooling, but no evidence of an interactive, runnable API reference (e.g., embedded 'try it' console or Swagger UI). The probe explicitly found no OpenAPI/Swagger spec exposed (all candidate paths 404), undermining any claim of an interactive reference.", "evidenceIds": [ "retell-docs-8", "retell-probe-2" ] }, { "productId": "retell", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe for standard OpenAPI/Swagger spec locations (openapi.json, swagger.json, etc.) on Retell's docs domain returned 404 for all candidates, and no evidence pack item shows a downloadable, machine-readable API spec being published or linked from docs. The CLI's version string mentions an internal 'OpenAPI 3.0.0' schema, but this is not shown to be an artifact users can download or fetch programmatically.", "evidenceIds": [ "retell-probe-2", "retell-probe-rt-1" ] }, { "productId": "retell", "storyId": "api-sandbox", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Retell's quick-start lets you click 'Test' to run a free web call with your agent before it ever touches a phone number or payment method, and new accounts get $10 trial credit to experiment risk-free — a lightweight way to try an agent without production telephony traffic. However, there's no documented dedicated sandbox environment, test-mode API flag, or explicit data isolation guarantee separating test calls from production records/analytics. Missing for 10: explicit sandbox/staging environment concept, test vs prod data isolation guarantees, and independent confirmation that test-call data doesn't mix with production analytics/QA.", "evidenceIds": [ "retell-docs-20", "retell-docs-31", "retell-docs-45" ] }, { "productId": "retell", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers SDKs, CLI, MCP server, and API endpoints extensively, but there is no documentation of API versioning scheme or a deprecation policy for breaking changes. The probe notes an OpenAPI version string (3.0.0, catalog v4) but this is not evidence of a documented deprecation/versioning policy for API consumers.", "evidenceIds": [] }, { "productId": "retell", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Retell exposes a full-coverage SDK, CLI, and MCP server that let an AI agent script repeated single-item operations (create/update/list agents, phone numbers, knowledge bases) programmatically, enabling scripted bulk-like loops, but there is no documented native batch/bulk endpoint operating on many items in a single call. missing for 10: explicit batch/bulk API endpoints or CLI commands operating on multiple items in one call, evidence of rate-limit-safe bulk workflows, and hands-on confirmation of bulk usage at scale", "evidenceIds": [ "retell-docs-3", "retell-docs-2", "retell-docs-4", "retell-probe-rt-1", "retell-probe-rt-2", "retell-docs-47" ] }, { "productId": "retell", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Retell supports event-driven automation via webhooks that fire on account events (retell-docs-12, retell-docs-24), function calling that lets agents automatically transfer calls, end calls, book appointments, or call external APIs based on conversation logic (retell-docs-5, retell-docs-35), and node-based flows with conditional branching (retell-docs-25) plus AI QA rules evaluated against configured metrics (retell-docs-43). However, this is scattered across call-flow logic and webhook notifications rather than a unified 'if event then action' rule-definition interface. Missing for 10: a dedicated rules/trigger engine UI for arbitrary account-wide events, documentation of webhook-to-action automation chains, and independent verification that rule-based automation works reliably (community reports note conversational logic errors, e.g. retell-comm-1, retell-comm-3, retell-comm-7).", "evidenceIds": [ "retell-docs-5", "retell-docs-12", "retell-docs-24", "retell-docs-25", "retell-docs-35", "retell-docs-43" ] }, { "productId": "retell", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Retell's docs cover webhooks, function calling, MCP tool-calling, and telephony integration, but nothing describes scheduling recurring jobs, campaigns, or workflows (e.g., cron-like recurring outbound call batches or repeating automations). This is a fair question for a voice-agent platform (buyers often want scheduled/recurring outbound campaigns), so the axis applies, but no evidence in the pack shows this capability. missing for 10: any mention of a scheduler, recurring campaign/job feature, or cron-style automation trigger.", "evidenceIds": [] }, { "productId": "retell", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Retell supports 'agent versions' with create/update/publish/list/fetch operations via CLI and MCP (retell-docs-47), implying some versioning and possibly rollback via republishing an older version, but there is no documented review workflow (diffs, approvals, changelogs) or explicit rollback command/UI. missing for 10: explicit rollback mechanism, version diff/review UI, changelog or audit trail, and any independent/hands-on confirmation that version history and rollback actually work as described.", "evidenceIds": [ "retell-docs-47", "retell-docs-3", "retell-docs-4" ] }, { "productId": "retell", "storyId": "barge-in-interruptions", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers transcription latency/accuracy trade-offs (retell-docs-9, retell-docs-39, retell-docs-48) but nowhere describes explicit barge-in/interruption handling — stopping agent speech, updating context, and recovering the turn when a caller interrupts. Community threads show turn-taking confusion (contradictions, loops, disconnects) but do not address interruption handling specifically. Missing for full/partial credit: any documentation or hands-on report confirming barge-in detection, speech-stop behavior, or turn recovery mechanics.", "evidenceIds": [] }, { "productId": "retell", "storyId": "batch-outbound-campaigns", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Docs describe outbound vs inbound concurrency reservation (retell-docs-11, retell-docs-41) and custom telephony/SIP integration, but there is no evidence of a batch outbound campaign feature — no mention of scheduling calls, uploading contact lists, or campaign-level throughput controls beyond general concurrency limits. Axis is plausible for a voice-agent platform but unsupported by the evidence pack.", "evidenceIds": [ "retell-docs-11", "retell-docs-41", "retell-docs-10" ] }, { "productId": "retell", "storyId": "build-phone-agent-fast", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs walk through the exact flow described: create account, pick/generate a template, set prompt and voice, test in dashboard, assign a phone number, and make a live call, with a 15-minute quickstart and free trial credits removing payment friction. Community feedback confirms the demo/agent works end-to-end for real calls, though it also shows occasional conversational glitches (contradictions, confusion) that are quality issues rather than build-flow blockers. missing for 10: independent third-party timing confirmation that a full build-to-first-call truly completes within an hour, and evidence of phone-number provisioning speed in practice.", "evidenceIds": [ "retell-docs-1", "retell-docs-20", "retell-docs-30", "retell-docs-31", "retell-docs-32", "retell-docs-45", "retell-docs-7", "retell-comm-7", "retell-comm-8" ] }, { "productId": "retell", "storyId": "call-analytics-dashboard", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Retell docs describe a built-in Analytics dashboard with charts, filters, and breakdowns for success rate, latency, cost, and concurrency, plus AI QA scoring for sentiment, hallucination, and tool usage—directly matching the founder's need for out-of-box call analytics. Missing for 10: independent/hands-on confirmation of dashboard usability and no evidence of exportable reports or deeper outcome breakdowns beyond what's documented.", "evidenceIds": [ "retell-docs-13", "retell-docs-42", "retell-docs-14", "retell-docs-43", "retell-docs-19" ] }, { "productId": "retell", "storyId": "call-transfer-escalation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Retell's function-calling docs list 'transfer calls' as a built-in agent action alongside ending calls, booking appointments, etc., which supports the general concept of escalating a call to a human. However, the evidence never distinguishes warm vs. blind transfer modes nor describes passing conversational context/metadata to the receiving human agent. missing for 10: explicit warm-transfer vs blind-transfer configuration, evidence of context/data hand-off to the human agent, and any hands-on confirmation of this feature working in practice.", "evidenceIds": [ "retell-docs-5", "retell-docs-35" ] }, { "productId": "retell", "storyId": "concurrency-scaling-limits", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Retell documents a concurrency configuration feature (reserved inbound concurrency vs. outbound limit) showing platform-level concurrency controls exist, but there is no documented default/numeric concurrency limits per plan, no described self-service process to raise limits without contacting support, and no evidence (docs or community) confirming scaling to many simultaneous calls happens without manual requests. missing for 10: published default concurrency limits, self-service limit-increase mechanism, and independent confirmation of large-scale concurrent call handling without support intervention.", "evidenceIds": [ "retell-docs-11", "retell-docs-41" ] }, { "productId": "retell", "storyId": "consent-recording-controls", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Retell documents per-agent configurable data retention that auto-deletes transcripts, recordings, and logs after a set period, directly addressing retention obligations. However, there is no evidence of explicit per-call recording enable/disable controls or built-in consent-disclosure features (e.g., automated recording announcements) that founders would need for two-party consent compliance. Missing for 10: per-call recording on/off toggle, consent-disclosure/announcement mechanism, and any documentation tying these to legal consent requirements.", "evidenceIds": [ "retell-docs-15", "retell-docs-27", "retell-docs-44" ] }, { "productId": "retell", "storyId": "custom-voice-cloning", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Retell documents voice cloning from uploaded audio files (up to 25 files) and lets you attach the cloned voice to an agent via the voice selector, covering the core 'clone and use a custom voice' capability. However, none of the evidence describes any documented consent-verification step (e.g., consent recording, rights attestation) as part of the cloning flow. Missing for 10: an explicit consent-collection/verification mechanism in the clone-voice API or dashboard docs, and any compliance guidance tying voice cloning to consent requirements.", "evidenceIds": [ "retell-docs-7", "retell-docs-8", "retell-docs-28", "retell-docs-37" ] }, { "productId": "retell", "storyId": "documented-latency-numbers", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Retell docs offer tuning guidance for latency (transcription mode trade-offs, endpointing settings) and let customers monitor latency via analytics dashboards and AI QA scoring, but there are no documented end-to-end latency benchmarks or numbers substantiating speed claims. missing for 10: published end-to-end latency figures/benchmarks, and independent verification of speed claims.", "evidenceIds": [ "retell-docs-9", "retell-docs-39", "retell-docs-48", "retell-docs-13", "retell-docs-42", "retell-docs-14", "retell-docs-43" ] }, { "productId": "retell", "storyId": "dtmf-ivr-voicemail", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers function calling, custom telephony/SIP trunking, transcription, and MCP tool integration, but nowhere mentions DTMF keypress sending, IVR menu navigation, or voicemail detection/leaving capabilities. This is a fair capability question for a telephony voice-agent platform, but no evidence confirms it is supported.", "evidenceIds": [] }, { "productId": "retell", "storyId": "dynamic-call-personalization", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers function calling, webhooks, custom telephony, knowledge base retrieval, CRM field sync, and SDK/CLI/MCP tooling, but nowhere describes injecting dynamic variables or per-caller context (e.g., a startCall/create-call parameter for passing caller-specific data into the prompt/LLM at runtime). This is a fair, plausible capability for a voice-agent platform, so absence of evidence yields 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "retell", "storyId": "hipaa-regulated-workloads", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only generic data-retention controls (docs-15/27/44) but no mention of HIPAA, BAA, SOC 2 certification, or data-residency options anywhere in the docs or community pack, despite this being a fair compliance axis for a voice-AI platform handling call recordings/transcripts.", "evidenceIds": [ "retell-docs-15", "retell-docs-27", "retell-docs-44" ] }, { "productId": "retell", "storyId": "knowledge-base-grounding", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Retell explicitly ships a knowledge base feature where you can crawl websites or upload documents for the agent to retrieve from, and AI QA scores 'knowledge base accuracy' as a call metric, confirming grounding is a first-class capability. However, evidence is thin — only brief homepage-level mentions rather than a dedicated deep-dive doc on KB architecture/RAG internals, and there's no independent/hands-on confirmation of retrieval quality. Missing for 10: dedicated knowledge-base documentation page detailing chunking/retrieval mechanics, and independent verification that answers are accurately grounded in uploaded content.", "evidenceIds": [ "retell-docs-17", "retell-docs-18", "retell-docs-14" ] }, { "productId": "retell", "storyId": "live-call-monitoring", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Retell provides real building blocks for production monitoring — a customizable analytics dashboard tracking success rate, latency, cost and concurrency (retell-docs-13/42), AI QA that scores calls for hallucination, sentiment and tool-usage issues (retell-docs-14/43), and webhooks that push real-time event notifications (retell-docs-12/24) which a platform engineer could wire into an alerting pipeline. However there is no documented native alerting/threshold system (e.g., automatic notification when error rates spike or an agent misbehaves) or a live in-call monitoring view — engineers must build that themselves on top of webhooks/dashboard APIs. Missing for 10: built-in threshold-based alerts or anomaly detection, a real-time 'in-progress calls' monitoring view, and any independent evidence that alerting/monitoring works reliably in production.", "evidenceIds": [ "retell-docs-12", "retell-docs-13", "retell-docs-14", "retell-docs-24", "retell-docs-42", "retell-docs-43" ] }, { "productId": "retell", "storyId": "mid-call-tool-calling", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Retell's docs clearly document function calling that lets agents call external APIs mid-conversation (retell-docs-5, retell-docs-35) and even connect to remote MCP servers to invoke tools live during a call (retell-docs-6, retell-docs-23, retell-docs-36), confirmed by a keyless runtime probe showing the hosted MCP server exposing invoke_api_endpoint tools (retell-probe-rt-2). However, no evidence specifically addresses the 'no awkward dead air' requirement — there's no documented filler-speech, streaming-response, or latency-masking mechanism specifically tied to function-call execution; transcription-mode docs (retell-docs-9/39/48) discuss general STT latency tradeoffs, not function-call dead-air handling. missing for 10: explicit filler/backchannel speech during function execution, hands-on demonstration of seamless mid-call tool latency, independent confirmation of no dead-air behavior.", "evidenceIds": [ "retell-docs-5", "retell-docs-35", "retell-docs-6", "retell-docs-23", "retell-docs-36", "retell-probe-rt-2" ] }, { "productId": "retell", "storyId": "multilingual-conversations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No documentation in the evidence pack describes multi-language conversation support, language detection, or mid-call language switching; the only related evidence is a founder comment stating multilingual support is merely 'on our roadmap' rather than shipped, indicating the capability does not currently exist.", "evidenceIds": [ "retell-comm-12" ] }, { "productId": "retell", "storyId": "noise-robustness", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item mentions noise suppression, background-noise filtering, or audio-preprocessing features for handling noisy real-world call environments; the closest related feature (transcription-mode trade-off between latency and accuracy) addresses turn-taking speed, not noise robustness. This is a plausible axis for a voice-agent platform, so absence of evidence yields 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "retell", "storyId": "openness-api-parity", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Retell exposes full API coverage via official Node/Python SDKs, a CLI, and a hosted MCP server that dynamically exposes the entire API (list/get/invoke endpoint tools), confirmed by keyless runtime probes actually reaching the CLI and MCP handshake — meaning nearly anything doable in the dashboard (agents, phone numbers, knowledge bases, function calling, voice cloning, analytics) is API/CLI/MCP accessible. missing for 10: no public OpenAPI spec was found (404s across candidate paths), and no independent hands-on confirmation that every single UI feature (e.g., analytics dashboards, AI QA config) has a 1:1 API equivalent.", "evidenceIds": [ "retell-docs-3", "retell-docs-2", "retell-docs-4", "retell-docs-22", "retell-probe-rt-1", "retell-probe-rt-2", "retell-probe-2", "retell-docs-47" ] }, { "productId": "retell", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Docs describe API/SDK/CLI access, data retention (auto-deletion) and analytics dashboards, but nowhere document a data-export or portability feature that would let a user pull all call/chat data, transcripts, and recordings in an open format to leave the platform; retention docs only cover automatic deletion, not export.", "evidenceIds": [ "retell-docs-15", "retell-docs-27", "retell-docs-44", "retell-docs-33", "retell-docs-3" ] }, { "productId": "retell", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Retell AI is a closed, proprietary SaaS platform; the evidence pack shows SDKs, CLI, MCP server and docs but no mention of source code being published under any open license, nor any GitHub repo for the core platform. Missing for 10: any open-source license grant, public source repository, or licensing terms for the core agent/voice engine.", "evidenceIds": [] }, { "productId": "retell", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Retell AI is presented entirely as a hosted SaaS (cloud dashboard, hosted APIs, hosted MCP server, usage-based pricing) with no mention anywhere in docs of a self-hosted or on-prem deployment option; all evidence points to a fully managed cloud product.", "evidenceIds": [ "retell-docs-16", "retell-docs-1", "retell-probe-rt-2", "retell-docs-10" ] }, { "productId": "retell", "storyId": "phone-number-provisioning", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Retell's docs show phone-number provisioning via native assignment (docs-32) or importing/SIP-trunking your own numbers from Twilio/Telnyx/Vonage (docs-10,40,49), and both inbound and outbound calling are explicitly supported and distinguished (reserved inbound concurrency vs. outbound traffic in docs-11/41). Full API/SDK coverage for voice endpoints (docs-2,33,46) and a CLI to manage phone numbers (docs-3,21) round out programmatic control, with a working keyless CLI/MCP probe corroborating API-level access (retell-probe-rt-1/2). Missing for 10: explicit documentation of a native 'buy a new number' provisioning endpoint (as opposed to importing existing numbers) and a concrete outbound-call code example.", "evidenceIds": [ "retell-docs-1", "retell-docs-10", "retell-docs-11", "retell-docs-40", "retell-docs-41", "retell-docs-49", "retell-docs-2", "retell-docs-33", "retell-docs-3", "retell-docs-21", "retell-probe-rt-1", "retell-probe-rt-2" ] }, { "productId": "retell", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack mentions data residency, region selection, or geographic storage options; the closest related feature is per-agent data retention/deletion (retell-docs-15/27/44), which addresses retention duration, not location of storage. This axis is applicable to a cloud AI voice platform handling call/chat data, so absence of evidence yields 'none'. Missing for 10: any documentation of selectable regions, data residency guarantees, or geo-specific hosting.", "evidenceIds": [ "retell-docs-15", "retell-docs-27", "retell-docs-44" ] }, { "productId": "retell", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence only covers configurable data retention (auto-deleting transcripts/recordings/logs after a set period) for compliance, not an explicit opt-out or guarantee against using customer data for AI model training. No docs mention training-data usage policy, opt-out toggles, or contractual no-train clauses.", "evidenceIds": [ "retell-docs-15", "retell-docs-27", "retell-docs-44" ] }, { "productId": "retell", "storyId": "privacy-retention-controls", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Retell's docs describe per-agent configurable data retention with automatic, permanent deletion of call/chat transcripts, recordings, and logs after a set period (retell-docs-15, retell-docs-27, retell-docs-44), which is solid first-party evidence for the retention-control axis. However, an independent community report describes the company refusing to delete a user's stored data (credit card) on request, forcing the user to escalate to bank disputes/consumer-affairs complaints — a concrete real-world case where deletion did not work as a customer expected (retell-comm-14). Missing for 10: independent verification that call/chat data deletion itself (not just billing data) works as documented, and no public response/resolution to the deletion complaint.", "evidenceIds": [ "retell-docs-15", "retell-docs-27", "retell-docs-44", "retell-comm-14" ] }, { "productId": "retell", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers data retention for call/chat data (transcripts, recordings, logs) but contains no mention of telemetry or usage-tracking opt-out controls for the product itself (e.g., CLI/SDK/dashboard analytics collection). Data retention (retell-docs-15/27/44) addresses deletion of customer call data, not opting out of Retell's own telemetry collection.", "evidenceIds": [] }, { "productId": "retell", "storyId": "realtime-transcription-quality", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Retell docs confirm real-time transcription with a documented latency/accuracy trade-off (interim vs context-aware endpointing), giving developers some control over accuracy tuning, and community notes hint at underlying provider choices (e.g., Deepgram) for voice pipelines. However, there is no documented ability to select or swap STT providers, choose an ASR language model, or configure custom vocabulary/key-term boosting for transcription accuracy. Missing for 10: explicit STT provider selection API, language model choice for transcription, custom vocabulary/key-term boosting support, and independent accuracy benchmarks.", "evidenceIds": [ "retell-docs-9", "retell-docs-39", "retell-docs-48", "retell-comm-10" ] }, { "productId": "retell", "storyId": "recordings-transcripts-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm that Retell stores per-call transcripts and recordings (referenced in data-retention docs) and that full voice/chat API endpoint coverage exists via official SDKs, implying programmatic retrieval, but no evidence pack item explicitly documents a 'get call' or 'list calls' API endpoint returning recording URLs/transcript text, nor examples of pulling them via SDK/CLI. Missing for 10: explicit API reference for call/recording/transcript retrieval endpoints, CLI/SDK code samples showing recording download or transcript fetch, and any independent confirmation of this working end-to-end.", "evidenceIds": [ "retell-docs-15", "retell-docs-27", "retell-docs-44", "retell-docs-2", "retell-docs-33", "retell-docs-12", "retell-docs-24" ] }, { "productId": "retell", "storyId": "self-host-oss-runtime", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Retell AI is a hosted SaaS voice-agent platform; all evidence points to cloud-hosted APIs, SDKs, CLI, and MCP server that call Retell's own servers, with no mention of open-source runtime code or self-hosting instructions. There is no evidence a platform-engineer can download and run the voice agent runtime on their own infrastructure.", "evidenceIds": [ "retell-docs-1", "retell-docs-16", "retell-probe-rt-1", "retell-probe-rt-2" ] }, { "productId": "retell", "storyId": "semantic-turn-detection", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The transcription-mode docs only describe tuning an 'endpointing setting' (low vs high) as a speed/accuracy tradeoff for VAD-style silence detection, not a distinct model-based semantic end-of-turn detector that understands incomplete utterances from slow speakers. No evidence pack item describes a dedicated turn-detection model or algorithm beyond adjusting silence-timeout thresholds.", "evidenceIds": [ "retell-docs-9", "retell-docs-39", "retell-docs-48" ] }, { "productId": "retell", "storyId": "simulated-agent-testing", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Retell's dashboard has a free 'Test' web-call button that lets developers try an agent before assigning it to a real phone number, and AI QA can score calls on hallucination, accuracy, sentiment, etc., which supports some testing/analytics workflow. However there's no documented feature for automated simulated-conversation test suites or eval scripts run pre-deployment — AI QA appears to operate on sampled real calls rather than synthetic scripted evals. Missing for 10: dedicated simulation/eval framework for scripted test conversations, batch eval tooling, and any independent verification that pre-call testing catches issues before production use.", "evidenceIds": [ "retell-docs-20", "retell-docs-31", "retell-docs-14", "retell-docs-43", "retell-docs-13" ] }, { "productId": "retell", "storyId": "sip-trunk-byoc", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Official docs explicitly describe elastic SIP trunking and importing numbers from Twilio, Telnyx, and Vonage, walking through setting up a SIP trunk and pointing/importing numbers into Retell — directly matching the story. This is first-party documentation without independent hands-on corroboration of the SIP flow itself. Missing for 10: independent/community verification of a real SIP trunk setup working end-to-end, and detail on carrier-specific edge cases (codecs, failover, latency).", "evidenceIds": [ "retell-docs-10", "retell-docs-40", "retell-docs-49" ] }, { "productId": "retell", "storyId": "structured-outcome-extraction", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Retell explicitly supports post-call structured extraction: built-in call success/sentiment scoring plus custom fields synced to CRM, delivered via real-time webhooks and viewable/aggregated in the analytics/AI QA dashboards. This directly covers outcomes (success rate), dispositions (sentiment), and entities (custom fields) delivered via API/webhook as the story requires. Missing for 10: no independent/hands-on confirmation of the specific post-call-analysis JSON schema or webhook payload structure, and no detail on how custom entity fields are defined/configured.", "evidenceIds": [ "retell-docs-19", "retell-docs-12", "retell-docs-24", "retell-docs-13", "retell-docs-42", "retell-docs-14", "retell-docs-43" ] }, { "productId": "retell", "storyId": "transparent-per-minute-pricing", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Retell's pricing page explicitly states pay-as-you-go pricing starting at $0 with no annual contracts, offers an interactive calculator to estimate approximate monthly cost by adjusting key parameters, and new accounts get $10 free trial credit to test before paying — directly enabling a founder to estimate call costs before committing. Community discussion also confirms a concrete per-minute rate (17¢/min) is being charged in practice, corroborating usage-based billing exists. missing for 10: the per-minute rate itself is not shown in first-party docs (only inferred from a community comment), and there's no independent verification of the calculator's accuracy.", "evidenceIds": [ "retell-docs-16", "retell-docs-29", "retell-docs-50", "retell-docs-45", "retell-comm-10", "retell-comm-13" ] }, { "productId": "retell", "storyId": "visual-flow-builder", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Retell's docs describe node-based, drag-and-drop conversation flows for 'structured, high-stakes calls' as an alternative to single-prompt agents, with function calling enabling transfers, call-ending, and API calls that serve as handoff points — directly matching the branching/states/handoff story without requiring code. The quick-start flow (pick template, test in dashboard, deploy) reinforces a no-code workflow. Missing for 10: independent/hands-on evidence of the visual builder's branching UI itself (community evidence only covers conversational behavior, not the builder), and no detail on how 'states' are represented/connected beyond the general node-based description.", "evidenceIds": [ "retell-docs-17", "retell-docs-25", "retell-docs-26", "retell-docs-5", "retell-docs-35", "retell-docs-1", "retell-docs-30" ] }, { "productId": "retell", "storyId": "voice-library-choice", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs confirm a searchable voice library (ElevenLabs community voices) plus voice cloning options (retell-docs-7, retell-docs-37, retell-docs-38), and a founder community comment independently confirms multiple underlying TTS providers (OpenAI TTS, Deepgram, ElevenLabs) with different pricing tiers (retell-comm-10), showing developers can indeed pick across providers/voices. Missing for 10: a first-party docs page enumerating all supported TTS providers and API-level provider-switching parameters beyond the voice-selector UI, and independent hands-on comparison of voice quality across providers.", "evidenceIds": [ "retell-docs-7", "retell-docs-37", "retell-docs-38", "retell-comm-10" ] }, { "productId": "telli", "storyId": "agent-consumes-mcp-tools", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "telli documents custom tools connecting to external APIs (telli-docs-20) and even an MCP server (telli-docs-22, telli-probe-3), but that MCP server lets external AI tools like Claude/ChatGPT talk to telli's Charlie assistant — it is telli acting as an MCP server, not the voice agent consuming external MCP servers as a tool source mid-call. There is no evidence the voice agent itself can plug in MCP servers to gain toolsets during a call.", "evidenceIds": [ "telli-docs-22", "telli-probe-3", "telli-docs-20" ] }, { "productId": "telli", "storyId": "agent-provisions-voice-agent", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "telli's REST API documents endpoints for scheduling calls and importing/attaching phone numbers (telli-docs-17, telli-docs-19), and the MCP server lets AI tools talk to Charlie, who can 'build and refine agents' (telli-docs-11, telli-docs-22), suggesting agent creation is possible without the dashboard. However, there is no explicit API/CLI 'create agent' endpoint documented, no CLI tool is mentioned at all, and the MCP-driven agent creation is only described as an assistant chat capability rather than a deterministic API call — so a full end-to-end create→attach-number→place-call pipeline via one consistent interface is not clearly evidenced. Missing for 10: a documented create-agent API/CLI endpoint, confirmation MCP/Charlie can fully provision agents without any dashboard step, and an example combining all three steps in one flow.", "evidenceIds": [ "telli-docs-11", "telli-docs-12", "telli-docs-17", "telli-docs-19", "telli-docs-22", "telli-probe-2", "telli-probe-3" ] }, { "productId": "telli", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "telli hosts a live llms.txt file (confirmed via probe, HTTP 200) plus machine-readable OpenAPI spec explicitly intended for coding agents, and docs also mention agent-oriented context for downloading API specs. missing for 10: no independent/third-party confirmation of an agent successfully consuming llms.txt in practice.", "evidenceIds": [ "telli-probe-1", "telli-docs-16", "telli-probe-2" ] }, { "productId": "telli", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "telli ships a built-in AI assistant (Charlie) that can 'analyze your calls' and help build/refine agents, plus dashboards for reach rate, call duration, success rate, and per-call transcript/outcome analysis, which together provide AI-driven insight into product data. However, the docs don't detail proactive, autonomous 'suggestions' surfaced to the user (e.g., recommended actions, anomaly detection) beyond on-demand analysis via Charlie or manual dashboard review. Missing for 10: concrete examples of Charlie proactively surfacing insights/recommendations, independent user corroboration of insight quality.", "evidenceIds": [ "telli-docs-11", "telli-docs-7", "telli-docs-8", "telli-docs-10" ] }, { "productId": "telli", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "telli supports Scheduled triggers so workflows run automatically without a call event, plus retry/dialing-window configuration, webhooks, and an AI assistant (Charlie) to build/automate workflows — together enabling background autonomous automations. Missing for 10: independent/hands-on verification of scheduled automations running reliably in production, and more detail on monitoring/error-handling for autonomous runs beyond docs claims.", "evidenceIds": [ "telli-docs-1", "telli-docs-9", "telli-docs-11", "telli-docs-13", "telli-docs-17" ] }, { "productId": "telli", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "telli documents \"Charlie,\" a built-in AI assistant that can build/refine agents, automate workflows, and analyze calls, and this assistant is also exposed via an MCP server for external AI tools to interact with. This directly matches delegating tasks to a built-in assistant within the product. Missing for 10: independent/hands-on verification of Charlie's task delegation in practice and more detail on the scope of tasks it can autonomously execute.", "evidenceIds": [ "telli-docs-11", "telli-docs-22", "telli-probe-3" ] }, { "productId": "telli", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "telli exposes a full REST API (with OpenAPI spec) to programmatically schedule calls, manage contacts, and trigger workflows, plus webhooks for event-driven automation and scheduled/recurring workflow triggers — all of which support headless, non-UI automation suitable for CI-like pipelines. However there is no explicit CI/CD example, CLI tool, or documented automated pipeline integration beyond the API/webhook primitives. Missing for 10: dedicated CI/CD documentation or examples, a CLI for scripted invocation, and independent confirmation of headless use in production pipelines.", "evidenceIds": [ "telli-docs-12", "telli-docs-17", "telli-docs-1", "telli-docs-21", "telli-probe-2" ] }, { "productId": "telli", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Telli's documented MCP support only runs in the opposite direction — it exposes an MCP server so external AI tools (Claude, ChatGPT, Codex) can talk to its Charlie assistant (telli-docs-22, telli-probe-3), not a mechanism for telli's own voice agents to consume external MCP servers' tools. Custom Tools (telli-docs-14, telli-docs-20) let agents call external APIs but are described as custom integrations, not MCP-based tool ingestion.", "evidenceIds": [ "telli-docs-22", "telli-probe-3", "telli-docs-14", "telli-docs-20" ] }, { "productId": "telli", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "telli documents an official MCP server that lets external AI tools (Claude, ChatGPT, Codex) connect over the Model Context Protocol to interact with Charlie, telli's assistant, confirmed both in docs and via a live probe of the integrations/mcp page. Missing for 10: no independent/hands-on corroboration beyond first-party docs and the probe, and no detail on server setup/auth specifics.", "evidenceIds": [ "telli-docs-22", "telli-probe-3" ] }, { "productId": "telli", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "telli offers Charlie, a built-in AI assistant that can build/refine agents, automate workflows, and analyze calls via natural language, plus an MCP server letting external AI tools (Claude, ChatGPT, Codex) converse with Charlie — both point to natural-language operation. However, most of the product surface (workflows, scheduling, dialing windows, phone number setup) is described as configured through UI/API rather than conversational commands, and there's no independent/hands-on evidence of Charlie's NL command scope or reliability. missing for 10: hands-on evidence of Charlie executing complex natural-language commands, documentation detailing the breadth of actions controllable purely via NL, independent corroboration beyond vendor docs.", "evidenceIds": [ "telli-docs-11", "telli-docs-22", "telli-probe-3" ] }, { "productId": "telli", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "telli offers a REST API, webhooks, and an MCP server for AI assistants, but there is no mention anywhere in the evidence of an official CLI tool for telli. missing for 10: any documentation or reference to a telli CLI, its commands, or installation instructions.", "evidenceIds": [] }, { "productId": "telli", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "telli documents a REST API with endpoints (schedule-call, import-phone-number), API key/bearer auth, webhooks, and a downloadable OpenAPI spec (confirmed live at openapi.json), plus an MCP server for AI tool access — enabling full programmatic/agentic control. missing for 10: independent third-party validation of API robustness beyond vendor docs.", "evidenceIds": [ "telli-docs-12", "telli-docs-13", "telli-docs-15", "telli-docs-16", "telli-docs-17", "telli-docs-19", "telli-docs-21", "telli-docs-22", "telli-probe-2", "telli-probe-3" ] }, { "productId": "telli", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "telli documents a single API key (bearer token) generated in Settings > Developer, but there's no evidence of scoped permissions, role-based keys, or least-privilege credential issuance for agents specifically.", "evidenceIds": [ "telli-docs-15" ] }, { "productId": "telli", "storyId": "agentic-sdks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "telli provides a REST API with an OpenAPI spec that can be used to generate client libraries, plus webhooks, custom tools, and an MCP server for agent integration, but there is no evidence of telli-maintained official SDKs in specific languages (e.g., Python/JS packages published by telli) — only self-generated clients from the OpenAPI spec. missing for 10: dedicated first-party SDK packages/repos, language-specific SDK documentation, versioning/release notes for such SDKs.", "evidenceIds": [ "telli-docs-12", "telli-docs-15", "telli-docs-16", "telli-probe-2" ] }, { "productId": "telli", "storyId": "agentic-webhooks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "telli documents a webhooks system that pushes real-time POST notifications about call events, status changes, and outcomes, explicitly described as event-driven subscription for external systems (telli-docs-13, telli-docs-21). Missing for 10: no evidence of configurable event-type filtering/subscription granularity or independent third-party confirmation of reliability.", "evidenceIds": [ "telli-docs-13", "telli-docs-21", "telli-docs-12" ] }, { "productId": "telli", "storyId": "ai-assisted-agent-authoring", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Charlie, telli's built-in AI assistant, is documented as helping to 'build and refine agents, automate workflows, and analyze your calls,' which supports the core idea of AI-assisted agent authoring, and the MCP server lets external AI tools interact with Charlie too. However there's no concrete evidence of generating test cases from a description, or detailed examples of prompt/flow generation output. missing for 10: documented examples of Charlie generating/improving prompts or flows from a natural-language description, evidence of AI-generated test case creation, and independent/hands-on confirmation of these capabilities in practice.", "evidenceIds": [ "telli-docs-11", "telli-docs-22", "telli-probe-3" ] }, { "productId": "telli", "storyId": "ai-call-qa-scoring", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "telli provides analytics like reach rate, success rate, call outcome fields, transcripts, and manual feedback submission for agent improvement, but there's no evidence of automated AI-driven quality scoring, failure flagging, or resolution analysis performed autonomously by the platform's AI — feedback and analysis appear human-driven rather than AI-automated review. missing for 10: automated AI scoring of call quality, automatic failure/anomaly flagging, AI-generated resolution analysis without human review.", "evidenceIds": [ "telli-docs-7", "telli-docs-8", "telli-docs-10" ] }, { "productId": "telli", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "telli publishes an OpenAPI spec (openapi.json) and API docs with downloadable schemas for generating client libraries or feeding coding agents, but there is no evidence of an interactive reference with runnable/'try it' examples in-browser. missing for 10: an interactive try-it console, live runnable code samples, or sandboxed request execution within the docs site.", "evidenceIds": [ "telli-docs-16", "telli-probe-2", "telli-docs-15" ] }, { "productId": "telli", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "telli provides a downloadable OpenAPI spec confirmed live at /openapi.json, and docs explicitly state it can be downloaded to generate client libraries, import into tools, or give coding agents machine-readable API context. Missing for 10: no independent third-party confirmation of spec completeness/versioning beyond first-party docs and probe.", "evidenceIds": [ "telli-docs-16", "telli-probe-2" ] }, { "productId": "telli", "storyId": "api-sandbox", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "telli's agent builder includes a 'Test call' feature that lets users dial a draft or saved agent version and review the recording/transcript before going live, which functions as a lightweight test mode (telli-docs-28, telli-docs-29). However, there is no documentation of a dedicated sandbox environment, separate test dataset, or explicit guarantee that test calls don't write to production contacts/analytics. missing for 10: explicit sandbox/staging environment separate from production, confirmation that test calls don't affect production data/metrics, and any mention of test API keys or isolated test accounts.", "evidenceIds": [ "telli-docs-28", "telli-docs-29" ] }, { "productId": "telli", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows an OpenAPI spec and API key auth exist, but nothing in the pack mentions API versioning (e.g., v1/v2 paths, version headers) or any documented deprecation/sunset policy for endpoints. Missing for 10: explicit API version scheme, changelog of breaking changes, and a stated deprecation/sunset policy.", "evidenceIds": [ "telli-docs-12", "telli-docs-15", "telli-probe-2" ] }, { "productId": "telli", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Telli's platform is built around calling large contact lists (dialing windows, calling strategies, retry behavior, reach-rate metrics) and its REST API lets users 'programmatically manage contacts, schedule calls' at scale, implying bulk campaign operations. However, there is no explicit documentation of a bulk-upload/bulk-schedule endpoint or UI for acting on many items simultaneously — the schedule-call endpoint documented is single-call oriented. Missing for 10: explicit bulk-contact import/schedule endpoint, documented batch size limits, and independent evidence of large-scale bulk execution.", "evidenceIds": [ "telli-docs-9", "telli-docs-7", "telli-docs-12", "telli-docs-17", "telli-probe-1" ] }, { "productId": "telli", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "telli's workflows can be triggered by call events (implied default) and now by a recurring schedule, and webhooks notify external systems on call events, showing some event-driven automation. However there's no documented rule-definition interface (conditions, event-type selection, if/then logic) for arbitrary events beyond calls/schedule. Missing for 10: explicit rule/condition builder, broader event-type coverage beyond calls, and evidence of user-defined trigger logic rather than fixed schedule/call triggers.", "evidenceIds": [ "telli-docs-1", "telli-docs-13", "telli-docs-21" ] }, { "productId": "telli", "storyId": "automation-scheduled-jobs", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "telli documents scheduled triggers that let workflows run recurringly on an interval/start time without a call event, directly matching the recurring-workflow story, and also supports scheduling individual calls via API. missing for 10: independent/hands-on corroboration of recurrence reliability and detail on editing/monitoring scheduled workflow runs.", "evidenceIds": [ "telli-docs-1", "telli-docs-17", "telli-docs-18" ] }, { "productId": "telli", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The only hint of versioning is in agent testing, where you can pick 'your draft with unsaved changes or any saved revision' (telli-docs-28), implying saved agent versions exist. However there is no documented review interface for comparing versions, no explicit rollback/restore action, and no versioning mentioned for broader automations/workflows (only agents). Missing for 10: explicit rollback UI/API, version diff/review tooling, and versioning coverage for workflows beyond agent test calls.", "evidenceIds": [ "telli-docs-28" ] }, { "productId": "telli", "storyId": "barge-in-interruptions", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses barge-in, interruption handling, speech-stopping, or turn recovery mechanics; the docs cover scheduling, workflows, analytics, and integrations but never mention conversational turn-taking behavior.", "evidenceIds": [] }, { "productId": "telli", "storyId": "batch-outbound-campaigns", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "telli documents dialing windows, calling strategies, and retry behavior (telli-docs-9) plus a schedule-call API endpoint that respects dialer windows or can bypass them (telli-docs-17, telli-docs-18) and reach-rate/success metrics (telli-docs-7), which together support scheduled, throughput-controlled outbound calling. However there is no explicit mention of a 'campaign' concept, bulk contact upload, or concurrency/rate limits for large batches. Missing for 10: explicit batch/campaign creation UI or endpoint, bulk contact import for calling, and documented concurrency/throughput limits beyond dialing windows.", "evidenceIds": [ "telli-docs-9", "telli-docs-17", "telli-docs-18", "telli-docs-7", "telli-docs-1" ] }, { "productId": "telli", "storyId": "build-phone-agent-fast", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "telli provides agent builder, test call feature, voice cloning, and phone number setup (forwarding or SIP trunk import) which together support building and testing a voice agent with prompt, voice, and number, but there's no explicit walkthrough or evidence confirming a full end-to-end setup can be completed within an hour or that phone number provisioning is instant/self-serve for new numbers. missing for 10: a documented quick-start/time-to-first-call benchmark, evidence of instant/native number provisioning (vs SIP/forwarding setup), and independent/hands-on confirmation of the full flow's speed.", "evidenceIds": [ "telli-docs-28", "telli-docs-29", "telli-docs-27", "telli-docs-23", "telli-docs-24", "telli-docs-19" ] }, { "productId": "telli", "storyId": "call-analytics-dashboard", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "telli's docs describe built-in dashboards tracking reach rate, call duration, and success rate, plus per-call analysis of transcripts and outcome fields, and a live monitoring overview—covering most of the story without needing custom tooling. However, there is no explicit mention of sentiment analysis in dashboards. missing for 10: sentiment scoring/analysis in dashboards, independent/hands-on corroboration of dashboard usability.", "evidenceIds": [ "telli-docs-7", "telli-docs-8", "telli-docs-32" ] }, { "productId": "telli", "storyId": "call-transfer-escalation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs confirm a transfer tool that can move a call to another agent or number, matching the core escalation-to-human idea, but there's no mention of distinct warm vs. blind transfer modes or explicit context-passing to the human recipient. missing for 10: explicit warm/blind transfer distinction, evidence of context/data hand-off during transfer, hands-on confirmation of the feature in practice.", "evidenceIds": [ "telli-docs-31" ] }, { "productId": "telli", "storyId": "concurrency-scaling-limits", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item documents concurrency limits, per-account call caps, or scaling guarantees for simultaneous calls; docs mention dialing windows/retry strategy but not concurrency ceilings or auto-scaling behavior. This is a fair question for a voice-calling platform (scaling many simultaneous calls is core to the product's value), so 'none' applies rather than 'na'.", "evidenceIds": [] }, { "productId": "telli", "storyId": "consent-recording-controls", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "telli documents conditional recording based on caller consent and a configurable data-retention policy that auto-deletes contacts and call data after a defined age, directly addressing the founder's compliance need. However, evidence lacks detail on per-call granular controls (e.g., pause/resume, jurisdiction-specific disclosure scripts), audit logging of consent events, or independent verification that these features work as described. Missing for 10: per-call recording toggle/audit trail details, jurisdiction-specific disclosure templates, retention configuration granularity (e.g., per-data-type settings), and independent/hands-on confirmation of compliance behavior.", "evidenceIds": [ "telli-docs-25", "telli-docs-26" ] }, { "productId": "telli", "storyId": "custom-voice-cloning", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Voice cloning is documented as a real feature (telli-docs-27) letting founders give agents a custom voice, but the pack's only consent-related documentation concerns call recording consent (telli-docs-26), not a consent process specifically for cloning someone's voice. Missing for 10: explicit documented consent/authorization workflow for the voice being cloned (e.g., proof of rights from the voice owner), and any legal/compliance guardrails for cloning third-party voices.", "evidenceIds": [ "telli-docs-27", "telli-docs-26" ] }, { "productId": "telli", "storyId": "documented-latency-numbers", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions latency numbers, turn-taking performance, or tuning guidance for voice response speed; documentation covers workflows, API, webhooks, and agent building but nothing on end-to-end latency benchmarks or tuning.", "evidenceIds": [] }, { "productId": "telli", "storyId": "dtmf-ivr-voicemail", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack mentions DTMF keypress sending, IVR menu navigation, or voicemail detection/leaving; telli's docs cover scheduling, transfers, web search tool, and webhooks but nothing about touch-tone or answering machine handling.", "evidenceIds": [] }, { "productId": "telli", "storyId": "dynamic-call-personalization", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes contact properties and collected values being used to select SMS/WhatsApp recipients (telli-docs-5) and APIs to manage contacts and schedule calls (telli-docs-12, telli-docs-17), but nowhere does it document injecting dynamic variables or per-caller context into an agent's voice call script/persona at call time. Missing for 10: explicit documentation of variable placeholders in agent prompts, a mechanism for passing per-call custom data via the schedule-call API, and any example of personalized call content driven by caller-specific fields.", "evidenceIds": [ "telli-docs-5", "telli-docs-12", "telli-docs-17" ] }, { "productId": "telli", "storyId": "hipaa-regulated-workloads", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions HIPAA, BAA, SOC 2, or data-residency options; only data retention and consent-based recording are covered, which don't establish formal compliance certifications.", "evidenceIds": [] }, { "productId": "telli", "storyId": "knowledge-base-grounding", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of a built-in knowledge base or RAG feature for grounding agents on uploaded documents; the docs describe custom tools for API integration and a web_search tool for public information, but nothing about ingesting documents or content for retrieval-based answers.", "evidenceIds": [] }, { "productId": "telli", "storyId": "live-call-monitoring", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "telli offers a real-time Live Monitoring call overview (telli-docs-32) and webhooks that push real-time notifications on call events/status/outcomes (telli-docs-13, telli-docs-21), which a platform engineer could wire into an alerting pipeline, plus metric tracking like success/reach rate (telli-docs-7). However there is no documented built-in alerting mechanism, error-rate-spike detection, or agent-misbehavior alerting system. Missing for 10: native alert/notification thresholds, anomaly or error-rate-spike detection, and evidence of alert delivery (email/Slack/PagerDuty) tied to live monitoring.", "evidenceIds": [ "telli-docs-32", "telli-docs-13", "telli-docs-21", "telli-docs-7" ] }, { "productId": "telli", "storyId": "mid-call-tool-calling", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm custom tools let agents call external APIs mid-call (telli-docs-20) and built-in tool calls like web_search operate during live calls (telli-docs-30), plus transfer tools show mid-call action execution (telli-docs-31). However, there is no documentation addressing latency masking or filler-speech to avoid 'dead air' while awaiting API responses. missing for 10: explicit evidence of latency-hiding/filler speech during tool calls, independent hands-on confirmation of smooth voice continuity.", "evidenceIds": [ "telli-docs-20", "telli-docs-30", "telli-docs-31" ] }, { "productId": "telli", "storyId": "multilingual-conversations", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions multilingual support, language detection, or mid-call language switching for telli's voice agents; all documented features concern scheduling, webhooks, tools, analytics, and MCP integration.", "evidenceIds": [] }, { "productId": "telli", "storyId": "noise-robustness", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions noise suppression, audio filtering, or any mechanism for handling noisy call environments; documentation covers scheduling, workflows, analytics, and voice cloning but not audio quality/noise handling.", "evidenceIds": [] }, { "productId": "telli", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "telli exposes a REST API (with OpenAPI spec) covering contacts, scheduling calls, phone number import, custom tools, and webhooks, and also offers an MCP server so external AI tools can interact with Charlie. However, several UI-native features—agent builder/test-call dialog, live monitoring dashboard, side-by-side transcript/recording view, voice cloning setup, workflow trigger/recurring schedule configuration UI, and analytics dashboards—are not documented as API-accessible endpoints, so full UI/API parity is not evidenced. missing for 10: API endpoints for agent building/testing, live monitoring, voice cloning, workflow scheduling configuration, and analytics/reporting equivalent to UI dashboards.", "evidenceIds": [ "telli-docs-12", "telli-docs-15", "telli-docs-16", "telli-docs-17", "telli-docs-19", "telli-docs-20", "telli-docs-21", "telli-docs-22", "telli-probe-2", "telli-probe-3" ] }, { "productId": "telli", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of a bulk data-export feature or open-format data portability; only API endpoints to manage/read specific records and a data-retention deletion policy are documented, not a full self-service export/leave capability. Missing for 10: documented full-account export tool/endpoint, open-format (CSV/JSON) bulk export, and any explicit data-portability statement.", "evidenceIds": [ "telli-docs-12", "telli-docs-25" ] }, { "productId": "telli", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "telli", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "telli", "storyId": "phone-number-provisioning", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "telli's docs show a REST API with a schedule-call endpoint for outbound calls, an import-phone-number endpoint to bring SIP-trunk numbers under API control, webhooks for call events, and an OpenAPI spec confirming the API surface. However, inbound handling is documented mainly as manual forwarding/SIP trunk setup rather than a clear 'provision a new number via API' flow, and there's no explicit doc of purchasing/creating a fresh telli-hosted number through the API. missing for 10: a documented endpoint to provision/purchase brand-new phone numbers (not just import existing SIP trunk numbers), and clearer API-driven inbound call handling beyond forwarding.", "evidenceIds": [ "telli-docs-12", "telli-docs-17", "telli-docs-19", "telli-docs-21", "telli-docs-23", "telli-docs-24", "telli-probe-2" ] }, { "productId": "telli", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, region selection, or data storage location controls; data retention (deletion timing) is covered but that is a different capability from choosing storage region.", "evidenceIds": [] }, { "productId": "telli", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses AI training data usage or opt-out policies; data retention docs cover deletion of contacts/call data but not model-training exclusion.", "evidenceIds": [] }, { "productId": "telli", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "telli documents automatic data retention deletion (contacts and call data purged once they reach a defined age) and conditional consent-based recording, giving users some control over data lifecycle and privacy. However, there's no evidence of user-initiated manual deletion, export/right-to-access tools, or granular per-record deletion controls beyond the age-based policy. missing for 10: manual/on-demand deletion controls, data export capability, independent verification of retention enforcement.", "evidenceIds": [ "telli-docs-25", "telli-docs-26" ] }, { "productId": "telli", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "telli is a voice-agent platform, not a developer tool where telemetry opt-out is typically discussed, but the axis is still fair to ask (does the product track usage, and can a user disable it). The evidence pack (data retention, consent-based recording, etc.) never mentions any telemetry/usage-tracking opt-out mechanism for the product itself. Missing for 10: any documentation of telemetry collection practices or an opt-out setting/toggle.", "evidenceIds": [] }, { "productId": "telli", "storyId": "realtime-transcription-quality", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "telli documents transcript viewing and side-by-side playback (telli-docs-29) but provides no evidence of developer-facing control over STT provider selection, choice of underlying language models, or custom key-term/vocabulary boosting for transcription accuracy.", "evidenceIds": [ "telli-docs-29" ] }, { "productId": "telli", "storyId": "recordings-transcripts-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm transcripts and recordings exist and can be reviewed in-app (side-by-side view, consent-based recording) and telli exposes a general REST API plus webhooks for call events, but no endpoint is documented that explicitly returns full call recordings or transcripts programmatically for every call. missing for 10: a documented GET/recording or GET/transcript API endpoint, confirmation of bulk/enumerable retrieval, and independent verification that recordings are accessible via the API rather than only in the UI.", "evidenceIds": [ "telli-docs-8", "telli-docs-29", "telli-docs-26", "telli-docs-12", "telli-docs-13", "telli-docs-21", "telli-probe-2" ] }, { "productId": "telli", "storyId": "self-host-oss-runtime", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "telli", "storyId": "semantic-turn-detection", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack discusses turn-taking, end-of-turn detection, VAD, or interruption handling in telli's voice agent; the docs cover scheduling, workflows, tools, and integrations but not conversational latency/turn-taking mechanics.", "evidenceIds": [] }, { "productId": "telli", "storyId": "simulated-agent-testing", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "telli's agent builder includes a 'Test call' feature to trial a draft or saved agent version and review the recording/transcript side-by-side before going live, which covers pre-production testing, and feedback submission helps iterate on agent quality. However, there is no evidence of simulated/synthetic conversation runs, automated eval suites, or scripted test scenarios independent of an actual phone call. Missing for 10: automated evals framework, non-call simulated conversation testing, batch/regression testing tools.", "evidenceIds": [ "telli-docs-28", "telli-docs-29", "telli-docs-10" ] }, { "productId": "telli", "storyId": "sip-trunk-byoc", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly support importing numbers from a customer's own SIP trunk provider via a dedicated API endpoint, plus general SIP trunk connection for incoming/outgoing calls and call forwarding from an existing PBX/provider number. This directly matches the platform-engineer's need to bring their own carrier rather than being locked into bundled telephony. Missing for 10: no explicit mention of native Twilio/Telnyx number import (only generic SIP trunk import), and no independent/hands-on confirmation of the SIP trunk setup working in practice.", "evidenceIds": [ "telli-docs-19", "telli-docs-23", "telli-docs-24" ] }, { "productId": "telli", "storyId": "structured-outcome-extraction", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "telli documents webhooks pushing real-time call event/status/outcome data and a REST API for programmatic access, plus conversation views showing appointment/outcome fields and call outcome analysis, supporting structured post-call data delivery. However, there's no explicit documentation of a structured schema for entities/dispositions extraction or example payloads showing the full breadth of structured fields promised. missing for 10: example webhook/API payload showing structured entities and disposition fields, documentation of a dedicated 'call outcomes' data model, independent/hands-on confirmation of data completeness.", "evidenceIds": [ "telli-docs-13", "telli-docs-21", "telli-docs-8", "telli-docs-2", "telli-docs-12" ] }, { "productId": "telli", "storyId": "transparent-per-minute-pricing", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions pricing, per-minute rates, usage costs, or any cost calculator/estimator; the docs cover features (workflows, appointments, API, MCP, etc.) but nothing about published pricing.", "evidenceIds": [] }, { "productId": "telli", "storyId": "visual-flow-builder", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "telli has an 'agent builder' with test-call and versioning, and workflow triggers (scheduled, SMS/WhatsApp) suggesting some no-code workflow configuration, plus tool-calls like transfer to another agent implying handoffs. But there's no evidence of a visual flow builder with branching/state diagrams — evidence only shows agent settings, tools, and triggers, not a drag-and-drop conversation flow canvas. missing for 10: explicit visual builder UI with branching/state nodes, evidence of designing multi-step flows with conditional paths, and documentation of handoff logic beyond simple 'transfer tool'.", "evidenceIds": [ "telli-docs-28", "telli-docs-31", "telli-docs-5", "telli-docs-1" ] }, { "productId": "telli", "storyId": "voice-library-choice", "verdict": "partial", "quality": 3, "confidence": "medium", "rationale": "Docs mention voice cloning to give an agent a custom voice, but there is no evidence of a curated voice library to choose from or the ability to plug in multiple external TTS providers as the story requires. missing for 10: evidence of a selectable voice library, multi-provider TTS integration, developer API/config for switching TTS engines.", "evidenceIds": [ "telli-docs-27" ] }, { "productId": "vapi", "storyId": "agent-consumes-mcp-tools", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows Vapi exposing its own APIs as an MCP *server* for external AI assistants (e.g., Claude Desktop) to manage calls/assistants — the reverse direction of the story. There is no documentation or probe showing a Vapi voice assistant can itself act as an MCP *client*, plugging in external MCP servers as tool sources mid-call; tool integration is instead described only via webhook-based custom tools and OpenAI-style function calling.", "evidenceIds": [ "vapi-docs-5", "vapi-docs-33", "vapi-docs-6", "vapi-docs-23", "vapi-probe-rt-2" ] }, { "productId": "vapi", "storyId": "agent-provisions-voice-agent", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Vapi documents API/SDK-based assistant creation, phone number attachment, and call placement (vapi-docs-1, vapi-docs-18, vapi-docs-19), a CLI for managing assistants, phone numbers, and calls entirely from the terminal (vapi-docs-2, vapi-docs-31, vapi-probe-rt-1 confirming keyless install/version), and an official MCP server exposing these same operations to any MCP-compatible agent, verified live and auth-gated in a runtime probe (vapi-docs-33, vapi-probe-rt-2). Together these three surfaces (API, CLI, MCP) cover the full agent-provision-number-call workflow without dashboard use. missing for 10: no single end-to-end hands-on trace showing one agent chaining create→attach→call purely via CLI/MCP/API in one session, and no independent (non-vendor) confirmation of the full workflow succeeding.", "evidenceIds": [ "vapi-docs-1", "vapi-docs-2", "vapi-docs-18", "vapi-docs-19", "vapi-docs-31", "vapi-docs-33", "vapi-probe-2", "vapi-probe-3", "vapi-probe-rt-1", "vapi-probe-rt-2" ] }, { "productId": "vapi", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Vapi confirms an llms.txt file exists and is served at docs.vapi.ai/llms.txt (HTTP 200), explicitly instructing agents on how to fetch clean Markdown per page, which directly satisfies the story. missing for 10: no independent/community corroboration of agents actually consuming llms.txt in practice, only first-party probe evidence.", "evidenceIds": [ "vapi-probe-1" ] }, { "productId": "vapi", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers building/testing assistants (Evals, Voice Test Suites), CLI/MCP management tools, and tool-calling, but nothing describes the product itself surfacing AI-generated insights or suggestions derived from a user's own call/usage data (e.g., analytics dashboards with AI-generated recommendations).", "evidenceIds": [] }, { "productId": "vapi", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Vapi assistants operate autonomously once a call starts—handling conversation flow, invoking custom tools via webhooks, and triggering server-side automations without further human input (vapi-docs-6, vapi-docs-17, vapi-docs-23, vapi-docs-34). However, this autonomy is scoped to an active call session; there is no evidence of scheduled/cron-style background jobs or agent loops that run independently of a triggered call or user interaction. Missing for 10: evidence of scheduled/background triggers outside live calls, persistent autonomous task queues, or proactive (non-call-triggered) automation runs.", "evidenceIds": [ "vapi-docs-6", "vapi-docs-17", "vapi-docs-23", "vapi-docs-34", "vapi-docs-20" ] }, { "productId": "vapi", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Vapi's docs describe building and configuring voice AI assistants for end-users (phone/web calls, tools, testing) and even an MCP server/CLI for managing those assistants, but there is no evidence of a built-in AI assistant/copilot inside the Vapi product itself that the AI-native user can delegate platform tasks to (e.g., an in-dashboard copilot that configures assistants or writes tools for you). The CLI mentions external IDE assistants (Cursor/Windsurf) gaining API knowledge, but that's a third-party tool, not a built-in in-product assistant.", "evidenceIds": [ "vapi-docs-36", "vapi-docs-2", "vapi-docs-5" ] }, { "productId": "vapi", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vapi ships an official CLI (vapi-docs-2/31/32, confirmed working keylessly in vapi-probe-rt-1) that can manage assistants, calls, and forward webhooks, which supports scripted/CI-style usage, and SDKs/REST APIs imply headless programmatic calls. However there is no explicit CI/automation documentation, no examples of running in a pipeline, and the MCP server requires bearer-key auth (vapi-probe-rt-2) which is unaddressed for CI contexts. missing for 10: explicit CI/automation examples or docs, non-interactive auth/service-account flow for CI, evidence of headless voice-testing/evals running in a pipeline.", "evidenceIds": [ "vapi-docs-2", "vapi-docs-31", "vapi-docs-32", "vapi-probe-rt-1", "vapi-docs-13", "vapi-docs-29", "vapi-probe-rt-2" ] }, { "productId": "vapi", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows Vapi exposes its own APIs as an MCP server (so external MCP clients like Claude Desktop can control Vapi), but this is the opposite direction from the story — there's no evidence that Vapi assistants can consume/plug-in external MCP servers to use their tools within a conversation. Vapi's tool docs describe webhook-based custom tools and OpenAI-style function calling only, with no mention of MCP-server ingestion.", "evidenceIds": [ "vapi-docs-5", "vapi-docs-33", "vapi-docs-6", "vapi-docs-34", "vapi-probe-2" ] }, { "productId": "vapi", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Vapi ships an official hosted MCP server (mcp.vapi.ai) exposing its assistant/phone/call APIs as MCP tools for any MCP-compatible agent (Claude Desktop, agent frameworks), documented in first-party docs and confirmed live via a runtime probe (401 bearer-key gate exactly as documented). Missing for 10: independent/community hands-on testimonials specifically about using the MCP server (community evidence only covers general demo/founder trivia, not MCP usage).", "evidenceIds": [ "vapi-docs-5", "vapi-docs-33", "vapi-probe-2", "vapi-probe-rt-2" ] }, { "productId": "vapi", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Vapi exposes an official MCP server that lets any MCP-compatible AI assistant (e.g. Claude Desktop) manage assistants, phone numbers, and calls via natural-language tool calls, and this endpoint is confirmed live and auth-gated in a runtime probe. It also ships a CLI and llms.txt docs optimized for AI agents to operate it. Missing for 10: no first-party evidence of a built-in chat/NL command console inside the Vapi product itself (beyond MCP/CLI proxies), and no independent hands-on report of an agent successfully performing multi-step tasks via MCP.", "evidenceIds": [ "vapi-docs-5", "vapi-docs-33", "vapi-docs-36", "vapi-probe-2", "vapi-probe-rt-2", "vapi-probe-rt-1", "vapi-probe-1" ] }, { "productId": "vapi", "storyId": "agentic-official-cli", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Vapi ships a documented official CLI for managing assistants, phone numbers, and calls from the terminal, with webhook forwarding, org/environment switching, and tech-stack auto-detection, and this was independently verified in a runtime probe (successful install and `--version` output). missing for 10: no independent third-party review or community discussion of the CLI's day-to-day usage beyond the vendor docs and single install probe.", "evidenceIds": [ "vapi-docs-2", "vapi-docs-3", "vapi-docs-4", "vapi-docs-27", "vapi-docs-31", "vapi-docs-32", "vapi-probe-3", "vapi-probe-rt-1" ] }, { "productId": "vapi", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Vapi ships a documented public API/SDK (assistant management, calls, tools), a CLI for terminal-driven workflows, and a hosted MCP server exposing the API as tools, all confirmed by runtime probes (CLI installs and runs, MCP endpoint live and auth-gated as documented). This directly satisfies programmatic/AI-native control via a documented public interface. Missing for 10: no independent third-party developer report deeply exercising the API beyond docs/probes.", "evidenceIds": [ "vapi-docs-1", "vapi-docs-2", "vapi-docs-5", "vapi-docs-33", "vapi-probe-1", "vapi-probe-2", "vapi-probe-3", "vapi-probe-rt-1", "vapi-probe-rt-2" ] }, { "productId": "vapi", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Vapi's enterprise docs mention Role Based Access Control (RBAC) and SSO, implying some access-scoping capability at the org level, and its MCP server/API require bearer API keys — but there is no documented mechanism for issuing per-agent, least-privilege scoped API keys or tokens tailored to a specific agent's permissions. missing for 10: explicit scoped API key creation/management UI or API, documentation of key-level permission granularity, and independent confirmation that RBAC restricts agent credentials rather than just human dashboard users.", "evidenceIds": [ "vapi-docs-11", "vapi-docs-25", "vapi-probe-rt-2" ] }, { "productId": "vapi", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Vapi documents an official JS/TS SDK usage pattern, official CLI (probe-confirmed installed and runnable), and official MCP server (probe-confirmed live and auth-gated), all clearly aimed at AI-native/agentic developer workflows including IDE assistant integration (Cursor/Windsurf/VSCode). missing for 10: independent third-party corroboration of SDK code quality and coverage across multiple languages beyond the docs snippets.", "evidenceIds": [ "vapi-docs-1", "vapi-docs-2", "vapi-docs-5", "vapi-docs-33", "vapi-docs-36", "vapi-probe-2", "vapi-probe-3", "vapi-probe-rt-1", "vapi-probe-rt-2" ] }, { "productId": "vapi", "storyId": "agentic-webhooks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Vapi supports webhook-based events via Server URL (tool-calls messages), custom webhook tools, and CLI webhook forwarding for local debugging, giving AI-native users a documented event subscription mechanism. missing for 10: independent/hands-on confirmation of webhook delivery reliability, a full event-type catalog/schema, and signature/verification documentation.", "evidenceIds": [ "vapi-docs-6", "vapi-docs-17", "vapi-docs-23", "vapi-docs-27", "vapi-docs-34" ] }, { "productId": "vapi", "storyId": "ai-assisted-agent-authoring", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Vapi provides CLI, MCP server, tool/webhook infrastructure, and testing frameworks (Evals, Voice Test Suites), plus a Customer Support template, but nothing indicates the platform itself uses AI to generate or improve prompts, conversation flows, or test cases from a natural-language description — testing tools require manually defined mock conversations/scripts rather than AI-authored ones.", "evidenceIds": [ "vapi-docs-12", "vapi-docs-13", "vapi-docs-28", "vapi-docs-29", "vapi-docs-16" ] }, { "productId": "vapi", "storyId": "ai-call-qa-scoring", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Vapi's Evals framework provides automated validation of assistant behavior via mock conversations and its Voice Test Suites use an AI tester to simulate calls and check outcomes, which is adjacent to automated call review, but neither is documented as automatically scoring, flagging failures in, or analyzing resolution of real production calls after the fact. Missing for 10: evidence of automatic post-call quality scoring on live/production transcripts, explicit failure-flagging analytics, and resolution-outcome analysis dashboards distinct from pre-production test suites.", "evidenceIds": [ "vapi-docs-13", "vapi-docs-29", "vapi-docs-12", "vapi-docs-28" ] }, { "productId": "vapi", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows static markdown documentation, code snippets, a CLI, and an MCP server, but no interactive API reference or runnable/try-it-out examples are described anywhere in the docs pages cited.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Vapi's CLI, MCP server, docs, and SDKs, but no citation mentions an OpenAPI spec, API reference schema, or any machine-readable spec file available for download. This is a fair axis for an API-first product, but no supporting evidence exists in the pack.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "api-sandbox", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Vapi offers dedicated Voice Test Suites (AI tester simulating conversations) and an Evals framework explicitly for creating mock conversations to 'validate your agents work correctly before production,' plus CLI webhook forwarding to localhost for local development/debugging — all separate from live production calls. Missing for 10: explicit vendor use of the term 'sandbox environment' and independent/hands-on verification that test runs are fully isolated from production data.", "evidenceIds": [ "vapi-docs-12", "vapi-docs-13", "vapi-docs-28", "vapi-docs-29", "vapi-docs-27", "vapi-docs-31" ] }, { "productId": "vapi", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item mentions API versioning scheme, version numbers in endpoints, or any documented deprecation/sunset policy for Vapi's APIs; the docs cover features (CLI, MCP, tools, voices) but not API lifecycle governance.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents CLI/SDK/MCP management of individual assistants, phone numbers, and calls, but nowhere shows bulk/batch endpoints or commands (e.g., batch-create, bulk-update, mass-delete) for operating across many items at once.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Vapi supports event-driven server-side tools/webhooks that fire on call events (tool-calls messages, server-url events) which act as a rule-trigger mechanism, but this is scoped to voice-call events only rather than a general-purpose automation/rules engine for arbitrary triggers. missing for 10: a documented general condition/trigger-action rules engine spanning non-call events, cross-system automation, or independent hands-on evidence of complex conditional automation chains.", "evidenceIds": [ "vapi-docs-17", "vapi-docs-23", "vapi-docs-34", "vapi-docs-6" ] }, { "productId": "vapi", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack shows Vapi supports scheduling recurring jobs, workflows, or automated recurring calls/tasks — the docs cover assistants, tools, CLI, MCP server, testing, and voice customization but nothing about cron-like scheduling or recurring automation triggers.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Vapi's docs cover assistant creation, tools, CLI, MCP server, and testing/evals, but there is no mention of versioning assistant configurations, review/approval workflows, or rollback to prior versions of automations. No evidence pack item addresses version history or rollback capability.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "barge-in-interruptions", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers CLI, MCP server, tools, voices, compliance, and testing, but contains no documentation or evidence about interruption handling, barge-in behavior, context updates during interruptions, or turn recovery mechanics. This is a core latency/turn-taking capability for voice agents, but nothing in the pack addresses it directly.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "batch-outbound-campaigns", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers assistant creation, tools, CLI, MCP server, testing, and voice customization, but nothing addresses batch/outbound campaign management, call scheduling, or throughput/concurrency controls for bulk dialing. No mention of a campaigns API, CSV/list upload, dialer pacing, or rate-limiting controls for outbound calling at scale.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "build-phone-agent-fast", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly walk through creating a voice assistant, connecting a phone number, and making a first call in under 5 minutes (vapi-docs-18, vapi-docs-19), with voice selection via the Voice Library (vapi-docs-8, vapi-docs-24) and dashboard-driven assistant creation with templates (vapi-docs-16); community evidence corroborates a working live-call demo (vapi-comm-1). Missing for 10: independent third-party benchmarking of the full 'within an hour' timing claim beyond vendor docs and a single anecdotal community comment.", "evidenceIds": [ "vapi-docs-18", "vapi-docs-19", "vapi-docs-16", "vapi-docs-8", "vapi-docs-24", "vapi-comm-1" ] }, { "productId": "vapi", "storyId": "call-analytics-dashboard", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents assistant creation, tools, testing (Evals, Voice Test Suites), CLI/MCP management, and voice library features, but contains no mention of a dashboard showing call success rates, durations, outcomes, or sentiment analytics — the core of this story.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "call-transfer-escalation", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions call transfer, warm/blind transfer, or passing context to a human agent; the evidence covers assistants, CLI, MCP server, tools/webhooks, voices, and compliance, but nothing about live-call escalation/transfer capabilities.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "concurrency-scaling-limits", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack items document concurrency limits, per-org call caps, autoscaling behavior, or capacity-request processes; the docs cover CLI, MCP server, tools, voices, and enterprise SSO but nothing about simultaneous-call scaling limits or how to raise them.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "consent-recording-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers HIPAA toggle, SSO/RBAC, testing/evals, and CLI/MCP tooling, but contains no mention of call-recording enable/disable controls, consent disclosure features, or configurable data retention policies for call recordings — a fair and applicable axis for a voice-calling platform.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "custom-voice-cloning", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Vapi docs show you can plug in a 'custom voice' by setting the voice property with a provider ID (vapi-docs-9, vapi-docs-26) and browse a Voice Library (vapi-docs-8, vapi-docs-35), but nothing describes an actual voice-cloning workflow or any documented consent/verification process required before cloning a brand voice. Missing for 10: a described voice-cloning feature/flow, a documented consent or identity-verification process, and any policy language governing voice cloning.", "evidenceIds": [ "vapi-docs-9", "vapi-docs-26", "vapi-docs-8", "vapi-docs-35" ] }, { "productId": "vapi", "storyId": "documented-latency-numbers", "verdict": "partial", "quality": 3, "confidence": "medium", "rationale": "Vapi's docs mention that 'Ultra Fast' vs 'High Intelligence' model settings trade off latency, giving conceptual tuning guidance, but there are no documented end-to-end latency numbers (e.g., ms benchmarks) or detailed tuning playbooks in the evidence pack. Missing for 10: concrete latency benchmark figures, component-level latency breakdowns (STT/LLM/TTS), and independent verification of speed claims.", "evidenceIds": [ "vapi-docs-15" ] }, { "productId": "vapi", "storyId": "dtmf-ivr-voicemail", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no mention of DTMF keypress sending, IVR menu navigation, or voicemail detection/leaving functionality anywhere in the docs or community items — these telephony-specific capabilities are entirely unevidenced despite being a plausible axis for a voice AI telephony product.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "dynamic-call-personalization", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers assistant creation, tools/webhooks, CLI, MCP server, voices, and testing, but contains no documentation of variable injection, assistantOverrides, or per-caller context personalization at call time. Missing for 10: docs on dynamic variable substitution (e.g., {{variableName}} templating), call-time overrides/metadata injection, and any example showing per-caller personalization.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "hipaa-regulated-workloads", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Vapi documents HIPAA compliance toggle in the dashboard and enterprise SSO/RBAC, indicating some regulated-workload support, but evidence lacks explicit SOC 2 certification details, BAA execution process, or data-residency region options. missing for 10: SOC 2 report/certification evidence, explicit BAA process details, data-residency/region selection options, independent audit confirmation.", "evidenceIds": [ "vapi-docs-10", "vapi-docs-11", "vapi-docs-25" ] }, { "productId": "vapi", "storyId": "knowledge-base-grounding", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Vapi is a voice-agent platform focused on assistants, phone numbers, custom webhook tools, and telephony/testing; the evidence pack shows no built-in knowledge base or RAG feature for grounding assistants on uploaded documents. Custom tools/webhooks could be used to build a workaround, but no document ingestion or retrieval capability is documented.", "evidenceIds": [ "vapi-docs-6", "vapi-docs-20", "vapi-docs-34" ] }, { "productId": "vapi", "storyId": "live-call-monitoring", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers pre-production testing (Voice Test Suites, Evals) and webhook debugging via the CLI, but contains no mention of live call monitoring dashboards, real-time alerting, or error-rate-spike detection for production traffic. This is a fair axis for a voice AI platform, but nothing in the pack demonstrates it.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "mid-call-tool-calling", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vapi's docs clearly show mid-call function/tool calling via webhook Server URLs and OpenAI-style tool-calls messages, letting the assistant fetch external API results and use them in conversation (vapi-docs-6, vapi-docs-7, vapi-docs-17, vapi-docs-20, vapi-docs-23, vapi-docs-34). However, the pack lacks explicit evidence about mechanisms for avoiding 'dead air' during the API call latency (e.g., async tool config, filler phrases, or interim speech) — only general latency-tuning docs for model selection are present (vapi-docs-15). Missing for 10: explicit documentation/demo of filler/interim speech or async tool handling during function execution, and independent hands-on confirmation that the conversation flow feels seamless during a live tool call.", "evidenceIds": [ "vapi-docs-6", "vapi-docs-7", "vapi-docs-17", "vapi-docs-20", "vapi-docs-23", "vapi-docs-34", "vapi-docs-15" ] }, { "productId": "vapi", "storyId": "multilingual-conversations", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers CLI, MCP server, tools, voices, compliance, and testing, but contains no mention of multi-language support, language detection, or mid-call language switching capabilities. Absence of evidence for this applicable axis (multilingual voice agents are a fair capability to expect from a voice AI platform) means it must be judged 'none'. missing for 10: any mention of multi-language conversation support, language auto-detection, or mid-call language switching features.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "noise-robustness", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items mention noise suppression, background noise filtering, or audio denoising features for handling noisy real-world calls; the pack covers voices, tools, CLI, MCP, latency modes, and testing but nothing about audio filtering.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "openness-api-parity", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Vapi's API/SDK covers assistant, phone number, and call management, backed by a full CLI (vapi-docs-2 through 4, 27, 31-32, verified live in vapi-probe-rt-1) and an official MCP server exposing the same operations for external agents (vapi-docs-5,33, confirmed live and auth-gated in vapi-probe-rt-2). Dashboard-only conveniences like Voice Library browsing/preview remain UI-centric, though voice IDs and custom voices can be set via API (vapi-docs-9,24,26,35). Missing for 10: explicit confirmation that every dashboard-only feature (e.g., visual Voice Library browsing, HIPAA toggle) has a full API equivalent, and independent (non-vendor) verification of full UI/API parity.", "evidenceIds": [ "vapi-docs-2", "vapi-docs-3", "vapi-docs-5", "vapi-docs-33", "vapi-docs-31", "vapi-docs-32", "vapi-probe-rt-1", "vapi-probe-rt-2", "vapi-docs-9", "vapi-docs-24" ] }, { "productId": "vapi", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of a data export feature, open-format data dumps, or account portability/deletion tooling; the docs cover CLI, MCP server, tools, and voice features but nothing about exporting assistants/call data/transcripts in open formats for user exit. Missing for 10: documented export functionality, open data formats supported, and any account/data portability guarantees.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Vapi is a closed, commercial SaaS platform; nothing in the evidence indicates its core source (server, dashboard, or model runtime) is published under an open license—only SDKs/CLI tooling and docs are mentioned, and the one open-source reference (pipecat) is a third-party project, not Vapi itself.", "evidenceIds": [ "vapi-comm-2" ] }, { "productId": "vapi", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Vapi is presented exclusively as a hosted cloud platform (dashboard, hosted MCP server, hosted API/CLI against cloud endpoints); nothing in the evidence pack mentions a self-hostable core engine, open-source repo for the core product, or on-prem deployment option (the mentioned pipecat is a different open-source project, not Vapi itself). This is a fair axis for a voice-AI platform, but no evidence supports self-hosting.", "evidenceIds": [ "vapi-comm-2", "vapi-probe-rt-2", "vapi-docs-33" ] }, { "productId": "vapi", "storyId": "phone-number-provisioning", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly cover creating an assistant, connecting it to a phone number, and making inbound/outbound calls via the quickstart, plus CLI/MCP support for managing phone numbers and calls, and SIP for advanced telephony integration, with runtime probes confirming the CLI and MCP endpoints work as documented. Missing for 10: independent hands-on confirmation of actual outbound call placement via raw API (only demo/inbound anecdote in community evidence) and explicit multi-number provisioning workflow details.", "evidenceIds": [ "vapi-docs-1", "vapi-docs-18", "vapi-docs-19", "vapi-docs-2", "vapi-docs-5", "vapi-docs-33", "vapi-docs-14", "vapi-docs-22", "vapi-probe-rt-1", "vapi-probe-rt-2", "vapi-comm-1" ] }, { "productId": "vapi", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no mention of data residency, regional storage options, or geographic data controls anywhere in Vapi's docs (only HIPAA toggle and SSO/RBAC are mentioned as compliance features). Missing for 10: any documentation of region selection, data residency guarantees, or storage location controls.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data-training opt-out, model-training data policies, or privacy controls preventing AI training use; only HIPAA and SSO/RBAC enterprise features are mentioned, which are unrelated to this specific claim. missing for 10: explicit data-training opt-out policy, DPA/ToS language on training use, dashboard/API setting to exclude data from model training.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers HIPAA, SSO/RBAC, tooling, CLI, and MCP server features but contains no documentation about data retention policies, call/recording deletion controls, or user-initiated data export/erasure mechanisms. Missing for 10: retention period settings, deletion/erasure APIs or dashboard controls, data export tools, and any policy documentation on how long call data/transcripts are stored.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item addresses telemetry opt-out or usage tracking controls; Vapi's docs cover HIPAA/SSO/RBAC compliance but not a telemetry toggle.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "realtime-transcription-quality", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation about selecting/configuring an STT provider, choosing language models for transcription, or defining custom key terms/vocabulary for real-time transcription — topics like 'transcriber', 'Deepgram', or keyword boosting are absent. While Vapi is clearly a voice AI platform where such controls are a fair axis, none of the provided docs, community items, or probes address it.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "recordings-transcripts-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation, API reference, or example demonstrating retrieval of call recordings or transcripts, despite this being a natural, expected capability for a voice AI platform; only tangential tooling (CLI, MCP server, custom tools, testing/evals) is covered.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "self-host-oss-runtime", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Vapi is presented entirely as a hosted SaaS platform with a CLI, MCP server, and dashboard for managing cloud-hosted assistants; there is no evidence of open-source runtime code or any self-hosting option. One community note even references that a different product (pipecat by daily.co) is the open-source alternative, implicitly contrasting with Vapi's closed hosted model.", "evidenceIds": [ "vapi-comm-2", "vapi-docs-31", "vapi-probe-rt-2" ] }, { "productId": "vapi", "storyId": "semantic-turn-detection", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack lacks any mention of model-based end-of-turn detection, turn-taking models, or semantic VAD alternatives; only latency tuning between 'Ultra Fast' and 'High Intelligence' model modes is documented, which does not address end-of-turn detection beyond silence-based VAD. Missing for 10: any documentation of a model-based or semantic end-of-turn detector, comparison to VAD-only timeout behavior, or configuration options for turn-taking sensitivity.", "evidenceIds": [ "vapi-docs-15" ] }, { "productId": "vapi", "storyId": "simulated-agent-testing", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Vapi documents two dedicated testing features directly matching the story: Voice Test Suites for simulated phone conversations via an AI tester following scripted customer behavior, and Evals, a testing framework for mock conversations with automated validation before production. Missing for 10: independent/hands-on developer corroboration of these specific testing features beyond vendor docs.", "evidenceIds": [ "vapi-docs-12", "vapi-docs-28", "vapi-docs-13", "vapi-docs-29" ] }, { "productId": "vapi", "storyId": "sip-trunk-byoc", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm SIP connectivity: platform engineers can point any SIP client/softphone at a Vapi SIP URI to make/receive calls (vapi-docs-14, vapi-docs-22), which supports self-hosted/carrier SIP trunking use cases. However, the evidence never explicitly documents PBX/carrier trunk configuration options or a number-import workflow for Twilio/Telnyx accounts. missing for 10: explicit documentation of BYO-carrier/PBX trunk setup steps, and evidence of importing existing Twilio/Telnyx phone numbers into Vapi.", "evidenceIds": [ "vapi-docs-14", "vapi-docs-22" ] }, { "productId": "vapi", "storyId": "structured-outcome-extraction", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Vapi's webhook/tool-calling system for live in-call actions (tool-calls messages, custom webhook tools, function calling) but contains no mention of a post-call structured-data/analysis feature (outcomes, entities, dispositions) delivered via API or webhook after the call ends. Missing for 10: any docs on end-of-call reports, structured data extraction schemas, call analysis/summary webhooks, or an API endpoint returning call outcome/entity data.", "evidenceIds": [ "vapi-docs-17", "vapi-docs-23", "vapi-docs-34", "vapi-docs-6" ] }, { "productId": "vapi", "storyId": "transparent-per-minute-pricing", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of pricing pages, per-minute rates, usage-based cost breakdowns, or any pricing calculator/estimator; all evidence is about docs, CLI, MCP server, tools, and testing features. Absence of evidence for this applicable axis yields none.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "visual-flow-builder", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers Vapi's assistant configuration, tools, CLI, MCP server, voices, and testing, but there is no mention of a visual flow builder with branching, states, or handoffs — Vapi appears configuration/code-driven (assistant config, function tools, webhooks) rather than a no-code visual conversation designer.", "evidenceIds": [] }, { "productId": "vapi", "storyId": "voice-library-choice", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm a Voice Library with many previewable voices and support for custom voices with any supported provider, indicating multi-provider TTS flexibility, but the evidence never names or lists specific TTS providers (e.g., ElevenLabs, PlayHT, Azure) or details plugging in third-party/custom TTS engines beyond voice ID selection. missing for 10: explicit list of supported TTS providers, documentation of custom/BYO TTS provider integration mechanics, independent hands-on confirmation of voice quality/variety.", "evidenceIds": [ "vapi-docs-8", "vapi-docs-9", "vapi-docs-24", "vapi-docs-26", "vapi-docs-35" ] } ]