[ { "productId": "chatgpt", "storyId": "agentic-agent-docs", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "ChatGPT's Browser and MCP features let a user direct the agent to fetch and use third-party documentation or websites (e.g. Context7 for developer docs, browsing arbitrary URLs), which would technically allow pointing it at an llms.txt or agent-oriented doc page. The evidence pack also shows ChatGPT's own docs site publishes an llms.txt (probe-1), indicating familiarity with the convention, but there is no explicit documented feature or example of a user instructing ChatGPT to consume an llms.txt file specifically. Missing for 10: explicit product support/example for llms.txt ingestion, confirmation that browsing normalizes/parses such files as agent-context rather than generic web content, independent hands-on verification.", "evidenceIds": [ "chatgpt-probe-1", "chatgpt-docs-9", "chatgpt-docs-38", "chatgpt-docs-93", "chatgpt-docs-42" ] }, { "productId": "chatgpt", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "ChatGPT can analyze uploaded/connected data (files, projects, local folders, connected apps like Google Drive/SharePoint/Salesforce) and generate insights, summaries, and suggestions (e.g. 'Search the web, browse websites, compare sources, read files, analyze data, and summarize findings' and 'Create a deck, analyze files, draft a report, build a project plan'), with projects giving persistent context for tailored suggestions. Missing for 10: independent hands-on evidence specifically validating data-insight quality/accuracy (community evidence largely covers unrelated features like web search accuracy issues, not data-analysis insight quality).", "evidenceIds": [ "chatgpt-docs-90", "chatgpt-docs-66", "chatgpt-docs-15", "chatgpt-docs-16", "chatgpt-docs-87", "chatgpt-docs-68" ] }, { "productId": "chatgpt", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "ChatGPT's Scheduled Tasks/Automations feature explicitly lets users schedule recurring or event-triggered tasks (Gmail, Slack, GitHub events) to run autonomously in the background, with a dashboard to review active, paused, and completed runs. This is well-documented first-party functionality directly matching the story. Missing for 10: independent hands-on verification of background automation reliability and no community corroboration specifically about the automations/scheduled-tasks feature.", "evidenceIds": [ "chatgpt-docs-11", "chatgpt-docs-12", "chatgpt-docs-37", "chatgpt-docs-47", "chatgpt-docs-26" ] }, { "productId": "chatgpt", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "ChatGPT ships extensive built-in agentic delegation: Work/Codex modes carry tasks through to reviewable results, scheduled/background tasks and event-triggered automations, computer use and browser control, subagent delegation, and voice-initiated task delegation — all first-party features, not third-party add-ons. Missing for 10: independent hands-on verification of these newer agentic features (Work, subagents, Computer Use) beyond vendor docs.", "evidenceIds": [ "chatgpt-docs-11", "chatgpt-docs-12", "chatgpt-docs-28", "chatgpt-docs-32", "chatgpt-docs-33", "chatgpt-docs-103", "chatgpt-docs-13", "chatgpt-docs-9", "chatgpt-docs-10", "chatgpt-docs-71" ] }, { "productId": "chatgpt", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "ChatGPT's Codex offers a CLI and SDK explicitly documented for headless/CI use ('Use the Codex SDK to automate coding tasks, including jobs in CI'), plus CLI flags (--search, /fast) and a bulk-scan command (npx @openai/codex-security bulk-scan) that support non-interactive automation workflows. missing for 10: independent/hands-on confirmation of actual CI pipeline runs, and more detail on authentication/headless setup specifics for CI environments.", "evidenceIds": [ "chatgpt-docs-91", "chatgpt-docs-92", "chatgpt-docs-97", "chatgpt-docs-98", "chatgpt-docs-96", "chatgpt-docs-61" ] }, { "productId": "chatgpt", "storyId": "agentic-mcp-client", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "ChatGPT has first-party documentation for MCP support, explicitly describing configuring MCP servers (e.g., Context7) so ChatGPT/Codex can access third-party tools like documentation, browser, or Figma, with config stored in config.toml and scoping options for trusted projects, plus cross-client portability once configured. missing for 10: independent/hands-on community corroboration of MCP server usage specifically (community evidence covers plugins/browsing but not MCP directly).", "evidenceIds": [ "chatgpt-docs-3", "chatgpt-docs-4", "chatgpt-docs-38", "chatgpt-docs-42", "chatgpt-docs-55", "chatgpt-docs-74", "chatgpt-docs-93" ] }, { "productId": "chatgpt", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "ChatGPT/Codex has official first-party documentation for MCP support: users can add official MCP servers (e.g., Context7 for docs, Figma, browser tools) to extend the agent, configuration is stored in config.toml, servers can be scoped per-project, and setup carries across clients without redoing it. This directly evidences 'connecting an agent via an official MCP server.' Missing for 10: independent/hands-on community corroboration of MCP server usage and broader detail on the range of officially supported/verified servers beyond the Context7 example.", "evidenceIds": [ "chatgpt-docs-3", "chatgpt-docs-4", "chatgpt-docs-38", "chatgpt-docs-42", "chatgpt-docs-55", "chatgpt-docs-74" ] }, { "productId": "chatgpt", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "ChatGPT is fundamentally a natural-language interface: users type or speak requests and it carries out tasks (browsing, coding, file edits, image generation, scheduling, computer use, voice) via plain conversational commands, per extensive first-party docs and corroborating community reports of natural-language driven capability (e.g. running Docker/filesystem commands via prompts). Missing for 10: independent hands-on verification specifically of newer agentic features (Work, Goal mode, subagents) beyond vendor docs.", "evidenceIds": [ "chatgpt-docs-9", "chatgpt-docs-10", "chatgpt-docs-11", "chatgpt-docs-51", "chatgpt-docs-79", "chatgpt-docs-103", "chatgpt-comm-1", "chatgpt-comm-6" ] }, { "productId": "chatgpt", "storyId": "agentic-official-cli", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "ChatGPT ships an official Codex CLI with documented commands (--search, --cd, /memories, /fast, MCP config via config.toml, codex-security bulk-scan), and the llms.txt probe confirms 'Codex CLI' as a first-party interface for building and automating dev workflows, satisfying the AI-native official CLI story. Missing for 10: independent/hands-on community verification of the CLI itself, since community evidence only covers the general ChatGPT web product, not Codex CLI usage.", "evidenceIds": [ "chatgpt-docs-35", "chatgpt-docs-36", "chatgpt-docs-61", "chatgpt-docs-69", "chatgpt-docs-91", "chatgpt-docs-92", "chatgpt-docs-94", "chatgpt-docs-95", "chatgpt-docs-96", "chatgpt-docs-97", "chatgpt-docs-98", "chatgpt-probe-1" ] }, { "productId": "chatgpt", "storyId": "agentic-public-api", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "There is documented evidence of programmatic/scriptable access via the Codex CLI and Codex SDK ('Use the Codex SDK to automate coding tasks, including jobs in CI', CLI flags like --search, --cd, /fast, config.toml), which lets an AI-native user drive parts of the product outside the chat UI. However, a probe for a standard documented public API (openapi.json/swagger) at the docs site returned 404 on all candidate paths, and none of the evidence describes a general, versioned public API for driving ChatGPT itself (as opposed to Codex-specific tooling or MCP client configuration). Missing for 10: an explicit REST/GraphQL API reference for ChatGPT product actions, OpenAPI/swagger spec, and independent corroboration that non-Codex ChatGPT features are API-drivable.", "evidenceIds": [ "chatgpt-docs-91", "chatgpt-docs-92", "chatgpt-docs-97", "chatgpt-docs-98", "chatgpt-probe-3" ] }, { "productId": "chatgpt", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "ChatGPT/Codex docs show some least-privilege mechanisms for agent tool access — MCP servers can be scoped to a trusted project via `.codex/config.toml` (chatgpt-docs-55), sandbox command rules can restrict what Codex can run outside the sandbox (chatgpt-docs-95), and workspace admins can control access to browser use, plugins, and network access (chatgpt-docs-88), plus GPT builders can configure scoped third-party API actions (chatgpt-docs-34, chatgpt-docs-44). However there is no evidence of a user-facing mechanism to actually issue/generate a scoped or least-privilege API credential/token specifically for an agent to use. missing for 10: explicit scoped API key/token issuance workflow, granularity/expiry controls on such credentials, and any documentation or independent confirmation that a user can mint a restricted-permission credential for agent use.", "evidenceIds": [ "chatgpt-docs-55", "chatgpt-docs-95", "chatgpt-docs-88", "chatgpt-docs-34", "chatgpt-docs-44" ] }, { "productId": "chatgpt", "storyId": "agentic-sdks", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The pack contains a single line noting 'Use the Codex SDK to automate coding tasks, including jobs in CI' (chatgpt-docs-91), indicating an official SDK exists for Codex/automation, but there is no linked documentation, API reference, code samples, or independent corroboration of this SDK's use. The openapi probe found no public API spec, reinforcing that developer-facing SDK documentation is thin in this evidence set. missing for 10: dedicated SDK documentation/reference pages, code examples, language support details, independent developer reports of building with the SDK.", "evidenceIds": [ "chatgpt-docs-91", "chatgpt-probe-3" ] }, { "productId": "chatgpt", "storyId": "agentic-webhooks", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "ChatGPT's Automations/Scheduled Tasks support event-driven triggers ('act on pull request feedback without polling on a fixed cadence') tied to Gmail, Slack, and GitHub events, which functions like a webhook-subscription mechanism, but this is not exposed as a general-purpose webhook subscription API — it's limited to a few pre-integrated services with no documented endpoint/config for arbitrary webhook URLs. Missing for 10: a generic webhook subscription API/config for third-party or custom events, documentation of payload/security handling, and any independent confirmation of reliability.", "evidenceIds": [ "chatgpt-docs-12", "chatgpt-docs-26", "chatgpt-docs-47" ] }, { "productId": "chatgpt", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation of an interactive API reference or runnable code examples; a direct probe for OpenAPI/swagger specs on the docs site returned 404 across all candidate paths, and no other citation mentions an API reference sandbox or runnable snippets.", "evidenceIds": [ "chatgpt-probe-3" ] }, { "productId": "chatgpt", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The probe explicitly checked for an OpenAPI/machine-readable spec at all standard locations and found only 404s, with no other evidence pack item showing a downloadable API spec for ChatGPT itself.", "evidenceIds": [ "chatgpt-probe-3" ] }, { "productId": "chatgpt", "storyId": "api-sandbox", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Codex documentation indicates commands run inside a sandbox by default, with rules governing what can run 'outside the sandbox' (chatgpt-docs-95), and Codex Security's scanning is scoped to repos you own/have permission to assess (chatgpt-docs-97/98), implying isolated execution rather than direct production access. However, there is no explicit documentation describing a dedicated 'test/staging' environment distinct from production data, no detail on how production systems/data are excluded, and no independent/hands-on confirmation of this isolation. Missing for 10: explicit sandbox-vs-production data separation docs, details on network/data isolation guarantees, and independent verification of the sandbox boundary holding in practice.", "evidenceIds": [ "chatgpt-docs-95", "chatgpt-docs-97", "chatgpt-docs-98" ] }, { "productId": "chatgpt", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of a versioned API or a documented deprecation policy is present; the OpenAPI probe explicitly returned 404s and none of the docs mention API versioning or deprecation practices.", "evidenceIds": [ "chatgpt-probe-3" ] }, { "productId": "chatgpt", "storyId": "app-directory", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "ChatGPT documents a Plugins/connectors ecosystem — installing named plugins (Slack, Codex Security), connecting to third-party tools like Google Drive, SharePoint, Salesforce, and Gong, and GPT builders choosing 'connected apps' or custom actions from an approved list — which together describe browsing and adding third-party integrations to the assistant. Missing for 10: independent/hands-on confirmation of the actual browse-and-add directory UI/UX and its breadth beyond the named examples.", "evidenceIds": [ "chatgpt-docs-64", "chatgpt-docs-77", "chatgpt-docs-87", "chatgpt-docs-34", "chatgpt-docs-99", "chatgpt-docs-44" ] }, { "productId": "chatgpt", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "ChatGPT/Codex docs show concrete bulk-style automation: 'bulk-scan' scanning many repositories in one campaign (chatgpt-docs-98), a CLI for scanning multiple repos over time (chatgpt-docs-97), running multiple projects in parallel (chatgpt-docs-32), and delegating work to subagents (chatgpt-docs-103), plus background scheduled tasks with multiple runs (chatgpt-docs-37). This covers meaningful bulk/parallel automation but is concentrated in coding/security contexts rather than general-purpose bulk operations across arbitrary item sets, and there's no independent/hands-on corroboration of these bulk claims. Missing for 10: evidence of bulk operations on non-code items (e.g., bulk document/data processing), and independent verification that bulk-scan/subagent delegation works reliably at scale.", "evidenceIds": [ "chatgpt-docs-98", "chatgpt-docs-97", "chatgpt-docs-32", "chatgpt-docs-103", "chatgpt-docs-37" ] }, { "productId": "chatgpt", "storyId": "automation-rules-engine", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "ChatGPT's automations/scheduled tasks feature explicitly supports event-driven triggers (Gmail, Slack, GitHub) instead of just fixed schedules, letting users define rules that fire actions on external events, and lets them review active/paused/completed runs. Missing for 10: independent or hands-on verification that these event triggers work reliably in practice, and detail on how flexible/general the rule definitions can be beyond the three named integrations.", "evidenceIds": [ "chatgpt-docs-11", "chatgpt-docs-12", "chatgpt-docs-26", "chatgpt-docs-37", "chatgpt-docs-47" ] }, { "productId": "chatgpt", "storyId": "automation-scheduled-jobs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "ChatGPT's Scheduled Tasks/Automations feature explicitly supports recurring background jobs ('Schedule recurring tasks to run in the background,' with review of active/paused/completed runs) and even event-based triggers (Gmail, Slack, GitHub) as an alternative to fixed cadence. This directly matches the story of scheduling recurring jobs/workflows with first-party docs. Missing for 10: independent/hands-on community corroboration of the scheduling feature specifically (evidence pack community items don't address automations/scheduling) and more detail on reliability/limits in practice.", "evidenceIds": [ "chatgpt-docs-11", "chatgpt-docs-37", "chatgpt-docs-12", "chatgpt-docs-47", "chatgpt-docs-26" ] }, { "productId": "chatgpt", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "ChatGPT supports scheduled/recurring automations with a review of active/paused/completed task runs (chatgpt-docs-11, chatgpt-docs-37, chatgpt-docs-47), which gives some review and status visibility, but there is no evidence of version history, diffing between automation versions, or a rollback mechanism to revert an automation to a prior state. missing for 10: explicit versioning of automations, diff/comparison between versions, and a documented rollback/revert feature.", "evidenceIds": [ "chatgpt-docs-11", "chatgpt-docs-37", "chatgpt-docs-47" ] }, { "productId": "chatgpt", "storyId": "autonomous-background-tasks", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "ChatGPT's Tasks/automations feature explicitly supports scheduling and running multi-step work in the background (including event-triggered runs from Gmail/Slack/GitHub), with a review interface for active/completed runs, and 'Work' mode is described as carrying larger tasks through to a reviewable result; Codex further supports background coding tasks and subagent delegation. Missing for 10: independent hands-on verification of background task reliability/quality and no third-party review of the review/approval workflow.", "evidenceIds": [ "chatgpt-docs-11", "chatgpt-docs-12", "chatgpt-docs-37", "chatgpt-docs-28", "chatgpt-docs-47", "chatgpt-docs-103", "chatgpt-docs-39" ] }, { "productId": "chatgpt", "storyId": "browser-agent", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "ChatGPT's Browser feature explicitly lets it open websites, gather information, and take action on the user's behalf while staying in control, and can complete multi-step tasks like comparing options or completing actions on a website; this is reinforced by Computer Use (operating GUIs) and Work/Codex browser tab integration for signed-in site tasks. missing for 10: independent hands-on verification of complex multi-step website task completion, and detail on reliability/success rate limits.", "evidenceIds": [ "chatgpt-docs-9", "chatgpt-docs-49", "chatgpt-docs-10", "chatgpt-docs-48", "chatgpt-docs-23", "chatgpt-docs-25", "chatgpt-docs-54" ] }, { "productId": "chatgpt", "storyId": "computer-use-desktop", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "ChatGPT's Computer Use feature explicitly lets it see and operate GUIs on macOS/Windows, including testing desktop/mobile app flows, backed by Browser and Work agent capabilities that carry out multi-step tasks on the user's behalf. Documentation is extensive and detailed across multiple first-party sources describing screen operation, browser control, and file/app interaction. Missing for 10: independent hands-on verification of Computer Use reliability in practice.", "evidenceIds": [ "chatgpt-docs-10", "chatgpt-docs-48", "chatgpt-docs-75", "chatgpt-docs-43", "chatgpt-docs-9", "chatgpt-docs-49", "chatgpt-docs-32", "chatgpt-docs-33", "chatgpt-docs-89" ] }, { "productId": "chatgpt", "storyId": "connect-drive-calendar", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show ChatGPT can connect to Google Drive/SharePoint via plugins and can trigger on Gmail events, giving cloud-drive and email connectivity, and MCP/plugins provide a general connector framework for third-party tools. However, there is no explicit mention of calendar integration, and all evidence is vendor documentation with no independent/hands-on confirmation that these connectors actually surface content in answers. Missing for 10: explicit calendar connector support, independent corroboration of connector functionality, and clearer 'search across all three data sources in one answer' evidence.", "evidenceIds": [ "chatgpt-docs-87", "chatgpt-docs-64", "chatgpt-docs-12", "chatgpt-docs-47", "chatgpt-docs-93", "chatgpt-docs-38" ] }, { "productId": "chatgpt", "storyId": "create-edit-documents", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs strongly support creating and iteratively editing documents, presentations, spreadsheets, and PDFs, with review/annotation and export/download workflows (chatgpt-docs-8, chatgpt-docs-30, chatgpt-docs-60, chatgpt-docs-66, chatgpt-docs-68, chatgpt-docs-70), plus a dedicated artifacts/canvas viewer supporting annotations and comments for iterative refinement (chatgpt-docs-7, chatgpt-docs-18, chatgpt-docs-24, chatgpt-docs-101). Missing for 10: independent hands-on corroboration specifically of document/presentation export workflows (community evidence pack focuses on other capabilities like coding/search, not file export).", "evidenceIds": [ "chatgpt-docs-8", "chatgpt-docs-30", "chatgpt-docs-60", "chatgpt-docs-66", "chatgpt-docs-68", "chatgpt-docs-70", "chatgpt-docs-7", "chatgpt-docs-18", "chatgpt-docs-101" ] }, { "productId": "chatgpt", "storyId": "custom-bots", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm a 'GPT builder' concept for creating custom GPTs with configurable actions/connected apps and a dedicated 'gpts-and-sharing' doc implying sharing, but no evidence details custom instructions or uploading a knowledge base, and no independent/hands-on corroboration of the sharing flow. Missing for 10: explicit mention of setting custom instructions, uploading knowledge files, and community/hands-on validation of building and sharing a custom GPT.", "evidenceIds": [ "chatgpt-docs-34", "chatgpt-docs-44", "chatgpt-docs-99" ] }, { "productId": "chatgpt", "storyId": "data-analysis-code", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "ChatGPT's Code Interpreter/Data Analysis capability (documented via file drafting, spreadsheet/PDF handling, interactive visualizations, and downloadable finished files) lets it write and execute code against uploaded data to produce charts and computed answers, then export results as documents/spreadsheets/PDFs. Docs explicitly describe analyzing files, building interactive visualizations, and downloading completed files, and community evidence corroborates real code/file execution capability (e.g., running code/containers, file interpretation plugins). Missing for 10: no dedicated first-party doc page specifically titled 'Code Interpreter/Data Analysis' in this pack, and no hands-on example showing a specific chart-from-CSV walkthrough.", "evidenceIds": [ "chatgpt-docs-30", "chatgpt-docs-57", "chatgpt-docs-66", "chatgpt-docs-90", "chatgpt-docs-60", "chatgpt-comm-6" ] }, { "productId": "chatgpt", "storyId": "deep-research-report", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show ChatGPT can search the web, browse multiple sources, compare them, and produce cited results in-chat (docs-86, docs-90), and can turn research into finished documents/reports (docs-66, docs-68, docs-60). However, there is no explicit mention of a dedicated 'Deep Research' autonomous multi-source research mode/run with a structured long-form cited report as its own distinct feature — the evidence only shows general web-search-with-citations and document drafting capabilities. missing for 10: an explicitly named deep-research mode, evidence of autonomous long-running multi-source research runs, and a structured final cited-report output format.", "evidenceIds": [ "chatgpt-docs-86", "chatgpt-docs-90", "chatgpt-docs-66", "chatgpt-docs-68", "chatgpt-docs-60", "chatgpt-docs-61" ] }, { "productId": "chatgpt", "storyId": "desktop-app", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "The evidence confirms an official ChatGPT desktop app (macOS/Windows) that can 'use your computer' and run 'Computer Use' to see and operate GUIs, including screenshot-based screen access, plus desktop-only features like floating Pets controls and multi-browser support. However, there is no documentation of OS-level global keyboard shortcuts (e.g., a system-wide hotkey to invoke the app) or of general 'what's on my screen' querying outside the specific Computer Use/testing use case. Missing for 10: explicit OS-level shortcut/hotkey documentation, general screen-content awareness beyond Computer Use testing scenarios, independent hands-on confirmation of these desktop-specific features.", "evidenceIds": [ "chatgpt-docs-32", "chatgpt-docs-14", "chatgpt-docs-10", "chatgpt-docs-48", "chatgpt-docs-75", "chatgpt-docs-89", "chatgpt-docs-105", "chatgpt-docs-43" ] }, { "productId": "chatgpt", "storyId": "export-my-data", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no mention of data export, account data download, or chat history export features anywhere in the docs, community, or probe results; there is only a passing reference to 'data controls' applying to Computer Use content, which does not address exporting complete chat history or account data. Missing for 10: any documentation of an export data feature, its scope (chats, files, settings), format, or process, and any independent corroboration it works.", "evidenceIds": [ "chatgpt-docs-89" ] }, { "productId": "chatgpt", "storyId": "file-upload-analysis", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs confirm document/spreadsheet/PDF upload and analysis (draft/refine documents, presentations, spreadsheets, PDFs; 'analyze files'; 'read files, analyze data, and summarize findings'), which directly matches the story. However, evidence is entirely first-party documentation with no independent hands-on validation of accuracy specifically for uploaded file analysis (community evidence covers unrelated web-search accuracy issues, not file analysis). Missing for 10: independent/hands-on verification of analysis accuracy on real uploaded documents/spreadsheets/PDFs, and detail on limits (file size, complex spreadsheet formulas, OCR quality).", "evidenceIds": [ "chatgpt-docs-8", "chatgpt-docs-30", "chatgpt-docs-60", "chatgpt-docs-66", "chatgpt-docs-90" ] }, { "productId": "chatgpt", "storyId": "image-generation", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "ChatGPT's docs explicitly cover image generation and editing from natural-language prompts, including editing via Comment/annotations, reference images, and Canvas view for reviewing multiple images, plus mobile app support for generating/transforming images and creating stickers. This directly matches the knowledge-worker story of generating and editing images conversationally. Missing for 10: no independent/hands-on community corroboration specifically for image generation quality or reliability.", "evidenceIds": [ "chatgpt-docs-5", "chatgpt-docs-6", "chatgpt-docs-29", "chatgpt-docs-45", "chatgpt-docs-56", "chatgpt-docs-78", "chatgpt-docs-82", "chatgpt-docs-58", "chatgpt-docs-59" ] }, { "productId": "chatgpt", "storyId": "image-understanding", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "First-party docs confirm ChatGPT accepts photo/screenshot uploads and interprets their content (e.g., transcribing a handwritten recipe or identifying a landmark from a photo), which directly matches the knowledge-worker use case of sharing images for analysis. Missing for 10: independent/hands-on evidence corroborating accuracy of image interpretation, and explicit documentation of screenshot-specific analysis (e.g., UI screenshots) rather than just general photo uploads.", "evidenceIds": [ "chatgpt-docs-20", "chatgpt-docs-52", "chatgpt-docs-58" ] }, { "productId": "chatgpt", "storyId": "memory-across-chats", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "ChatGPT's Memories feature explicitly carries useful context from earlier chats into future conversations, with dedicated docs on memory controls (personalize.md, customization/memories.md) and even CLI-level controls (/memories) for whether a chat can use or contribute to memories. Missing for 10: independent/hands-on community corroboration of cross-chat memory recall working reliably, and detail on limits/scope of what memories retain.", "evidenceIds": [ "chatgpt-docs-19", "chatgpt-docs-63", "chatgpt-docs-35", "chatgpt-docs-80" ] }, { "productId": "chatgpt", "storyId": "mobile-apps", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "The iOS App Store listing (chatgpt-docs-20/21/41/52/58/59) documents rich mobile features—voice mode, photo upload, image generation, stickers—confirming a full-featured iOS app, and doc references (chatgpt-docs-67) explicitly mention continuing tasks 'in ChatGPT Work on the web, iOS, or Android,' indicating Android parity. Missing for 10: no dedicated Android app store evidence pack, no independent hands-on reviews of the mobile apps' feature completeness or performance.", "evidenceIds": [ "chatgpt-docs-20", "chatgpt-docs-21", "chatgpt-docs-41", "chatgpt-docs-52", "chatgpt-docs-58", "chatgpt-docs-59", "chatgpt-docs-67" ] }, { "productId": "chatgpt", "storyId": "openness-api-parity", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "There is evidence of programmatic access via the Codex CLI/SDK for coding automation and CI jobs, and a security-scan CLI, but nothing shows that the broad set of UI-only capabilities (Canvas/image editing, Computer Use, Voice, Work desktop automations, Sites, MCP-in-chat, memories, scheduled tasks) are exposed through a general ChatGPT API with parity to the UI. A probe for an OpenAPI/swagger spec on the docs site also returned 404s, reinforcing the lack of a documented full API surface. missing for 10: documented API endpoints mirroring Canvas/image tools, Computer Use, Voice, Work automations, Sites, and MCP-in-chat; independent confirmation of API-UI feature parity.", "evidenceIds": [ "chatgpt-docs-91", "chatgpt-docs-92", "chatgpt-docs-97", "chatgpt-probe-3" ] }, { "productId": "chatgpt", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack of a data export feature producing open/portable formats, nor any mention of account data export or deletion workflow. Evidence only covers general product features (Codex, Work, MCP, image gen, etc.) with nothing about exporting personal chat/data history or leaving the platform with your data intact.", "evidenceIds": [] }, { "productId": "chatgpt", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "ChatGPT is a closed-source, proprietary SaaS product; open-sourcing its source code is not a plausible axis for this kind of product (unlike an open-source framework or tool), so this is a category mismatch rather than a missing capability.", "evidenceIds": [] }, { "productId": "chatgpt", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "ChatGPT is a closed, hosted proprietary product with no evidence of a self-hostable core model or server; self-hosting is not a fair axis for this SaaS product category (it does not ship open weights or an installable core).", "evidenceIds": [] }, { "productId": "chatgpt", "storyId": "persistent-instructions", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "ChatGPT supports persistent custom instructions/personalization and Memories that carry context across chats, plus Codex's global AGENTS.md for persistent personal instructions in coding workflows, and fine-grained control via /memories. Missing for 10: independent hands-on verification that custom instructions reliably shape *every* response over long-term use, and no detail on limits/scope conflicts between memories and per-chat overrides.", "evidenceIds": [ "chatgpt-docs-19", "chatgpt-docs-63", "chatgpt-docs-72", "chatgpt-docs-80", "chatgpt-docs-35" ] }, { "productId": "chatgpt", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses data residency or regional storage choice for ChatGPT; the evidence only touches general data controls (e.g., Computer Use screenshots) without any region-selection capability.", "evidenceIds": [] }, { "productId": "chatgpt", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no documentation or mention of data-training opt-out controls, privacy settings, or 'improve the model for everyone' toggles; only tangential data-controls mention (chatgpt-docs-89) refers to Computer Use screenshots, not training opt-out generally. Missing for 10: any reference to training-data opt-out settings, business/API data-usage policies, or privacy dashboard controls.", "evidenceIds": [ "chatgpt-docs-89" ] }, { "productId": "chatgpt", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The only relevant evidence is a passing reference that 'ChatGPT data controls apply to content processed through ChatGPT, including screenshots taken by Computer Use,' implying some data-control/retention settings exist, but no documentation details how to view, export, or delete data, set retention periods, or manage memory deletion. Missing for 10: explicit data retention/export/delete documentation, memory deletion controls, and independent confirmation that these controls work as described.", "evidenceIds": [ "chatgpt-docs-89" ] }, { "productId": "chatgpt", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "None of the evidence addresses telemetry/usage-tracking opt-out controls for ChatGPT; the docs cover data controls tangentially (e.g., data usage for Computer Use screenshots) but nothing about disabling telemetry or usage analytics.", "evidenceIds": [] }, { "productId": "chatgpt", "storyId": "projects-workspaces", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "ChatGPT's Projects feature explicitly lets users organize related chats, share context/instructions, and even attach local folders/files for shared context, directly matching the story. Memories also reinforce carrying context across chats. Missing for 10: independent/hands-on corroboration of the Projects feature working as described, and detail on instruction-sharing UI beyond docs claims.", "evidenceIds": [ "chatgpt-docs-15", "chatgpt-docs-16", "chatgpt-docs-19", "chatgpt-docs-63" ] }, { "productId": "chatgpt", "storyId": "scheduled-tasks", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "ChatGPT's Automations/Scheduled Tasks feature explicitly supports scheduling recurring or one-off tasks that run in the background and report back, plus event-triggered tasks from Gmail/Slack/GitHub, with a UI to review active, paused, and completed runs. missing for 10: independent/hands-on community corroboration of scheduled task reliability and no detail on notification/result-delivery mechanics beyond docs.", "evidenceIds": [ "chatgpt-docs-11", "chatgpt-docs-12", "chatgpt-docs-37", "chatgpt-docs-47", "chatgpt-docs-26" ] }, { "productId": "chatgpt", "storyId": "training-opt-out", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers ChatGPT's agentic/feature capabilities (Codex, Work, MCP, Computer Use, etc.) but contains no documentation or mention of data controls, training opt-out settings, or 'Improve the model for everyone' toggles that let a user control whether their conversations are used for model training.", "evidenceIds": [] }, { "productId": "chatgpt", "storyId": "voice-conversation", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "ChatGPT ships Advanced Voice Mode for real-time spoken conversation on mobile ('Tap the soundwave icon to have a real-time convo on the go') and ChatGPT Voice on desktop that lets users talk through ideas, start work, check progress, or change direction without switching to typing. This directly matches the knowledge-worker story of natural, real-time voice conversation. Missing for 10: independent hands-on corroboration of voice quality/latency and any community verification beyond vendor docs.", "evidenceIds": [ "chatgpt-docs-21", "chatgpt-docs-41", "chatgpt-docs-51", "chatgpt-docs-76", "chatgpt-docs-81", "chatgpt-docs-13" ] }, { "productId": "chatgpt", "storyId": "web-search-citations", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "ChatGPT's web search docs explicitly state that search results and citations appear in chat when web search is used, and describe searching, browsing, comparing sources, and summarizing findings — directly matching the story. Community feedback shows mixed satisfaction with search accuracy (e.g., comparisons to Perplexity, a weather inaccuracy) but doesn't concretely show citations failing to appear, so this doesn't rise to a dispute. Missing for 10: independent verification of citation accuracy/consistency across many queries and no first-party detail on citation formatting/source diversity.", "evidenceIds": [ "chatgpt-docs-86", "chatgpt-docs-90", "chatgpt-docs-61", "chatgpt-comm-9", "chatgpt-comm-10", "chatgpt-comm-11" ] }, { "productId": "chatgpt", "storyId": "workspace-admin-controls", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Only a single explicit doc line notes that a workspace administrator can control access to ChatGPT Work, plugins, browser use, and network access, plus general mentions of GPT builders using 'approved workspace apps' and data controls applying to processed content. There is no evidence of a full admin console covering member invitation/removal, granular role-based permissions, or explicit data retention/training-opt-out policy controls. missing for 10: admin console/member management UI, granular role/permission settings, explicit data-retention and training-opt-out policy controls, independent corroboration of these admin features.", "evidenceIds": [ "chatgpt-docs-88", "chatgpt-docs-34", "chatgpt-docs-44", "chatgpt-docs-89" ] }, { "productId": "claude", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "A probe confirms Claude's support site serves a working llms.txt (HTTP 200) listing topic links, and numerous individual documentation pages are available in clean .md format (e.g. claude-docs-1 through 63 all resolve as .md URLs), making the docs directly consumable by an agent. Missing for 10: confirmation that llms.txt/agent-readable docs exist on the main claude.com domain (only support.claude.com was probed), and independent/community evidence that agents actually consume these successfully rather than just first-party doc structure.", "evidenceIds": [ "claude-probe-1", "claude-docs-2", "claude-docs-45", "claude-probe-2" ] }, { "productId": "claude", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Claude can ingest user data (files, connectors to Gmail/Drive/Calendar, Projects knowledge bases) and generate AI-driven insights, analysis, visualizations, and reports directly from that data, including agentic multi-step research that synthesizes findings with citations. This is well documented across file analysis, Research mode, Cowork task automation, and artifact/document generation features. Missing for 10: independent hands-on benchmarking specifically validating insight quality/accuracy from user data (community evidence is mixed/general rather than about this specific data-insight capability).", "evidenceIds": [ "claude-docs-14", "claude-docs-15", "claude-docs-16", "claude-docs-20", "claude-docs-27", "claude-docs-39", "claude-docs-55", "claude-docs-19", "claude-docs-62" ] }, { "productId": "claude", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Claude Cowork's scheduled tasks let users describe a task once and have Claude execute it autonomously on a recurring or on-demand basis, delivering finished outputs like reports and briefings without further user input, which directly matches background automation. Missing for 10: independent/hands-on verification of scheduled task reliability and no detail on failure handling or notification mechanisms.", "evidenceIds": [ "claude-docs-5", "claude-docs-38", "claude-docs-58", "claude-docs-4", "claude-docs-37", "claude-docs-57" ] }, { "productId": "claude", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Claude's built-in Cowork feature explicitly lets users delegate multi-step tasks ('describe an outcome, step away, come back to finished work'), with agentic execution including browser/computer use, scheduling, and research that autonomously plans next steps. This is well-documented first-party functionality directly matching task delegation to a built-in assistant. Missing for 10: independent hands-on verification of Cowork's delegation reliability (community evidence covers Claude Code/coding use more than Cowork specifically).", "evidenceIds": [ "claude-docs-4", "claude-docs-5", "claude-docs-37", "claude-docs-38", "claude-docs-57", "claude-docs-27", "claude-docs-39" ] }, { "productId": "claude", "storyId": "agentic-headless", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes Claude Code as a terminal tool and mentions CLI/print output structure, but there is no documentation or claim of headless/non-interactive execution, CI pipeline integration, or scriptable automation mode for Claude products.", "evidenceIds": [ "claude-docs-52", "claude-docs-53", "claude-comm-15" ] }, { "productId": "claude", "storyId": "agentic-mcp-client", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Claude documents connecting to both remote MCP servers via custom connectors and installing local MCP servers on Claude Desktop as easily as browser extensions, plus a unified directory for finding/installing connectors, letting Claude use their tools. Missing for 10: independent/hands-on verification of MCP tool usage reliability beyond vendor docs.", "evidenceIds": [ "claude-docs-18", "claude-docs-31", "claude-docs-36", "claude-docs-51", "claude-docs-64", "claude-docs-17", "claude-probe-4" ] }, { "productId": "claude", "storyId": "agentic-mcp-server", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Claude is itself an AI agent/assistant (client role); the evidence only shows Claude connecting to or building remote MCP servers as a client (claude-docs-18, claude-docs-31, claude-docs-36, claude-probe-4), which is the separate MCP-client story. There is no evidence Claude itself runs as an MCP server that other agents could connect to, so this server-role axis does not apply to this product.", "evidenceIds": [] }, { "productId": "claude", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Claude's entire interface is natural-language chat/voice, and documentation shows this extends to agentic actions—generating files, browsing/clicking the web, using the computer, scheduling recurring tasks, and building artifacts—all triggered by describing the desired outcome in plain language (claude-docs-4,6,7,15,37,38,54). Community evidence corroborates real-world agentic use (claude-comm-1,10,12), though some report friction with CLI usability. Missing for 10: independent benchmark of NL command reliability across all surfaces, and no rebuttal to the CLI unresponsiveness anecdote being addressed.", "evidenceIds": [ "claude-docs-4", "claude-docs-6", "claude-docs-7", "claude-docs-15", "claude-docs-37", "claude-docs-38", "claude-docs-54", "claude-comm-1", "claude-comm-10", "claude-comm-12" ] }, { "productId": "claude", "storyId": "agentic-official-cli", "verdict": "disputed", "quality": 6, "confidence": "medium", "rationale": "Anthropic ships an official CLI, Claude Code, well documented for terminal-based coding, deployment, debugging, and even screen-reader accessibility (claude-docs-8, claude-docs-23, claude-docs-52, claude-docs-65). However, a hands-on community report describes it as an 'empty unresponsive terminal' upon first use, contradicting the polished experience implied by docs, and another notes serious code-quality issues in the CLI's own codebase (claude-comm-13, claude-comm-15). missing for 10: broader independent corroboration of reliable day-to-day CLI usage, resolution of reported unresponsiveness, and evidence the code-quality issues have been fixed.", "evidenceIds": [ "claude-docs-8", "claude-docs-23", "claude-docs-52", "claude-docs-65", "claude-comm-13", "claude-comm-15" ] }, { "productId": "claude", "storyId": "agentic-public-api", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Evidence only hints at API-like surfaces (an Enterprise Compliance API for audit/chat data access, and references to using the Claude API/Console to power products) but never surfaces the actual general-purpose public API documentation for driving Claude's core capabilities; automated probes for an OpenAPI/swagger spec on the docs site returned 404s. Missing for 10: direct citation of the main Claude API reference docs, authentication/quickstart guides, and confirmation of comprehensive public API coverage beyond compliance/audit data.", "evidenceIds": [ "claude-docs-11", "claude-docs-26", "claude-docs-44", "claude-docs-56", "claude-probe-3" ] }, { "productId": "claude", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence describes scoped or least-privilege API credential issuance for agents; enterprise features mention audit logs and compliance API access but nothing about creating scoped/restricted API keys or credentials for agent use. Missing for 10: any documentation of API key scoping, permission tiers, or least-privilege credential management for agents.", "evidenceIds": [] }, { "productId": "claude", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Claude's consumer/enterprise features (Cowork, Projects, connectors, MCP, Chrome extension, file handling) but contains no documentation of official language SDKs (e.g., Python/TypeScript client libraries) for building applications against the Claude API. Missing for 10: any first-party SDK docs, API reference, or code samples showing programmatic API usage.", "evidenceIds": [] }, { "productId": "claude", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item describes webhooks or event subscription mechanisms for Claude; the closest agentic integrations are MCP connectors and scheduled tasks, which are not webhook-based event subscriptions.", "evidenceIds": [] }, { "productId": "claude", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "This evidence pack is entirely about Claude's consumer/product features (Cowork, connectors, memory, artifacts, etc.); there is no evidence of an interactive API reference with runnable examples. The probe explicitly found no OpenAPI spec at the checked endpoints, and no docs page describing an interactive API playground is present.", "evidenceIds": [ "claude-probe-3", "claude-probe-2" ] }, { "productId": "claude", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack includes explicit probes checking for a machine-readable API spec (openapi.json, swagger.json, etc.) on Claude's support domain, all returning 404, and no docs item anywhere references an OpenAPI/Swagger spec or downloadable schema for the Claude API. Nothing in the docs list an API reference format for AI-native consumption.", "evidenceIds": [ "claude-probe-3", "claude-probe-2" ] }, { "productId": "claude", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Claude's connectors, Cowork, computer use, and browser automation acting directly on real accounts (Gmail, Drive, live websites) with no mention of a sandbox, staging, or test-mode environment that isolates actions from production data.", "evidenceIds": [] }, { "productId": "claude", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Claude's consumer features, connectors, Cowork, and MCP integrations, but contains no mention of API versioning schemes or a documented deprecation policy for any Claude/Anthropic API. Since Claude does expose developer-facing APIs (e.g., for Console, Claude Code, MCP), this axis is applicable, but no evidence supports it.", "evidenceIds": [] }, { "productId": "claude", "storyId": "app-directory", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Claude documents a unified directory that brings skills, connectors, and plugins together in one place to find and install everything that customizes Claude, plus specific connectors like Google Workspace and remote/local MCP servers to extend the assistant. Missing for 10: no independent hands-on review confirming the browsing/discovery UX of the directory itself.", "evidenceIds": [ "claude-docs-17", "claude-docs-30", "claude-docs-19", "claude-docs-61", "claude-docs-51", "claude-docs-64", "claude-docs-31" ] }, { "productId": "claude", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents agentic task automation (Cowork, scheduled tasks, Research) and single-document file handling, but no feature is described for processing or acting on many items at once (e.g., batch file processing, bulk edit across records, or a batch API). Automation-depth stories like this are plausible for Claude given its agentic tooling, but nothing in the pack shows bulk/multi-item operation support.", "evidenceIds": [] }, { "productId": "claude", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Claude Cowork supports scheduled/recurring tasks that run automatically on a time-based schedule or on demand (claude-docs-38, claude-docs-58), which is a form of automation, but there's no evidence of user-defined rules triggered by external events (e.g., 'when an email arrives' or 'when a file changes, do X') as opposed to calendar/time-based scheduling. missing for 10: event-driven trigger conditions (webhooks, connector-based event listeners), a rules engine for conditional automation, and any documentation of non-time-based triggers.", "evidenceIds": [ "claude-docs-38", "claude-docs-58", "claude-docs-5", "claude-docs-37" ] }, { "productId": "claude", "storyId": "automation-scheduled-jobs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Claude Cowork explicitly supports scheduled recurring tasks, letting users describe a workflow once and have Claude execute it automatically on a recurring or on-demand basis, delivering outputs like reports and briefings — this directly matches the story. Missing for 10: independent/hands-on verification of scheduling reliability and details on scheduling granularity/limits beyond vendor docs.", "evidenceIds": [ "claude-docs-5", "claude-docs-38", "claude-docs-58", "claude-docs-4" ] }, { "productId": "claude", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence describes versioning, reviewing, or rolling back automations like scheduled Cowork tasks or Skills; docs cover creating/scheduling tasks but not history, diffs, or rollback capability.", "evidenceIds": [] }, { "productId": "claude", "storyId": "autonomous-background-tasks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Claude Cowork is documented as letting users describe a multi-step outcome, step away, and return to finished work, with scheduled/recurring tasks and background agentic execution (browser/computer use, file generation) explicitly designed for delegation and later review. Missing for 10: independent hands-on validation of Cowork's reliability/quality for complex delegated tasks and clearer detail on review/approval workflow beyond docs.", "evidenceIds": [ "claude-docs-4", "claude-docs-5", "claude-docs-37", "claude-docs-38", "claude-docs-58", "claude-docs-66", "claude-docs-6", "claude-docs-7" ] }, { "productId": "claude", "storyId": "browser-agent", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Claude has multiple documented browser-control capabilities: the built-in browser in Cowork that opens sites, reads pages, clicks, types, and fills forms autonomously (claude-docs-6/67), the Claude in Chrome extension that reads/clicks/navigates websites (claude-docs-63), and computer-use navigation for on-screen actions (claude-docs-7/22/59), enabling research and task completion on websites. Missing for 10: independent hands-on validation of browser-task success rates and reliability under real-world site complexity.", "evidenceIds": [ "claude-docs-6", "claude-docs-67", "claude-docs-63", "claude-docs-7", "claude-docs-22", "claude-docs-59", "claude-docs-37" ] }, { "productId": "claude", "storyId": "computer-use-desktop", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Claude Cowork/computer use lets Claude navigate directly to the user's screen—clicking, typing, opening apps, filling forms—while the user watches, plus a built-in browser for opening sites and interacting with pages, directly matching the story of operating applications to complete work. Community evidence corroborates agentic coding/task use though notes mixed quality perceptions unrelated to this specific capability. Missing for 10: independent hands-on verification of computer-use reliability/accuracy and broader third-party benchmarking beyond vendor docs.", "evidenceIds": [ "claude-docs-7", "claude-docs-22", "claude-docs-59", "claude-docs-6", "claude-docs-67", "claude-docs-4", "claude-docs-37", "claude-docs-63" ] }, { "productId": "claude", "storyId": "connect-drive-calendar", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Claude has a documented Google Workspace connector letting users connect Gmail, Google Calendar, and Google Drive so Claude can search emails, manage calendar, and work with documents/files directly in conversation, plus a privacy commitment not to train on this data. This directly satisfies the story's cloud drive/email/calendar connection use case. Missing for 10: no independent hands-on report corroborating real-world reliability of these specific connectors.", "evidenceIds": [ "claude-docs-19", "claude-docs-61", "claude-docs-42", "claude-docs-17" ] }, { "productId": "claude", "storyId": "create-edit-documents", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly describe generating and editing Excel, PowerPoint, Word, and PDF files via natural-language prompts, plus Artifacts for documents/code/interactive content that can be iteratively refined and downloaded, and uploading/persisting files in Projects for ongoing editing. Missing for 10: independent hands-on review confirming export fidelity/iteration quality across file types.", "evidenceIds": [ "claude-docs-15", "claude-docs-55", "claude-docs-13", "claude-docs-35", "claude-docs-62", "claude-docs-14" ] }, { "productId": "claude", "storyId": "custom-bots", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Claude supports Projects (self-contained workspaces with custom knowledge bases/files) and Skills (teach Claude repeatable instructions like brand guidelines), which together let a power-user build a persona-like assistant with instructions and knowledge; Projects can be shared with team members. However, there's no dedicated 'custom GPT'-style public sharing/marketplace for assistants, and no independent evidence of end-to-end sharing outside an org. missing for 10: evidence of public/marketplace sharing of custom assistants, explicit persona/system-instruction configuration UI, and independent hands-on confirmation of building and sharing such assistants.", "evidenceIds": [ "claude-docs-16", "claude-docs-60", "claude-docs-70", "claude-docs-62" ] }, { "productId": "claude", "storyId": "data-analysis-code", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly describe generating downloadable Excel/PowerPoint/Word/PDF files, producing reports with charts and visualizations, and using artifacts to build code-driven visualizations and interactive components from uploaded data files (CSV, XLSX, etc.). This directly matches the power-user story of writing/running code on data to produce charts, computed answers, and downloadable outputs.\nmissing for 10: independent/hands-on community corroboration specifically validating the data-analysis/code-execution-to-chart workflow (community evidence in the pack focuses on coding agent quality, not this analysis feature).", "evidenceIds": [ "claude-docs-15", "claude-docs-55", "claude-docs-41", "claude-docs-14", "claude-docs-29", "claude-docs-13", "claude-docs-35", "claude-docs-28", "claude-docs-54", "claude-docs-62" ] }, { "productId": "claude", "storyId": "deep-research-report", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Claude's Research feature is well documented: it operates agentically running multiple searches that build on each other, determines what to investigate next, and delivers thorough answers in minutes with easy-to-check citations. Community evidence is mixed on research depth (some say it lags ChatGPT/Gemini), which caps quality but the core capability is clearly delivered. Missing for 10: independent hands-on benchmark showing citation accuracy/report quality, and clearer detail on breadth of sources searched.", "evidenceIds": [ "claude-docs-20", "claude-docs-27", "claude-docs-39", "claude-docs-68", "claude-comm-8", "claude-comm-9" ] }, { "productId": "claude", "storyId": "desktop-app", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Claude ships an official desktop app (Mac, Windows, Linux via apt) and a Cowork/computer-use feature that lets Claude 'navigate to your screen directly—clicking, typing, and opening apps' while the user watches, which shows real screen access. However there is no evidence of dedicated OS-level keyboard shortcuts (e.g., a global hotkey to invoke Claude with current screen context) — the screen access described is agentic task automation rather than a power-user shortcut workflow. Missing for 10: documented OS-level global shortcuts/hotkeys, and evidence that a user can quickly summon Claude to see the current screen via keypress rather than launching a Cowork/computer-use task.", "evidenceIds": [ "claude-docs-9", "claude-docs-24", "claude-docs-7", "claude-docs-22", "claude-docs-59", "claude-docs-66" ] }, { "productId": "claude", "storyId": "export-my-data", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Claude's docs explicitly state individual users can export user information and chat history from Settings > Privacy, and that exports include both conversation data and account/user data; Enterprise adds a Compliance API for programmatic access to chat histories and file content. Missing for 10: independent/hands-on confirmation of export completeness or format details beyond first-party docs.", "evidenceIds": [ "claude-docs-12", "claude-docs-40", "claude-docs-11" ] }, { "productId": "claude", "storyId": "file-upload-analysis", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs confirm Claude supports uploading PDFs, DOCX, CSV, XLSX, TXT, HTML, ODT, RTF, EPUB, JSON, with analysis of both text and visual elements in PDFs up to 100 pages, plus persistent Project file storage for cross-conversation reference and generation of derived documents/reports. Missing for 10: independent hands-on benchmarking of analysis accuracy across large/complex spreadsheets or long PDFs beyond the 100-page limit.", "evidenceIds": [ "claude-docs-14", "claude-docs-29", "claude-docs-48", "claude-docs-62", "claude-docs-15", "claude-docs-55" ] }, { "productId": "claude", "storyId": "image-generation", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Claude can understand and analyze uploaded/pasted images (JPEG, PNG, GIF, WebP) and can produce SVG images, diagrams, and flowcharts as artifacts from natural-language descriptions, but there is no evidence of true raster image generation or photo-editing capability comparable to dedicated image models. Editing of generated visual artifacts is possible via iterative prompting, but this is limited to code-rendered graphics rather than general image generation/editing. Missing for 10: dedicated raster image-generation model, photo editing/inpainting features, and any independent confirmation of image-generation quality.", "evidenceIds": [ "claude-docs-35", "claude-docs-48", "claude-docs-49", "claude-docs-50", "claude-docs-54" ] }, { "productId": "claude", "storyId": "image-understanding", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Claude's docs confirm native image upload/paste support (JPEG, PNG, GIF, WebP) and clipboard paste, plus PDF analysis that includes visual elements like images and charts, directly supporting screenshot/photo interpretation for knowledge workers. Missing for 10: independent/hands-on evidence validating accuracy of image interpretation, and no explicit mention of screenshot-specific use cases (e.g., UI screenshots, photos of documents) beyond general image/PDF support.", "evidenceIds": [ "claude-docs-48", "claude-docs-49", "claude-docs-50", "claude-docs-14" ] }, { "productId": "claude", "storyId": "memory-across-chats", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Claude has explicit first-party memory/chat-search docs: it can search previous conversations and \"remember context from your chats and carry it into new conversations and Cowork tasks,\" plus a dedicated article on how memory works, what's remembered, and how to review/edit it; the product page also advertises \"Memory across conversations\" as a core feature. This directly matches the story of remembering context across chats and applying it in new ones. Missing for 10: independent/hands-on corroboration of memory quality or limitations in practice beyond vendor docs.", "evidenceIds": [ "claude-docs-2", "claude-docs-21", "claude-docs-45", "claude-docs-69" ] }, { "productId": "claude", "storyId": "mobile-apps", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Official iOS and Android apps are documented (App Store install instructions, Chrome/mobile chat parity across web/iOS/Android/desktop, voice mode explicitly available on Claude Mobile), indicating full-featured mobile apps rather than a bare wrapper. Missing for 10: no independent hands-on review of mobile app feature parity/quality, and no detail on which advanced features (Cowork, computer use) are available on mobile vs desktop-only.", "evidenceIds": [ "claude-docs-10", "claude-docs-25", "claude-docs-21", "claude-docs-32", "claude-docs-43" ] }, { "productId": "claude", "storyId": "openness-api-parity", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack details many UI-exclusive Claude.ai features (Cowork, voice mode, computer use, memory/chat search, artifacts, browser extension, scheduled tasks) but contains no documentation that these capabilities are exposed through the Claude API, nor any statement of API/UI feature parity. Enterprise API mentions are limited to a Compliance API for logs, not general feature parity.", "evidenceIds": [ "claude-docs-4", "claude-docs-3", "claude-docs-6", "claude-docs-7", "claude-docs-11", "claude-docs-16", "claude-docs-45" ] }, { "productId": "claude", "storyId": "openness-full-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Claude documents a data-export feature covering conversation and account data, letting users take their data with them ([claude-docs-12], [claude-docs-40]). However, there is no evidence specifying the export format is an open/standard one (e.g., JSON/portable), nor documentation of easy migration/interoperability with other tools, so the 'open format' and full portability aspects of the story are unconfirmed. missing for 10: explicit statement of open/standard export format, evidence of full interoperability/reuse elsewhere, and independent confirmation of export completeness.", "evidenceIds": [ "claude-docs-12", "claude-docs-40" ] }, { "productId": "claude", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Claude is closed-source; there is no evidence of an open-license source release for the model or app, and community evidence even criticizes it as closed/opaque compared to FOSS alternatives like Codex CLI (claude-comm-6).", "evidenceIds": [ "claude-comm-6" ] }, { "productId": "claude", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Claude is a closed, hosted proprietary model/service with no self-hosting option; self-hosting the core product is a category error for this type of SaaS/AI assistant offering, not an unmet applicable axis.", "evidenceIds": [] }, { "productId": "claude", "storyId": "persistent-instructions", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Claude supports persistent memory/context (chat search and memory, Projects with knowledge bases, custom instructions implied via Skills and Projects) that carries into new conversations, but there is no explicit documented feature for setting global 'custom instructions' that shape every response the way ChatGPT's system prompt does. missing for 10: dedicated persistent custom-instructions/preferences UI applying to all chats, independent/hands-on verification that memory reliably shapes every response, and clarity on scope/limits of what's remembered.", "evidenceIds": [ "claude-docs-2", "claude-docs-45", "claude-docs-69", "claude-docs-16", "claude-docs-70" ] }, { "productId": "claude", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, regional storage options, or geographic controls for where account/chat data is stored; the closest items concern export, audit logs, and training opt-outs, none of which address residency choice.", "evidenceIds": [] }, { "productId": "claude", "storyId": "privacy-no-training", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Claude documents explicit user controls to prevent training use: incognito chats are never used to improve Claude even with Model Improvement enabled, users can toggle the 'Model Improvement' privacy setting, and connector data (Gmail/Drive/Calendar) is explicitly excluded from training. This directly satisfies the ai-native privacy-posture story of preventing data from being used for model training. Missing for 10: independent/third-party verification of these claims and explicit default policy for Enterprise/Team plans beyond connectors.", "evidenceIds": [ "claude-docs-46", "claude-docs-47", "claude-docs-42" ] }, { "productId": "claude", "storyId": "privacy-retention-controls", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Claude provides explicit user-facing data controls: exporting chat history/user data (claude-docs-12, 40), reviewing/editing/turning off memory (claude-docs-45), incognito chats excluded from training (claude-docs-46), and enterprise audit logs plus a compliance API for data governance (claude-docs-11, 44). Missing for 10: explicit self-service account/data deletion flow documentation and independent verification that deletion requests are honored.", "evidenceIds": [ "claude-docs-12", "claude-docs-40", "claude-docs-45", "claude-docs-46", "claude-docs-47", "claude-docs-11", "claude-docs-44" ] }, { "productId": "claude", "storyId": "privacy-telemetry-optout", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Claude documents a 'Model Improvement' privacy setting that can be toggled off and 'incognito chats' that are excluded from training even if Model Improvement is enabled, giving users control over whether their conversations are used to improve the model (claude-docs-46, claude-docs-47). This addresses opt-out of usage-for-training, and data export/deletion options exist (claude-docs-12, claude-docs-40), but there is no explicit documentation of a broader telemetry/analytics opt-out (e.g., product usage metrics, crash reporting) beyond model-training data use. missing for 10: explicit telemetry/analytics tracking opt-out settings, independent confirmation that toggling actually stops all usage tracking, documentation of what non-training telemetry data is collected.", "evidenceIds": [ "claude-docs-46", "claude-docs-47", "claude-docs-12", "claude-docs-40" ] }, { "productId": "claude", "storyId": "projects-workspaces", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Projects docs confirm self-contained workspaces with shared chat histories and knowledge bases, persistent file uploads scoped to a project for cross-conversation reference, and memory/chat search to build on prior context. missing for 10: no evidence of custom instructions/system prompt configuration per project beyond files, and no independent/hands-on corroboration of the project workflow in practice.", "evidenceIds": [ "claude-docs-16", "claude-docs-62", "claude-docs-45", "claude-docs-21" ] }, { "productId": "claude", "storyId": "scheduled-tasks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Claude Cowork explicitly supports scheduled tasks that run on a recurring or one-off basis and deliver finished outputs like reports and summaries back to the user, matching the story closely. Missing for 10: independent/hands-on validation of scheduling reliability and no detail on notification/delivery mechanisms beyond docs.", "evidenceIds": [ "claude-docs-38", "claude-docs-58", "claude-docs-5", "claude-docs-4", "claude-docs-37" ] }, { "productId": "claude", "storyId": "training-opt-out", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Claude provides explicit privacy controls: incognito chats are excluded from model training even with Model Improvement enabled, a 'Model Improvement' opt-in/out toggle exists, and connector data (Gmail/Drive/Calendar) is explicitly excluded from training. Missing for 10: independent/hands-on verification that the training opt-out is actually honored in practice, and clearer documentation of the toggle's exact location/scope for all plan tiers.", "evidenceIds": [ "claude-docs-46", "claude-docs-47", "claude-docs-42" ] }, { "productId": "claude", "storyId": "voice-conversation", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Claude ships an explicit Voice mode enabling complete spoken conversations, available across web, desktop, iOS and Android, positioned to work best on phone. Missing for 10: independent hands-on reviews of voice latency/naturalness and confirmation it's out of beta.", "evidenceIds": [ "claude-docs-3", "claude-docs-43" ] }, { "productId": "claude", "storyId": "web-search-citations", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Claude's Research feature is documented to perform agentic multi-step web searches and deliver answers with 'easy-to-check citations' (claude-docs-20, 27, 39, 68), directly matching the story. However, independent community feedback suggests mixed real-world quality—commenters say Claude's general research 'is fine' but that ChatGPT and Gemini 'run circles around it' in depth and presentation (claude-comm-8, claude-comm-9), tempering confidence in how well-grounded/comprehensive the citations truly are. missing for 10: independent hands-on verification of citation accuracy/source quality, and confirmation research draws from live/current web data versus stale index.", "evidenceIds": [ "claude-docs-20", "claude-docs-27", "claude-docs-39", "claude-docs-68", "claude-comm-8", "claude-comm-9" ] }, { "productId": "claude", "storyId": "workspace-admin-controls", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence confirms Enterprise plan admin/security features like audit logs and a Compliance API for programmatic access to usage data, implying some org-level governance, but there is no documentation of member management, role/permission assignment, or granular data policy controls for team admins. missing for 10: member invitation/removal workflows, role-based permission management, workspace-level data retention/policy settings, and independent corroboration of admin console functionality.", "evidenceIds": [ "claude-docs-11", "claude-docs-26", "claude-docs-44" ] }, { "productId": "copilot", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Probes confirm no llms.txt, no markdown docs endpoint, and no OpenAPI spec exposed, and none of the docs mention agent-oriented documentation formats for Copilot; this is not a developer/agentic tool ecosystem story being served here. Missing for 10: llms.txt file, agent-readable docs endpoint, any mention of machine-consumable documentation format.", "evidenceIds": [ "copilot-probe-1", "copilot-probe-2", "copilot-probe-3" ] }, { "productId": "copilot", "storyId": "agentic-ai-insights", "verdict": "disputed", "quality": 6, "confidence": "medium", "rationale": "Docs strongly claim Copilot surfaces AI-generated insights from user data (Excel 'Analyst' turning raw numbers into insight, Outlook summaries, Researcher synthesizing information, connectors that let Copilot 'find, summarize, and act on information' from OneDrive/Gmail/Drive) [copilot-docs-2,6,4,39,7]. However hands-on community reports concretely contradict the usefulness of these insight features in practice — Word Copilot 'couldn't do anything except append to the end of the document' and failed basic tasks, Outlook's summarize feature is described as 'so bad', and users report never finding a real use case despite daily AI use [copilot-comm-4,5,9], while one users praises cross-document data fetching [copilot-comm-7]. Missing for 10: independent benchmarking of insight quality/accuracy, resolution of the mixed hands-on experience, and clearer first-party detail on how proactive suggestions are generated beyond marketing copy.", "evidenceIds": [ "copilot-docs-2", "copilot-docs-6", "copilot-docs-4", "copilot-docs-39", "copilot-docs-7", "copilot-docs-33", "copilot-comm-4", "copilot-comm-5", "copilot-comm-7", "copilot-comm-9" ] }, { "productId": "copilot", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Copilot offers agentic features like Cowork for multi-step task completion, proactive research suggestions, and Browse-with-Copilot actions that only pause for user approval on sensitive steps, suggesting some autonomous multi-step execution, but there is no documented evidence of scheduled or trigger-based automations that run unattended in the background over time. Community feedback also shows integrated Copilot features (Word/Outlook) are seen as thin/incomplete for automation. Missing for 10: explicit scheduling/triggers, evidence of unattended background execution over time, independent hands-on confirmation of autonomous automation working as advertised.", "evidenceIds": [ "copilot-docs-10", "copilot-docs-33", "copilot-docs-34", "copilot-docs-17", "copilot-comm-4" ] }, { "productId": "copilot", "storyId": "agentic-builtin-assistant", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Microsoft documents rich task-delegation capabilities: Cowork for multi-step tasks and completed to-dos (copilot-docs-10), automating repetitive work without coding (copilot-docs-17, copilot-docs-12), Researcher/Analyst agents (copilot-docs-5/6), and Copilot in Word/Excel/PowerPoint/Outlook executing tasks from prompts. However, hands-on community reports concretely contradict this for at least one major surface: a user describes Copilot in Word as unable to do anything except append text, unable to create tables, search/replace, or maintain formatting ('What a failure!' — copilot-comm-4), and another calls Outlook's summarize feature 'so bad' (copilot-comm-5), while other users report positive experiences fetching data across documents (copilot-comm-7). This is a genuine dispute: vendor claims robust in-app task delegation, but specific hands-on failure reports show it breaking down for core document-editing delegation. Missing for 10: independent benchmark of Cowork/agentic task completion, resolution of the Word editing failure reports, and clearer evidence the failures are outdated/fixed.", "evidenceIds": [ "copilot-docs-10", "copilot-docs-17", "copilot-docs-5", "copilot-docs-6", "copilot-docs-1", "copilot-comm-4", "copilot-comm-5", "copilot-comm-7" ] }, { "productId": "copilot", "storyId": "agentic-headless", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Microsoft Copilot is a consumer-facing chat/assistant product designed for interactive use in apps, browser, and Office; there is no evidence of a headless mode, CLI, or CI-compatible API for automation. This axis is more appropriate for developer tools/agent frameworks, not a consumer chat assistant—no first-party CI/headless invocation is documented or implied.", "evidenceIds": [] }, { "productId": "copilot", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Copilot supports proprietary 'connectors' to services like OneDrive, Gmail, Google Drive, etc., but there is no mention of MCP (Model Context Protocol) server support or ability to plug in arbitrary MCP tool servers. Missing for 10: any documentation of MCP protocol compatibility, MCP server connection UI/settings, or third-party MCP tool integration.", "evidenceIds": [ "copilot-docs-39", "copilot-docs-7", "copilot-docs-37" ] }, { "productId": "copilot", "storyId": "agentic-mcp-server", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Microsoft Copilot is itself an AI assistant/agent (client role) with connectors to services like OneDrive and Gmail, not a platform exposing an official MCding for other agents to consume; no evidence shows it running as an MCP server. This axis applies to the client-consumption side, not to Copilot's own agent role.", "evidenceIds": [] }, { "productId": "copilot", "storyId": "agentic-nl-commands", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Docs extensively describe natural-language operation—prompting Copilot in Word/Excel/PowerPoint, natural-language file/email/calendar search, and browser action commands (copilot-docs-1,2,3,8,14,20-22,26,27). However, hands-on community evidence directly contradicts this for at least one core surface: a user reports Copilot in Word 'couldn't do anything except append to the end of the document... couldn't create tables! couldn't search and replace! couldn't maintain formatting' (copilot-comm-4), and others found the Outlook summarize feature and overall NL assistant underwhelming or unused (copilot-comm-5, copilot-comm-9). This is a concrete documented failure of the claimed NL command capability, not mere skepticism. Missing for 10: independent third-party benchmarking of NL command reliability across all Office apps, and resolution of the reported Word functional gaps.", "evidenceIds": [ "copilot-docs-1", "copilot-docs-8", "copilot-docs-14", "copilot-docs-20", "copilot-docs-26", "copilot-comm-4", "copilot-comm-5", "copilot-comm-9" ] }, { "productId": "copilot", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an official CLI for Microsoft Copilot; the product is presented as a chat/web/office-integrated assistant with browser and app surfaces, and probes for docs/API endpoints returned 404s. missing for 10: any mention of a Copilot CLI tool, command-line installation instructions, or terminal-based usage.", "evidenceIds": [ "copilot-probe-1", "copilot-probe-2", "copilot-probe-3" ] }, { "productId": "copilot", "storyId": "agentic-public-api", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a documented public API for driving Microsoft Copilot programmatically; probes for llms.txt, docs.md, and OpenAPI specs all returned 404, and all docs describe consumer-facing chat/connector features, not an API for AI-native/agentic control.", "evidenceIds": [ "copilot-probe-1", "copilot-probe-2", "copilot-probe-3" ] }, { "productId": "copilot", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence describes issuing scoped or least-privilege API credentials for an agent; Copilot's docs discuss connecting consumer services and confirmation prompts for actions, but nothing about credential scoping, API keys, or granular permission grants for agents. missing for 10: any mention of API credential issuance, scoping/least-privilege controls, or agent-specific access tokens.", "evidenceIds": [] }, { "productId": "copilot", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers only consumer-facing Copilot features (Word, Excel, browsing, connectors) with no mention of any official SDK for developers to build against; probes for API/OpenAPI specs on the Microsoft support domain all returned 404s, indicating no evidence of an SDK offering.", "evidenceIds": [ "copilot-probe-1", "copilot-probe-2", "copilot-probe-3" ] }, { "productId": "copilot", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of webhook subscription capability; Copilot's documentation covers connectors, chat, and consumer app integrations but nothing about outbound event/webhook subscriptions, and probes for API/OpenAPI specs returned 404s.", "evidenceIds": [ "copilot-probe-1", "copilot-probe-2", "copilot-probe-3" ] }, { "productId": "copilot", "storyId": "api-interactive-docs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Microsoft Copilot is a consumer/assistant product, not a developer platform with an API reference; this axis is a category error for its type. Probe evidence confirms no OpenAPI/docs endpoints exist, but the story itself doesn't apply to this kind of product.", "evidenceIds": [ "copilot-probe-1", "copilot-probe-2", "copilot-probe-3" ] }, { "productId": "copilot", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Probes for an OpenAPI/machine-readable spec all returned 404s, and no documentation references a downloadable API spec for Copilot as a consumer AI assistant.", "evidenceIds": [ "copilot-probe-3", "copilot-probe-1", "copilot-probe-2" ] }, { "productId": "copilot", "storyId": "api-sandbox", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Microsoft Copilot is a consumer/productivity AI assistant, not a developer platform with a sandbox/test environment concept; the evidence pack contains no mention of sandbox testing or staging environments, and this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "copilot", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a public versioned API or documented deprecation policy for Microsoft Copilot; probes for API/OpenAPI specs returned 404s and all documentation is consumer feature-focused rather than developer/API documentation. missing for 10: any public API reference, versioning scheme, or deprecation policy documentation.", "evidenceIds": [ "copilot-probe-1", "copilot-probe-2", "copilot-probe-3" ] }, { "productId": "copilot", "storyId": "app-directory", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Copilot documents 'connectors' to link services like OneDrive, Outlook, Google Drive, Gmail/Calendar/Contacts, but this is a short fixed list of Microsoft/Google services rather than a browsable directory of many third-party apps/connectors that power-users could explore and add. No evidence of a marketplace-style catalog with search/browse UI or a broad ecosystem of third-party integrations. missing for 10: evidence of a searchable directory/marketplace UI, breadth of third-party (non-Microsoft/Google) apps and connectors, and independent confirmation of the add/browse experience.", "evidenceIds": [ "copilot-docs-7", "copilot-docs-13", "copilot-docs-19", "copilot-docs-25", "copilot-docs-37", "copilot-docs-39" ] }, { "productId": "copilot", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs vaguely gesture at automation depth (\"automate repetitive tasks\", Cowork's \"bigger, multi-step tasks\") and connectors for finding/acting on files, but there is no concrete evidence of Copilot performing bulk operations across many items (e.g., batch-editing many files, mass email actions, applying an operation to a whole dataset) — most citations are single-item lookups (\"find my spreadsheet\", \"show me this email\"). Community reports even describe failures on basic single-document tasks in Word, further undercutting confidence in larger-scale batch automation. Missing for 10: explicit bulk/batch operation examples, independent hands-on confirmation of multi-item automation, and detail on scale/limits of Cowork's multi-step tasks.", "evidenceIds": [ "copilot-docs-10", "copilot-docs-17", "copilot-docs-39", "copilot-comm-4" ] }, { "productId": "copilot", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes chat-based prompting, connectors for search/retrieval, and browser automation with human confirmation steps, but nothing about user-defined rules/triggers that fire automatically on events (e.g., 'when X happens, do Y'). No scheduling, event-listener, or conditional automation feature is documented.", "evidenceIds": [] }, { "productId": "copilot", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence describes scheduling recurring jobs, workflows, or automations that run on a timer or trigger; Copilot is documented as a chat/response tool with connectors, browsing, and multi-step task assistance (Cowork), but nothing about recurring/scheduled automation.", "evidenceIds": [] }, { "productId": "copilot", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence describes versioning, review workflows, or rollback capabilities for automations built with Copilot; the evidence only covers task execution, connectors, and privacy controls for conversation history, not automation version control.", "evidenceIds": [] }, { "productId": "copilot", "storyId": "autonomous-background-tasks", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs describe 'Cowork' explicitly for 'bigger, multi-step tasks and deliver completed to-dos' and 'Browse with Copilot' which acts across steps with real-time visibility and pauses for user attention on sensitive actions (buying, sending email, deleting events), matching the delegate-and-review pattern. However these are thin vendor descriptions with no detail on true background/async execution, no hands-on or independent confirmation of Cowork's autonomy, and community feedback (though about general Office Copilot features, not Cowork specifically) shows skepticism about Copilot's practical task-completion reliability. Missing for 10: independent/hands-on validation of Cowork's autonomous background execution, detail on task duration/async return-for-review workflow, and confirmation the feature works reliably beyond marketing copy.", "evidenceIds": [ "copilot-docs-10", "copilot-docs-12", "copilot-docs-9", "copilot-docs-15", "copilot-docs-24", "copilot-docs-34" ] }, { "productId": "copilot", "storyId": "browser-agent", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Microsoft documents a dedicated 'Browse with Copilot' capability that lets Copilot click, type, and navigate tabs in Edge to complete web tasks, with real-time visibility and required user confirmation for sensitive actions like purchases or bookings, plus a 'Cowork' mode for multi-step task completion. However, all supporting evidence is first-party docs with no independent/hands-on confirmation of browser-operation success, and community feedback in the pack focuses on Office-integrated Copilot rather than this browsing agent feature. Missing for 10: independent/hands-on validation of the browse-and-complete-task capability, more detail on reliability/success rate on real websites.", "evidenceIds": [ "copilot-docs-9", "copilot-docs-15", "copilot-docs-24", "copilot-docs-34", "copilot-docs-10" ] }, { "productId": "copilot", "storyId": "computer-use-desktop", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Microsoft documents 'Browse with Copilot' letting the assistant click, type, and navigate tabs directly in Edge, plus 'Cowork' for multi-step task completion and Vision for seeing images/screens, with human-in-the-loop confirmation for sensitive actions. However, this operating capability is scoped mainly to the browser and Office apps rather than general desktop application control, and community reports describe weak real-world execution (e.g., Copilot in Word failing at basic edits, Outlook integration being minimal), undercutting the strength of the claim. Missing for 10: broad OS-level/desktop app automation beyond Edge/Office, and independent hands-on validation that agentic actions reliably complete tasks.", "evidenceIds": [ "copilot-docs-9", "copilot-docs-15", "copilot-docs-10", "copilot-docs-34", "copilot-comm-4", "copilot-comm-5" ] }, { "productId": "copilot", "storyId": "connect-drive-calendar", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "First-party docs explicitly describe Copilot connectors for OneDrive, Outlook.com (email/calendar/contacts), Google Drive, and Gmail/Calendar/Contacts, with concrete example queries for finding files, emails, and calendar events (copilot-docs-7/13/19/25/26/27/40/43). Community comment confirms cross-document data fetching works well (copilot-comm-7), while another clarifies this connector feature is scoped to the standalone Copilot app rather than Office-integrated Copilot (copilot-comm-1), a scope caveat rather than a failure. Missing for 10: deeper independent hands-on verification specifically of the connector search/answer flow (most community feedback concerns Office in-app editing, not the connector-search feature) and clarity on rollout/availability limits.", "evidenceIds": [ "copilot-docs-7", "copilot-docs-13", "copilot-docs-19", "copilot-docs-25", "copilot-docs-26", "copilot-docs-27", "copilot-docs-40", "copilot-docs-43", "copilot-docs-39", "copilot-comm-7", "copilot-comm-1" ] }, { "productId": "copilot", "storyId": "create-edit-documents", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Microsoft's docs claim Copilot in Word/Excel/PowerPoint can create content from a prompt and be refined via chat (e.g., copilot-docs-1, copilot-docs-3, copilot-docs-16), which matches the story's ask for document/presentation creation and iterative editing. However, a hands-on community report directly contradicts this, stating the Word Copilot 'couldn't do anything except append to the end of the document,' couldn't create tables, search/replace, or maintain formatting (copilot-comm-4), and another user found the Outlook summarizer 'so bad' (copilot-comm-5). Missing for 10: reliable hands-on corroboration of iterative multi-turn editing, and no evidence at all about exporting finished files to standard formats.", "evidenceIds": [ "copilot-docs-1", "copilot-docs-3", "copilot-docs-16", "copilot-comm-4", "copilot-comm-5" ] }, { "productId": "copilot", "storyId": "custom-bots", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers built-in Copilot features (Word, Excel, Outlook, connectors, memory, browsing) but contains no mention of a feature letting power-users build or share custom assistants with their own instructions and knowledge base (e.g., Copilot Studio-style custom GPTs). This is a reasonable ask for an AI assistant platform, but no supporting evidence is present.", "evidenceIds": [] }, { "productId": "copilot", "storyId": "data-analysis-code", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm Copilot can run code for calculations and generate graphs, and 'Analyst' turns raw data into insights, supporting the core of the story. However, there is no explicit documentation of producing downloadable output files, and community feedback shows some Office-integrated Copilot features (Word/Excel/Outlook) underperform in practice, raising doubts about reliability of this capability in production use. Missing for 10: explicit evidence of downloadable file output, independent hands-on verification of chart/code generation working reliably.", "evidenceIds": [ "copilot-docs-31", "copilot-docs-6", "copilot-docs-16", "copilot-comm-4", "copilot-comm-2" ] }, { "productId": "copilot", "storyId": "deep-research-report", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Copilot explicitly ships a 'Researcher' feature ('Find, synthesize, and understand information faster with Researcher') and Copilot's general web-grounded answers include hyperlinked citations, which together map onto autonomous multi-source research with citations. However, there's no first-party detail on how many sources Researcher searches, its autonomy/duration, or report structure, and no hands-on/community corroboration of a deep-research run producing a cited report. Missing for 10: detailed documentation of Researcher's multi-source search depth and autonomous workflow, evidence of a full cited report output, and independent/hands-on verification of the feature working as described.", "evidenceIds": [ "copilot-docs-5", "copilot-docs-28" ] }, { "productId": "copilot", "storyId": "desktop-app", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes Copilot integrations inside Office apps, a browser automation feature (Edge), and connectors to cloud services, but never mentions an official standalone desktop app with OS-level keyboard shortcuts or an ability to see/read the user's current screen contents outside the browser. Vision is described only for analyzing uploaded images/photos, not live screen capture, and no docs reference a Windows/macOS desktop client or shortcut.", "evidenceIds": [] }, { "productId": "copilot", "storyId": "export-my-data", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence documents the ability to delete individual or entire conversation history (copilot-docs-38/41/44) and opt out of training use (copilot-docs-35/42), but no citation describes an export function for chat history or account data. Absence of evidence for an applicable trust-controls capability means this is 'none' rather than 'na'.", "evidenceIds": [ "copilot-docs-38", "copilot-docs-41", "copilot-docs-44", "copilot-docs-35", "copilot-docs-42" ] }, { "productId": "copilot", "storyId": "file-upload-analysis", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Microsoft's docs describe first-party file-analysis capabilities: Copilot in Word/Excel/PowerPoint working with uploaded notes and data, Analyst for data analysis, Researcher for synthesis, connectors that can find and act on Word docs, PDFs, and spreadsheets, and code execution for calculations. Community evidence is mixed: one hands-on report praises cross-document data fetching in 365 while another reports Word Copilot failing at basic tasks like tables/formatting, suggesting inconsistent real-world quality rather than a clean contradiction of the analysis claim itself. Missing for 10: independent accuracy benchmarks specifically for PDF/spreadsheet content analysis, and no hands-on confirmation that complex PDFs or large spreadsheets are analyzed accurately rather than just searched/summarized.", "evidenceIds": [ "copilot-docs-1", "copilot-docs-2", "copilot-docs-6", "copilot-docs-20", "copilot-docs-31", "copilot-docs-40", "copilot-comm-7", "copilot-comm-4" ] }, { "productId": "copilot", "storyId": "image-generation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack mentions Copilot's Vision feature for analyzing and understanding images (copilot-docs-30) and general multimodal capabilities (voice, code execution), but no citation describes generating or editing images from natural-language prompts (e.g., an image-creation/DALL·E-style feature). Missing for 10: any documentation of text-to-image generation, in-chat image editing, or independent confirmation of such a feature.", "evidenceIds": [ "copilot-docs-30", "copilot-docs-31" ] }, { "productId": "copilot", "storyId": "image-understanding", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm Copilot has a 'Vision' capability to analyze images and photos ([copilot-docs-30]), directly supporting the story, but this is a single first-party mention with no elaboration on screenshot-specific use cases or accuracy, and no independent/hands-on evidence corroborates real-world interpretation quality. Missing for 10: detailed documentation on screenshot vs photo handling, accuracy benchmarks, and independent/community verification of correct image interpretation.", "evidenceIds": [ "copilot-docs-30" ] }, { "productId": "copilot", "storyId": "memory-across-chats", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "First-party docs confirm a memory feature: Copilot can store facts users ask it to remember and use that context in future responses, and can proactively surface suggestions based on past queries and memory (copilot-docs-32, copilot-docs-33). However, this is opt-in/explicit fact storage rather than full automatic context carry-over, and there is no independent/hands-on evidence confirming it works reliably across conversations. missing for 10: independent verification of cross-conversation recall, detail on scope/limits of automatic memory use, and confirmation it isn't just facts explicitly asked to be remembered.", "evidenceIds": [ "copilot-docs-32", "copilot-docs-33", "copilot-docs-38" ] }, { "productId": "copilot", "storyId": "mobile-apps", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm official Copilot mobile apps exist for iOS and Android and that connectors work there (copilot-docs-37), and broader Copilot capabilities like voice, vision, and chat (copilot-docs-29,30) are described as core features generally available across app surfaces. However, no evidence details mobile-specific UI parity, app store ratings, or hands-on mobile app reviews confirming full feature parity with desktop/web Copilot. Missing for 10: mobile-specific feature parity documentation, independent hands-on mobile app reviews, and confirmation that all Office-integrated Copilot features (Word, Excel, PowerPoint) are fully usable in the mobile app.", "evidenceIds": [ "copilot-docs-37", "copilot-docs-29", "copilot-docs-30", "copilot-docs-39" ] }, { "productId": "copilot", "storyId": "openness-api-parity", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a public API for Microsoft Copilot at all; the evidence pack only shows UI-based consumer features (chat, connectors, browse-with-copilot) and probes confirm no llms.txt/openapi/API discovery endpoints exist. Missing for 10: any documented API surface, parity claims between API and UI, or developer API docs.", "evidenceIds": [ "copilot-probe-1", "copilot-probe-2", "copilot-probe-3" ] }, { "productId": "copilot", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a bulk data export feature or open-format export/portability tool for Copilot; documentation only covers deleting conversation history, not exporting it, and no data-export capability is mentioned anywhere in the pack.", "evidenceIds": [] }, { "productId": "copilot", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Microsoft Copilot is closed-source proprietary software; no evidence in the pack indicates any open-license source availability, and probes for machine-readable docs/openapi all 404. missing for 10: any open-source license, public repository, or source availability statement.", "evidenceIds": [ "copilot-probe-1", "copilot-probe-2", "copilot-probe-3" ] }, { "productId": "copilot", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Microsoft Copilot is a cloud-hosted SaaS service; no evidence anywhere in the pack mentions self-hosting, on-prem deployment, or open-source release of the core product. All documentation describes cloud connectors, apps, and privacy controls, not deployment options.", "evidenceIds": [] }, { "productId": "copilot", "storyId": "persistent-instructions", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm Copilot has a memory feature that stores user-requested facts and uses them to personalize future responses/suggestions (docs-32, docs-33), which supports persistent context, but there's no evidence of an explicit 'custom instructions' or persona-setting panel that shapes every response, and no independent corroboration of how consistently memory is applied. missing for 10: explicit custom-instructions/persona settings UI, evidence of consistent application across all responses, independent/hands-on confirmation of memory reliability.", "evidenceIds": [ "copilot-docs-32", "copilot-docs-33" ] }, { "productId": "copilot", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data residency, regional storage selection, or data locality controls for Microsoft Copilot; the docs focus on features, connectors, and privacy opt-outs but never mention region/residency choice.", "evidenceIds": [] }, { "productId": "copilot", "storyId": "privacy-no-training", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Microsoft documents an explicit privacy control letting users 'opt out of use of your conversation activity for model training' and delete conversation history, directly matching the story's ask. However, a community report shows real confusion about whether documents opened in Word/Office are also sent for training and whether that data path can be opted out, and the opt-out documentation only covers 'conversation activity' rather than the full range of Copilot integrations (Excel, Outlook, connectors). Missing for 10: independent verification the opt-out is honored, clarity on whether Office-integrated Copilot documents/data are covered by the same toggle, and confirmation for enterprise/consumer parity.", "evidenceIds": [ "copilot-docs-35", "copilot-docs-42", "copilot-docs-44", "copilot-comm-6" ] }, { "productId": "copilot", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Microsoft documents clear controls to delete individual or full conversation history and opt out of using conversation data for model training (copilot-docs-38/41/44/35/36/42), which directly addresses retention/deletion. However, community evidence notes these privacy FAQ controls apply only to the standalone Copilot chat/app, not the Copilot integrations embedded in Office apps like Word/Outlook (copilot-comm-1), and a user reported uncertainty about whether Word documents were being sent for training with no clear indicator (copilot-comm-6), showing the retention story is inconsistent across the product surface. Missing for 10: unified retention/deletion controls across all Copilot surfaces (Office, Edge, connectors), enterprise/admin-level data retention policy documentation, and independent verification that deletion requests are honored end-to-end.", "evidenceIds": [ "copilot-docs-38", "copilot-docs-41", "copilot-docs-44", "copilot-docs-35", "copilot-docs-36", "copilot-docs-42", "copilot-comm-1", "copilot-comm-6" ] }, { "productId": "copilot", "storyId": "privacy-telemetry-optout", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Microsoft's documentation explicitly offers an opt-out for using conversation activity in model training and lets users delete conversation history (copilot-docs-35, copilot-docs-42, copilot-docs-44), but community reports show this opt-out doesn't extend to Copilot embedded in Office apps like Word for Mac, where users report 'no way of turning it off' and uncertainty about whether their documents are used for training (copilot-comm-3, copilot-comm-6). A HN commenter also clarifies the opt-out is scoped only to the standalone Copilot chat/app, not to Office-integrated Copilot (copilot-comm-1), directly contradicting a blanket telemetry opt-out claim. Missing for 10: a documented telemetry/usage-tracking opt-out (beyond model-training) covering all Copilot surfaces including Office integrations, and confirmation that the Word/Outlook Copilot experiences honor an opt-out toggle.", "evidenceIds": [ "copilot-docs-35", "copilot-docs-42", "copilot-docs-44", "copilot-comm-1", "copilot-comm-3", "copilot-comm-6" ] }, { "productId": "copilot", "storyId": "projects-workspaces", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Copilot's memory of user-stated facts, connectors to files/email/calendar, and per-app integrations (Word, Excel, PowerPoint, Outlook), but nothing describing a 'project' or 'space' construct that groups related chats and files with shared instructions/context. Missing for 10: any documentation of a project/space organizational feature, ability to scope chats to a folder/workspace, or shared custom instructions per project.", "evidenceIds": [ "copilot-docs-32", "copilot-docs-33", "copilot-docs-39", "copilot-docs-7" ] }, { "productId": "copilot", "storyId": "scheduled-tasks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes Copilot's chat, in-app assistance, connectors, and multi-step 'Cowork' task completion, but nothing about scheduling recurring or one-off tasks that run automatically and report back later. Missing for 10: any mention of a task scheduler, recurring triggers, or proactive automated check-ins/results delivery.", "evidenceIds": [] }, { "productId": "copilot", "storyId": "training-opt-out", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Microsoft's own privacy FAQ explicitly states users can opt out of conversation data being used for model training and can delete conversation history at any time (copilot-docs-35, copilot-docs-36, copilot-docs-42, copilot-docs-38, copilot-docs-44), directly delivering the trust-control story. However, community evidence notes this training opt-out control specifically applies only to the standalone Copilot chat/app/website and not to Copilot integrated into Office 365 apps like Word (copilot-comm-1), and a user reported uncertainty over whether their Word document was being auto-pushed to Copilot for training (copilot-comm-6), indicating the control's scope and clarity are inconsistent across the product's surface. Missing for 10: independent verification the opt-out functions as documented, clear documentation of training-data controls for Office-integrated Copilot (not just standalone), and enterprise/commercial data protection specifics.", "evidenceIds": [ "copilot-docs-35", "copilot-docs-36", "copilot-docs-42", "copilot-docs-38", "copilot-docs-44", "copilot-comm-1", "copilot-comm-6" ] }, { "productId": "copilot", "storyId": "voice-conversation", "verdict": "full", "quality": 6, "confidence": "medium", "rationale": "Microsoft's transparency docs explicitly describe Copilot Voice mode: users send natural-language voice prompts and receive audio responses, with citations shown in conversation transcripts, confirming real-time spoken interaction. Missing for 10: independent/hands-on corroboration of voice mode's real-time quality, latency, or interruption handling, and no community evidence specifically validates or disputes the voice experience.", "evidenceIds": [ "copilot-docs-29", "copilot-docs-28" ] }, { "productId": "copilot", "storyId": "web-search-citations", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Microsoft's own transparency documentation states Copilot 'centers its response on high-ranking content from the web and provides hyperlinked citations following generated text responses,' directly matching the story, and is reinforced by 'Ask questions and get answers fast' and the Researcher feature for finding/synthesizing information. Missing for 10: independent/hands-on verification that citations are accurate and consistently sourced, and no community evidence specifically evaluates citation quality (only tangential complaints about other Copilot features).", "evidenceIds": [ "copilot-docs-28", "copilot-docs-23", "copilot-docs-5" ] }, { "productId": "copilot", "storyId": "workspace-admin-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers only consumer-facing Copilot features (personal chat, Office app integrations, individual privacy settings like deleting conversation history or opting out of training) with no mention of an admin console, member/role management, or organization-wide data policy controls. Team-admin workspace governance is a plausible axis for a product used in organizations, but nothing in the evidence demonstrates it.", "evidenceIds": [ "copilot-docs-35", "copilot-docs-38", "copilot-docs-41", "copilot-docs-42" ] }, { "productId": "gemini", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence that Gemini can be pointed at llms.txt or agent-oriented documentation formats; the evidence only covers uploading files, GitHub repos, and connecting to apps, none of which address consuming llms.txt or agent-facing docs conventions. Missing for 10: any mention of llms.txt support, agent-oriented doc parsing, or a documented workflow for feeding agent-discovery docs to Gemini.", "evidenceIds": [] }, { "productId": "gemini", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Gemini's docs show robust support for generating insights from user data: uploading documents/spreadsheets/photos/videos for 'answers, summaries, and insights' (gemini-docs-25), Notebooks for organizing and querying sources (gemini-docs-11, gemini-docs-24, gemini-docs-29), Deep Research for in-depth analysis (gemini-docs-7), and Connected Apps that summarize Gmail and productivity data (gemini-docs-30). Independent hands-on reports corroborate real usage: navigating large codebases effortlessly (gemini-comm-13), replacing an OCR vendor for document ingestion with strong accuracy (gemini-comm-17), and NotebookLM being 'really powerful for comprehending large corpuses' (gemini-comm-19), though one report notes weaker accuracy on visual bounding-box tasks (gemini-comm-18). missing for 10: broader independent benchmarking specifically on insight-generation quality across data types, and resolution of the noted accuracy gaps in some visual tasks.", "evidenceIds": [ "gemini-docs-25", "gemini-docs-11", "gemini-docs-24", "gemini-docs-7", "gemini-docs-29", "gemini-docs-30", "gemini-comm-13", "gemini-comm-17", "gemini-comm-19", "gemini-comm-18" ] }, { "productId": "gemini", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Google's docs explicitly describe scheduling recurring actions so Gemini becomes part of daily/weekly/monthly routines (gemini-docs-8), and the Daily Brief feature autonomously delivers a personalized digest without user prompting (gemini-docs-32/41), supported by action-taking integrations with other apps (gemini-docs-10/17/23/30). However, evidence is limited to first-party docs with no independent/hands-on confirmation of reliability, no detail on trigger conditions, failure handling, or scope of what 'recurring actions' can actually do autonomously. Missing for 10: independent verification that scheduled automations run reliably unattended, documentation on trigger/condition logic, and evidence of broader autonomous multi-step task execution beyond simple digests.", "evidenceIds": [ "gemini-docs-8", "gemini-docs-32", "gemini-docs-41", "gemini-docs-10", "gemini-docs-17", "gemini-docs-23", "gemini-docs-30" ] }, { "productId": "gemini", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Gemini's docs show extensive task delegation: connected apps to take actions in Gmail/Calendar/Maps/Flights (gemini-docs-10, gemini-docs-23, gemini-docs-30), scheduled recurring actions (gemini-docs-8), custom Gems for specific goals (gemini-docs-9), Deep Research for autonomous multi-step research (gemini-docs-7), and Daily Brief proactive digests (gemini-docs-32, gemini-docs-41). Community evidence corroborates real-world delegated use cases like large codebase analysis and audio transcription (gemini-comm-13, gemini-comm-10). missing for 10: independent hands-on verification specifically of agentic task delegation/action-taking (vs. just Q&A), and evidence of reliability/success rate when Gemini executes multi-step delegated tasks.", "evidenceIds": [ "gemini-docs-7", "gemini-docs-8", "gemini-docs-9", "gemini-docs-10", "gemini-docs-23", "gemini-docs-30", "gemini-docs-32", "gemini-comm-13", "gemini-comm-10" ] }, { "productId": "gemini", "storyId": "agentic-headless", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack is dominated by consumer-facing Gemini Apps documentation (chat, image generation, Gemini Live, notebooks) and contains no first-party documentation of a headless mode, CI integration, or automation-friendly API/CLI usage. The only hint of a CLI tool appears in community complaints (gemini-comm-14, gemini-comm-15) about login failures for Workspace accounts trying to use 'Gemini CLI' for software development, which does not establish that headless/CI automation works.", "evidenceIds": [ "gemini-comm-14", "gemini-comm-15" ] }, { "productId": "gemini", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence covers Gemini Apps' 'Connected Apps' feature (Gmail, Calendar, Maps, Flights) and coding-related uploads (GitHub repos, folders), but nothing describes plugging in Model Context Protocol (MCP) servers as a tool-connection mechanism. Missing for 10: any documentation or mention of MCP server support, MCP client configuration, or third-party tool registration via MCP standard.", "evidenceIds": [ "gemini-docs-10", "gemini-docs-17", "gemini-docs-23", "gemini-docs-30", "gemini-docs-33" ] }, { "productId": "gemini", "storyId": "agentic-mcp-server", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gemini is itself an AI assistant/agent product; the evidence shows only that it can connect outward to other apps (Connected Apps, code import) as a client, not that it exposes an official MCP server for other agents to connect to it. Per the agent-role exception, this axis does not apply unless Gemini documents running as an MCP server, which no evidence here shows.", "evidenceIds": [ "gemini-docs-10", "gemini-docs-17", "gemini-docs-23", "gemini-docs-33" ] }, { "productId": "gemini", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Gemini is fundamentally operated via natural-language chat and voice (Gemini Live's free-flowing conversation, 'Hey Google' voice commands, text prompts for image generation, scheduling recurring actions, custom Gems, and connected-app actions) rather than rigid commands or GUIs, per extensive first-party docs. Missing for 10: independent/hands-on verification that complex natural-language action requests (e.g., connected-app task execution) reliably work as described, and community evidence is mixed on model quality/accuracy for specific tasks rather than confirming the NL interface itself.", "evidenceIds": [ "gemini-docs-12", "gemini-docs-22", "gemini-docs-34", "gemini-docs-40", "gemini-docs-10", "gemini-docs-17", "gemini-docs-23", "gemini-docs-8", "gemini-docs-9", "gemini-docs-30", "gemini-docs-6" ] }, { "productId": "gemini", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Community discussion confirms the existence of an official 'Gemini CLI' used for software development (comm-13, comm-14), but there is no first-party documentation of it in the evidence pack, and a hands-on report shows it fails to work for Workspace accounts ('Failed to login... now as a Workspace account I cant use Gemini CLI for my work') (comm-15). Missing for 10: official CLI documentation/spec, installation and usage guides, and evidence that login/account restrictions have been resolved.", "evidenceIds": [ "gemini-comm-13", "gemini-comm-14", "gemini-comm-15" ] }, { "productId": "gemini", "storyId": "agentic-public-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Community evidence confirms a public Gemini API exists and is actively used by developers (pricing tiers, model versions, Google AI Studio parameters like temperature and top_k), but the evidence pack contains no first-party API reference documentation, and one community report notes undocumented/hidden parameter behavior (fixed top_k, capped temperature) that limits full programmatic control. missing for 10: first-party API reference/docs citation, SDK/auth documentation, confirmation that documented parameters match actual behavior.", "evidenceIds": [ "gemini-comm-8", "gemini-comm-9", "gemini-comm-10", "gemini-comm-11", "gemini-comm-12" ] }, { "productId": "gemini", "storyId": "agentic-scoped-keys", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gemini is an AI product/app, not a platform issuing API credentials with scoped permissions for agents; the evidence pack covers consumer features (imports, images, connected apps) with no mention of credential scoping or API key/permission management for agents, so this axis is a category error for this product's evidenced role.", "evidenceIds": [] }, { "productId": "gemini", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack only covers Gemini's consumer-facing app features (uploads, Gemini Live, Gems, connected apps) and does not document any official SDKs for building applications (e.g., Python/Node/Go SDKs for the Gemini API). Community mentions reference API pricing and a 'Gemini CLI' having login issues, but no concrete SDK documentation or hands-on developer usage evidence is present.", "evidenceIds": [ "gemini-comm-8", "gemini-comm-15" ] }, { "productId": "gemini", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack describes Gemini offering a webhook subscription mechanism for events; the docs cover imports, connected apps, Gems, Live, Deep Research, etc., but nothing about outbound webhooks or event subscriptions for AI-native/agentic integration.", "evidenceIds": [] }, { "productId": "gemini", "storyId": "api-interactive-docs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "This story asks for an interactive API reference with runnable examples, which is a developer-portal/API-docs concern, not a fair axis for Gemini Apps (the consumer product surfaced in this evidence pack). The evidence pack covers Gemini's app features (Gems, Connected Apps, Deep Research, Live) but contains no mention of an interactive API reference, making this a wrong-axis question for the product as evidenced.", "evidenceIds": [] }, { "productId": "gemini", "storyId": "api-machine-spec", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gemini here refers to the consumer AI assistant app/product; the story asks for a downloadable machine-readable API spec (OpenAPI), which is a developer-platform artifact, not a consumer app feature. This axis applies to API/platform products, not to the Gemini Apps experience described in the evidence.", "evidenceIds": [] }, { "productId": "gemini", "storyId": "api-sandbox", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gemini is a consumer/enterprise assistant product, not a dev platform with sandbox/production data environments; the evidence pack shows no sandbox testing environment concept, making this axis a category error for this product type.", "evidenceIds": [] }, { "productId": "gemini", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack is entirely consumer-focused Gemini Apps documentation (image generation, Gemini Live, connected apps, notebooks) and general community sentiment about model quality/pricing; there is no mention of API versioning, version lifecycle, or a documented deprecation policy anywhere in the pack.", "evidenceIds": [] }, { "productId": "gemini", "storyId": "app-directory", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Gemini docs describe a 'Connected Apps' feature that lets users link Gmail, Calendar, Maps, Flights and 'a variety of productivity apps' so Gemini can search/create content and take actions, which matches the connectors-directory concept ([gemini-docs-10],[gemini-docs-17],[gemini-docs-23],[gemini-docs-30],[gemini-docs-33],[gemini-docs-19]). However, the evidence never shows a browsable app-store-like directory UI, doesn't clarify how many third-party (non-Google) apps are included, and there's no independent/hands-on confirmation of the browse-and-add experience. Missing for 10: explicit directory/marketplace UI screenshots or docs, breadth of third-party (non-Google) app listings, independent user corroboration of adding connectors.", "evidenceIds": [ "gemini-docs-10", "gemini-docs-17", "gemini-docs-23", "gemini-docs-30", "gemini-docs-33", "gemini-docs-19" ] }, { "productId": "gemini", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Gemini supports uploading whole code folders/repos and large codebases (a form of bulk ingestion) and can take actions across connected apps or run scheduled recurring actions, but there is no explicit documentation of a bulk/batch operation feature (e.g., applying an action to many items at once like bulk-editing files, mass-tagging, or batch task execution) as an AI-native automation capability. missing for 10: explicit bulk-action/batch-processing feature, evidence of applying one instruction across many discrete items simultaneously, independent corroboration of this specific capability.", "evidenceIds": [ "gemini-docs-2", "gemini-docs-14", "gemini-docs-8", "gemini-docs-10", "gemini-comm-13" ] }, { "productId": "gemini", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Gemini supports 'scheduling recurring actions' as part of daily/weekly/monthly routines (gemini-docs-8), which is a form of automation, but this is time-based scheduling rather than user-defined conditional rules that trigger on arbitrary events. Connected Apps let Gemini take actions in other apps (gemini-docs-10, gemini-docs-23, gemini-docs-30) but only in response to a direct user request, not an automatic event trigger. Missing for 10: evidence of event-based (not just time-based) trigger conditions, ability to define custom 'if X happens then do Y' rules, and any confirmation of conditional logic beyond fixed recurring schedules.", "evidenceIds": [ "gemini-docs-8", "gemini-docs-10", "gemini-docs-23", "gemini-docs-30" ] }, { "productId": "gemini", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Gemini docs explicitly confirm scheduling recurring actions ('daily, weekly, and monthly routines') via a dedicated feature (gemini-docs-8), and Daily Brief shows an example of a recurring automated digest (gemini-docs-32/41). However, evidence lacks detail on workflow complexity, chaining multiple steps/tools, error handling, or independent/community validation of this scheduling feature. missing for 10: detail on multi-step workflow scheduling, chaining across apps, reliability/error handling, and independent hands-on confirmation.", "evidenceIds": [ "gemini-docs-8", "gemini-docs-32", "gemini-docs-41" ] }, { "productId": "gemini", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of version control, review workflows, or rollback capabilities for automations (e.g., scheduled actions, Gems, or connected app actions). The evidence only covers scheduling recurring actions and connecting apps, with no mention of versioning, diffing, review approval, or rollback of automations.", "evidenceIds": [] }, { "productId": "gemini", "storyId": "autonomous-background-tasks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Deep Research lets users delegate an open-ended research task that Gemini works on autonomously and returns a report for review, and scheduled/recurring actions show background task automation; Connected Apps/Gems add some autonomous action-taking on the user's behalf. However there's no clear evidence of a general-purpose multi-step 'agent mode' that plans and executes across arbitrary apps end-to-end with an explicit review/approval step, nor independent hands-on confirmation of this workflow. Missing for 10: evidence of a broad autonomous agent (beyond Deep Research/scheduling) handling arbitrary multi-step tasks across apps, and independent/community validation of such background task delegation working reliably.", "evidenceIds": [ "gemini-docs-7", "gemini-docs-8", "gemini-docs-9", "gemini-docs-10", "gemini-docs-23", "gemini-docs-30" ] }, { "productId": "gemini", "storyId": "browser-agent", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Deep Research lets Gemini conduct 'in-depth and real-time research' across web sources, and Connected Apps let it take actions in other apps (Gmail, Calendar, Maps, Flights) with permission — showing some research/task-automation capability. However, there is no evidence of Gemini actually driving a web browser (navigating pages, clicking, filling forms) to complete tasks on arbitrary websites, only sanctioned first-party app integrations. Missing for 10: explicit browser-control/agentic navigation evidence, hands-on demonstration of completing a multi-step website task, independent corroboration of this capability.", "evidenceIds": [ "gemini-docs-7", "gemini-docs-29", "gemini-docs-10", "gemini-docs-23", "gemini-docs-30" ] }, { "productId": "gemini", "storyId": "computer-use-desktop", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Gemini's Connected Apps feature lets it take actions in other apps (Gmail, Calendar, Maps, Flights) with permission, and on Android it can see what's on-screen to help with context, but this is limited to a curated set of app integrations/APIs rather than general computer-use/desktop automation across arbitrary applications. Missing for 10: broader OS-level screen perception, ability to click/type across arbitrary desktop apps, and independent hands-on evidence of an agentic computer-use mode.", "evidenceIds": [ "gemini-docs-10", "gemini-docs-17", "gemini-docs-23", "gemini-docs-30", "gemini-docs-6", "gemini-docs-33" ] }, { "productId": "gemini", "storyId": "connect-drive-calendar", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Google's docs describe first-party 'Connected Apps' letting Gemini summarize Gmail, create Calendar events, search across productivity apps, and use personal Gmail/Drive as sources for Deep Research, directly matching the story's cloud drive/email/calendar connector requirement. Missing for 10: independent/hands-on verification of search quality and confirmation that calendar reading (not just event creation) works reliably.", "evidenceIds": [ "gemini-docs-10", "gemini-docs-17", "gemini-docs-23", "gemini-docs-29", "gemini-docs-30", "gemini-docs-33" ] }, { "productId": "gemini", "storyId": "create-edit-documents", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence documents Gemini's ability to generate and iteratively edit images (Nano Banana) with export/download at set resolutions, and general writing assistance, but there is no evidence of a dedicated document or presentation creation/editing workflow (e.g., a canvas-style doc/slide editor) with export to file formats. Missing for 10: explicit documentation of creating/iterating on text documents or slide decks, and exporting them to standard file formats (docx/pptx/pdf).", "evidenceIds": [ "gemini-docs-3", "gemini-docs-4", "gemini-docs-5", "gemini-docs-27", "gemini-docs-28", "gemini-docs-20" ] }, { "productId": "gemini", "storyId": "custom-bots", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Gemini's \"Gems\" feature (gemini-docs-9) lets power-users create a custom Gem with its own instructions and requirements, and Gems can be given knowledge via uploaded files/sources (gemini-docs-25), directly matching the core of this story. However, evidence does not confirm a straightforward sharing/publishing mechanism for Gems to other users, nor independent hands-on validation of the feature's robustness. Missing for 10: documentation of sharing/publishing custom Gems with other users, evidence of knowledge-base scope/limits, and independent community corroboration of Gems in practice.", "evidenceIds": [ "gemini-docs-9", "gemini-docs-25" ] }, { "productId": "gemini", "storyId": "data-analysis-code", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Gemini can import/upload code folders and GitHub repos and get 'insights' from uploaded documents/spreadsheets, but there is no evidence of an actual code-execution/sandbox feature that writes and runs code to produce charts, computed answers, or downloadable output files. No docs or community reports describe this specific data-analysis-with-code-execution workflow.", "evidenceIds": [ "gemini-docs-1", "gemini-docs-2", "gemini-docs-25" ] }, { "productId": "gemini", "storyId": "deep-research-report", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Gemini Apps has a documented Deep Research feature that conducts in-depth, real-time research across sources and allows adding personal sources (Gmail, Drive, uploads, NotebookLM notebooks) for the research, matching the autonomous multi-source research story. missing for 10: explicit documentation of citation formatting in the final report and independent hands-on evidence of report quality/citation accuracy from third parties.", "evidenceIds": [ "gemini-docs-7", "gemini-docs-29" ] }, { "productId": "gemini", "storyId": "desktop-app", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm an official Mac desktop app with a global OS-level keyboard shortcut (Option+Space) to summon Gemini without opening a browser [gemini-docs-15, gemini-docs-36, gemini-docs-20], but the only 'see what's on your screen' capability documented is for the Android mobile app via 'Hey Google' [gemini-docs-6], not the desktop app. Missing for 10: evidence of desktop screen-reading/screenshare capability, and independent/hands-on confirmation of the desktop shortcut and screen access working together.", "evidenceIds": [ "gemini-docs-15", "gemini-docs-20", "gemini-docs-36", "gemini-docs-6" ] }, { "productId": "gemini", "storyId": "export-my-data", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence only covers reviewing and deleting Gemini activity via My Activity (gemini-docs-31) and privacy settings for Live audio (gemini-docs-37, gemini-docs-38), but there is no mention of exporting chat history or account data (e.g., via Google Takeout or a dedicated export tool). missing for 10: any documentation of an export/download feature for chat history or account data, independent confirmation of export completeness.", "evidenceIds": [ "gemini-docs-31" ] }, { "productId": "gemini", "storyId": "file-upload-analysis", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "First-party docs explicitly state users can upload documents, spreadsheets, and more to get answers, summaries, and insights (gemini-docs-25), and independent hands-on evidence corroborates accurate document/PDF analysis (a fintech user reporting 96% OCR accuracy on PDFs, comm-17) plus praise for NotebookLM's ability to comprehend large source corpora (comm-19). Missing for 10: dedicated evidence on spreadsheet-specific accuracy and broader independent benchmarking across document types beyond the single fintech PDF case.", "evidenceIds": [ "gemini-docs-25", "gemini-comm-17", "gemini-comm-19" ] }, { "productId": "gemini", "storyId": "image-generation", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Gemini docs explicitly describe generating and editing images via natural-language prompts (Nano Banana 2), including local edits, character consistency, blending images, and resolution export options. missing for 10: independent hands-on corroboration of image editing quality and no community evidence testing this specific feature.", "evidenceIds": [ "gemini-docs-3", "gemini-docs-4", "gemini-docs-5", "gemini-docs-16", "gemini-docs-27", "gemini-docs-28" ] }, { "productId": "gemini", "storyId": "image-understanding", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Docs confirm users can upload photos/screenshots and get insights (gemini-docs-25) and get screen-context help via 'Hey Google' (gemini-docs-6), but hands-on reports show real accuracy failures: Gemini generated 'wildly inaccurate bounding boxes' on images (gemini-comm-18) and failed to correctly interpret a manipulated photo, missing key details even in the latest version (gemini-comm-12). A positive OCR use case exists (gemini-comm-17) but that's for PDFs, not general screenshot/photo scene interpretation, so the core 'accurately interpret' claim is contradicted by concrete hands-on failures. Missing for 10: independent benchmarks specifically on screenshot/UI interpretation accuracy, and resolution of the documented bounding-box/detail-recognition failures.", "evidenceIds": [ "gemini-docs-25", "gemini-docs-6", "gemini-comm-18", "gemini-comm-12", "gemini-comm-17" ] }, { "productId": "gemini", "storyId": "memory-across-chats", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Gemini docs explicitly describe a personalization feature where 'the memory of your past Gemini chats' informs new conversations, plus user controls to review/delete that activity, confirming both the capability and its persistence across sessions. Missing for 10: independent hands-on validation of cross-chat memory recall accuracy and more detail on scope/duration of retained context.", "evidenceIds": [ "gemini-docs-18", "gemini-docs-31" ] }, { "productId": "gemini", "storyId": "mobile-apps", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm a full-featured Gemini mobile app with rich functionality (Gemini Live voice conversations, screen-aware Android assistant via 'Hey Google', Maps/Flights integration, daily briefs, scheduling, image generation, coding help) showing strong Android support, and generic mobile-app language implies broader availability, but the evidence never explicitly names or details an iOS app or iOS-specific parity. missing for 10: explicit iOS app documentation/feature parity, App Store presence, and any cross-platform comparison confirming equal functionality on iOS vs Android.", "evidenceIds": [ "gemini-docs-6", "gemini-docs-12", "gemini-docs-19", "gemini-docs-20", "gemini-docs-26", "gemini-docs-32", "gemini-docs-35", "gemini-docs-41" ] }, { "productId": "gemini", "storyId": "openness-api-parity", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack details many Gemini Apps UI-only features (Gems, Deep Research, Connected Apps, Daily Brief, Live conversations, Notebooks, scheduling) with no corresponding API documentation showing these are exposed programmatically. Community evidence (e.g., gemini-comm-11) even notes the API/AI Studio path is more restricted (capped temperature, hidden top_k) than the full product, reinforcing that API and UI capability sets are not equivalent.", "evidenceIds": [ "gemini-docs-7", "gemini-docs-9", "gemini-docs-10", "gemini-docs-12", "gemini-docs-32", "gemini-comm-11" ] }, { "productId": "gemini", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers activity deletion/review (gemini-docs-31) but no mention of exporting data in open/portable formats or data-export tooling for leaving the platform. Missing for 10: any documentation of a data export/download feature (e.g., Google Takeout integration), open-format export options, or independent confirmation of data portability.", "evidenceIds": [ "gemini-docs-31" ] }, { "productId": "gemini", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gemini is a closed, proprietary AI product; no evidence suggests its source code is available under an open license, and open-sourcing the model itself is not a plausible axis for this kind of commercial closed product.", "evidenceIds": [] }, { "productId": "gemini", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gemini is a closed, proprietary hosted AI service with no evidence of any self-hostable model weights or on-prem deployment option; self-hosting is not a plausible axis for this cloud-only consumer/enterprise product category.", "evidenceIds": [] }, { "productId": "gemini", "storyId": "persistent-instructions", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Gemini offers a 'memory of past chats' personalization feature (gemini-docs-18) and custom Gems that let users pre-set instructions/goals (gemini-docs-9), which together approximate persistent customization, but there's no documented single 'custom instructions' setting that explicitly shapes every response across all chats. Missing for 10: explicit persistent instruction/preference settings applied globally to every response, independent verification of memory consistency, and detail on how memory interacts with different chat sessions.", "evidenceIds": [ "gemini-docs-18", "gemini-docs-9", "gemini-docs-32" ] }, { "productId": "gemini", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers Gemini's features and privacy/data-use settings (e.g., activity retention, human review opt-outs) but contains no mention of data residency or regional storage controls for AI-native/enterprise users. missing for 10: any documentation of region selection, data residency guarantees, or enterprise data-location controls.", "evidenceIds": [] }, { "productId": "gemini", "storyId": "privacy-no-training", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Google's support docs state that chats and uploaded files in Gemini Apps are not reviewed by humans or used to improve generative AI models, and users can manage/delete their activity via 'My Activity' and toggle 'Keep Activity' off (gemini-docs-38, gemini-docs-31). However, a separate doc notes that audio/video from Gemini Live can be used to train models unless the user opts out, showing the training-data control is feature-specific rather than a single unified setting (gemini-docs-37). Missing for 10: a clear, centralized opt-out control covering all Gemini surfaces (not just chat text), independent/hands-on verification that opted-out data is truly excluded from training, and clarity for enterprise/Workspace accounts.", "evidenceIds": [ "gemini-docs-38", "gemini-docs-37", "gemini-docs-31" ] }, { "productId": "gemini", "storyId": "privacy-retention-controls", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Google's support docs explicitly describe controls to review prompts, delete Gemini Apps activity, and turn off 'Keep Activity' (gemini-docs-31), plus explicit statements that chats/files are not used for model training by default (gemini-docs-38) while audio/video in Gemini Live can be opted in/out of training use (gemini-docs-37) — directly matching retention/deletion control for an AI-native user. Missing for 10: independent/hands-on verification that deletion actually removes data, and detail on data export or full account-level deletion workflows.", "evidenceIds": [ "gemini-docs-31", "gemini-docs-37", "gemini-docs-38" ] }, { "productId": "gemini", "storyId": "privacy-telemetry-optout", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show Gemini provides some privacy controls: reviewing/deleting prompts and turning off 'Keep Activity' (gemini-docs-31), and settings around whether audio/video/chats are used to improve models (gemini-docs-37, gemini-docs-38). This addresses usage-history and training-data opt-out but not a full 'telemetry/usage tracking' opt-out (e.g., diagnostic/product analytics separate from chat history) and there's no independent verification of these controls' effectiveness. Missing for 10: explicit documentation of a general telemetry/analytics opt-out toggle beyond activity/training-data settings, and independent/hands-on confirmation that opting out actually stops tracking.", "evidenceIds": [ "gemini-docs-31", "gemini-docs-37", "gemini-docs-38" ] }, { "productId": "gemini", "storyId": "projects-workspaces", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Gemini's 'Notebooks' feature explicitly provides a continuous, focused space that remembers sources, instructions, and ongoing chat history, and lets users upload documents, code, and other files into that same organized context (gemini-docs-11, gemini-docs-24, gemini-docs-25, gemini-docs-29). Custom Gems (gemini-docs-9) further let users set persistent instructions for recurring tasks, and independent commentary confirms NotebookLM/Notebooks are genuinely useful for organizing large source sets into a coherent project workspace (gemini-comm-19), though that same source flags UI/display shortcomings. Missing for 10: no first-party doc detailing multi-chat organization within one persistent 'project' the way competitors define it, and no hands-on evidence of shared-instruction consistency across many chats in the same notebook.", "evidenceIds": [ "gemini-docs-11", "gemini-docs-24", "gemini-docs-25", "gemini-docs-29", "gemini-docs-9", "gemini-comm-19" ] }, { "productId": "gemini", "storyId": "scheduled-tasks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Gemini docs confirm scheduling recurring actions (gemini-docs-8) and Daily Brief personalized digest (gemini-docs-32/41), showing one-off/recurring automated tasks that return results. However, evidence lacks detail on task complexity, multi-step agent tasks, notification delivery mechanisms, or independent/community corroboration of this specific feature working reliably. missing for 10: detailed documentation on scheduling flexibility, agentic multi-step task execution, notification/result delivery mechanisms, and independent hands-on validation of the scheduled-tasks feature.", "evidenceIds": [ "gemini-docs-8", "gemini-docs-32", "gemini-docs-41" ] }, { "productId": "gemini", "storyId": "training-opt-out", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Google's own support docs state that by default chats/uploaded files aren't used to improve generative AI models unless a user opts in via 'Keep Activity', and that users can review, delete activity, or turn off Keep Activity at any time; separately, users must explicitly consent to let Google use Gemini Live audio/video for model training. This gives knowledge-workers clear, documented control over whether conversations feed model training. Missing for 10: independent/hands-on verification that these settings work as described and clarity on enterprise/Workspace account defaults.", "evidenceIds": [ "gemini-docs-31", "gemini-docs-37", "gemini-docs-38" ] }, { "productId": "gemini", "storyId": "voice-conversation", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Gemini Live is explicitly documented to support natural, free-flowing, interruptible voice conversations where Gemini talks back verbally in real time, directly matching the story. Missing for 10: independent/hands-on community corroboration specifically validating the voice conversation experience (community evidence covers other Gemini features/models but not Live voice interactions).", "evidenceIds": [ "gemini-docs-12", "gemini-docs-22", "gemini-docs-34", "gemini-docs-40" ] }, { "productId": "gemini", "storyId": "web-search-citations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Gemini's Deep Research feature explicitly supports 'in-depth and real-time research on almost any subject,' implying grounded, current-web-based answers, and users can add further sources for research context. However, the evidence pack never explicitly documents inline citation links back to specific web sources, nor independent confirmation that citations are reliably shown. Missing for 10: explicit documentation of citation/source-linking UI, independent verification that citations are accurate and consistently displayed.", "evidenceIds": [ "gemini-docs-7", "gemini-docs-29" ] }, { "productId": "gemini", "storyId": "workspace-admin-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence only shows end-user features (image generation, Live conversations, notebooks, connected apps) and passing mentions that a 'work or school account' administrator can 'enable Gemini,' but there is no evidence of an admin console for managing members, setting permissions, or configuring org-wide data policies.", "evidenceIds": [] }, { "productId": "grok", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "The docs site serves a discoverable llms.txt (HTTP 200, describing itself as official xAI developer documentation) and every doc page is also available in plain markdown form, which is exactly the agent-oriented docs pattern the story asks for. Missing for 10: independent/community confirmation that agents actually consume this llms.txt successfully in practice.", "evidenceIds": [ "grok-probe-1", "grok-docs-1", "grok-docs-20" ] }, { "productId": "grok", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Grok's connectors (Gmail/Calendar, Google Drive, Microsoft Teams, Salesforce, custom APIs/MCP) let it read and act on a user's own data inside the chat, and features like Realtime Multi-agent Research suggest Grok can analyze and synthesize that data into insights. However, the evidence is mostly first-party docs describing data access rather than concrete examples of proactive 'insights and suggestions,' and one hands-on report notes Grok has 'real trouble with agentic tool calling,' undercutting confidence in reliable in-context analysis. Missing for 10: independent hands-on demonstration of Grok surfacing unprompted insights/suggestions from connected data, and confirmation that tool-calling reliability issues don't undermine this in practice.", "evidenceIds": [ "grok-docs-11", "grok-docs-19", "grok-docs-35", "grok-docs-36", "grok-docs-37", "grok-docs-26", "grok-comm-6" ] }, { "productId": "grok", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Grok Bot is described as an AI teammate running on a 'persistent cloud computer' and Grok supports multi-agent orchestration for research tasks, hinting at some background/autonomous agent capability, but there is no documentation of scheduling, triggers, or recurring/timed automation setup. Missing for 10: explicit scheduling/trigger mechanism, details on how automations run without user prompting, and independent confirmation of autonomous background execution.", "evidenceIds": [ "grok-docs-15", "grok-docs-26" ] }, { "productId": "grok", "storyId": "agentic-builtin-assistant", "verdict": "disputed", "quality": 6, "confidence": "medium", "rationale": "xAI docs describe strong built-in agentic delegation — Grok Bot hands work to 'AI teammates on a persistent cloud computer,' Realtime Multi-agent Research orchestrates multiple agents for research tasks, and connectors let Grok act on email/calendar/Salesforce from chat (grok-docs-15, grok-docs-26, grok-docs-11, grok-docs-16, grok-docs-19). However, hands-on community testing directly contradicts smooth agentic delegation: a user reports Grok 'had real trouble with agentic tool calling - the model felt like it hadn't been trained to think of tool calling as one of its primary modalities' (grok-comm-6), and another notes the lack of a proper GUI for Grok Build limiting usability (grok-comm-7). Missing for 10: independent verification that Grok Bot/multi-agent research reliably completes delegated tasks end-to-end, and resolution of the reported tool-calling weakness.", "evidenceIds": [ "grok-docs-15", "grok-docs-26", "grok-docs-11", "grok-docs-16", "grok-docs-19", "grok-comm-6", "grok-comm-7" ] }, { "productId": "grok", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "The xAI API (REST, OpenAPI spec confirmed via probe, streaming, structured outputs, quickstart script) provides a clear headless programmatic interface suitable for automation/CI, and community mentions a 'terminal' mode for Grok Build. However there is no explicit CI/CD documentation, GitHub Actions example, or non-interactive automation guide. missing for 10: dedicated CI/CD integration docs, GitHub Actions or pipeline examples, explicit 'headless mode' documentation.", "evidenceIds": [ "grok-docs-20", "grok-docs-27", "grok-docs-22", "grok-docs-24", "grok-probe-3", "grok-comm-7" ] }, { "productId": "grok", "storyId": "agentic-mcp-client", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "xAI docs explicitly describe adding custom MCP connectors and tunneling a local MCP server so Grok can reach it and use its tools, alongside built-in OAuth connectors for other tools (grok-docs-2, grok-docs-3, grok-docs-38, grok-docs-19). Missing for 10: independent/hands-on confirmation that custom MCP tool-calling works reliably in practice, and more detail on protocol/version compatibility.", "evidenceIds": [ "grok-docs-2", "grok-docs-3", "grok-docs-38", "grok-docs-19" ] }, { "productId": "grok", "storyId": "agentic-mcp-server", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Grok is itself an agent/assistant (chat + Grok Build coding agent), and the MCP-related evidence (grok-docs-2,3,19,38) describes Grok acting as an MCP *client* connecting outbound to a user's custom MCP server — not Grok exposing itself as an MCP server for other agents to connect to. Per the rules, client-side MCP connectivity does not make this server-role axis applicable, and there is no evidence of a first-party 'Grok as MCP server' endpoint.", "evidenceIds": [ "grok-docs-2", "grok-docs-3", "grok-docs-19", "grok-docs-38" ] }, { "productId": "grok", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Grok's docs describe a chat/voice interface where natural-language input drives text generation, image/video creation, and connector actions like creating Salesforce records or searching email 'directly from chat' (grok-docs-11, grok-docs-16, grok-docs-35, grok-docs-40), which is core evidence for NL-command operation. However, hands-on community testing reports real trouble with agentic tool calling, saying the model 'hadn't been trained to think of tool calling as one of its primary modalities' (grok-comm-6), which tempers confidence that NL commands reliably trigger tool/connector actions. Missing for 10: independent benchmarks or user reports specifically confirming reliable NL-to-action execution across the built-in connectors, and no clear resolution of the tool-calling reliability concern.", "evidenceIds": [ "grok-docs-11", "grok-docs-16", "grok-docs-35", "grok-docs-40", "grok-docs-19", "grok-comm-6" ] }, { "productId": "grok", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "There's no official CLI documentation for Grok, but a community comment references a terminal-based interface for 'Grok Build' ('terminal is nice but codex desktop app is very useful'), suggesting some CLI-like coding tool exists even though it isn't documented in the official docs pack. missing for 10: official CLI documentation/reference, installation instructions, and command examples confirming a first-party CLI product.", "evidenceIds": [ "grok-comm-7", "grok-docs-27" ] }, { "productId": "grok", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "xAI publishes a full public REST API with documented endpoints for text, image, video, tool use, structured outputs, and an OpenAPI spec confirmed live by probe, plus a quickstart guide, satisfying the AI-native/agentic driving story. missing for 10: independent third-party benchmarking of API reliability/completeness beyond xAI's own docs and anecdotal community chatter about tool-calling quirks.", "evidenceIds": [ "grok-docs-20", "grok-docs-27", "grok-docs-2", "grok-docs-24", "grok-docs-22", "grok-probe-3", "grok-probe-1" ] }, { "productId": "grok", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The xAI API is a developer product with API keys, so scoped/least-privilege credential issuance is a fair capability to expect, but the evidence pack contains no mention of API key scopes, roles, granular permissions, or least-privilege credential controls — only OAuth-based connector authentication (grok-docs-19) for built-in tools, which is about connecting Grok to external services, not issuing scoped agent credentials.", "evidenceIds": [ "grok-docs-19", "grok-docs-2" ] }, { "productId": "grok", "storyId": "agentic-sdks", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The evidence shows an official xAI developer API with a published OpenAPI spec, quickstart docs, and text-generation/tool-use guides (grok-docs-20, grok-docs-27, grok-probe-3), which implies programmatic access an AI-native builder could use, but none of the evidence explicitly names or documents an official SDK (e.g., Python/JS client library) as opposed to raw REST/OpenAPI usage. Missing for 10: explicit documentation of a first-party SDK package, install instructions, or code samples showing SDK usage, plus any independent/community confirmation of SDK quality or adoption.", "evidenceIds": [ "grok-docs-20", "grok-docs-27", "grok-probe-3", "grok-probe-1" ] }, { "productId": "grok", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of any webhook subscription mechanism for Grok events; the docs describe connectors, tools, image/video generation, and inbound MCP tunneling, but nothing about outbound event notifications or webhook endpoints. Missing for 10: any webhook subscription API, event types, or delivery/registration mechanism.", "evidenceIds": [] }, { "productId": "grok", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "xAI publishes machine-readable docs (llms.txt, openapi.json) and a quickstart with example prompts/code, showing a structured API reference exists, but there is no evidence of an interactive 'try it now' console or in-browser runnable examples typical of tools like Swagger UI or Postman-style explorers. Missing for 10: interactive in-browser code execution/sandbox, live request/response try-it widget, independent confirmation of an interactive reference UI.", "evidenceIds": [ "grok-probe-1", "grok-probe-3", "grok-docs-27", "grok-docs-20" ] }, { "productId": "grok", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A direct probe confirms https://docs.x.ai/openapi.json returns HTTP 200 and contains an 'openapi' key, i.e., a downloadable, machine-readable OpenAPI spec for the xAI API, and this is corroborated by extensive first-party API docs describing the full endpoint surface (text, images, video, tools). Missing for 10: independent/community confirmation that developers actually consume this spec (e.g., via SDK codegen) beyond the automated probe.", "evidenceIds": [ "grok-probe-3", "grok-probe-1", "grok-docs-20" ] }, { "productId": "grok", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a sandbox/staging environment or test mode that isolates from production data; docs cover connectors, tunneling for custom tools, and general product features but nothing about a dedicated sandbox for testing without touching production data. missing for 10: sandbox/test environment documentation, data isolation guarantees, staging API keys or environments.", "evidenceIds": [] }, { "productId": "grok", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows model version slugs (grok-4.6, grok-4.5) and an OpenAPI spec, but no documentation of a formal deprecation policy, sunset timelines, or versioning guarantees for the API. Release notes describe new features and model updates but never state how/when older model versions are retired.", "evidenceIds": [ "grok-docs-1", "grok-docs-23", "grok-probe-3" ] }, { "productId": "grok", "storyId": "app-directory", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show Grok exposes a set of built-in, OAuth-authenticated connectors (Salesforce, Gmail, Google Calendar, Google Drive, Microsoft Teams) plus custom/MCP connectors that a user can add to reach external tools (grok-docs-16,19,35,36,37,2,3). This matches the 'add third-party apps/connectors' part of the story, but there's no explicit evidence of a browsable directory/marketplace UI, ratings, or a curated catalog page for discovering connectors. missing for 10: a documented directory/marketplace UI for browsing connectors, independent user confirmation of the browse experience.", "evidenceIds": [ "grok-docs-19", "grok-docs-16", "grok-docs-35", "grok-docs-36", "grok-docs-37", "grok-docs-2", "grok-docs-3" ] }, { "productId": "grok", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "The docs mention batch generation of multiple images and support for up to 5 source images in a single edit, which is a narrow form of bulk operation, but there's no evidence of general-purpose bulk operations (e.g., bulk file processing, bulk record updates, batch text API) across many items or arbitrary data types. Missing for 10: a true batch/bulk API for text or file processing, bulk actions on connector data (e.g., mass email/calendar updates), and independent confirmation that these batch features work reliably at scale.", "evidenceIds": [ "grok-docs-28", "grok-docs-8", "grok-docs-30" ] }, { "productId": "grok", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Grok's docs describe connectors, tool integrations, and a persistent 'Grok Bot' cloud agent, but nothing describes user-defined rules that trigger actions automatically on external events (e.g., webhooks, triggers, if-this-then-that automations, or scheduled event listeners). The axis is plausible for an AI-native product with tool/connector access, so this is 'none' rather than 'na'.", "evidenceIds": [ "grok-docs-15", "grok-docs-19", "grok-docs-2" ] }, { "productId": "grok", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of scheduling, cron-like recurrence, or persistent recurring workflow features anywhere in the docs; Grok Bot ('persistent cloud computer') and connectors describe on-demand agents/tools, not scheduled recurring jobs.", "evidenceIds": [] }, { "productId": "grok", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence describes any versioning, review, diffing, or rollback mechanism for automations/agents/connectors built with Grok; the docs cover connectors, generation capabilities, and workspace management but nothing about tracking or reverting automation history.", "evidenceIds": [] }, { "productId": "grok", "storyId": "autonomous-background-tasks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs describe 'Grok Bot' as handing 'real work to AI teammates on a persistent cloud computer' and a 'Realtime Multi-agent Research' feature that orchestrates multi-step research autonomously, directly matching the story's core idea of background delegation and review. However, evidence is vendor-doc only with no independent confirmation of the async review workflow (notifications, task state, resuming), and community feedback (grok-comm-6) notes real trouble with agentic tool calling, suggesting reliability caveats. Missing for 10: independent hands-on validation of the background-task lifecycle, detail on how results are surfaced for review, and confirmation tool-calling reliability issues don't undermine multi-step autonomy.", "evidenceIds": [ "grok-docs-15", "grok-docs-26", "grok-comm-6", "grok-docs-20" ] }, { "productId": "grok", "storyId": "browser-agent", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows connectors for APIs, email, calendar, Salesforce, and a vague 'Grok Bot' cloud computer teammate, but nothing documents Grok actually driving a web browser to navigate sites or complete tasks (no browsing/computer-use agent feature is described). Multi-agent research (grok-docs-26) is about research orchestration, not website interaction, so this applicable-but-unevidenced capability is 'none'.", "evidenceIds": [ "grok-docs-15", "grok-docs-19", "grok-docs-26" ] }, { "productId": "grok", "storyId": "computer-use-desktop", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Grok offers connectors (Gmail, Drive, Teams, Salesforce, custom MCP) and a 'Grok Bot' that operates on a persistent *cloud* computer, but there is no evidence of Grok seeing or controlling applications running on the user's own local machine/desktop (e.g., screen/GUI control akin to computer-use agents). Missing for 10: any documented local-computer screen/GUI control, first-party or hands-on evidence of operating a user's own desktop apps.", "evidenceIds": [ "grok-docs-15", "grok-docs-11", "grok-docs-19", "grok-docs-38" ] }, { "productId": "grok", "storyId": "connect-drive-calendar", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Grok docs explicitly describe built-in OAuth connectors for Gmail, Google Calendar, and Google Drive that let it search emails, read files, and manage calendar items right inside chat, plus a broader connectors framework for other SaaS tools. This directly matches the story of connecting cloud drive, email, and calendar for search/use in answers. Missing for 10: independent/hands-on confirmation of these specific connectors working reliably in practice (community evidence largely covers coding/chat quality, not connector behavior).", "evidenceIds": [ "grok-docs-11", "grok-docs-19", "grok-docs-35", "grok-docs-36" ] }, { "productId": "grok", "storyId": "create-edit-documents", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Grok connectors for reading/searching Google Drive, Gmail, Calendar, and Salesforce, plus image/video generation and coding via Grok Build, but nothing describes a document/presentation authoring or iterative-editing feature (e.g., a canvas-style doc editor) with export to file formats like Word/PowerPoint/PDF.", "evidenceIds": [ "grok-docs-36", "grok-docs-15", "grok-docs-27", "grok-docs-11" ] }, { "productId": "grok", "storyId": "custom-bots", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers connectors, tool integrations, MCP, image/video generation, and enterprise features, but there is no mention of a feature to create and share custom assistants/personas with their own instructions and knowledge base (akin to OpenAI's GPTs or Claude Projects).", "evidenceIds": [] }, { "productId": "grok", "storyId": "data-analysis-code", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes Grok's coding assistant (Grok Build), connectors for email/calendar/files, image/video generation, and multi-agent research, but nothing documents a code-interpreter style capability where Grok writes and executes code against user-supplied data to produce charts, computed answers, or downloadable output files. Community comments discuss coding quality and agentic tool-calling issues but not this specific data-analysis/file-output workflow.", "evidenceIds": [ "grok-docs-27", "grok-docs-11", "grok-docs-36", "grok-comm-6" ] }, { "productId": "grok", "storyId": "deep-research-report", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Grok's docs describe a dedicated 'Realtime Multi-agent Research' capability that orchestrates multiple AI agents to perform deep, multi-step research tasks, matching the core of the story [grok-docs-26]. However, the docs don't explicitly describe the output format (a cited report with sources) and there is no independent/hands-on corroboration of this specific feature in the evidence pack. Missing for 10: explicit documentation of citation/report formatting in the research output, and independent user validation that the feature works as described.", "evidenceIds": [ "grok-docs-26" ] }, { "productId": "grok", "storyId": "desktop-app", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows only cross-platform sign-in sync and hands-free voice, but no mention of an official desktop app with OS-level shortcuts or screen-reading/access capability; a community comment even notes the lack of a proper GUI for Grok's coding product ('still waiting for a proper gui for grok build ... codex desktop app is very useful'), suggesting no such desktop tool exists.", "evidenceIds": [ "grok-docs-41", "grok-docs-40", "grok-comm-7" ] }, { "productId": "grok", "storyId": "export-my-data", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack mentions a data export or account-data download feature for Grok; docs cover connectors, models, billing, SSO, and workspace features but never a 'export my chat history/data' capability. This is a fair trust-controls axis for a consumer chat product, so absence of evidence yields 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "grok", "storyId": "file-upload-analysis", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs show Grok can ingest images and connect to Google Drive/Gmail to search and read files, and boasts a 500k context window for knowledge work, implying some document handling, but there is no explicit documentation of native PDF or spreadsheet upload/parsing in the core chat UI, nor any accuracy benchmarks or hands-on validation of document analysis. Missing for 10: explicit PDF/spreadsheet upload feature docs, accuracy testing/validation, independent user reports confirming reliable document analysis.", "evidenceIds": [ "grok-docs-1", "grok-docs-11", "grok-docs-25", "grok-docs-36" ] }, { "productId": "grok", "storyId": "image-generation", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Grok Imagine and the API both explicitly support text-to-image generation and image editing (including multi-image edits up to 5 sources, aspect ratio control) with docs describing prompt-driven creation and editing directly relevant to knowledge-worker use. Missing for 10: independent hands-on community corroboration specifically of image generation/editing quality (community evidence pack focuses on coding/text/voice, not images).", "evidenceIds": [ "grok-docs-7", "grok-docs-28", "grok-docs-29", "grok-docs-30", "grok-docs-8", "grok-docs-9", "grok-docs-6" ] }, { "productId": "grok", "storyId": "image-understanding", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs confirm image understanding is a first-class capability: models accept image input and 'consider the image context when generating the response' (grok-docs-25), plus related image editing docs show the model 'understands the image content' (grok-docs-29). No independent hands-on report specifically tests screenshot/photo interpretation accuracy. missing for 10: independent/community corroboration of accuracy on real screenshots or photos, detail on multi-image or OCR-style interpretation limits.", "evidenceIds": [ "grok-docs-25", "grok-docs-29", "grok-docs-30" ] }, { "productId": "grok", "storyId": "memory-across-chats", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows conversation storage (grok-docs-21) and cross-device sync of conversations/settings (grok-docs-41), but nothing describing a memory feature that recalls relevant context from past chats and applies it to new, unrelated conversations. No docs or community reports confirm persistent cross-chat memory recall.", "evidenceIds": [] }, { "productId": "grok", "storyId": "mobile-apps", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs mention voice mode ('Talk to Grok hands-free with voice') and cross-platform sync of conversations/settings/subscriptions, implying mobile app support, but there is no explicit confirmation of dedicated full-featured iOS and Android apps, no app-store links, and no community/hands-on feedback about mobile app quality or feature parity with desktop/web. Missing for 10: explicit iOS/Android app documentation, feature-parity details, and independent user reviews of the mobile apps themselves.", "evidenceIds": [ "grok-docs-40", "grok-docs-41" ] }, { "productId": "grok", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The xAI API exposes core Grok model capabilities (text, images, video, tools, structured outputs, multi-agent research, connectors) documented alongside an OpenAPI spec, so much of the UI functionality is API-accessible. However, UI-only features like Grok Bot persistent cloud computer, team workspace/SSO/billing management, and built-in OAuth connectors (Gmail, Calendar, Teams, Salesforce) appear tied to the consumer/business app rather than exposed as general API endpoints, and there's no evidence of full parity. Missing for 10: confirmation that workspace/team management, SSO, billing, and native connector orchestration are API-accessible rather than UI-exclusive, plus independent corroboration of full feature parity.", "evidenceIds": [ "grok-docs-20", "grok-docs-2", "grok-docs-19", "grok-docs-15", "grok-docs-17", "grok-docs-42", "grok-probe-3" ] }, { "productId": "grok", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item mentions any data export feature, open format (e.g., JSON/Markdown conversation export) or account data portability tool for Grok; documentation covers connectors, models, and billing but never data export/leave-with-your-data capability.", "evidenceIds": [] }, { "productId": "grok", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains only proprietary API/product documentation and community commentary; there is no mention of any open-source license, public source repository, or open-weights release for Grok. As a mainstream capability question applicable to AI products, absence of evidence yields 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "grok", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Grok is a closed, proprietary API/cloud service; no evidence anywhere in the pack of open-sourced weights or a self-hostable core model/product, only hosted APIs, connectors, and SaaS features. Absence of any self-hosting capability for an applicable axis (models can in principle be released for self-hosting, as some competitors do) yields none rather than na.", "evidenceIds": [ "grok-docs-20", "grok-docs-21", "grok-docs-42" ] }, { "productId": "grok", "storyId": "persistent-instructions", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack items describe a settings feature for persistent custom instructions or personality/preference profiles that shape all future responses; only per-request system prompts, connectors, and server-side storage of past prompts/responses are documented.", "evidenceIds": [] }, { "productId": "grok", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack mentions data residency, region selection, or storage location choices for Grok; docs only confirm data is stored on xAI's servers without any residency options (grok-docs-21). Missing for 10: any mention of regional data storage options, residency controls, or compliance certifications tied to geography.", "evidenceIds": [ "grok-docs-21" ] }, { "productId": "grok", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents data retention (prompts and responses stored on xAI's servers, grok-docs-21) and vague 'enterprise-grade privacy protections' for team workspaces (grok-docs-13), but nowhere mentions an opt-out toggle, training-data exclusion setting, or policy statement about excluding user conversations from model training. This axis clearly applies to a consumer/enterprise AI chat product, so absence of evidence yields 'none' rather than 'na'.", "evidenceIds": [ "grok-docs-21", "grok-docs-13", "grok-docs-39" ] }, { "productId": "grok", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Docs confirm that inputs, reasoning, and responses are stored on xAI's servers (grok-docs-21) and mention vague 'enterprise-grade privacy protections' for team workspaces (grok-docs-13), but there is no documented mechanism for users to view, control, or delete their stored data/conversation history. Missing for 10: explicit data retention settings, a user-facing deletion/export tool, and any policy detailing retention periods or opt-out controls.", "evidenceIds": [ "grok-docs-21", "grok-docs-13" ] }, { "productId": "grok", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack confirms xAI stores prompts, reasoning, and responses on its servers (grok-docs-21) but contains no documentation of any telemetry/usage-tracking opt-out setting, privacy toggle, or data-collection preference control for individual or ai-native users.", "evidenceIds": [ "grok-docs-21" ] }, { "productId": "grok", "storyId": "projects-workspaces", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes team 'workspaces' for licensing, SSO, and sharing conversations among teammates, plus connectors for external tools, but there is no mention of a project/space construct that groups related chats and files together with shared custom instructions or context, unlike dedicated 'Projects' features in other chat products.", "evidenceIds": [ "grok-docs-13", "grok-docs-14", "grok-docs-39", "grok-docs-19" ] }, { "productId": "grok", "storyId": "scheduled-tasks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows connectors, tool integrations, and a 'Grok Bot' persistent cloud agent for handing off work, but no mention of scheduling recurring or one-off tasks that run automatically and report back results.", "evidenceIds": [] }, { "productId": "grok", "storyId": "training-opt-out", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Docs mention 'enterprise-grade privacy protections' for team workspaces and that prompts are stored on xAI servers, but there is no documented setting or toggle letting a knowledge-worker control whether their conversations are used to train models. Community commentary raises general distrust but does not confirm or deny a specific training opt-out control.", "evidenceIds": [ "grok-docs-13", "grok-docs-21", "grok-comm-16" ] }, { "productId": "grok", "storyId": "voice-conversation", "verdict": "full", "quality": 6, "confidence": "medium", "rationale": "xAI documents hands-free voice conversations directly in Grok ('Talk to Grok hands-free with voice') and separately documents real-time streaming speech-to-text with VAD tuning, indicating a live voice-conversation feature exists. However, missing for 10: independent/hands-on corroboration of voice conversation naturalness or latency, and no detail on turn-taking/interruption handling beyond the VAD parameter.", "evidenceIds": [ "grok-docs-40", "grok-docs-4", "grok-comm-13" ] }, { "productId": "grok", "storyId": "web-search-citations", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence shows Grok has a 'Realtime Multi-agent Research' capability for deep, multi-step research tasks and community notes suggest it can discuss current events from live X/web data, implying some grounding, but nothing in the pack explicitly documents inline citations or source links back to web results in Grok's answers. missing for 10: explicit documentation of citation formatting/source links, first-party description of real-time web search grounding, and independent verification that citations are reliably attached to responses.", "evidenceIds": [ "grok-docs-26", "grok-comm-13" ] }, { "productId": "grok", "storyId": "workspace-admin-controls", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show a business/team workspace admin story: inviting members via email (grok-docs-18), SSO integration with IdP (grok-docs-17), workspace switching with business-plan-protected conversations (grok-docs-39), license-gated sharing (grok-docs-14), and 'enterprise-grade privacy protections' (grok-docs-13). This covers member invitation, authentication/SSO, and some data-policy framing for admins. Missing for 10: detailed role/permission granularity (e.g., admin vs member roles, per-connector access control), explicit data retention/DLP policy configuration UI, and any independent/community corroboration of the admin console working as documented.", "evidenceIds": [ "grok-docs-17", "grok-docs-18", "grok-docs-39", "grok-docs-14", "grok-docs-13" ] }, { "productId": "martin", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "martin-probe-1 confirms a live llms.txt at docs.trymartin.com/llms.txt returning HTTP 200 with a structured index of docs, and the .md-suffixed doc URLs throughout the evidence pack (e.g. martin-docs-1 through martin-docs-24) show agent-oriented markdown documentation is directly accessible. Missing for 10: no independent/hands-on confirmation that an agent successfully consumed llms.txt to complete a task.", "evidenceIds": [ "martin-probe-1", "martin-docs-1", "martin-docs-24" ] }, { "productId": "martin", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Martin generates daily/weekly briefings synthesized from your calendar and inbox data, proactively drafts emails, and triages/labels emails with follow-up suggestions—all forms of AI-generated insights/suggestions surfaced inside the product. However, these are framed as autonomous actions rather than an explicit 'insights' or analytics layer, and there's no dedicated dashboard-style insight generation from broader personal data. Missing for 10: no independent/hands-on evidence corroborating quality of the 'insight' content itself, and no explicit analytics/trend-surfacing feature distinct from task-execution suggestions.", "evidenceIds": [ "martin-docs-9", "martin-docs-10", "martin-docs-11", "martin-docs-8" ] }, { "productId": "martin", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Martin's docs describe multiple always-on background automations—daily/weekly briefings, proactive email drafting, email triage, wake-up calls, cc-to-schedule meeting booking, and custom multi-step shortcuts—all of which run without direct user prompting, and community reports (e.g., using Martin as a persistent background agent for todos) corroborate real autonomous use. Missing for 10: independent verification of scheduling/configuration UI for these background jobs and long-term reliability data beyond one HN comment about failure rates improving.", "evidenceIds": [ "martin-docs-7", "martin-docs-9", "martin-docs-10", "martin-docs-11", "martin-docs-12", "martin-docs-13", "martin-comm-4", "martin-comm-6" ] }, { "productId": "martin", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Martin IS the built-in AI assistant, and users delegate tasks to it across calendar, email, Slack, reminders, todos, notes, and multi-step shortcuts via chat, phone, SMS, email, or Slack, with hands-on community accounts confirming real task delegation and reliability improvements. missing for 10: independent quality/reliability benchmarking beyond anecdotal HN comments, and no evidence of failure-mode transparency or task-success metrics.", "evidenceIds": [ "martin-docs-1", "martin-docs-5", "martin-docs-6", "martin-docs-7", "martin-docs-24", "martin-comm-1", "martin-comm-4", "martin-comm-6" ] }, { "productId": "martin", "storyId": "agentic-headless", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Martin is a personal AI assistant for consumer communication channels (email, Slack, phone, SMS), not a developer tool or agent framework meant to be run headlessly in CI pipelines; there's no API, CLI, or programmatic invocation model evidenced, and CI automation is not a fair category expectation for this kind of product.", "evidenceIds": [] }, { "productId": "martin", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Martin's docs describe a fixed set of built-in integrations (calendar, email, Slack, todos, notes) but there is no mention of MCP protocol support or any way for users to plug in arbitrary MCP servers for Martin to use as tools.", "evidenceIds": [ "martin-docs-1", "martin-docs-2", "martin-docs-3", "martin-docs-5", "martin-docs-6" ] }, { "productId": "martin", "storyId": "agentic-mcp-server", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Martin is itself a personal AI agent/assistant (client role acting on behalf of the user), not a platform or service that exposes tools to other agents. Serving as an official MCP server is a different product role, and no evidence shows Martin operating in that mode.", "evidenceIds": [] }, { "productId": "martin", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Martin is explicitly designed around natural-language commands across calendar, email, Slack, reminders, todos, notes, shortcuts, and phone/SMS interfaces, with docs and taglines like 'Text Jon my arrival time' or 'Cc Martin on an email to schedule a meeting' showing conversational control, and community feedback (HN) corroborates real usage of natural-language task delegation. missing for 10: independent hands-on benchmarking of NL command breadth/accuracy beyond anecdotal HN praise, and no formal API/spec confirming NLU robustness.", "evidenceIds": [ "martin-docs-1", "martin-docs-4", "martin-docs-5", "martin-docs-13", "martin-docs-16", "martin-docs-24", "martin-comm-1", "martin-comm-4" ] }, { "productId": "martin", "storyId": "agentic-official-cli", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Martin is a personal AI assistant reachable via phone/SMS/Slack/email interfaces, not a developer tool or platform with a CLI-native workflow; no evidence of any CLI exists, and the product category doesn't fit an 'official CLI' axis.", "evidenceIds": [] }, { "productId": "martin", "storyId": "agentic-public-api", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Martin is a personal-assistant product controlled via chat/email/Slack/phone interfaces, but there is no evidence of a documented public API for developers to programmatically drive it — probes for OpenAPI/Swagger specs all returned 404. missing for 10: a documented public API, SDK, or endpoint reference; any developer-facing API authentication or usage docs.", "evidenceIds": [ "martin-probe-3" ] }, { "productId": "martin", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Martin is a personal assistant that connects to full-access integrations (email, calendar, Slack) but there is no evidence of scoped or least-privilege credential issuance, API key management, or granular permissioning for agent access — no OpenAPI/API docs even exist per probe results.", "evidenceIds": [ "martin-probe-3", "martin-docs-19", "martin-docs-20" ] }, { "productId": "martin", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Martin's evidence pack covers only end-user integrations (calendar, email, Slack, etc.) and consumer interfaces, with no mention of an SDK, API, or developer library; probes for an OpenAPI/Swagger spec all returned 404s, indicating no public API/SDK surface exists to build against.", "evidenceIds": [ "martin-probe-3", "martin-docs-24" ] }, { "productId": "martin", "storyId": "agentic-webhooks", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Martin is a personal AI assistant product, not a developer platform exposing a webhook subscription API; no evidence of any webhook/event subscription mechanism exists, and this is not a fair axis for this kind of consumer-facing assistant.", "evidenceIds": [] }, { "productId": "martin", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Martin is a personal assistant product with docs pages but no evidence of any interactive API reference or runnable examples; probes explicitly show no OpenAPI/swagger spec found at any candidate path. missing for 10: interactive API reference UI, runnable code examples, OpenAPI spec availability.", "evidenceIds": [ "martin-probe-3" ] }, { "productId": "martin", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Martin is a personal assistant product, not a developer API platform, but the axis of publishing a machine-readable API spec still applies as a fair question; the probe explicitly checked all standard OpenAPI/swagger paths and found only 404s, with no evidence of any downloadable spec.", "evidenceIds": [ "martin-probe-3" ] }, { "productId": "martin", "storyId": "api-sandbox", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Martin is a personal AI assistant that operates directly on a user's real calendar, inbox, Slack, etc.; there is no concept of a sandbox/test environment separate from production data in its evidence or product category. This axis fits developer-facing platforms/APIs, not a consumer assistant like Martin.", "evidenceIds": [] }, { "productId": "martin", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Martin is a personal-assistant product with no evidence of any public API, versioning scheme, or deprecation policy; the openapi probe returned 404 on all candidate paths and no docs mention API versioning. missing for 10: any public API reference, version numbers, changelog, deprecation policy documentation.", "evidenceIds": [ "martin-probe-3" ] }, { "productId": "martin", "storyId": "app-directory", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Martin ships a small fixed set of first-party integrations (Calendar, Inbox, Slack) rather than a browsable directory/marketplace of third-party apps and connectors that a power-user can explore and add. No docs, UI, or community mentions describe an app directory or connector marketplace.", "evidenceIds": [ "martin-docs-1", "martin-docs-2", "martin-docs-3" ] }, { "productId": "martin", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Martin's docs show a few multi-item behaviors — e.g. forwarding an email with several events and having 'all of them' added to the calendar (martin-docs-23), and automatic email labeling/triage across an inbox (martin-docs-11) — which imply some batch-like processing, but there is no explicit bulk-operations feature (e.g. batch todo updates, mass note edits, multi-item scheduling command) described anywhere in the docs. missing for 10: explicit bulk-action tooling/API for operating on many items at once, first-party documentation of bulk operations, and independent confirmation of bulk-scale reliability.", "evidenceIds": [ "martin-docs-23", "martin-docs-11" ] }, { "productId": "martin", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Martin ships several automatic, event-triggered behaviors (email triage that auto-labels and follows up, cc-to-schedule meeting creation, forwarded-email calendar parsing, proactive drafts, wake-up calls, briefings) which act like predefined automation rules, and 'custom shortcuts' let users define multi-step action sequences. However, there's no evidence of a general user-defined rule/trigger engine (e.g., 'if email from X arrives then do Y' configurable conditions) — the automations shown are fixed built-in behaviors rather than arbitrary user-authored event rules. Missing for 10: a documented custom rule/condition builder, evidence of arbitrary event-trigger definitions beyond built-in background tasks, and independent confirmation these automations fire reliably in practice.", "evidenceIds": [ "martin-docs-7", "martin-docs-11", "martin-docs-13", "martin-docs-10", "martin-docs-12", "martin-docs-23" ] }, { "productId": "martin", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Martin ships several built-in recurring background jobs — daily/weekly briefings, daily wake-up calls, and automatic email triage — which are documented recurring workflows, and shortcuts allow custom multi-step actions. However, there's no evidence of a general-purpose, user-defined recurring job scheduler (e.g., custom cadence for arbitrary tasks/workflows beyond the fixed built-in recurring features). Missing for 10: user-configurable custom recurring schedules beyond the fixed briefing/wake-up-call cadence, and independent confirmation these recurring jobs run reliably at scale.", "evidenceIds": [ "martin-docs-9", "martin-docs-12", "martin-docs-11", "martin-docs-7", "martin-docs-21" ] }, { "productId": "martin", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Martin is a personal assistant agent (email, calendar, reminders, shortcuts) with no evidence of version history, review workflows, or rollback for its automations/shortcuts; docs describe creating shortcuts but nothing about versioning or undoing them. Missing for 10: version history UI, review/approval workflow, rollback mechanism.", "evidenceIds": [ "martin-docs-7" ] }, { "productId": "martin", "storyId": "autonomous-background-tasks", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Martin's docs describe multiple background/autonomous capabilities (proactive email drafts, email triage with follow-up actions, cc-to-schedule meeting booking, custom multi-step shortcuts, daily briefings) that match delegating a task for autonomous background work with results delivered back to the user, and community posts corroborate real usage of the cloud-based agent for ongoing tasks (todos) across sessions. However, evidence doesn't clearly show a distinct 'review before finalizing' step for these autonomous actions (e.g., whether drafts are held for approval vs. auto-sent), and independent hands-on validation of the full delegate→work→review loop is thin. Missing for 10: explicit review/approval workflow evidence, and richer independent hands-on accounts of multi-step autonomous task completion.", "evidenceIds": [ "martin-docs-7", "martin-docs-9", "martin-docs-10", "martin-docs-11", "martin-docs-13", "martin-comm-4", "martin-comm-6" ] }, { "productId": "martin", "storyId": "browser-agent", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Martin's documented capabilities cover calendar, email, Slack, notes, reminders, todos, and search-engine lookups, but nothing in the evidence describes it operating a web browser, navigating websites, or completing multi-step tasks on arbitrary sites on the user's behalf.", "evidenceIds": [ "martin-docs-8", "martin-docs-1", "martin-docs-2", "martin-docs-3", "martin-docs-24" ] }, { "productId": "martin", "storyId": "computer-use-desktop", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Martin can act inside specific applications via integrations (calendar, email, Slack, notes, to-dos) rather than through general screen/GUI-level control of arbitrary desktop apps — it operates named services via API-like connections, not a computer-use style visual agent. A desktop app exists (martin-comm-4) but evidence only shows it syncing to-dos, not controlling arbitrary programs. Missing for 10: evidence of generic screen/vision-based computer control, ability to operate arbitrary (non-integrated) applications, and independent hands-on proof of this broader 'operate any app' capability.", "evidenceIds": [ "martin-docs-1", "martin-docs-2", "martin-docs-3", "martin-docs-6", "martin-comm-4" ] }, { "productId": "martin", "storyId": "connect-drive-calendar", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Martin has well-documented calendar and email (inbox) integrations letting it read, draft, reply, and add events on the user's behalf, plus a general search capability across connected sources. However, no evidence mentions a cloud drive (e.g., Google Drive/Dropbox) connector, so the story's full connector set is only partially covered. missing for 10: cloud drive integration, independent verification that search actually spans all connected sources.", "evidenceIds": [ "martin-docs-1", "martin-docs-2", "martin-docs-8", "martin-docs-18", "martin-docs-19", "martin-docs-23" ] }, { "productId": "martin", "storyId": "create-edit-documents", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Martin is a personal-assistant product focused on email, calendar, Slack, notes, todos, and reminders — not a document/presentation creation or file-editing tool. No evidence in the pack mentions creating or iteratively editing documents, presentations, or exportable files, so this axis is a category mismatch for Martin's product type.", "evidenceIds": [] }, { "productId": "martin", "storyId": "custom-bots", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Martin is a personal AI assistant with fixed integrations (calendar, email, Slack, todos, notes, reminders) and shortcuts, but there is no evidence of a feature to build a custom assistant with its own distinct instructions/persona and knowledge base, nor any sharing mechanism for such an assistant. missing for 10: custom assistant creation with configurable instructions/persona, custom knowledge base attachment, and sharing/publishing of a built assistant.", "evidenceIds": [] }, { "productId": "martin", "storyId": "data-analysis-code", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Martin is a personal-assistant agent for email, calendar, messaging, reminders and notes; there is no evidence of a code execution/data analysis capability for producing charts, computed answers, or downloadable files, which is outside its product category as a personal-life-admin assistant.", "evidenceIds": [] }, { "productId": "martin", "storyId": "deep-research-report", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Martin's docs mention a basic search capability that 'syncs with multiple search engines to find relevant information' (martin-docs-8), but there is no evidence of an autonomous multi-source deep-research mode that returns a structured, cited report — no mention of citation formatting, report generation, or a dedicated 'deep research' feature.", "evidenceIds": [ "martin-docs-8" ] }, { "productId": "martin", "storyId": "desktop-app", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Community evidence confirms a Martin desktop app exists (martin-comm-4), and docs mention in-app 'shortcuts' for multi-step actions (martin-docs-7), but there is no evidence of OS-level global keyboard shortcuts or any capability for Martin to read/access what's on the user's screen. missing for 10: evidence of OS-level hotkey integration, evidence of screen-reading/context capture, any documentation describing desktop-native system integration beyond a generic app shell.", "evidenceIds": [ "martin-comm-4", "martin-docs-7" ] }, { "productId": "martin", "storyId": "export-my-data", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data export, chat history download, or account data portability features; community comments even raise privacy concerns without any mention of export tools. Missing for 10: any documentation or feature reference to exporting chat history or account data, data portability API, or GDPR-style export mechanism.", "evidenceIds": [] }, { "productId": "martin", "storyId": "file-upload-analysis", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Martin is a personal-assistant product focused on calendar, email, Slack, reminders, notes, and todos; no evidence anywhere in the pack mentions document, spreadsheet, or PDF upload/analysis capabilities. This is a plausible axis for an AI assistant but there's no supporting documentation or community mention of file/document analysis features.", "evidenceIds": [] }, { "productId": "martin", "storyId": "image-generation", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Martin is a personal-assistant agent for calendar, email, messaging, notes, and tasks; there is no evidence of any image generation or editing capability, which is entirely outside its product category.", "evidenceIds": [] }, { "productId": "martin", "storyId": "image-understanding", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions image, screenshot, or photo interpretation capabilities; Martin's documented features focus on text-based channels (email, Slack, SMS, calendar, notes, reminders) with no mention of vision/multimodal input handling.", "evidenceIds": [] }, { "productId": "martin", "storyId": "memory-across-chats", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Martin's notes, to-dos, and reminders (martin-docs-6, martin-docs-22, martin-docs-5) provide persistent storage that Martin can read back across interfaces, giving it some ability to carry information forward. However, there is no explicit documentation of the assistant automatically recalling relevant context from prior conversations and applying it in new, unrelated chats — memory here is user-invoked (explicit notes) rather than an automatic contextual-memory feature. Missing for 10: explicit memory/context-recall feature description, evidence of automatic application of past-conversation context in new sessions, independent confirmation of this behavior.", "evidenceIds": [ "martin-docs-6", "martin-docs-22", "martin-docs-5", "martin-docs-21" ] }, { "productId": "martin", "storyId": "mobile-apps", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Martin's docs describe channel-based access (phone, SMS, WhatsApp, email, Slack, desktop) but no evidence of dedicated, full-featured native iOS/Android apps; a community comment explicitly asks 'I would try this if it had Android support! Is that planned?' indicating no Android app exists, and no iOS app is mentioned anywhere.", "evidenceIds": [ "martin-docs-24", "martin-comm-4", "martin-comm-5" ] }, { "productId": "martin", "storyId": "openness-api-parity", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Martin's documented surface is entirely conversational interfaces (email, Slack, phone, SMS, WhatsApp) with no mention of a public/developer API; explicit probes for an OpenAPI/swagger spec at all standard paths returned 404, indicating no programmatic API exists to mirror these UI capabilities.", "evidenceIds": [ "martin-probe-3", "martin-docs-24" ] }, { "productId": "martin", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack of a data export feature or open-format data portability; Martin's docs focus on integrations, reminders, and briefings but never mention exporting user data or account deletion/export tools.", "evidenceIds": [] }, { "productId": "martin", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence Martin's source code is available under any license; a community comment explicitly calls for Martin to 'make that system open source' as a trust-building step, implying it currently is not.", "evidenceIds": [ "martin-comm-3" ] }, { "productId": "martin", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Martin is offered exclusively as a cloud-based assistant (email, Slack, calendar interfaces, cloud-based agent per martin-comm-4); no evidence of a self-hostable core product, open-source release, or on-prem deployment option. One community comment (martin-comm-3) even calls for open-sourcing the data-handling system, implying it is not currently available.", "evidenceIds": [ "martin-comm-4", "martin-comm-3" ] }, { "productId": "martin", "storyId": "persistent-instructions", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack describes a persistent custom-instructions or preferences feature that shapes all of Martin's responses; the docs cover integrations, reminders, notes, and background tasks but nothing about setting persona/style preferences persistently.", "evidenceIds": [] }, { "productId": "martin", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack mentions data residency, region selection, or storage location controls; docs focus entirely on assistant features and integrations. Community comments raise general privacy/security concerns but do not address data residency options. missing for 10: any mention of region selection, data residency policy, or storage location controls.", "evidenceIds": [ "martin-comm-2", "martin-comm-3" ] }, { "productId": "martin", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item addresses AI-training data opt-out or data-usage policy; community comments explicitly raise privacy/trust concerns without any documented answer or control from Martin. Missing for 10: any privacy policy statement, opt-out setting, or documented no-training-data commitment.", "evidenceIds": [ "martin-comm-2", "martin-comm-3" ] }, { "productId": "martin", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence describes data retention policies, deletion controls, export tools, or privacy settings; community comments even raise unaddressed concerns about lack of transparency around data handling. Missing for 10: any documentation of data retention periods, user-initiated deletion mechanism, or privacy controls.", "evidenceIds": [ "martin-comm-2", "martin-comm-3" ] }, { "productId": "martin", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any telemetry opt-out, privacy settings, or data collection controls; community comments even raise unresolved privacy/trust concerns (martin-comm-2, martin-comm-3) with no documented response or opt-out mechanism.", "evidenceIds": [ "martin-comm-2", "martin-comm-3" ] }, { "productId": "martin", "storyId": "projects-workspaces", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Martin is a personal-assistant agent operating over email, calendar, Slack, notes, and to-dos for an individual user — it has no concept of 'projects' or 'spaces' grouping chats and files with shared instructions, which is a workspace/organization construct outside its product category.", "evidenceIds": [] }, { "productId": "martin", "storyId": "scheduled-tasks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Martin supports recurring background tasks (daily/weekly briefings, wake-up calls, email triage) and one-off scheduled reminders that ping the user via chosen interface, matching the core of the story. However, there's no unified 'task scheduler' concept with arbitrary custom recurring workflows beyond the specific built-in background tasks listed, and no independent verification of reliability for scheduled/recurring runs. missing for 10: evidence of general-purpose custom recurring task scheduling (not just fixed briefings/wake-up-calls), independent hands-on confirmation of reminder/briefing reliability, and any dashboard/management UI for viewing scheduled tasks.", "evidenceIds": [ "martin-docs-4", "martin-docs-9", "martin-docs-11", "martin-docs-12", "martin-docs-21", "martin-comm-6" ] }, { "productId": "martin", "storyId": "training-opt-out", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions any setting, policy, or statement about whether user conversations/data are used for model training; community comments explicitly raise privacy/trust concerns without any documented opt-out or training-control mechanism (martin-comm-2, martin-comm-3).", "evidenceIds": [ "martin-comm-2", "martin-comm-3" ] }, { "productId": "martin", "storyId": "voice-conversation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Martin supports real phone-call interactions — users can call Martin to dictate notes, and Martin can call users for wake-up briefings — indicating a live voice channel exists (martin-docs-22, martin-docs-24, martin-docs-12). However, there is no documentation or independent evidence describing the naturalness, latency, or conversational fluidity of these voice interactions, and community feedback focuses on text/action reliability rather than voice quality. Missing for 10: hands-on/independent evidence of real-time conversational voice quality, documentation of voice-specific features like interruption handling or natural turn-taking.", "evidenceIds": [ "martin-docs-22", "martin-docs-24", "martin-docs-12" ] }, { "productId": "martin", "storyId": "web-search-citations", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Martin claims to sync with multiple search engines to find relevant information (martin-docs-8), suggesting web-grounded answers, but there is no evidence of citation formatting, source linking, or hands-on verification that answers include traceable citations back to web sources. Missing for 10: documentation or examples showing inline citations/source links, independent corroboration of search-grounded answer quality, and detail on which search engines/sources are used.", "evidenceIds": [ "martin-docs-8" ] }, { "productId": "martin", "storyId": "workspace-admin-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Martin's evidence pack is entirely about individual personal-assistant features (calendar, email, Slack, reminders, to-dos) with no mention of organization workspaces, member management, permission controls, or data-policy settings for teams. missing for 10: any admin console, member/role management, workspace-level permissions, or data governance policy documentation.", "evidenceIds": [] }, { "productId": "muse", "storyId": "agentic-agent-docs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Muse is a consumer personal AI agent app, not a docs/API platform with an agent-facing surface; pointing an agent at llms.txt or agent-oriented docs is a wrong axis for this product category.", "evidenceIds": [] }, { "productId": "muse", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Muse links personal data sources (email, calendar, social, health, finances) and proactively tracks tasks/goals, implying some data-driven suggestions, but the evidence never explicitly describes AI-generated insights or recommendations derived from that data—most documentation focuses on agentic task execution and tool-building rather than analytic insight generation. missing for 10: explicit description of insight/recommendation generation from user data, independent hands-on validation of this specific capability.", "evidenceIds": [ "muse-docs-11", "muse-docs-5", "muse-docs-7" ] }, { "productId": "muse", "storyId": "agentic-autonomous-automation", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Muse's Goals tab, activity log, and approval-card workflow (muse-docs-5, muse-docs-8, muse-docs-13) show first-party support for autonomous background task tracking and execution, and muse-docs-7 explicitly describes agents building tools for unattended tasks. However, an independent hands-on stress test found the agent's control plane starts timing out under load, exposing concrete failure points in its background-automation architecture (muse-comm-1), directly undercutting reliability claims for autonomous operation. missing for 10: independent confirmation that background automations run reliably at scale, more detail on scheduling/triggering mechanisms for autonomous runs.", "evidenceIds": [ "muse-docs-5", "muse-docs-7", "muse-docs-8", "muse-docs-13", "muse-comm-1" ] }, { "productId": "muse", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Muse is explicitly a built-in AI agent with its own compute, terminal, and file system to build tools and complete delegated tasks (goals, side chats, autonomous tool creation), with approval flows and activity logs for oversight. Missing for 10: independent hands-on verification of delegation quality beyond stress-test edge cases and more detail on task breadth/reliability at scale.", "evidenceIds": [ "muse-docs-1", "muse-docs-5", "muse-docs-7", "muse-docs-8", "muse-docs-13" ] }, { "productId": "muse", "storyId": "agentic-headless", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Muse is a consumer personal-AI-agent app (mobile/avatar-based, personal tasks, permissions), not a developer tool or framework meant to be invoked headlessly in CI pipelines; no evidence pack content relates to CLI, API, or automation-pipeline usage.", "evidenceIds": [] }, { "productId": "muse", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence Muse supports plugging in external MCP servers; it only describes its agent building its own tools internally, which is a different mechanism.", "evidenceIds": [ "muse-docs-7", "muse-docs-1" ] }, { "productId": "muse", "storyId": "agentic-mcp-server", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Muse is itself an AI agent product (consumer personal agent), not a service/platform exposing an MCP server for other agents to connect to; this axis is a category error for its role, akin to the agent-vs-server distinction.", "evidenceIds": [] }, { "productId": "muse", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Muse is designed around a chat interface where users talk to it in natural language to manage goals, tasks, and tool-building ('talk to it about these tasks in chat', side chats for topic-specific NL interaction). This is corroborated by first-party docs describing conversational control as the primary interface. missing for 10: independent hands-on confirmation of broad natural-language command coverage beyond goals/chat, and detail on command reliability at scale (comm-1 notes control-plane timeouts under stress).", "evidenceIds": [ "muse-docs-5", "muse-docs-6", "muse-docs-8", "muse-comm-1" ] }, { "productId": "muse", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of an official CLI for Muse; all evidence describes a consumer app/avatar/chat interface with an internal terminal for the agent itself, not a CLI for AI-native users to interact with Muse.", "evidenceIds": [] }, { "productId": "muse", "storyId": "agentic-public-api", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a documented public API for developers to programmatically drive Muse; all evidence describes end-user chat/app features and internal agent behavior, not an external API surface.", "evidenceIds": [] }, { "productId": "muse", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Muse has approval cards, activity logs, and permission tracking for connected accounts, but nothing about issuing scoped or least-privilege API credentials/tokens for an agent to use with external services — this is a consumer personal-agent product, not a developer credential-management tool.", "evidenceIds": [] }, { "productId": "muse", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any official SDK for developers to build against; evidence only covers end-user features (avatars, goals, side chats, agent building its own tools) rather than a developer-facing SDK.", "evidenceIds": [] }, { "productId": "muse", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions webhooks or any event-subscription API for developers; Muse's documented features focus on agent tasks, permissions, and integrations, not outbound webhook subscriptions. missing for 10: any webhook/event API documentation, developer subscription mechanism, or third-party confirmation of webhook support.", "evidenceIds": [] }, { "productId": "muse", "storyId": "api-interactive-docs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Muse is a personal AI agent consumer product, not a developer API/SDK platform; the evidence pack shows no API reference or developer documentation concept at all, making this axis a category mismatch rather than a gap in an applicable capability.", "evidenceIds": [] }, { "productId": "muse", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence Muse publishes any machine-readable API spec (OpenAPI or equivalent) for developers; all evidence concerns end-user agent features, not a documented API surface.", "evidenceIds": [] }, { "productId": "muse", "storyId": "api-sandbox", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Muse is a consumer personal AI agent (linking real email, calendar, social accounts) rather than a developer/testing product with sandbox vs production environments; the story's axis of testing against a sandbox without touching production data is a category error for this kind of product.", "evidenceIds": [] }, { "productId": "muse", "storyId": "api-versioning-policy", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Muse is a consumer personal AI agent product, not a developer platform exposing versioned APIs; the story concerns API versioning/deprecation policy, which is a wrong axis for this end-user app.", "evidenceIds": [] }, { "productId": "muse", "storyId": "app-directory", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Muse lets users link specific third-party data sources like email, calendar, Instagram, Facebook, health and finance data (muse-docs-11), which shows some connector-adding capability, but there is no evidence of a browsable directory or marketplace of third-party apps/connectors to explore and add. Missing for 10: an actual app/connector directory UI, discovery/browsing mechanism, and independent confirmation of a marketplace beyond ad-hoc account linking.", "evidenceIds": [ "muse-docs-11" ] }, { "productId": "muse", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence describes Muse performing bulk operations across many items; docs mention goals tracking, avatars, side chats, and tool-building, but nothing about batch/bulk processing of items. The community stress-test post even highlights the agent timing out under load rather than handling bulk tasks smoothly.", "evidenceIds": [] }, { "productId": "muse", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes Muse's goal tracking, tool-building, and approval workflows, but there is no mention of user-defined conditional rules or event-triggered automations (e.g., 'when X happens, do Y'). Missing for 10: any documentation of a rules/trigger engine, event-based automation configuration, or examples of users setting conditional actions.", "evidenceIds": [ "muse-docs-5", "muse-docs-7" ] }, { "productId": "muse", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While Muse has a Goals tab that tracks ongoing tasks and an activity log, there is no evidence describing recurring/scheduled job execution, cron-like automation, or repeat-workflow triggers. missing for 10: explicit scheduling/recurrence feature, evidence of periodic execution, workflow automation configuration.", "evidenceIds": [] }, { "productId": "muse", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Muse provides an activity log/audit trail and approval cards that support reviewing what the agent has done and plans to do (muse-docs-8, muse-docs-13, muse-docs-10), which covers the 'review' portion of the story. However, there is no evidence of explicit versioning of automations/skills or a rollback mechanism to revert an automation to a prior state — the 'Forget' skill only removes stored information, not automation history. missing for 10: explicit versioning of automations, a rollback/undo feature for agent actions, independent confirmation these review tools work reliably in practice.", "evidenceIds": [ "muse-docs-8", "muse-docs-13", "muse-docs-10", "muse-docs-12" ] }, { "productId": "muse", "storyId": "autonomous-background-tasks", "verdict": "disputed", "quality": 6, "confidence": "medium", "rationale": "Muse's docs describe autonomous background task handling via the Goals tab, self-built tools, activity logs and approval cards for review — matching the delegate-and-review story [muse-docs-5][muse-docs-7][muse-docs-8][muse-docs-13][muse-docs-10]. However, an independent hands-on stress test found the agent's control plane timing out under load, exposing failure points in the very autonomous-task architecture being claimed [muse-comm-1], and another commenter notes the assistant is less capable than power-user expectations for building/using tools [muse-comm-2]. Missing for 10: reproducible evidence of reliable long-running multi-step task completion, and resolution/acknowledgment of the reported control-plane timeout issue.", "evidenceIds": [ "muse-docs-5", "muse-docs-7", "muse-docs-8", "muse-docs-13", "muse-docs-10", "muse-comm-1", "muse-comm-2" ] }, { "productId": "muse", "storyId": "browser-agent", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence describes Muse having its own computer/terminal to build tools, task tracking (Goals tab), and permissions/approval cards, but nothing in the pack mentions browser automation or web navigation on the user's behalf. Missing for 10: any mention of browser control, web research execution, or site-task completion capability.", "evidenceIds": [] }, { "productId": "muse", "storyId": "computer-use-desktop", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Muse describes having its own computer/terminal to build tools and links external accounts (email, calendar, social, health, finance) via structured approvals, which lets it act on the user's behalf across services, but the evidence never shows it seeing or directly operating applications on the user's own device (e.g., screen/GUI control) — it operates its own sandboxed environment, not the user's computer. Missing for 10: evidence of direct screen/GUI control of the user's local applications, and independent confirmation that this cross-app automation reliably works beyond linked-account integrations.", "evidenceIds": [ "muse-docs-1", "muse-docs-7", "muse-docs-10", "muse-docs-11" ] }, { "productId": "muse", "storyId": "connect-drive-calendar", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Muse docs explicitly support linking email and calendar (and other data sources) with approval/permission controls, but no evidence mentions a cloud drive connector (e.g., Google Drive/Dropbox), and there's no hands-on confirmation that these connected sources are actually searched/used in answers beyond permission approval cards. missing for 10: explicit cloud drive integration, independent evidence of search/use in answers.", "evidenceIds": [ "muse-docs-11", "muse-docs-10", "muse-docs-8" ] }, { "productId": "muse", "storyId": "create-edit-documents", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Muse's evidence focuses on its file system/terminal, tool-building, goals, avatars, and privacy controls, but nothing describes creating or iteratively editing documents, presentations, or exportable files for knowledge-work tasks. missing for 10: any mention of document/presentation creation, editing UI, or export functionality.", "evidenceIds": [] }, { "productId": "muse", "storyId": "custom-bots", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Muse supports personal customization (avatar naming/style, side chats for separate contexts) but nothing about creating a distinct 'assistant' with its own instructions/knowledge base that can be shared with other users, which is the core of this story.", "evidenceIds": [] }, { "productId": "muse", "storyId": "data-analysis-code", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Muse's docs claim it 'has its own computer, with a file system and a terminal' and can 'write its own code and build the tools a task needs,' which supports general code-execution capability, but there's no explicit mention of producing charts, computed answers, or downloadable files for user data analysis. A community stress-test report also notes the agent's control plane can time out under load, raising doubts about reliability for heavier computational tasks. Missing for 10: explicit documentation or examples of chart generation, data analysis outputs, downloadable file creation, and independent verification of successful code-execution results.", "evidenceIds": [ "muse-docs-1", "muse-docs-7", "muse-comm-1" ] }, { "productId": "muse", "storyId": "deep-research-report", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence describes Muse performing a deep research run that autonomously searches multiple sources and returns a cited report; the pack focuses on avatar personalization, goals/side chats, permissions, and tool-building, none of which document a research/citation workflow. Missing for 10: any mention of multi-source search, report generation, or citation output.", "evidenceIds": [] }, { "productId": "muse", "storyId": "desktop-app", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence describes Muse as an AI agent with its own computer/file system, chat, and permission tracking, but there is no mention of an official desktop app, OS-level keyboard shortcuts, or screen access/observation capability. This appears to be a mobile/app-based assistant rather than a desktop tool with screen access.", "evidenceIds": [] }, { "productId": "muse", "storyId": "export-my-data", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence shows a 'Forget' skill and audit trail/activity log, but there is no mention of a data export or account data download feature for chat history or account data. missing for 10: export/download tool for chat history, account data export functionality, documentation of data portability process.", "evidenceIds": [ "muse-docs-12", "muse-docs-13" ] }, { "productId": "muse", "storyId": "file-upload-analysis", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack describes Muse handling document/spreadsheet/PDF uploads or analyzing their contents; the material focuses on agent features like avatars, goals, side chats, permissions, and tool-building, not file ingestion or analysis accuracy.", "evidenceIds": [] }, { "productId": "muse", "storyId": "image-generation", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no mention of image generation, editing, or any visual/multimodal content creation capability for Muse; it focuses on agent/task automation, avatars, chat organization, and privacy controls. missing for 10: any documentation of image generation from prompts, image editing tools, or multimodal output examples.", "evidenceIds": [] }, { "productId": "muse", "storyId": "image-understanding", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack discusses image, screenshot, or photo interpretation capabilities of Muse; all citations focus on agent architecture, avatars, goals, side chats, and privacy controls. Missing for 10: any mention of image/vision input support, screenshot sharing, or accuracy of visual interpretation.", "evidenceIds": [] }, { "productId": "muse", "storyId": "memory-across-chats", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Muse's docs imply persistent memory (a 'Forget skill' to remove remembered info about topics/people, plus 'side chats' for separate context threads and a Goals tab tracking ongoing tasks) suggesting it retains and reuses context across conversations, but there is no first-party or hands-on description of how memory is surfaced or applied in new chats. Missing for 10: explicit documentation of cross-chat memory recall/application, independent verification that remembered context actually surfaces in new conversations, and detail on memory scope/limits.", "evidenceIds": [ "muse-docs-12", "muse-docs-6", "muse-docs-9", "muse-docs-5" ] }, { "productId": "muse", "storyId": "mobile-apps", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "There is clear evidence of an iOS app (App Store reviews cited via apps.apple.com), but no evidence at all of an Android app, and no detailed description of feature parity between platforms. missing for 10: Android app existence, feature-completeness claims for either platform, independent verification of app quality.", "evidenceIds": [ "muse-docs-3", "muse-docs-4" ] }, { "productId": "muse", "storyId": "openness-api-parity", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack mentions a public API for Muse at all; all evidence describes UI features (avatars, Goals tab, side chats, activity log) with no mention of programmatic access. Absence of evidence for an applicable capability is 'none'.", "evidenceIds": [] }, { "productId": "muse", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "There is no evidence of a data export feature or open-format data portability in Muse; the pack only mentions a 'Forget' privacy skill and audit trail, not export/download of user data.", "evidenceIds": [] }, { "productId": "muse", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence indicates Muse's source code is published under any open license; all evidence describes product features and community discussion, not source availability.", "evidenceIds": [] }, { "productId": "muse", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack indicates Muse can be self-hosted; it's described as a cloud/app-based agent product with no mention of open-source release or self-hosting options.", "evidenceIds": [] }, { "productId": "muse", "storyId": "persistent-instructions", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence describes a persistent custom-instructions or preference-setting feature that shapes all responses; docs mention forget skill, avatars, side chats, and permissions but nothing about setting standing instructions/preferences.", "evidenceIds": [] }, { "productId": "muse", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions data residency, region selection, or storage location controls for Muse users.", "evidenceIds": [] }, { "productId": "muse", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers Muse's agent features, permissions, forget skill, and audit trail, but contains no mention of any opt-out or control over whether user data is used to train AI models. missing for 10: explicit opt-out/consent setting for model training, any privacy policy language addressing training-data usage.", "evidenceIds": [] }, { "productId": "muse", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Muse offers a 'Forget' skill to remove information about specific topics/people and provides an activity log/audit trail of actions, giving users some visibility and control, but there's no documentation of full account/data deletion, export, or granular retention policy settings. Missing for 10: explicit data export/deletion controls, retention period settings, and independent verification that 'Forget' fully purges underlying data rather than just suppressing recall.", "evidenceIds": [ "muse-docs-12", "muse-docs-13", "muse-docs-8" ] }, { "productId": "muse", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items mention a telemetry/usage-tracking opt-out setting; there is a 'Forget skill' for topic memory and privacy page references, but nothing about disabling telemetry or usage analytics collection. Missing for 10: any documentation of a telemetry/analytics opt-out toggle, privacy policy language on usage data collection, or user reports confirming such a control exists.", "evidenceIds": [ "muse-docs-12", "muse-docs-13" ] }, { "productId": "muse", "storyId": "projects-workspaces", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Muse offers 'side chats' for separate context on topics as work grows complex and a Goals tab for tracking tasks, suggesting some contextual organization, but there is no explicit 'project/space' construct that bundles chats+files+shared instructions together. Missing for 10: explicit project/workspace container, file-attachment grouping, shared custom instructions across chats, and independent verification of how side chats share or isolate context.", "evidenceIds": [ "muse-docs-5", "muse-docs-6", "muse-docs-9" ] }, { "productId": "muse", "storyId": "scheduled-tasks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Muse's Goals tab and activity log suggest it can track and manage ongoing tasks with an audit trail of completed/planned actions, implying some form of persistent task tracking, but there is no explicit documentation of recurring/scheduled triggers or a scheduling UI. Community evidence also shows the agent's control plane can time out under stress, raising doubts about reliability for automated recurring runs. missing for 10: explicit scheduling/recurrence configuration docs, evidence of one-off vs recurring task setup, and independent confirmation that scheduled tasks reliably complete and report back.", "evidenceIds": [ "muse-docs-5", "muse-docs-13", "muse-docs-7", "muse-comm-1" ] }, { "productId": "muse", "storyId": "training-opt-out", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows privacy-adjacent features like a 'Forget skill' for memory and audit trails/permissions, but nothing addresses whether users can opt their conversations out of model training specifically. This is a fair axis for a personal AI agent product, but no evidence documents a training-data control. Missing for 10: any documentation of a training opt-out setting, data-usage policy toggle, or explicit statement about whether conversations are used for model training.", "evidenceIds": [ "muse-docs-12", "muse-docs-13" ] }, { "productId": "muse", "storyId": "voice-conversation", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence describes real-time voice conversation capability for Muse; the evidence pack focuses on agent tool-building, goals, side chats, avatar, and privacy features, with no mention of voice interaction or speech interface. Missing for 10: any documentation of voice input/output, real-time conversational voice mode, or hands-on account of speaking with Muse.", "evidenceIds": [] }, { "productId": "muse", "storyId": "web-search-citations", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack describes Muse performing web search with cited sources for research answers; the material focuses on agent architecture, avatars, permissions, and goals rather than grounded citation-backed answers.", "evidenceIds": [] }, { "productId": "muse", "storyId": "workspace-admin-controls", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Muse is a personal consumer AI agent for individuals, not a team/organization workspace product; there is no evidence of team-admin roles, member management, or org-wide permission/data policy controls. This is a category mismatch rather than a missing feature.", "evidenceIds": [] }, { "productId": "perplexity", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Perplexity's docs site directly serves an llms.txt file (HTTP 200) plus per-page .md variants that explicitly point agents to the llms.txt index for full documentation, confirming a genuine agent-oriented docs surface; this is corroborated by an OpenAPI spec, official MCP server, and CLI docs also structured for agent consumption. Missing for 10: independent third-party confirmation that agents actually consume this llms.txt successfully in practice.", "evidenceIds": [ "perplexity-probe-1", "perplexity-probe-2", "perplexity-probe-3", "perplexity-docs-6" ] }, { "productId": "perplexity", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Perplexity's Assistant (drafting emails, scheduling, booking) and the browser extension's 'Contextual Understanding' (answers tied to current page/domain) and 'Your Library' (curation of past discoveries) show the product surfacing suggestions tied to a user's own activity/context, but there's no explicit feature for uploading or connecting personal datasets (docs, CRM, files) and generating proactive analytical insights from them — the core product is web search/answers, not user-data analytics. missing for 10: dedicated personal-data ingestion/analysis capability, proactive insight-generation dashboards, independent evidence these 'insights' features work well in practice (community reports flag research features stalling and answer quality issues).", "evidenceIds": [ "perplexity-docs-9", "perplexity-docs-13", "perplexity-docs-25", "perplexity-comm-7" ] }, { "productId": "perplexity", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes on-demand features (Assistant drafting emails/booking, Computer/MCP multi-step delegation, Agent API) that execute when invoked, but nothing documents scheduled, recurring, or persistent background automations that run autonomously without a user actively initiating each session. Missing for 10: any documented scheduling/trigger mechanism, recurring task execution, or evidence of an automation running unattended over time.", "evidenceIds": [ "perplexity-docs-9", "perplexity-docs-17", "perplexity-docs-29", "perplexity-docs-38" ] }, { "productId": "perplexity", "storyId": "agentic-builtin-assistant", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Perplexity ships a built-in 'Assistant' and 'Computer' agent that can draft emails, book reservations, browse the web, run code, and connect to 400+ services (perplexity-docs-9, -17, -19, -29, -30, -38), which is a genuine in-product delegation capability. However, hands-on community reports say the Computer/agentic research feature 'often fails to trigger via the UI, and when it does work, sessions frequently stall without returning an answer' (perplexity-comm-7), directly contradicting the vendor's smooth-delegation claims. Missing for 10: independent corroboration that the assistant reliably completes delegated multi-step tasks, and resolution of the reported stalling/failure-to-trigger issue.", "evidenceIds": [ "perplexity-docs-9", "perplexity-docs-17", "perplexity-docs-19", "perplexity-docs-29", "perplexity-docs-30", "perplexity-docs-38", "perplexity-comm-7" ] }, { "productId": "perplexity", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Perplexity provides an OpenAI-compatible Agent API with structured JSON output, model fallback, sandboxed code execution, and reusable profiles, plus a dedicated `pplx` CLI explicitly designed for shell pipelines and coding agents that need current web results — all suitable for headless/CI automation. missing for 10: no independent/hands-on report confirming real-world CI pipeline usage, and no documented rate-limit/quota guidance specifically for automated batch workloads.", "evidenceIds": [ "perplexity-docs-6", "perplexity-docs-20", "perplexity-docs-23", "perplexity-docs-31", "perplexity-docs-33", "perplexity-docs-34", "perplexity-docs-35", "perplexity-docs-37", "perplexity-probe-3", "perplexity-probe-5" ] }, { "productId": "perplexity", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All MCP evidence describes Perplexity acting as an MCP *server* (Computer MCP server, Perplexity MCP Server) that other AI assistants can connect to, not Perplexity itself consuming external MCP servers as a client to gain new tools. There is no documentation of a settings/config surface where a user adds arbitrary third-party MCP servers for Perplexity to call.", "evidenceIds": [ "perplexity-docs-5", "perplexity-docs-17", "perplexity-docs-19", "perplexity-docs-29", "perplexity-docs-36", "perplexity-docs-38" ] }, { "productId": "perplexity", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Perplexity documents two official MCP servers—the general 'Perplexity MCP Server' for search/reasoning and the 'Computer' MCP server for delegating multi-step browsing/coding tasks—explicitly stating AI assistants can connect via MCP to access these capabilities, corroborated by a probe confirming the docs page exists. Missing for 10: independent/hands-on corroboration that the MCP server works reliably in practice (community reports note the related 'Computer' feature sometimes stalls in the UI, though that's not MCP-specific).", "evidenceIds": [ "perplexity-docs-36", "perplexity-docs-38", "perplexity-docs-5", "perplexity-docs-17", "perplexity-docs-19", "perplexity-docs-29", "perplexity-probe-4" ] }, { "productId": "perplexity", "storyId": "agentic-nl-commands", "verdict": "disputed", "quality": 6, "confidence": "medium", "rationale": "Perplexity's core product is a natural-language chat/search interface (voice and text queries, cited answers) and its 'Assistant'/Computer agent claims to handle draft emails, scheduling, and multi-step web tasks purely via NL commands. However, hands-on community reports say the agentic 'Computer' research feature frequently fails to trigger via the UI and stalls without returning results, directly contradicting the docs' claims of reliable NL-driven task execution. missing for 10: independent verification that the Assistant reliably completes booking/scheduling tasks, and resolution of the reported reliability failures.", "evidenceIds": [ "perplexity-docs-9", "perplexity-docs-17", "perplexity-docs-19", "perplexity-docs-26", "perplexity-comm-7" ] }, { "productId": "perplexity", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Perplexity documents an official `pplx` CLI that returns structured JSON from the Search API, explicitly designed for shell pipelines, terminal work, and coding agents, with an independent probe confirming the docs page exists. Missing for 10: independent hands-on community review of the CLI itself (evidence is entirely first-party docs/probe) and detail on install/auth flows.", "evidenceIds": [ "perplexity-docs-6", "perplexity-docs-20", "perplexity-docs-23", "perplexity-docs-32", "perplexity-probe-5" ] }, { "productId": "perplexity", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Perplexity publishes a documented public API (Agent API, Search API, embeddings) with an OpenAPI spec, supports structured output, model fallback, sandbox tool use, and profiles, plus a CLI and MCP integrations for programmatic/agentic access, confirmed by both docs and live probes returning valid OpenAPI/docs endpoints. Missing for 10: independent third-party developer corroboration of API reliability/robustness in production use.", "evidenceIds": [ "perplexity-docs-31", "perplexity-docs-33", "perplexity-docs-34", "perplexity-docs-35", "perplexity-docs-37", "perplexity-docs-6", "perplexity-probe-3", "perplexity-probe-1", "perplexity-probe-5" ] }, { "productId": "perplexity", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers API endpoints, MCP servers, CLI, and browser/app features, but contains no mention of API key scoping, granular permissions, or least-privilege credential issuance for agents. This is a fair capability to expect from an API platform, so absence of evidence means 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "perplexity", "storyId": "agentic-sdks", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Perplexity documents a REST-based Agent API with an OpenAI-compatible endpoint and publishes an openapi.json spec, which supports SDK-style integration, but there is no explicit evidence of first-party official SDK libraries (e.g., a Perplexity Python/Node/Go client package) in the pack. missing for 10: explicit official SDK packages/libraries, language-specific SDK docs, independent developer corroboration of SDK usage.", "evidenceIds": [ "perplexity-docs-1", "perplexity-probe-3", "perplexity-docs-31" ] }, { "productId": "perplexity", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack of a webhook subscription mechanism for events (e.g., completed jobs, search results, agent tasks); documentation covers APIs, CLI, MCP servers, and apps but no event-driven webhook system.", "evidenceIds": [] }, { "productId": "perplexity", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Perplexity publishes a machine-readable OpenAPI spec (openapi.json) and structured API docs (quickstart, building-agents guides) that would typically power an interactive reference, but the evidence pack never explicitly shows a 'try it' playground or runnable code examples within the docs UI. missing for 10: explicit interactive 'try it now' console screenshots/docs, confirmation of runnable code snippets executed in-browser, independent user report of using the interactive reference.", "evidenceIds": [ "perplexity-probe-3", "perplexity-docs-31", "perplexity-docs-33", "perplexity-probe-1", "perplexity-probe-2" ] }, { "productId": "perplexity", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A probe confirms a live, machine-readable OpenAPI spec at https://docs.perplexity.ai/openapi.json returning HTTP 200 with an 'openapi' key, alongside an llms.txt documentation index, directly satisfying the story. Missing for 10: independent/community confirmation of the spec's completeness or usage in the wild.", "evidenceIds": [ "perplexity-probe-3", "perplexity-probe-1", "perplexity-probe-2" ] }, { "productId": "perplexity", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows a code-execution sandbox (isolated Linux container) used by Perplexity's agent/Computer tools for running code within an answer, not a separate staging/test environment that lets a developer exercise the Perplexity API or product without touching live production data or billing. No mention of test API keys, staging endpoints, or a sandbox mode for the platform itself.", "evidenceIds": [ "perplexity-docs-33", "perplexity-docs-17", "perplexity-docs-18" ] }, { "productId": "perplexity", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Perplexity has an API with an OpenAPI spec and 'versioned' profiles for agent configurations, but nothing documents API version numbering or a deprecation/sunset policy for endpoints or models. Missing for 10: explicit API versioning scheme (e.g., v1/v2 paths), documented deprecation/sunset timeline, and any changelog or migration guide referenced in the evidence pack.", "evidenceIds": [ "perplexity-docs-37", "perplexity-probe-3" ] }, { "productId": "perplexity", "storyId": "app-directory", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs mention Perplexity's Computer feature can 'connect to 400+ services through a managed connector framework with one-click OAuth,' implying a connector catalog exists, but there is no evidence of an actual browsable directory UI, categories, or user-facing app marketplace, and no independent/hands-on confirmation of this experience. Missing for 10: dedicated connector/app directory UI evidence, list of example connectors, independent user reports of browsing/adding connectors.", "evidenceIds": [ "perplexity-docs-19", "perplexity-docs-38" ] }, { "productId": "perplexity", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of a batch/bulk API, batch endpoint, or multi-item processing mode; the CLI and Agent API are described as single-call/pipeline-oriented tools that a user could script externally, but Perplexity itself does not document any bulk-operation capability.", "evidenceIds": [ "perplexity-docs-20", "perplexity-docs-31", "perplexity-docs-33" ] }, { "productId": "perplexity", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a rules/trigger/automation engine (event-based conditions triggering actions) anywhere in the docs — Perplexity's evidence covers search, chat, APIs, MCP/Computer agent tasks, and CLI, but nothing about defining persistent rules that fire automatically on events.", "evidenceIds": [] }, { "productId": "perplexity", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of scheduling, cron-like triggers, or recurring workflow automation anywhere in the docs, apps, or API references; the Agent API, CLI, MCP servers, and assistant features described are all single-invocation or on-demand tools with no persistent job scheduler.", "evidenceIds": [] }, { "productId": "perplexity", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The Agent API's 'profiles' feature is described as a reusable, versioned configuration that can be saved and managed, offering a thin sliver of version control for automation configs, but there is no documented review workflow or rollback mechanism for automations. missing for 10: rollback/restore functionality, review/diff UI, versioning for Computer/MCP-based automations, evidence of actual usage beyond a single doc line.", "evidenceIds": [ "perplexity-docs-37" ] }, { "productId": "perplexity", "storyId": "autonomous-background-tasks", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Perplexity's 'Computer' (Labs/Assistant) is documented as an autonomous background agent that can browse the web, execute code, use 400+ connectors, and complete tasks like drafting emails or booking reservations, returning structured results — matching the delegate-and-review story (perplexity-docs-9,10,17-19,29,30,38). However, a hands-on community report says the Computer/research feature 'often fails to trigger via the UI, and when it does work, sessions frequently stall without returning an answer,' directly contradicting reliable autonomous completion (perplexity-comm-7). missing for 10: independent corroboration that Computer reliably completes tasks and returns results, resolution of the reported stalling/failure-to-trigger issue.", "evidenceIds": [ "perplexity-docs-5", "perplexity-docs-9", "perplexity-docs-10", "perplexity-docs-17", "perplexity-docs-19", "perplexity-docs-29", "perplexity-docs-30", "perplexity-docs-38", "perplexity-comm-7" ] }, { "productId": "perplexity", "storyId": "browser-agent", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Perplexity documents a 'Computer' agent with a full cloud browser that can search, browse, fill forms, and access login-gated pages, launched automatically for such tasks and accessible via MCP (perplexity-docs-17, -29, -30, -38) — squarely matching the story. However, hands-on community reporting says this same 'Computer' feature 'often fails to trigger via the UI, and when it does work, sessions frequently stall without returning an answer' (perplexity-comm-7), directly contradicting reliable operation. missing for 10: independent corroboration that browser automation completes tasks reliably, and no rebuttal to the stall/failure reports.", "evidenceIds": [ "perplexity-docs-17", "perplexity-docs-29", "perplexity-docs-30", "perplexity-docs-38", "perplexity-comm-7" ] }, { "productId": "perplexity", "storyId": "computer-use-desktop", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Perplexity's 'Computer' (via MCP) and Assistant app claim to browse the web, execute code in a sandbox, connect to 400+ services, and perform tasks like booking/scheduling — a form of 'operate applications' — but this is a cloud browser/sandbox, not literal control of apps on the user's own computer, and community hands-on reports say the Computer research feature 'often fails to trigger via the UI, and when it does work, sessions frequently stall without returning an answer.' This directly contradicts the vendor's claims of reliable task completion. missing for 10: evidence of actually operating local desktop applications (not just cloud browser/sandbox), reliability data contradicting the stalling reports, independent success case studies.", "evidenceIds": [ "perplexity-docs-17", "perplexity-docs-18", "perplexity-docs-19", "perplexity-docs-29", "perplexity-docs-30", "perplexity-docs-38", "perplexity-comm-7" ] }, { "productId": "perplexity", "storyId": "connect-drive-calendar", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Perplexity's Computer/MCP docs claim a 'managed connector framework' connecting to 400+ services via one-click OAuth, and the Assistant feature claims it can draft emails and schedule appointments, which implies email/calendar integration, but no evidence names specific connectors like Google Drive, Gmail, or Google/Outlook Calendar, nor shows search results being grounded in those personal sources. Community reports also note the 'Computer' feature (where this connector framework lives) often fails to trigger via the UI and stalls without returning an answer, raising doubts about reliability. Missing for 10: named cloud-drive/email/calendar connector examples, documentation of how connected-source data is surfaced in answers, and independent confirmation the connector framework works reliably.", "evidenceIds": [ "perplexity-docs-19", "perplexity-docs-9", "perplexity-docs-28", "perplexity-comm-7" ] }, { "productId": "perplexity", "storyId": "create-edit-documents", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Perplexity Labs is described as helping 'bring your reports and projects to life,' hinting at document/report generation, but the evidence pack gives no detail on iterative editing, presentation creation, or exporting files to common formats. Missing for 10: iterative document/presentation editing workflow, export format support (PDF/DOCX/PPT), and any independent confirmation these outputs are usable/editable.", "evidenceIds": [ "perplexity-docs-10" ] }, { "productId": "perplexity", "storyId": "custom-bots", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Perplexity's search products, apps, CLI, MCP servers, and Agent API (including 'profiles' as reusable API configs), but nothing describes a user-facing feature for building a custom assistant with its own persistent instructions plus an attached knowledge base that can be shared with others (e.g., a Spaces/custom-GPT-like capability).", "evidenceIds": [] }, { "productId": "perplexity", "storyId": "data-analysis-code", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Perplexity documents a code-execution sandbox (Computer MCP / Agent API 'sandbox' tool) that can write and run Python/Node code and return outputs, and Labs is pitched for 'reports and projects,' suggesting some support for computing answers/files from data. However, hands-on community reporting says the 'Computer' feature 'often fails to trigger via the UI, and when it does work, sessions frequently stall without returning an answer,' directly contradicting reliable delivery of this workflow. missing for 10: explicit evidence of chart generation, working with user-uploaded datasets, downloadable file outputs, and independent confirmation the sandbox reliably works in the consumer product.", "evidenceIds": [ "perplexity-docs-17", "perplexity-docs-18", "perplexity-docs-33", "perplexity-docs-10", "perplexity-comm-7" ] }, { "productId": "perplexity", "storyId": "deep-research-report", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Perplexity's docs explicitly advertise 'Pro Search & Deep Research: Guided AI search for deeper exploration' with citations built into every answer, matching the story's core capability. However, hands-on community reports directly contradict reliable delivery: one user found citations/references 'frequently mismatched to the text' and quality declining over time, and another reports the 'Computer' research feature 'often fails to trigger via the UI, and when it does work, sessions frequently stall without returning an answer,' leading them to prefer other tools for actual research. Missing for 10: independent verification of successful autonomous multi-source runs, resolution of the stalling/citation-mismatch complaints, and clearer documentation distinguishing 'Deep Research' reliability from the separate 'Computer' agent.", "evidenceIds": [ "perplexity-docs-8", "perplexity-docs-27", "perplexity-docs-12", "perplexity-docs-31", "perplexity-comm-6", "perplexity-comm-7", "perplexity-comm-9" ] }, { "productId": "perplexity", "storyId": "desktop-app", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack only documents Perplexity's mobile apps (iOS/Android), a Chrome browser extension, a CLI, and MCP/API integrations — none describe an official desktop app with OS-level keyboard shortcuts or screen-content access.", "evidenceIds": [ "perplexity-docs-8", "perplexity-docs-14", "perplexity-docs-22", "perplexity-docs-6" ] }, { "productId": "perplexity", "storyId": "export-my-data", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions a data export feature, account data download, or chat history export capability for Perplexity's consumer product; documentation focuses on API, MCP, CLI, and app features with no trust-controls/export functionality described.", "evidenceIds": [] }, { "productId": "perplexity", "storyId": "file-upload-analysis", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of document, spreadsheet, or PDF upload/analysis features anywhere in Perplexity's docs, API, CLI, MCP server, or app store listings — coverage focuses on web search, citations, sandboxed code execution, and browser automation. Missing for 10: any documentation of file/PDF/spreadsheet upload, parsing, or analysis capability, and any user or hands-on evidence of accurate document Q&A.", "evidenceIds": [] }, { "productId": "perplexity", "storyId": "image-generation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack shows Perplexity is focused on web-grounded search, citations, code sandboxes, MCP/CLI integrations, and browser automation via 'Computer,' but contains no mention of image generation or editing capabilities from natural-language prompts anywhere in the docs, app store listings, or community reports.", "evidenceIds": [] }, { "productId": "perplexity", "storyId": "image-understanding", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no mention of image/screenshot upload or visual interpretation capability anywhere in the docs, app store listings, or API references—only text search, voice, browsing, and coding features are described.", "evidenceIds": [] }, { "productId": "perplexity", "storyId": "memory-across-chats", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only within-conversation context ('conversation context across turns', 'Thread Follow-Ups') and a searchable history/library, but no documented feature where the assistant recalls or applies context from previous separate chats into new conversations. Missing for 10: any explicit cross-session memory feature, opt-in/opt-out memory controls, or evidence of persisted user facts being reused in new threads.", "evidenceIds": [ "perplexity-docs-31", "perplexity-docs-39", "perplexity-docs-13" ] }, { "productId": "perplexity", "storyId": "mobile-apps", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Perplexity has official iOS and Android apps (App Store and Play Store listings) with a broad feature set: Pro Search/Deep Research, Assistant tasks, voice, citations, library/history sync across devices, and cross-model access — indicating full-featured rather than bare-bones apps. Missing for 10: independent hands-on app-store review/rating evidence and no direct community commentary specifically on mobile app quality (community feedback cited is about the web/desktop product and Computer feature, not mobile apps specifically).", "evidenceIds": [ "perplexity-docs-8", "perplexity-docs-9", "perplexity-docs-10", "perplexity-docs-11", "perplexity-docs-12", "perplexity-docs-13", "perplexity-docs-16", "perplexity-docs-22", "perplexity-docs-24", "perplexity-docs-39" ] }, { "productId": "perplexity", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The API/Agent API exposes web-grounded search, citations, sandboxed code execution, structured output, model fallback, and CLI/MCP integrations, covering much of what the UI offers, but UI-only consumer features like Assistant task automation (email drafting, scheduling, reservations), Voice, Library/Discover, and browser extension page-context features have no documented API equivalents. missing for 10: API parity for Assistant actions (email/scheduling/booking), Voice interface, Library/Discover/curation features, browser-extension contextual page understanding, and independent confirmation that all UI capabilities are API-accessible.", "evidenceIds": [ "perplexity-docs-31", "perplexity-docs-33", "perplexity-docs-34", "perplexity-docs-35", "perplexity-docs-9", "perplexity-docs-11", "perplexity-docs-13", "perplexity-docs-16", "perplexity-docs-25" ] }, { "productId": "perplexity", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of a data export/portability feature for user account data (chat history, library, settings) in open formats; the docs cover API access to models/search but nothing about exporting personal user data or account deletion workflows. missing for 10: any documented data export tool, format spec (JSON/CSV/etc.), or account-deletion-with-export flow.", "evidenceIds": [] }, { "productId": "perplexity", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Perplexity is a closed, proprietary product; the evidence pack shows API docs, apps, CLI, and MCP integrations but no mention of open-sourcing the product itself or any open-license source code repository.", "evidenceIds": [] }, { "productId": "perplexity", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Perplexity is offered exclusively as a hosted SaaS (web app, mobile apps, browser extension, cloud API/CLI/MCP) with no evidence of any self-hostable core product, open-source release, or on-prem deployment option.", "evidenceIds": [] }, { "productId": "perplexity", "storyId": "persistent-instructions", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The only related evidence is the Agent API 'profiles' feature (a reusable, versioned configuration you save and manage), which could function as persistent settings shaping responses, but this is a developer/API concept, not documented as a consumer-facing custom-instructions or persona feature. Missing for 10: explicit evidence of user-facing custom instructions/preferences in the consumer app, confirmation that profiles persist across all conversations/responses, and independent corroboration of this working as claimed.", "evidenceIds": [ "perplexity-docs-37", "perplexity-docs-13" ] }, { "productId": "perplexity", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, regional storage options, or any user-selectable region controls; Perplexity's docs focus on models, search, MCP/CLI integrations, and app features but never address data storage location or residency choices.", "evidenceIds": [] }, { "productId": "perplexity", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack of any opt-out setting, privacy policy statement, or documented control letting users prevent their data from being used for AI training; the evidence pack only covers product features (search, CLI, MCP, browser extension) and unrelated community complaints about crawling/robots.txt behavior. Missing for 10: any documentation of a training opt-out toggle, privacy settings page, or data-use policy addressing model training.", "evidenceIds": [] }, { "productId": "perplexity", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation of data retention settings, deletion controls, export options, or privacy dashboard features for Perplexity users; the only privacy-adjacent items concern the company's own web-crawling/robots.txt controversy, which is off-topic for user data control. This axis clearly applies to a consumer AI assistant, so absence of evidence yields 'none'.", "evidenceIds": [] }, { "productId": "perplexity", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack describes any telemetry opt-out, privacy settings, or usage-tracking controls; the community items discuss crawling/robots.txt disputes and product quality, not telemetry opt-out. Missing for 10: any documentation of a privacy settings page, data-collection opt-out toggle, or telemetry policy.", "evidenceIds": [] }, { "productId": "perplexity", "storyId": "projects-workspaces", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack mentions a 'Library' for curating search history but nothing about a 'Spaces'/'Projects' feature that groups chats and files with shared context or custom instructions, which is the core of this story. Missing for 10: any mention of collections/spaces, file upload grounding within a project, or persistent custom instructions per project.", "evidenceIds": [] }, { "productId": "perplexity", "storyId": "scheduled-tasks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows search, Q&A, Computer/MCP agent capabilities, and an Assistant that can draft emails or book reservations, but nothing describes a mechanism for scheduling recurring or one-off tasks that run automatically in the background and return results later. Community evidence even notes the 'Computer' research feature often fails to trigger or stalls without returning an answer, but there's no documented scheduling/task-runner feature at all.", "evidenceIds": [ "perplexity-docs-9", "perplexity-docs-17", "perplexity-docs-29", "perplexity-comm-7" ] }, { "productId": "perplexity", "storyId": "training-opt-out", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses privacy settings, opt-out of training, or data-usage controls for conversations; all citations concern product features (search, MCP, CLI, apps) or unrelated crawler/quality controversies. This axis applies to any consumer AI product handling user conversations, so absence of evidence yields 'none'.", "evidenceIds": [] }, { "productId": "perplexity", "storyId": "voice-conversation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Perplexity's mobile app docs mention a 'Voice' feature ('Instant, up-to-date answers whether you type or say it'), indicating voice input/output exists, but there's no evidence describing a natural, continuous, real-time back-and-forth voice conversation mode (like a dedicated voice-chat UI) or its quality/latency. Missing for 10: details on conversational turn-taking, interruption handling, voice mode UX specifics, and independent hands-on validation of voice conversation quality.", "evidenceIds": [ "perplexity-docs-11", "perplexity-docs-26" ] }, { "productId": "perplexity", "storyId": "web-search-citations", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Perplexity's docs and consumer apps clearly document web-grounded answers with inline citations across the API, apps, and browser extension (perplexity-docs-2, -12, -24, -31), and this is a core, heavily marketed feature. However, hands-on community reports concretely contradict citation quality: users report citations/links frequently mismatched to the text and Perplexity citing AI-generated 'best software' listicle spam, undermining trust in the grounding (perplexity-comm-6, perplexity-comm-11), with some users switching to Claude/ChatGPT for research quality (perplexity-comm-9). missing for 10: independent verification that citation accuracy holds up at scale, and resolution of reported citation/source mismatch issues.", "evidenceIds": [ "perplexity-docs-2", "perplexity-docs-12", "perplexity-docs-24", "perplexity-docs-31", "perplexity-comm-6", "perplexity-comm-9", "perplexity-comm-11" ] }, { "productId": "perplexity", "storyId": "workspace-admin-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions team/workspace admin controls, member management, permission roles, or data policy settings; all evidence covers API features, apps, CLI, and MCP integrations. This is a fair axis for a team-oriented product like Perplexity (which offers Enterprise/Team plans), but absence of any admin-console or policy documentation means it cannot be credited.", "evidenceIds": [] }, { "productId": "poke", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A live llms.txt file is confirmed via direct probe (HTTP 200) with actual agent-oriented content describing Poke, and Poke also maintains structured docs pages that agents/tools can be pointed at. missing for 10: no docs.md or openapi.json fallback (both 404), and no independent report of an agent successfully consuming llms.txt end-to-end.", "evidenceIds": [ "poke-probe-1", "poke-docs-1", "poke-probe-2", "poke-probe-3" ] }, { "productId": "poke", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Poke can extract and visualize data from shared images/spreadsheets into tables and charts, and integrations let it access and act on connected app data conversationally, which implies some data-driven insight generation. However, there's no explicit documentation of proactive AI-generated insights, trend analysis, or suggestions surfaced from a user's own accumulated data/history. missing for 10: dedicated insights/suggestions feature documentation, evidence of proactive analysis of user data patterns, independent/hands-on confirmation of this capability.", "evidenceIds": [ "poke-docs-21", "poke-docs-7", "poke-docs-19" ] }, { "productId": "poke", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Poke's docs show building blocks for automation—an API that external tools (CI/CD, monitoring) can call to trigger Poke, and a 'recipes' system that can be created and published—plus positioning as a 'proactive AI assistant.' However there is no explicit documentation of scheduled/autonomous background jobs running without an external trigger or user prompt. missing for 10: documented scheduling/cron-like triggers, evidence of recipes running unattended in the background, independent confirmation of autonomous execution.", "evidenceIds": [ "poke-docs-1", "poke-docs-2", "poke-docs-6", "poke-docs-17", "poke-probe-1" ] }, { "productId": "poke", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Poke IS the built-in AI assistant itself — users delegate tasks via natural conversation, integrations, MCP tools, and even real-human task execution (reservations, calls, deploying websites), all documented in first-party docs. Missing for 10: independent hands-on validation of task delegation beyond the security prompt-injection incident, which is off-topic here.", "evidenceIds": [ "poke-docs-7", "poke-docs-19", "poke-docs-10", "poke-docs-11", "poke-docs-3", "poke-probe-1" ] }, { "productId": "poke", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Poke's docs explicitly state its API can be called from scripts, CI/CD pipelines, and monitoring tools to trigger messages programmatically, which supports headless/automation use (poke-docs-1, poke-docs-2). However, there's no evidence of a full headless run mode (e.g., batch execution, structured output for CI parsing) and probes show no OpenAPI/swagger spec is exposed, suggesting the API's automation surface is thin and undocumented beyond message-sending. Missing for 10: independent/hands-on CI integration examples, a public OpenAPI schema, and evidence of structured/machine-readable responses needed for reliable CI pipelines.", "evidenceIds": [ "poke-docs-1", "poke-docs-2", "poke-probe-3" ] }, { "productId": "poke", "storyId": "agentic-mcp-client", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Poke's docs explicitly describe MCP server support, with CLI commands to tunnel local MCP servers and add remote ones ('poke mcp add'), plus a recipe/marketplace ecosystem for sharing MCP-based tools. This is first-party documented and corroborated by a probe confirming the docs page exists, though there's no independent hands-on report of the MCP integration actually working. Missing for 10: independent/hands-on verification of MCP tool usage in practice, and more detail on tool invocation reliability.", "evidenceIds": [ "poke-docs-3", "poke-docs-4", "poke-docs-5", "poke-docs-16", "poke-probe-4" ] }, { "productId": "poke", "storyId": "agentic-mcp-server", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Poke is itself an AI agent product; all MCP evidence (poke-docs-3,4,16) shows Poke acting as an MCP *client*—connecting to or tunneling external MCP servers to extend its own capabilities—not Poke exposing an official MCP server endpoint for other agents to connect to. Per the agent-role rule, this client-side capability does not make the 'official MCP server' axis applicable.", "evidenceIds": [] }, { "productId": "poke", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Poke's entire interface is natural-language texting across WhatsApp/Apple Messages/Telegram, and its docs explicitly state integrations let it 'access information and perform actions across apps and services through natural conversation' (poke-docs-7/19), corroborated by the llms.txt description of Poke texting like a human (poke-probe-1). Missing for 10: independent hands-on user reviews confirming natural-language command reliability beyond vendor docs, and no detail on limits/edge cases of NL command parsing.", "evidenceIds": [ "poke-docs-7", "poke-docs-19", "poke-probe-1", "poke-docs-10", "poke-docs-21" ] }, { "productId": "poke", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Poke ships an official CLI invoked via `npx poke@latest` with subcommands like `login`, `tunnel`, and `mcp add`, confirming a real CLI exists for developer/MCP workflows. However, the documented CLI scope is narrow (mainly MCP server registration/tunneling and login) rather than a general-purpose CLI for broader AI-native automation tasks. Missing for 10: evidence of a fuller command set (e.g. messaging, recipe management, account/config commands) and any independent/hands-on verification of the CLI's reliability.", "evidenceIds": [ "poke-docs-4", "poke-docs-6", "poke-docs-16", "poke-docs-3" ] }, { "productId": "poke", "storyId": "agentic-public-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Poke documents a public API for sending messages programmatically and integrating with external triggers (CI/CD, monitoring tools), plus CLI/MCP tooling for extending functionality, showing genuine programmatic access. However, no OpenAPI/Swagger spec or formal API reference was found (404s on standard discovery paths), so the API's documentation depth and completeness are unclear. Missing for 10: a full API reference/spec (OpenAPI schema), authentication details, rate limits, and independent developer corroboration of building against the API.", "evidenceIds": [ "poke-docs-1", "poke-docs-2", "poke-probe-3" ] }, { "productId": "poke", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Poke supports API access, integrations, and MCP servers, but there's no mention of scoped or least-privilege API credential/key issuance for agents — no API key management, permission scoping, or token-based access control documentation. The prompt-injection incident (poke-comm-1) raises security concerns but is unrelated to credential scoping specifically.", "evidenceIds": [] }, { "productId": "poke", "storyId": "agentic-sdks", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Poke exposes a documented API for sending messages programmatically and an official CLI (`npx poke@latest`) for managing MCP servers and recipes, which developers can build against, but there is no evidence of dedicated official SDK packages (e.g., Python/JS client libraries) — probes for OpenAPI/swagger specs and docs.md all returned 404, suggesting no formal SDK artifacts beyond the CLI/API docs. missing for 10: language-specific SDK packages, published OpenAPI/schema spec, independent developer corroboration of SDK usage.", "evidenceIds": [ "poke-docs-1", "poke-docs-2", "poke-docs-6", "poke-docs-16", "poke-probe-2", "poke-probe-3" ] }, { "productId": "poke", "storyId": "agentic-webhooks", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Poke's API docs mention that external services like GitHub, CI/CD, or monitoring tools can trigger Poke via webhook-style calls, and the API can send messages programmatically, showing some webhook-adjacent integration. However, there's no explicit documentation of Poke exposing an outbound webhook subscription system for AI-native users to subscribe to Poke's own events. missing for 10: explicit outbound webhook/event subscription API, event types list, subscription management docs, independent confirmation of webhook reliability.", "evidenceIds": [ "poke-docs-1", "poke-docs-2" ] }, { "productId": "poke", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an interactive API reference or runnable examples; probes for OpenAPI/Swagger specs and docs.md all returned 404, and only basic prose docs about the messaging API exist.", "evidenceIds": [ "poke-probe-2", "poke-probe-3", "poke-docs-1" ] }, { "productId": "poke", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Direct probes for OpenAPI/swagger spec files all returned 404, and no documentation references a downloadable machine-readable API spec; only an llms.txt file exists, which is not an API spec.", "evidenceIds": [ "poke-probe-2", "poke-probe-3", "poke-probe-1" ] }, { "productId": "poke", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack mentions any sandbox environment, test/staging mode, or way to isolate testing from production data; Poke's docs focus on integrations, MCP servers, and messaging features with no mention of sandboxing.", "evidenceIds": [] }, { "productId": "poke", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "There is no evidence of API versioning scheme, changelog for breaking changes, or a documented deprecation policy; probes show no OpenAPI spec available (404s), and docs only describe basic API usage without versioning or deprecation commitments.", "evidenceIds": [ "poke-docs-1", "poke-probe-3", "poke-probe-2" ] }, { "productId": "poke", "storyId": "app-directory", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Poke docs describe poke.com/recipes as a directory of first-party and approved third-party recipes/integrations that users can browse and add, plus a separate Integrations page for connecting apps/services. This matches the power-user story of browsing and adding connectors. missing for 10: independent/hands-on evidence of the directory's breadth or UX, and no confirmation of search/filtering or ratings within the directory.", "evidenceIds": [ "poke-docs-18", "poke-docs-7", "poke-docs-19", "poke-docs-15", "poke-docs-3" ] }, { "productId": "poke", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Poke can message, integrate apps, and run recipes/MCP tools, but nothing describes performing bulk operations across many items at once (e.g., batch processing, multi-item automation loops); this remains unevidenced.", "evidenceIds": [] }, { "productId": "poke", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Poke supports external triggers via its API (services like GitHub/CI/CD can trigger Poke) and integrations that let it act across apps, which enables event-driven automation, but there's no documented user-facing rules engine (e.g., 'if X happens, do Y' conditional automation builder) — triggers rely on third-party services calling the API rather than Poke itself defining declarative rules. missing for 10: a native rule/condition definition UI or DSL, evidence of multi-step conditional logic, and independent confirmation of reliability of triggered automations.", "evidenceIds": [ "poke-docs-1", "poke-docs-2", "poke-docs-7", "poke-docs-19" ] }, { "productId": "poke", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence describes a scheduling or recurring-job/workflow feature; docs cover API messaging, integrations, MCP servers, and recipes but nothing about setting up recurring or time-based automations. The 'proactive assistant' description in the probe snippet is too vague to confirm recurring job scheduling. missing for 10: any documentation of cron-like scheduling, recurring triggers, or persistent workflow automation.", "evidenceIds": [ "poke-docs-1", "poke-docs-2", "poke-probe-1" ] }, { "productId": "poke", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence describes versioning, review workflows, or rollback capability for Poke's automations/recipes; docs mention creating, publishing, and monetizing recipes but nothing about version history or reverting changes.", "evidenceIds": [] }, { "productId": "poke", "storyId": "autonomous-background-tasks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Poke's docs show it can be triggered by external events (GitHub, CI/CD) and can even delegate real-world tasks to humans (reservations, calls) and act across integrated apps 'through natural conversation,' plus the llms.txt describes it as a 'proactive AI assistant.' However, there is no explicit documentation of a structured multi-step task queue, autonomous background execution status, or a dedicated review/approval step for complex delegated tasks. Missing for 10: explicit multi-step task orchestration docs, background task status/progress tracking, and a defined 'review results' UX beyond ad-hoc chat replies.", "evidenceIds": [ "poke-docs-2", "poke-docs-7", "poke-docs-10", "poke-probe-1" ] }, { "productId": "poke", "storyId": "browser-agent", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Poke's docs describe integrations, MCP tool connections, and even outsourcing tasks to humans, but there is no evidence Poke itself drives a web browser to navigate sites or complete web-based research/tasks autonomously. missing for 10: any mention of browser automation, web navigation, or headless browsing capability.", "evidenceIds": [ "poke-docs-7", "poke-docs-10", "poke-docs-19" ] }, { "productId": "poke", "storyId": "computer-use-desktop", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Poke's evidence shows it connects to cloud services/integrations and MCP tool servers (poke-docs-7, poke-docs-19, poke-docs-3) but there is no evidence of it seeing a user's screen or directly operating local desktop applications — its automation is limited to API/webhook-based integrations and remote 'recipes,' not computer-use style control of a user's machine.", "evidenceIds": [ "poke-docs-7", "poke-docs-19", "poke-docs-3" ] }, { "productId": "poke", "storyId": "connect-drive-calendar", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Poke's docs describe a general 'Integrations' system that lets users connect apps/services (with Webflow given as the only named example) and mention an email-address feature, but no evidence names concrete cloud-drive (e.g., Google Drive), email (Gmail/Outlook), or calendar connectors, nor demonstrates search/use of such data in answers. Missing for 10: named cloud-drive integration, named email inbox integration, named calendar integration, and evidence of search/retrieval across them.", "evidenceIds": [ "poke-docs-7", "poke-docs-19", "poke-docs-15", "poke-docs-12" ] }, { "productId": "poke", "storyId": "create-edit-documents", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Poke can create websites and generate tables/charts from data, but there is no evidence of creating or iteratively editing documents/presentations that can be exported (e.g. Word, PDF, PowerPoint files). Missing for 10: document creation/editing evidence, presentation creation, export functionality for standard file formats.", "evidenceIds": [ "poke-docs-11", "poke-docs-21" ] }, { "productId": "poke", "storyId": "custom-bots", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Poke supports 'recipes' that can be created, published, shared via referral link, and monetized, plus MCP servers/integrations that extend its knowledge and behavior — this maps loosely to custom assistants with instructions/knowledge. However, there's no explicit documentation of defining a persistent 'assistant persona' with custom instructions in the way tools like GPTs allow; recipes appear more tool/automation-centric than persona-centric. missing for 10: explicit support for custom system-prompt/persona instructions per assistant, evidence of knowledge-base attachment beyond MCP tool integration, independent user validation of sharing custom assistants.", "evidenceIds": [ "poke-docs-5", "poke-docs-6", "poke-docs-17", "poke-docs-18", "poke-docs-3", "poke-docs-7" ] }, { "productId": "poke", "storyId": "data-analysis-code", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Poke's docs show it can create tables and charts from described data or shared images (poke-docs-21), but there's no evidence of an underlying code-execution environment, computed numeric answers, or generation of downloadable files. missing for 10: evidence of actual code execution/sandbox, computed-answer capability, downloadable file output.", "evidenceIds": [ "poke-docs-21" ] }, { "productId": "poke", "storyId": "deep-research-report", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Poke's evidence pack covers messaging integrations, MCP tool extensibility, recipes, and various assistant features, but nothing describes an autonomous multi-source deep research capability that returns a cited report. No mention of web search breadth, citation generation, or research-report output.", "evidenceIds": [] }, { "productId": "poke", "storyId": "desktop-app", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Poke is a messaging-based AI assistant (WhatsApp/Telegram/Apple Messages/API/MCP integrations), not a desktop application with OS-level shortcuts or screen access; this axis is a category error for its product type.", "evidenceIds": [] }, { "productId": "poke", "storyId": "export-my-data", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions any data export, chat history download, or account data portability feature; documentation covers integrations, MCP, recipes, and messaging channels but nothing about exporting user data.", "evidenceIds": [] }, { "productId": "poke", "storyId": "file-upload-analysis", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The only relevant evidence is that Poke can 'extract and visualize' data from shared images of spreadsheets to create tables/charts, which is a narrow slice of the story. There is no documentation of direct document or PDF upload, parsing, or accurate content analysis for knowledge-worker use cases. Missing for 10: explicit PDF/document upload support, evidence of accuracy on structured/text-heavy files, and any hands-on validation of analysis quality.", "evidenceIds": [ "poke-docs-21" ] }, { "productId": "poke", "storyId": "image-generation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Poke can create tables/charts from data and extract data from images (poke-docs-21), and can create websites, but there is no mention of generating or editing images from natural-language prompts anywhere in the docs or release notes.", "evidenceIds": [] }, { "productId": "poke", "storyId": "image-understanding", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Poke's release notes explicitly state it can 'share images of existing data (like spreadsheets) for Poke to extract and visualize,' showing some image-interpretation capability, but this is limited to structured data extraction rather than general photo/screenshot understanding across varied content. Missing for 10: evidence of accurate interpretation of arbitrary photos/screenshots (e.g., UI screenshots, handwritten notes, general scenes), independent hands-on verification of accuracy, and any first-party documentation describing general vision/OCR capability beyond the spreadsheet example.", "evidenceIds": [ "poke-docs-21" ] }, { "productId": "poke", "storyId": "memory-across-chats", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs mention Poke's Memory as a concept tied to account merging (poke-docs-20), implying persistent memory exists across the assistant, but there's no detailed documentation of how memory is captured, retained, or applied across separate conversations, nor independent confirmation of its effectiveness. Missing for 10: dedicated memory documentation explaining retention/retrieval mechanics, examples of cross-chat context recall, and independent/hands-on verification that it works reliably.", "evidenceIds": [ "poke-docs-20" ] }, { "productId": "poke", "storyId": "mobile-apps", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Poke operates through existing messaging channels (Apple Messages, WhatsApp, Telegram) rather than shipping its own official iOS/Android app; no documentation or probe evidence mentions a dedicated Poke mobile app.", "evidenceIds": [ "poke-docs-13", "poke-docs-14", "poke-probe-1" ] }, { "productId": "poke", "storyId": "openness-api-parity", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Poke exposes a documented API for sending messages programmatically and triggering it from external systems (poke-docs-1, poke-docs-2), plus an MCP server/tunnel mechanism for extending tools (poke-docs-3/4/16). However, there's no evidence of API parity for major UI capabilities like managing integrations, recipes, human-task requests, website deployment, or memory/account management, and no public OpenAPI/spec was found (poke-probe-2, poke-probe-3 both 404). missing for 10: API coverage for integrations/recipes/human-tasks/website-deploy features, published OpenAPI spec, independent confirmation of API-UI parity.", "evidenceIds": [ "poke-docs-1", "poke-docs-2", "poke-docs-3", "poke-docs-4", "poke-docs-16", "poke-probe-2", "poke-probe-3" ] }, { "productId": "poke", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of any data export feature or open-format data portability in Poke's docs; the pack covers messaging API, MCP servers, integrations, and account merging but nothing about exporting user data (memory, conversation history, recipes) in open formats or account closure/data takeout.", "evidenceIds": [] }, { "productId": "poke", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence Poke's source code is published under any open license; Poke is presented as a closed SaaS product with docs, APIs, and integrations but no repository or licensing information is mentioned anywhere in the evidence pack.", "evidenceIds": [] }, { "productId": "poke", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Poke is a hosted SaaS assistant with no evidence of any self-hostable core product, open-source release, or on-prem deployment option; all docs describe a cloud service accessed via API/integrations/MCP.", "evidenceIds": [] }, { "productId": "poke", "storyId": "persistent-instructions", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The docs mention a 'Memory' feature tied to a user's account (carried over when merging accounts), implying some persistent context storage, but there is no documentation of a dedicated custom-instructions/preferences setting that explicitly shapes every response. Missing for 10: explicit UI/API for setting persistent custom instructions, examples of how such preferences influence responses, and independent confirmation of this behavior.", "evidenceIds": [ "poke-docs-20" ] }, { "productId": "poke", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack mentions data residency, region selection, or storage location controls; Poke's docs cover integrations, MCP, messaging channels, and pricing but nothing about where user data is stored or any regional choice.", "evidenceIds": [] }, { "productId": "poke", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses data-training opt-out or privacy controls regarding AI model training; the docs focus on messaging, integrations, MCP, and recipes with no mention of data usage policies or opt-out settings.", "evidenceIds": [] }, { "productId": "poke", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item addresses data retention policies, deletion controls, or export/delete-my-data mechanisms; docs cover memory merging and integrations but not retention/deletion controls, and the prompt-injection report is unrelated to this axis.", "evidenceIds": [] }, { "productId": "poke", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item discusses telemetry, analytics opt-out, or usage-tracking controls for Poke users; docs cover API, MCP, integrations, pricing, and release notes but never privacy/telemetry settings.", "evidenceIds": [] }, { "productId": "poke", "storyId": "projects-workspaces", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Poke's docs describe messaging, integrations, MCP servers, and recipes, but there is no mention of organizing chats/files into projects or spaces with shared context/instructions, which is a distinct memory-organization feature not evidenced here.", "evidenceIds": [] }, { "productId": "poke", "storyId": "scheduled-tasks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Poke's API/integrations for triggering messages from external events (CI/CD, monitoring) and MCP tool extensions, but nothing documents a user-facing scheduler for recurring or one-off tasks that return results later. Missing for 10: any explicit scheduling/reminder feature, recurrence configuration, or hands-on proof of a task running and reporting back autonomously.", "evidenceIds": [ "poke-docs-1", "poke-docs-2", "poke-docs-7" ] }, { "productId": "poke", "storyId": "training-opt-out", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item addresses model-training data usage or opt-out controls for conversation data; documentation covers integrations, MCP, pricing, and release notes but nothing about training data policy. Missing for 10: any privacy/data-training policy page, opt-out toggle, or enterprise data-use agreement.", "evidenceIds": [] }, { "productId": "poke", "storyId": "voice-conversation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Poke is described as a text-based assistant across messaging platforms (WhatsApp, Telegram, Apple Messages) with no evidence of real-time voice call or voice conversation capability; all documented interactions are text/message-based.", "evidenceIds": [] }, { "productId": "poke", "storyId": "web-search-citations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence describes Poke performing web search, retrieval-augmented answers, or providing citations back to sources; docs focus on messaging, integrations, MCP tooling, and account features, not grounded research answers.", "evidenceIds": [] }, { "productId": "poke", "storyId": "workspace-admin-controls", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Poke is a consumer personal AI assistant (individual account, messaging channels, integrations, recipes) with no evidence of organizational/team workspace concepts like member management, role-based permissions, or org-wide data policies. This is a category mismatch rather than a missing feature.", "evidenceIds": [] } ]