[ { "productId": "consensus", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A live probe confirms Consensus serves an llms.txt file at its root (HTTP 200) with a structured summary of the product, directly enabling agents to be pointed at agent-oriented docs. Missing for 10: broader agent-oriented doc formats (e.g. .md endpoints) return 404, and no independent third-party confirmation of llms.txt usage exists.", "evidenceIds": [ "consensus-probe-1", "consensus-probe-2" ] }, { "productId": "consensus", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Consensus generates AI-driven synthesis, summaries, the Consensus Meter, PICO extraction, and literature review synthesis directly from the papers in its corpus/library, with citations tracing insights back to sources. This is core native functionality (not a bolt-on), covering search, synthesis, and structured insight generation. Missing for 10: independent/hands-on third-party verification of insight quality beyond vendor docs.", "evidenceIds": [ "consensus-docs-10", "consensus-docs-11", "consensus-docs-18", "consensus-docs-22", "consensus-docs-23", "consensus-docs-12", "consensus-docs-9" ] }, { "productId": "consensus", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Consensus is a research search/synthesis engine with an API and MCP server for on-demand retrieval, but there is no evidence of scheduled or event-triggered automations that run autonomously in the background without user invocation. Missing for 10: any scheduling/trigger mechanism, background job execution, or autonomous recurring workflow capability.", "evidenceIds": [] }, { "productId": "consensus", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Consensus ships a built-in \"Research Agent\" that chains citation crawling, DOI lookup, author search and similar-paper discovery on top of its search engine, and its core AI assistant performs search, screen, extract, and synthesize workflows with cited answers — this is essentially delegating research tasks to an in-product AI assistant. missing for 10: independent/hands-on validation of the agent's autonomy and reliability, and more detail on the scope/limits of delegable tasks beyond literature discovery.", "evidenceIds": [ "consensus-docs-7", "consensus-docs-8", "consensus-docs-12", "consensus-docs-10", "consensus-docs-11", "consensus-probe-1" ] }, { "productId": "consensus", "storyId": "agentic-headless", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Consensus offers an API for integrating its search into custom workflows and running automated searches (consensus-docs-1, consensus-docs-15), which implies some programmatic/headless usability. However, there is no explicit documentation of CI integration, headless execution modes, CLI tooling, or automation pipeline examples. Missing for 10: CI/CD integration examples, headless mode documentation, CLI or SDK for automation, and independent evidence of running in automated pipelines.", "evidenceIds": [ "consensus-docs-1", "consensus-docs-15" ] }, { "productId": "consensus", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Consensus is a research/search product, not an AI agent; the evidence shows an API for integration but nothing about MCP server plug-in support to consume external tools. This axis (agent-side MCP client capability) is a category error for this type of product.", "evidenceIds": [] }, { "productId": "consensus", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Consensus is not an agent product itself, so the MCP-server axis applies as an ecosystem/API capability, but evidence only shows a REST API and llms.txt file — no mention of an official MCP server for connecting agents. missing for 10: any documented MCP server endpoint, MCP spec compliance, or third-party confirmation of MCP support.", "evidenceIds": [ "consensus-docs-1", "consensus-docs-15", "consensus-probe-1" ] }, { "productId": "consensus", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Consensus's core interaction model is natural-language research queries (search, synthesize, Consensus Meter for yes/no questions) rather than rigid query syntax, and it exposes this same NL-driven retrieval/synthesis surface via an MCP server and REST API for programmatic/agentic use. Missing for 10: independent hands-on evidence of natural-language command execution quality, and no detailed example transcripts showing complex multi-step NL commands being interpreted.", "evidenceIds": [ "consensus-docs-10", "consensus-docs-16", "consensus-docs-18", "consensus-docs-22", "consensus-docs-5" ] }, { "productId": "consensus", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack documents a REST API and an MCP server (consensus-docs-16) but no official command-line interface is mentioned anywhere in the docs or probes. Missing for 10: any mention of a CLI tool, CLI installation instructions, or CLI command reference.", "evidenceIds": [ "consensus-docs-16" ] }, { "productId": "consensus", "storyId": "agentic-public-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Consensus advertises a documented API for integrating citations and running automated searches into custom workflows, and its site provides an llms.txt for AI-agent discovery, showing basic public-API and agent-friendliness. However, the evidence pack only shows marketing/landing pages, not actual API reference documentation, authentication, endpoints, or example requests/responses, and there's no independent or hands-on corroboration that the API works as described. Missing for 10: full API reference/spec details, code/SDK examples, and independent verification of API usage.", "evidenceIds": [ "consensus-docs-1", "consensus-docs-15", "consensus-probe-1" ] }, { "productId": "consensus", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows an API and MCP server exist, but there is no mention of scoped or least-privilege API keys, permission scopes, or credential management for agents — just generic API access. missing for 10: scoped/least-privilege credential issuance, API key permission controls, agent-specific auth documentation.", "evidenceIds": [ "consensus-docs-1", "consensus-docs-15", "consensus-docs-16" ] }, { "productId": "consensus", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Consensus documents an API for integration (consensus-docs-1, consensus-docs-15) but no evidence pack item mentions official SDKs (Python, JS, etc.) or client libraries for AI-native development — only the raw API and llms.txt discovery file are shown.", "evidenceIds": [ "consensus-docs-1", "consensus-docs-15", "consensus-probe-1" ] }, { "productId": "consensus", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack mentions webhooks or event subscriptions; Consensus's API/MCP surface is described only as REST retrieval/synthesis, not event-driven push notifications.", "evidenceIds": [] }, { "productId": "consensus", "storyId": "agreement-disagreement", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "The Consensus Meter explicitly classifies each relevant paper as supporting, refuting, or mixed/inconclusive on a given question and displays the distribution, directly surfacing agreement/disagreement across sources rather than a single answer, and every response includes citations back to originals. Missing for 10: independent/hands-on corroboration of the Meter's accuracy and no worked example showing disagreement handling in practice.", "evidenceIds": [ "consensus-docs-18", "consensus-docs-22", "consensus-docs-11" ] }, { "productId": "consensus", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence confirms Consensus offers a REST API and MCP server (consensus-docs-1, consensus-docs-15, consensus-docs-16), but there is no mention of an interactive API reference, sandbox, or runnable code examples anywhere in the pack.", "evidenceIds": [ "consensus-docs-1", "consensus-docs-15", "consensus-docs-16" ] }, { "productId": "consensus", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Consensus documents a REST API and MCP server (consensus-docs-15, consensus-docs-16) but no evidence pack item mentions an OpenAPI spec, Swagger file, or any downloadable machine-readable API schema; the llms.txt probe returns a plain-text description, not an API spec. missing for 10: OpenAPI/Swagger file, machine-readable schema download link, independent confirmation of spec availability.", "evidenceIds": [ "consensus-docs-15", "consensus-docs-16", "consensus-probe-1" ] }, { "productId": "consensus", "storyId": "api-sandbox", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Consensus is a research/literature-search engine over academic papers, not a data-producing or transactional system where 'sandbox vs production data' is a meaningful distinction; there is no concept of production data being modified. This axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "consensus", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "There's an API and MCP server mentioned, but no evidence of API versioning scheme or a documented deprecation policy anywhere in the pack. missing for 10: versioning scheme documentation, deprecation policy, changelog/migration guides.", "evidenceIds": [ "consensus-docs-16", "consensus-docs-15" ] }, { "productId": "consensus", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show bulk-style capabilities: one-click import of thousands of papers into a library, an API/MCP server for automated bulk searches, and Deep Searches across many studies — supporting bulk operations for an AI-native/automation persona. missing for 10: independent/hands-on verification of bulk API throughput or rate limits, explicit batch-processing endpoints (e.g., bulk extract/export across many items in one call), and any third-party confirmation of scale performance.", "evidenceIds": [ "consensus-docs-3", "consensus-docs-13", "consensus-docs-15", "consensus-docs-16", "consensus-docs-17", "consensus-docs-14" ] }, { "productId": "consensus", "storyId": "automation-rules-engine", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Consensus is a research/literature search and synthesis engine, not an automation/workflow-rules platform; there is no concept of user-defined trigger-action rules for events in its product category.", "evidenceIds": [] }, { "productId": "consensus", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions scheduling, recurring jobs, alerts, or automated re-running of searches/workflows over time; the API and MCP server are described as on-demand retrieval/synthesis interfaces, not schedulable automation. missing for 10: any scheduling/cron feature, recurring alert or saved-search re-run capability, or workflow automation trigger.", "evidenceIds": [ "consensus-docs-15", "consensus-docs-16" ] }, { "productId": "consensus", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Consensus is a research/literature-search engine, not an automation-building platform; there is no concept of 'automations' to version, review, or roll back. This axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "consensus", "storyId": "autonomous-research-run", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Consensus advertises a 'Research Agent' that chains citation crawling, DOI lookup, author search, and similar-papers search on top of its search engine, plus a literature-review feature that searches, screens, extracts, and synthesizes evidence — both suggesting multi-step, not single-pass, investigation. However, evidence is limited to marketing feature pages with no walkthrough, example transcript, or independent corroboration of true autonomous multi-step reasoning over a posed question. Missing for 10: a documented end-to-end example of the agent autonomously chaining steps for a specific question, independent/hands-on verification, and detail on how far it goes without user intervention.", "evidenceIds": [ "consensus-docs-7", "consensus-docs-8", "consensus-docs-12" ] }, { "productId": "consensus", "storyId": "export-reference-formats", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only documents importing papers into Consensus (from Zotero, BibTeX, PDF, RIS) but contains no mention of exporting results to documents, spreadsheets, or reference-manager formats. Missing for 10: any export-to-Word/PDF, export-to-CSV/spreadsheet, or export-to-Zotero/EndNote/BibTeX functionality.", "evidenceIds": [ "consensus-docs-14", "consensus-docs-20", "consensus-docs-21" ] }, { "productId": "consensus", "storyId": "free-tier-trial", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack references a pricing page (consensus-docs-17) but only quotes a single line about 'Deep Searches' feature tiering; there is no description of a free tier, trial period, usage caps, or sign-up-free access that a researcher could evaluate. No first-party or independent evidence confirms Consensus offers a meaningful free/trial experience.", "evidenceIds": [ "consensus-docs-17" ] }, { "productId": "consensus", "storyId": "inline-citations", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Consensus documents that every AI-generated response includes citations tracing back to the original source paper, and features like the Consensus Meter classify individual papers (supporting/refuting) with traceable provenance, directly matching the researcher's need to verify claims against sources. Missing for 10: independent/hands-on verification of citation accuracy and completeness beyond vendor docs.", "evidenceIds": [ "consensus-docs-11", "consensus-docs-18", "consensus-docs-22", "consensus-docs-10" ] }, { "productId": "consensus", "storyId": "long-horizon-runs", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence shows Deep Searches/Lit Reviews and a research agent chaining searches, but there is no mention of async job submission, background/unattended execution, or notification when a long-running job completes.", "evidenceIds": [] }, { "productId": "consensus", "storyId": "monitoring-alerts", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of standing searches, saved-search alerts, or notification features when new relevant papers appear; the evidence pack covers search, library import, citation graph, API/MCP retrieval, and literature review synthesis but nothing about recurring/alert-based monitoring of new sources.", "evidenceIds": [] }, { "productId": "consensus", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "The API/MCP server is documented to expose 'the same retrieval and synthesis surface that powers the web app' (consensus-docs-16), and supports automated search (consensus-docs-15), suggesting broad parity for core search/synthesis. However, UI-specific workflows like Library import/reference management (Zotero/BibTeX/RIS import), Citation Graph, and Consensus Meter visualizations are not explicitly confirmed as API-accessible endpoints. Missing for 10: explicit API documentation confirming library management, citation graph, and meter features are callable via API, plus independent/hands-on verification of claimed parity.", "evidenceIds": [ "consensus-docs-16", "consensus-docs-15", "consensus-docs-9", "consensus-docs-14", "consensus-docs-2", "consensus-docs-22" ] }, { "productId": "consensus", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only import capabilities (Zotero, BibTeX, PDF, RIS) into the Consensus Library, with no mention of exporting a user's library, annotations, or account data back out in open formats. Data portability/export is a fair axis for a reference-manager-style product, but no evidence supports it.", "evidenceIds": [ "consensus-docs-14", "consensus-docs-21", "consensus-docs-9", "consensus-docs-13" ] }, { "productId": "consensus", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Consensus is a closed, commercial SaaS research search engine; there is no indication its source code is open-licensed or expected to be, making this axis a category error for this product type.", "evidenceIds": [] }, { "productId": "consensus", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Consensus is a hosted SaaS research engine/API, not open-source software; self-hosting is a wrong-axis question for this type of product and no evidence suggests otherwise.", "evidenceIds": [] }, { "productId": "consensus", "storyId": "own-corpus-upload", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Consensus's Library feature explicitly supports importing PDFs, BibTeX, RIS, and Zotero corpora and turns them into a 'searchable, AI-powered workspace' for finding gaps and using the collection, which matches the story's upload+research intent. However, evidence doesn't detail how deeply the AI synthesis/agent features (Meter, PICO extraction, literature review synthesis) operate specifically over a user's uploaded corpus versus the general 200M-paper index. Missing for 10: explicit documentation of agent-style synthesis/Q&A running directly over an uploaded private corpus, and independent/hands-on confirmation of this workflow.", "evidenceIds": [ "consensus-docs-3", "consensus-docs-13", "consensus-docs-14", "consensus-docs-9", "consensus-docs-20" ] }, { "productId": "consensus", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, regional storage options, or any data-location controls for Consensus; all evidence concerns search, citation, and library features. Missing for 10: any mention of region selection, data residency policy, or storage location controls.", "evidenceIds": [] }, { "productId": "consensus", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item addresses data-training opt-out, privacy controls, or AI training data policies for Consensus; all citations concern search, library, and API features. missing for 10: any privacy policy statement, opt-out mechanism, or data usage/training disclosure.", "evidenceIds": [] }, { "productId": "consensus", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data retention policies, deletion controls, or privacy settings for user data/library content; all citations focus on search, citation, library, and API features. Missing for 10: any documentation on data retention windows, user-initiated deletion, export/erasure workflows, or privacy policy specifics.", "evidenceIds": [] }, { "productId": "consensus", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions telemetry, usage tracking, or any opt-out/privacy settings for Consensus; all citations are about search, citation, and library features unrelated to telemetry controls.", "evidenceIds": [] }, { "productId": "consensus", "storyId": "scholarly-sources", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Consensus is explicitly built as a scholarly-search engine over 200M+ academic papers, including full-text and paywalled content, positioned as an AI-native alternative to Google Scholar, with citation tracing back to original sources. Missing for 10: independent third-party verification of corpus quality/coverage beyond vendor claims.", "evidenceIds": [ "consensus-docs-4", "consensus-docs-5", "consensus-docs-10", "consensus-docs-11", "consensus-probe-1" ] }, { "productId": "consensus", "storyId": "share-collaborate", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack items mention sharing sessions, reports, collaborators, team accounts, or collaborative viewing/editing features—only individual research, library import, and API/agent capabilities are documented. missing for 10: any mention of sharing links, collaborator invites, team workspaces, or comment/build-on functionality.", "evidenceIds": [] }, { "productId": "consensus", "storyId": "steer-effort-scope", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack describes search, citation graph, library, and research-agent features but nowhere mentions controls for adjusting depth, effort, or scope of a research run before or during execution — no parameters, modes, or configuration options are documented.", "evidenceIds": [] }, { "productId": "consensus", "storyId": "structured-reports", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Consensus offers 'Literature Review' and 'Deep Search' features that synthesize evidence across papers, extract structured fields like PICO, and provide citations—suggesting output with some structure and sourcing suitable for sharing. However, there's no explicit evidence of a polished 'report' format with distinct sections, tables, and an executive summary designed for stakeholder sharing (e.g., export to PDF/Word, formatted report templates). Missing for 10: explicit documentation of report formatting/export (sections, tables, summary), evidence of stakeholder-sharing features like PDF export or presentation-ready output, and independent confirmation of report quality.", "evidenceIds": [ "consensus-docs-8", "consensus-docs-12", "consensus-docs-17", "consensus-docs-23", "consensus-docs-18" ] }, { "productId": "consensus", "storyId": "systematic-screening", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Consensus offers literature-review features (search, screen, extract, synthesize per docs-8/12), library import at scale, PICO extraction, and filters by study type/year/discipline that support systematic screening with consistent criteria. However, there is no evidence of documented inter-rater reliability, exportable screening decision logs, or PRISMA-style workflow tracking that a systematic review would require. missing for 10: evidence of structured screening criteria configuration/audit trail, PRISMA-compliant workflow support, independent validation of extraction consistency across large paper sets.", "evidenceIds": [ "consensus-docs-8", "consensus-docs-12", "consensus-docs-13", "consensus-docs-19", "consensus-docs-23", "consensus-docs-17" ] }, { "productId": "consensus", "storyId": "transparent-pricing", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items mention pricing plans, tiers, free/paid limits, or usage quotas — the pack is entirely about product features (citation graph, library, API capabilities). Absence of any pricing/limits documentation for an applicable axis yields none.", "evidenceIds": [] }, { "productId": "elicit", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Elicit hosts a live llms.txt at support.elicit.com/llms.txt (HTTP 200, confirmed via probe) listing agent-oriented docs, and it also exposes an official MCP server for agent access, showing genuine agent-oriented documentation infrastructure. missing for 10: no independent/community corroboration of an agent actually consuming llms.txt successfully, and no OpenAPI/agent-doc spec beyond the llms.txt itself.", "evidenceIds": [ "elicit-probe-1", "elicit-docs-28", "elicit-docs-35" ] }, { "productId": "elicit", "storyId": "agentic-ai-insights", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Elicit's Research Agent, columns, chat-with-papers, and Systematic Review reports are documented to generate AI insights and suggestions from ingested papers (elicit-docs-11, elicit-docs-40, elicit-docs-41), and one HN user found topic analysis genuinely useful (elicit-comm-1). However, independent hands-on reports concretely contradict reliability: users found mostly incorrect summaries, missed key papers, and fabricated/hallucinated facts even when directly quoting sources (elicit-comm-3, elicit-comm-4, elicit-comm-7), and Elicit's own docs admit ~10% inaccuracy requiring manual verification (elicit-comm-2). missing for 10: independent corroboration that generated insights are consistently accurate rather than frequently hallucinated, and resolution of the documented factual-error reports.", "evidenceIds": [ "elicit-docs-11", "elicit-docs-40", "elicit-docs-41", "elicit-comm-1", "elicit-comm-2", "elicit-comm-3", "elicit-comm-4", "elicit-comm-7" ] }, { "productId": "elicit", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Elicit's Alerts feature lets users set up a background process that automatically monitors for new relevant papers and notifies them (e.g., via instant email alerts), which is a real autonomous background automation, and the API/MCP server also enables scripted automated report generation from external workflows. However, this is narrow (limited to paper-discovery alerts) rather than a general-purpose scheduling/automation system for arbitrary agentic tasks, and there's no evidence of recurring scheduled jobs, triggers, or workflow orchestration beyond alerts. Missing for 10: evidence of a general automation/scheduling engine, ability to chain multi-step autonomous tasks, and independent confirmation that alerts reliably run unattended over time.", "evidenceIds": [ "elicit-docs-3", "elicit-docs-15", "elicit-docs-26", "elicit-docs-34", "elicit-docs-35" ] }, { "productId": "elicit", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Elicit ships a built-in Research Agent that users can delegate tasks to directly (search, screening, extraction, column creation, chat with papers, skills), with effort-level control and iterative outputs, all inside the product interface (elicit-docs-2,7,8,11,12,21,40,42). Community evidence corroborates real task delegation working in practice (elicit-comm-1) though also raises accuracy concerns that temper trust in outputs (elicit-comm-2,3,4). Missing for 10: independent quality benchmarking beyond anecdotal HN threads and more recent hands-on validation of the newer effort-level/skills features.", "evidenceIds": [ "elicit-docs-2", "elicit-docs-7", "elicit-docs-8", "elicit-docs-11", "elicit-docs-12", "elicit-docs-21", "elicit-docs-40", "elicit-docs-42", "elicit-comm-1", "elicit-comm-3" ] }, { "productId": "elicit", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Elicit documents an API (and MCP server) explicitly for running its search/report/systematic-review capabilities 'from your own code, scripts, and workflows,' which supports headless/automated use outside the UI. However, there's no explicit CI/pipeline example, and API access appears gated as a paid plan feature rather than a fully documented automation-first workflow. Missing for 10: explicit CI/automation examples or tutorials, rate-limit/auth details for unattended use, and independent confirmation that the API works reliably in automated pipelines.", "evidenceIds": [ "elicit-docs-34", "elicit-docs-36", "elicit-docs-6", "elicit-docs-35" ] }, { "productId": "elicit", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows Elicit exposes itself AS an MCP server for other clients (e.g., Claude Desktop) to consume its research tools (elicit-docs-28, elicit-docs-35), not the reverse capability of Elicit acting as an MCP client that plugs in external MCP servers to use their tools. No evidence exists that Elicit can connect to and use third-party MCP servers.", "evidenceIds": [ "elicit-docs-28", "elicit-docs-35" ] }, { "productId": "elicit", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Elicit provides an official documented MCP server endpoint (claude mcp add --transport http elicit https://elicit.com/api/mcp) exposing full API functionality for use from Claude Desktop, Claude Code, and other MCP-compatible clients. missing for 10: no independent/hands-on corroboration of the MCP server working, and no detail on auth/tool-list scope beyond the docs.", "evidenceIds": [ "elicit-docs-28", "elicit-docs-35" ] }, { "productId": "elicit", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Elicit's Research Agent and semantic search are explicitly natural-language driven (e.g., asking questions in plain language, adding columns via natural-language commands like 'Add a column for study type'), and skills let users reference natural-language instructions instead of re-typing prompts. However, community reports raise accuracy/hallucination concerns that temper confidence in reliability of NL command execution. Missing for 10: independent hands-on verification of complex multi-step NL command chains, and no evidence of NL support outside the research/agent workflows (e.g., no broader command-line or API NL interface).", "evidenceIds": [ "elicit-docs-11", "elicit-docs-20", "elicit-docs-27", "elicit-docs-16", "elicit-docs-12", "elicit-comm-3", "elicit-comm-4" ] }, { "productId": "elicit", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows Elicit offers an API and an MCP server for programmatic/agentic access, but there is no mention anywhere of an official command-line interface (CLI) tool. Since API-based products could plausibly ship a CLI, absence of evidence means this axis is unmet rather than inapplicable.", "evidenceIds": [ "elicit-docs-34", "elicit-docs-35", "elicit-docs-28" ] }, { "productId": "elicit", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Elicit documents a public API with keys/auth management and specific programmatic endpoints (search 138M+ papers, automated report generation, full systematic review workflow control), plus an MCP server exposing the same functionality for agentic clients. Missing for 10: an actual OpenAPI/swagger spec was not found (404s on candidate paths) and no independent/hands-on developer report validates real-world API usage.", "evidenceIds": [ "elicit-docs-6", "elicit-docs-34", "elicit-docs-35", "elicit-docs-36", "elicit-docs-28", "elicit-probe-3" ] }, { "productId": "elicit", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Elicit does offer API keys and an MCP server for programmatic/agent access (elicit-docs-34, elicit-docs-35, elicit-docs-28), so the axis of credential management applies, but there is no evidence of scoped or least-privilege permissions, roles, or restricted-scope API keys — the docs only describe managing API keys generically, not limiting their scope.", "evidenceIds": [ "elicit-docs-34", "elicit-docs-35", "elicit-docs-28" ] }, { "productId": "elicit", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Elicit documents an official API (with managed API keys) for programmatic access to search and automated research reports, plus full API functionality exposed via an MCP server for Claude Desktop/Code integration, which supports agentic, code-driven workflows. However, evidence shows only a REST-style API and API-key docs, not a dedicated official SDK/client library in specific languages, nor code samples or independent developer corroboration. Missing for 10: named client SDKs (e.g., Python/JS packages), quickstart code examples, and independent/hands-on developer confirmation of API reliability.", "evidenceIds": [ "elicit-docs-34", "elicit-docs-35", "elicit-docs-36", "elicit-docs-6", "elicit-docs-28" ] }, { "productId": "elicit", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Elicit offers email alerts, an API, and an MCP server, but no evidence anywhere in the pack mentions webhook subscriptions or event-driven callbacks for programmatic integration.", "evidenceIds": [ "elicit-docs-3", "elicit-docs-34", "elicit-docs-35" ] }, { "productId": "elicit", "storyId": "agreement-disagreement", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Elicit's Chat/Compare feature explicitly lets analysts 'compare and contrast papers' and 'summarize multiple papers along specific dimensions,' and its column/table extraction lets you see each paper's data side-by-side, which supports spotting agreement/disagreement across sources. However, there is no dedicated feature that explicitly flags or highlights when sources conflict versus concur (no consensus/disagreement indicator), and community reports note the model can hallucinate quotes that contradict its own summaries, undermining confidence in cross-source synthesis. Missing for 10: an explicit contradiction/agreement-detection UI, and independent verification that comparisons are reliably accurate rather than hallucination-prone.", "evidenceIds": [ "elicit-docs-40", "elicit-docs-18", "elicit-docs-11", "elicit-docs-22", "elicit-comm-4", "elicit-comm-3" ] }, { "productId": "elicit", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Elicit documents an API and MCP server (elicit-docs-6, elicit-docs-34, elicit-docs-35, elicit-docs-36) but there is no evidence of an interactive API reference/playground with runnable examples; probes for OpenAPI/swagger specs all returned 404 (elicit-probe-3), suggesting no such interactive reference exists.", "evidenceIds": [ "elicit-docs-6", "elicit-docs-34", "elicit-docs-35", "elicit-docs-36", "elicit-probe-3" ] }, { "productId": "elicit", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Elicit documents a REST API (elicit-docs-34,36) and an MCP server (elicit-docs-28,35), but no evidence of a downloadable OpenAPI/Swagger spec exists; direct probes for openapi.json/swagger.json all returned 404 (elicit-probe-3). Missing for 10: any published OpenAPI/Swagger file, machine-readable schema, or API reference page listing such a spec.", "evidenceIds": [ "elicit-docs-34", "elicit-docs-35", "elicit-docs-36", "elicit-probe-3" ] }, { "productId": "elicit", "storyId": "api-sandbox", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Elicit is a research/literature review tool, not a system with production data pipelines or deployment environments; the sandbox-vs-production testing story is a category error for this product type.", "evidenceIds": [] }, { "productId": "elicit", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence confirms Elicit has an API and MCP server (elicit-docs-34, elicit-docs-35, elicit-docs-36) but nothing documents API versioning or a deprecation policy, and probes for an OpenAPI/spec file returned 404s (elicit-probe-3).", "evidenceIds": [ "elicit-docs-34", "elicit-docs-35", "elicit-docs-36", "elicit-probe-3" ] }, { "productId": "elicit", "storyId": "automation-bulk-operations", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Elicit's core Systematic Reviews and Tables/Columns workflows explicitly apply extraction and screening operations across many papers at once (docs-11, docs-18, docs-22, docs-45), with bulk import (RIS/BIB, Zotero) and bulk export (CSV/Excel/RIS/BIB) of entire tables (docs-25, docs-38, docs-47), plus API/MCP access to run full systematic reviews programmatically at scale (docs-34, docs-36). Missing for 10: independent/hands-on evidence specifically validating bulk-scale accuracy or performance (community evidence addresses general accuracy, not bulk-operation mechanics).", "evidenceIds": [ "elicit-docs-11", "elicit-docs-18", "elicit-docs-22", "elicit-docs-25", "elicit-docs-36", "elicit-docs-38", "elicit-docs-47", "elicit-docs-34" ] }, { "productId": "elicit", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Elicit's 'Alerts' feature lets a user set a topic and receive an automatic email notification when a new relevant paper is found, which is a narrow event→action automation, but there is no general rule-builder allowing arbitrary triggers/conditions/actions across the product. Missing for 10: user-defined trigger conditions beyond 'new paper found', support for actions besides email alerts, and any workflow/automation engine tying events to custom actions.", "evidenceIds": [ "elicit-docs-3", "elicit-docs-15", "elicit-docs-26" ] }, { "productId": "elicit", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Elicit's Alerts feature lets users get recurring updates when new relevant papers matching a saved search appear (via instant email alerts), which is a limited form of a recurring job, but there is no evidence of general scheduling of arbitrary Research Agent workflows, systematic reviews, or API-driven jobs on a recurring cadence. Missing for 10: ability to schedule/repeat full Research Agent or Systematic Review workflows, cron-like or interval-based automation beyond paper alerts, and confirmation this works via API/MCP for programmatic recurring runs.", "evidenceIds": [ "elicit-docs-3", "elicit-docs-15", "elicit-docs-26", "elicit-docs-34", "elicit-docs-35" ] }, { "productId": "elicit", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Elicit provides skills, columns, and projects for automation, but there is no evidence of versioning, review history, or rollback capability for these automations/skills/workflows anywhere in the evidence pack.", "evidenceIds": [] }, { "productId": "elicit", "storyId": "autonomous-research-run", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Elicit's Research Agent and Systematic Review workflows are explicitly documented as multi-step (effort-level slider from Fastest to Smartest, pulling from multiple source types, iterating until output is complete, and separate search/screen/extract/report phases) rather than single-pass answers (elicit-docs-2, elicit-docs-12/13, elicit-docs-21/33, elicit-docs-22/45). However, hands-on community reports describe results arriving quickly and resembling a single pass over topic clusters rather than deep autonomous investigation, and multiple independent accounts report missed papers, hallucinated conclusions, and shallow reasoning that undercut confidence in true multi-step depth (elicit-comm-1, elicit-comm-3, elicit-comm-4, elicit-comm-5). Missing for 10: independent verification that the agent performs genuinely autonomous multi-step reasoning (not just sequential fixed workflow steps) and evidence rebutting the accuracy/depth complaints.", "evidenceIds": [ "elicit-docs-2", "elicit-docs-12", "elicit-docs-13", "elicit-docs-21", "elicit-docs-22", "elicit-docs-45", "elicit-comm-1", "elicit-comm-3", "elicit-comm-4", "elicit-comm-5" ] }, { "productId": "elicit", "storyId": "export-reference-formats", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Elicit documents export of tables/reports to CSV, Excel, PDF, DOCX, RIS, and BIB, covering documents, spreadsheets, and reference-manager formats, and also supports Zotero/EndNote/Mendeley import/export via RIS. Missing for 10: independent hands-on verification of export fidelity beyond vendor docs.", "evidenceIds": [ "elicit-docs-25", "elicit-docs-31", "elicit-docs-47", "elicit-docs-5", "elicit-docs-19", "elicit-docs-38" ] }, { "productId": "elicit", "storyId": "free-tier-trial", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack includes pricing-page references (e.g., elicit-docs-4, elicit-docs-6, elicit-docs-31) but none describe a free tier's scope, limits, or a trial period — no content confirms what a researcher could do without paying. Axis clearly applies to a SaaS research tool, but no evidence substantiates a meaningful free/trial experience.", "evidenceIds": [] }, { "productId": "elicit", "storyId": "inline-citations", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Elicit's search/columns features are built around pulling data from papers with links back to sources (elicit-docs-18, elicit-docs-20), and a hands-on user confirms getting 'a list of concepts with citations and links to papers' (elicit-comm-1). However, independent hands-on reports directly contradict the claim that citations reliably let you verify claims: users found Elicit 'hallucinates just as much and then gives a quote that directly contradicts its statement' (elicit-comm-4), produced 'mostly incorrect summaries' and missed key papers (elicit-comm-3), and Elicit itself warns only ~90% accuracy with a need to 'check the work in Elicit closely' (elicit-comm-2). Missing for 10: evidence that citation/quote extraction is reliably accurate, independent verification benchmarks, and resolution of the hallucination-despite-quoting complaints.", "evidenceIds": [ "elicit-docs-18", "elicit-docs-20", "elicit-comm-1", "elicit-comm-2", "elicit-comm-3", "elicit-comm-4" ] }, { "productId": "elicit", "storyId": "long-horizon-runs", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Elicit documents an alerts feature that emails you when new relevant papers are found (elicit-docs-3, elicit-docs-15, elicit-docs-26), and long-running workflows like Systematic Reviews and Research Agent effort levels ('Smartest' mode) imply tasks that can take longer to complete (elicit-docs-2, elicit-docs-13, elicit-docs-22). However, alerts are for ongoing topic monitoring, not notification of a specific job's completion, and there's no evidence of a 'start and walk away, get notified when this specific job is done' async job model. Missing for 10: explicit documentation of background/async execution of a research job plus a completion notification (vs. topic-monitoring alerts), and any independent confirmation this works as described.", "evidenceIds": [ "elicit-docs-3", "elicit-docs-15", "elicit-docs-26", "elicit-docs-2", "elicit-docs-13", "elicit-docs-22" ] }, { "productId": "elicit", "storyId": "monitoring-alerts", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Elicit's Alerts feature explicitly lets researchers set up standing topic alerts with instant email notifications when new relevant papers are found, adding them to a Library for future use — directly matching the standing-search/alert story. Missing for 10: independent/hands-on verification of alert accuracy or timeliness, and detail on how alert relevance/topics are configured beyond docs claims.", "evidenceIds": [ "elicit-docs-3", "elicit-docs-15", "elicit-docs-26" ] }, { "productId": "elicit", "storyId": "openness-api-parity", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Elicit's API and MCP server expose search and end-to-end Systematic Review functionality (search, screening, extraction, reporting), giving programmatic access to core research capabilities, but there's no evidence that UI-only features like the interactive Research Agent chat, Skills, real-time collaboration, columns customization, alerts, or the browser extension are exposed via the API/MCP surface. Missing for 10: explicit documentation of full feature parity, API/MCP access to Research Agent conversational sessions, skills, collaboration, and alerts.", "evidenceIds": [ "elicit-docs-34", "elicit-docs-35", "elicit-docs-36", "elicit-docs-6", "elicit-docs-28" ] }, { "productId": "elicit", "storyId": "openness-full-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Elicit supports exporting Library, tables, and reports in open formats (RIS, CSV, BIB, PDF, DOCX) and offers API/MCP access for programmatic retrieval, which supports data portability. However, export of core artifacts like screening/extraction tables is gated behind Pro/Scale/Enterprise plans, and there's no evidence of full account data export (e.g., all research agent sessions, projects, skills, chat history) in open formats, nor an explicit 'delete account and take everything' workflow. missing for 10: full-account/session export beyond tables and library, confirmation of free-tier export ability, independent verification of export completeness/fidelity.", "evidenceIds": [ "elicit-docs-5", "elicit-docs-19", "elicit-docs-25", "elicit-docs-31", "elicit-docs-47", "elicit-docs-34", "elicit-docs-35" ] }, { "productId": "elicit", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Elicit is a closed, proprietary SaaS research tool; no evidence in the pack mentions an open-source repository, source code availability, or an open license for its codebase. This axis applies (a product could plausibly open its source), but no evidence supports it.", "evidenceIds": [] }, { "productId": "elicit", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Elicit is a hosted SaaS research product with no evidence of any self-hostable core offering (only API/MCP access to the hosted service is documented); self-hosting is not a plausible axis for this type of cloud-only product, so this is a category mismatch rather than a missing capability.", "evidenceIds": [] }, { "productId": "elicit", "storyId": "own-corpus-upload", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Elicit supports importing/uploading a user's own corpus (RIS/BIB import, Zotero integration, Collections) and running research operations (chat, columns/data extraction, systematic reviews) over those uploaded papers, with the browser extension auto-fetching full text for extraction. Community feedback raises accuracy/hallucination concerns about summarization quality, which tempers reliability but does not contradict the upload/research capability itself. Missing for 10: independent hands-on verification of accuracy when researching over a user-uploaded corpus, and clearer documentation of raw multi-PDF drag-and-drop upload versus reference-manager import formats.", "evidenceIds": [ "elicit-docs-37", "elicit-docs-38", "elicit-docs-18", "elicit-docs-49", "elicit-docs-14", "elicit-docs-24", "elicit-docs-40", "elicit-comm-3", "elicit-comm-4" ] }, { "productId": "elicit", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions data residency, regional storage, or the ability to choose where data is stored; Elicit's docs cover exports, imports, API, and workflows but not data residency options. missing for 10: any mention of region/data-residency controls, enterprise data-locality options, or storage location settings.", "evidenceIds": [] }, { "productId": "elicit", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items address opt-out from AI training, data usage policies for model training, or any privacy/data-control settings related to training data; all evidence covers product features (search, review workflows, exports, API/MCP) with no mention of training-data privacy controls. missing for 10: any documentation of a training opt-out setting, data usage/privacy policy statement, or enterprise data handling terms addressing model training.", "evidenceIds": [] }, { "productId": "elicit", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items address data retention policies, deletion controls, or account/data deletion mechanisms; docs cover export/import formats but not retention or deletion of user data.", "evidenceIds": [] }, { "productId": "elicit", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions telemetry, usage tracking, or an opt-out mechanism; only unrelated docs about search, exports, and API/MCP features appear. This is a reasonable privacy-posture question for a SaaS AI product, but nothing in the evidence pack supports Elicit offering telemetry opt-out.", "evidenceIds": [] }, { "productId": "elicit", "storyId": "scholarly-sources", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Elicit's docs clearly show it searches scholarly literature (138M+ academic papers via semantic and keyword search), clinical trials, and journal-restricted queries, plus API/MCP access to the same corpus and systematic-review workflows built around paper screening/extraction rather than general web search. This directly matches the story of searching scholarly/primary sources rather than the open web. Missing for 10: independent corroboration specifically about breadth/quality of the scholarly corpus (community evidence addresses answer accuracy/hallucination, not source scope, so it doesn't contradict this particular axis).", "evidenceIds": [ "elicit-docs-20", "elicit-docs-34", "elicit-docs-43", "elicit-docs-44", "elicit-docs-30", "elicit-docs-48", "elicit-docs-36" ] }, { "productId": "elicit", "storyId": "share-collaborate", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Elicit has a documented feature to invite team members to collaborate live in Research Agent sessions, with edit-access collaborators able to ask the agent questions, produce new artifacts, and edit others' work, plus reports/tables can be exported as PDF/Word/CSV for sharing. missing for 10: no independent/hands-on corroboration of the live collaboration feature working smoothly, and no detail on view-only/read-access sharing permissions.", "evidenceIds": [ "elicit-docs-9", "elicit-docs-17", "elicit-docs-46", "elicit-docs-47" ] }, { "productId": "elicit", "storyId": "steer-effort-scope", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Elicit's Research Agent lets users set an effort level (Fastest→Smartest) via a slider before sending a query, and users can shape scope through columns, skills, and iterative follow-up prompts within a session; the API also exposes control over search strategy, screening criteria, and extraction parameters for Systematic Reviews. However, evidence only shows steering before/between turns, not genuine mid-execution adjustment of an in-flight run. Missing for 10: documentation of pausing/adjusting effort or scope while a run is actively executing, and independent verification that scope/effort controls meaningfully change output depth.", "evidenceIds": [ "elicit-docs-2", "elicit-docs-13", "elicit-docs-11", "elicit-docs-18", "elicit-docs-21", "elicit-docs-36" ] }, { "productId": "elicit", "storyId": "structured-reports", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Elicit's Systematic Reviews workflow produces a research report summarizing papers, includes data extraction and screening tables, and both reports and tables can be exported as PDF/Word/CSV/Excel for sharing with stakeholders (elicit-docs-1, elicit-docs-22, elicit-docs-47, elicit-docs-19, elicit-docs-25). The Research Agent can also produce documents, tables, and figures within a session (elicit-docs-9, elicit-docs-21, elicit-docs-33). Missing for 10: independent/hands-on corroboration that the exported report format is polished enough for external stakeholder sharing, and no evidence of customizable report sectioning beyond the standard systematic-review structure.", "evidenceIds": [ "elicit-docs-1", "elicit-docs-22", "elicit-docs-47", "elicit-docs-19", "elicit-docs-25", "elicit-docs-9", "elicit-docs-21" ] }, { "productId": "elicit", "storyId": "systematic-screening", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Elicit documents a dedicated Systematic Reviews workflow covering setup, search/gather, title-abstract screening, automated full-text screening, and data extraction via consistent custom columns applied across all papers, with export of screening/extraction tables — directly matching the story. Community evidence raises general accuracy/hallucination concerns about Elicit's paper analysis (not specifically the systematic review pipeline), which tempers confidence in perfect consistency at scale. Missing for 10: independent hands-on validation of the systematic review workflow's accuracy/consistency specifically (vs. general chat/summarization complaints), and no third-party benchmarking of screening reliability across large paper sets.", "evidenceIds": [ "elicit-docs-1", "elicit-docs-22", "elicit-docs-45", "elicit-docs-18", "elicit-docs-11", "elicit-docs-19", "elicit-docs-47", "elicit-docs-39", "elicit-comm-3", "elicit-comm-4" ] }, { "productId": "elicit", "storyId": "transparent-pricing", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Evidence confirms a public pricing page exists (elicit.com/pricing) and reveals plan tier names (Pro, Scale, Enterprise) tied to feature gating like table exports, but no evidence pack content shows actual price points, free-tier limits, or usage caps that a researcher would need to compare plans before committing. Missing for 10: actual price figures per tier, usage/query limits, free-plan restrictions, and any independent confirmation of pricing transparency.", "evidenceIds": [ "elicit-docs-4", "elicit-docs-6", "elicit-docs-31", "elicit-docs-19", "elicit-docs-47" ] }, { "productId": "futurehouse", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "A working llms.txt file is directly confirmed via probe at futurehouse.gitbook.io/llms.txt returning HTTP 200 with structured agent-oriented documentation links, satisfying the core ask of pointing an agent at llms.txt. missing for 10: markdown-per-page fallback (docs-md probe returned 404), no OpenAPI/machine-readable schema confirmation, and no independent third-party corroboration of agents actually consuming this file successfully.", "evidenceIds": [ "futurehouse-probe-1", "futurehouse-probe-2", "futurehouse-probe-3" ] }, { "productId": "futurehouse", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "The platform's core agents (Kosmos, Precedent, Molecules) are documented as autonomously generating insights, hypotheses, and cited reports directly from user-supplied data and literature, matching the story closely (e.g., Kosmos 'autonomously reads literature, writes and executes analysis code, generates hypotheses, and produces a comprehensive cited report'). Missing for 10: independent/hands-on corroboration beyond vendor docs, and some doc links returned 404 in probes, weakening confidence in documentation completeness.", "evidenceIds": [ "futurehouse-docs-5", "futurehouse-docs-13", "futurehouse-docs-7", "futurehouse-docs-8", "futurehouse-docs-9", "futurehouse-docs-6", "futurehouse-probe-2" ] }, { "productId": "futurehouse", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes an API client for submitting tasks/queries and autonomous agents that complete a single task end-to-end, but there is no mention of scheduling, triggers, or background automations that run without explicit invocation. missing for 10: scheduling/cron mechanism, event-driven triggers, persistent background job management, and any docs describing recurring or unattended automation setup.", "evidenceIds": [ "futurehouse-docs-4", "futurehouse-docs-12", "futurehouse-docs-5" ] }, { "productId": "futurehouse", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "The platform ships built-in autonomous agents (Kosmos, Precedent, Molecules, literature review agent) that users can delegate research tasks to via API/client, with docs describing task submission and autonomous execution producing cited reports. Missing for 10: independent/hands-on corroboration beyond vendor docs, and clearer in-product (non-API) delegation UI evidence.", "evidenceIds": [ "futurehouse-docs-4", "futurehouse-docs-5", "futurehouse-docs-7", "futurehouse-docs-9", "futurehouse-docs-10", "futurehouse-docs-12", "futurehouse-docs-13" ] }, { "productId": "futurehouse", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "The Edison client is a pip-installable Python SDK (`edison-client`) authenticated via API key, with a scriptable interface (`client.run_tasks_until_done(task_data)`) that requires no UI, making it straightforward to invoke headlessly or from CI pipelines. Missing for 10: explicit CI/automation examples (e.g., GitHub Actions workflow), documentation of exit codes/async polling patterns for pipelines, and independent confirmation of real-world CI usage.", "evidenceIds": [ "futurehouse-docs-1", "futurehouse-docs-2", "futurehouse-docs-4", "futurehouse-docs-12" ] }, { "productId": "futurehouse", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "FutureHouse Platform is a science-agent product (Kosmos, Precedent, Molecules, etc.) accessed via API/client, not an MCP-client tool for plugging in external MCP servers; no evidence discusses MCP integration at all, and the product's role is as an agent/service rather than an MCP-consuming host.", "evidenceIds": [] }, { "productId": "futurehouse", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack only shows a Python client (edison-client) for calling FutureHouse agents via API key, plus probes confirming no OpenAPI/MCP-related endpoints were found; there is no mention of an official MCP server for connecting agents.", "evidenceIds": [ "futurehouse-docs-1", "futurehouse-probe-3" ] }, { "productId": "futurehouse", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "The platform's core interaction model is natural-language queries (e.g., the query string \"Which neglected diseases had a treatment developed by artificial intelligence?\") passed to autonomous agents like Kosmos, Precedent, and Molecules, and docs explicitly say users can 'ask it a complex scientific question' or 'task it with a deep literature review.' However, the demonstrated flow is via a Python client (task_data dict + run_tasks_until_done), not a conversational chat UI, so it's NL-as-input-parameter rather than full conversational natural-language operation of the whole platform (e.g., no evidence of multi-turn dialogue or a chat interface). Missing for 10: evidence of a conversational/chat UI, multi-turn NL interaction, or NL control over platform features beyond the initial query.", "evidenceIds": [ "futurehouse-docs-4", "futurehouse-docs-5", "futurehouse-docs-7", "futurehouse-docs-13", "futurehouse-docs-19" ] }, { "productId": "futurehouse", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows only a Python client library (edison-client, installed via pip, used programmatically with client.run_tasks_until_done) rather than a command-line interface; no CLI tool, command syntax, or terminal usage is documented anywhere in the pack.", "evidenceIds": [ "futurehouse-docs-1", "futurehouse-docs-12" ] }, { "productId": "futurehouse", "storyId": "agentic-public-api", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs describe an installable Python client (edison-client) with API-key/token auth and a documented workflow (client.run_tasks_until_done) to submit tasks like literature review, matching a genuine public API for driving the platform programmatically. Missing for 10: no OpenAPI/swagger spec found (404s), and no independent/hands-on corroboration beyond first-party docs.", "evidenceIds": [ "futurehouse-docs-1", "futurehouse-docs-2", "futurehouse-docs-3", "futurehouse-docs-4", "futurehouse-docs-12", "futurehouse-probe-3" ] }, { "productId": "futurehouse", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows only a single, account-wide API token creation flow with no mention of scopes, permissions, or least-privilege controls for agents; no evidence of scoped or restricted credential issuance.", "evidenceIds": [ "futurehouse-docs-2", "futurehouse-docs-3", "futurehouse-docs-11" ] }, { "productId": "futurehouse", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "There is a documented official Python client (edison-client) with install, auth via API key, and task submission/run examples, showing a real SDK path for programmatic/agentic use. However, no OpenAPI spec was found (all probed paths 404), and the docs page itself returned 404 on a direct fetch, suggesting incomplete or unstable documentation coverage. Missing for 10: verified OpenAPI/REST spec, broader SDK language coverage, independent developer corroboration of SDK reliability.", "evidenceIds": [ "futurehouse-docs-1", "futurehouse-docs-2", "futurehouse-docs-3", "futurehouse-docs-4", "futurehouse-docs-12", "futurehouse-probe-3", "futurehouse-probe-2" ] }, { "productId": "futurehouse", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No mention of webhooks, event subscriptions, or callback mechanisms anywhere in the docs; the client is a polling/run-tasks style API and OpenAPI probe returned 404s, giving no evidence of webhook support.", "evidenceIds": [ "futurehouse-docs-12", "futurehouse-probe-3" ] }, { "productId": "futurehouse", "storyId": "agreement-disagreement", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs mention tasking the platform with 'a deep literature review synthesizing conflicting evidence across hundreds of papers' and full auditability back to source passages, suggesting some capability to surface disagreement across sources, but there is no dedicated UI/feature shown for explicitly contrasting agreeing vs disagreeing sources, no example output demonstrating this, and no independent/hands-on verification. missing for 10: concrete example output showing conflicting evidence highlighted, dedicated UI feature for source agreement/disagreement, independent corroboration beyond vendor docs.", "evidenceIds": [ "futurehouse-docs-7", "futurehouse-docs-19", "futurehouse-docs-6", "futurehouse-docs-16" ] }, { "productId": "futurehouse", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows only a quickstart guide with basic client code snippets, not an interactive API reference with runnable examples; probes for OpenAPI/swagger specs and doc endpoints all returned 404s, indicating no interactive reference exists.", "evidenceIds": [ "futurehouse-probe-3", "futurehouse-probe-2", "futurehouse-docs-1", "futurehouse-docs-12" ] }, { "productId": "futurehouse", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Probes for OpenAPI/swagger spec paths all returned 404, and no docs reference a machine-readable API spec; only a Python client library and manual REST auth flow are documented. Missing for 10: any OpenAPI/Swagger endpoint or downloadable spec file, any documentation mentioning API schema export.", "evidenceIds": [ "futurehouse-probe-3" ] }, { "productId": "futurehouse", "storyId": "api-sandbox", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "FutureHouse is a research-agent API/platform for scientific literature and analysis tasks, not a data-processing or transactional system where sandbox-vs-production data separation is a relevant concept; the evidence pack contains no mention of environments, sandboxes, or production data at all.", "evidenceIds": [] }, { "productId": "futurehouse", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of API versioning scheme, changelog, or deprecation policy anywhere in the docs; OpenAPI spec probes all 404 and no versioning documentation is present. missing for 10: versioned API endpoints, changelog/release notes, documented deprecation policy or timelines.", "evidenceIds": [ "futurehouse-probe-3" ] }, { "productId": "futurehouse", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The docs show only single-task submission via task_data/run_tasks_until_done, with no mention of batch endpoints, bulk task lists, or parallel multi-item processing APIs; missing for 10: batch/bulk submission API, documented loop or list-based task queuing, evidence of concurrent multi-item execution.", "evidenceIds": [ "futurehouse-docs-4", "futurehouse-docs-12" ] }, { "productId": "futurehouse", "storyId": "automation-rules-engine", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "FutureHouse is a scientific research agent platform (literature review, hypothesis generation, data analysis) invoked via API/client calls, not an event-driven automation/rules engine; no concept of user-defined triggers or event-based rules appears anywhere in the evidence, and this is not the kind of capability the product's category would offer.", "evidenceIds": [] }, { "productId": "futurehouse", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers running one-off tasks via the client (run_tasks_until_done) but there is no mention of scheduling, cron-like recurrence, or persistent workflow automation anywhere in the docs or probes.", "evidenceIds": [ "futurehouse-docs-4", "futurehouse-docs-12" ] }, { "productId": "futurehouse", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "FutureHouse is a research/agent platform for scientific tasks (literature review, data analysis), not an automation-builder with versioned workflows; there is no concept of automations to version, review, or roll back in the evidence.", "evidenceIds": [] }, { "productId": "futurehouse", "storyId": "autonomous-research-run", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs describe Kosmos and other agents autonomously reading literature, writing/executing analysis code, generating hypotheses, and producing cited multi-step reports (not single-pass summaries), with auditable traceability of findings and multi-agent capabilities (Precedent, Molecules, literature review across hundreds of papers). This directly matches the story of posing a question and getting an autonomous multi-step investigation. Missing for 10: independent/hands-on verification of the multi-step autonomy claims (probe shows docs page 404'd, reducing corroboration strength) and no third-party case study confirming real-world multi-step behavior.", "evidenceIds": [ "futurehouse-docs-5", "futurehouse-docs-13", "futurehouse-docs-6", "futurehouse-docs-7", "futurehouse-docs-19", "futurehouse-docs-9" ] }, { "productId": "futurehouse", "storyId": "export-reference-formats", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes cited reports and analyses produced by Kosmos/agents but nowhere mentions exporting to document, spreadsheet, or reference-manager formats (e.g., PDF/DOCX, CSV/XLSX, BibTeX/RIS); the docs-cookbook page itself is even reported as not found in the probe.", "evidenceIds": [ "futurehouse-docs-5", "futurehouse-docs-6", "futurehouse-docs-8", "futurehouse-probe-2" ] }, { "productId": "futurehouse", "storyId": "free-tier-trial", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs explicitly state Edison maintains a generous free tier for academics, with paid plans only needed for higher rate limits/additional features, and the quickstart shows a full API key + client workflow to run real tasks (literature review job). Missing for 10: independent/hands-on confirmation of free tier limits and no clarity on exact quota/duration restrictions.", "evidenceIds": [ "futurehouse-docs-20", "futurehouse-docs-15", "futurehouse-docs-2", "futurehouse-docs-3", "futurehouse-docs-4", "futurehouse-docs-12" ] }, { "productId": "futurehouse", "storyId": "inline-citations", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs state that outputs are cited and auditable, with claims traceable back to specific literature passages or code, and that literature review/answer agents produce cited responses. This is first-party documentation only, with no independent/hands-on corroboration of citation accuracy or completeness. missing for 10: independent verification of citation accuracy, example of an actual cited report output, detail on citation format/traceability mechanism.", "evidenceIds": [ "futurehouse-docs-6", "futurehouse-docs-16", "futurehouse-docs-7", "futurehouse-docs-5", "futurehouse-docs-13" ] }, { "productId": "futurehouse", "storyId": "long-horizon-runs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The client SDK exposes a task submission and `run_tasks_until_done` pattern (docs-4, docs-12) plus descriptions of Kosmos autonomously running long, multi-step research (literature review, code execution, hypothesis generation) unattended (docs-5, docs-13, docs-19), supporting the 'long unattended job' half of the story. However there is no evidence of any notification mechanism (email, webhook, push) when a job completes — the only documented pattern is a client-side polling call that blocks until done, not an async notify-on-completion flow. Missing for 10: explicit notification/webhook/email-on-completion feature, evidence of true async fire-and-forget job submission separate from polling.", "evidenceIds": [ "futurehouse-docs-4", "futurehouse-docs-12", "futurehouse-docs-5", "futurehouse-docs-13", "futurehouse-docs-19" ] }, { "productId": "futurehouse", "storyId": "monitoring-alerts", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence describes on-demand literature search, review, and analysis agents (Kosmos, Precedent, etc.) invoked via API calls, but nothing describes standing searches, saved alerts, or recurring notifications when new relevant sources appear.", "evidenceIds": [] }, { "productId": "futurehouse", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "The edison-client library and quickstart docs show that core agent tasks (e.g., JobNames.LITERATURE) can be invoked via API with authentication and task submission (docs-1–4,12), and the cookbook describes multiple agents (Kosmos, Precedent, Molecules) that presumably map to API job types, suggesting broad functional overlap with the UI. However, there is no explicit documentation stating full feature parity between UI and API, no OpenAPI/swagger spec discoverable (probe-3 shows 404s across all candidate paths), and no confirmation that all UI-exposed capabilities (e.g., account management, dataset uploads, report viewing) are also API-accessible. missing for 10: explicit parity statement, discoverable OpenAPI schema, evidence covering non-task UI features (billing, dataset management, report exports) via API.", "evidenceIds": [ "futurehouse-docs-4", "futurehouse-docs-12", "futurehouse-docs-5", "futurehouse-docs-9", "futurehouse-docs-10", "futurehouse-probe-3" ] }, { "productId": "futurehouse", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of any data export capability, open-format export, or account portability/deletion feature; documentation covers agent/task usage and API access but nothing about exporting user data or leaving with it.", "evidenceIds": [] }, { "productId": "futurehouse", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items reference an open-source license or public repository for the FutureHouse Platform itself; only client library install instructions and product docs are provided, with no license information. missing for 10: evidence of open-source license, public source repo link, license file or statement.", "evidenceIds": [] }, { "productId": "futurehouse", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "FutureHouse Platform is a hosted API/SaaS agent service (Edison) accessed via client library and API keys, not a distributable self-hostable stack; no evidence of any self-hosting option exists or is claimed.", "evidenceIds": [] }, { "productId": "futurehouse", "storyId": "own-corpus-upload", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The docs describe FutureHouse's agents (Kosmos, literature review, Precedent) searching a fixed 175M+ paper corpus and analyzing 'datasets' for experiments, but nowhere describe a mechanism for a researcher to upload their own PDFs or personal corpus for the agent to search/reason over. Without evidence of a file-upload or custom-corpus ingestion feature, this capability is not shown.", "evidenceIds": [ "futurehouse-docs-13", "futurehouse-docs-17", "futurehouse-docs-18", "futurehouse-docs-19" ] }, { "productId": "futurehouse", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses data residency, regional storage options, or geographic control over where data is processed/stored; documentation only covers API usage, agents, and pricing tiers.", "evidenceIds": [] }, { "productId": "futurehouse", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data usage for AI training opt-out, data privacy controls, or any training-data policy; the docs focus entirely on product features and API usage.", "evidenceIds": [] }, { "productId": "futurehouse", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses data retention policies, deletion controls, or privacy/data lifecycle management; documentation focuses on client setup, agents, and use cases only.", "evidenceIds": [] }, { "productId": "futurehouse", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions telemetry, usage tracking, analytics collection, or an opt-out mechanism anywhere in the FutureHouse docs or probes; the documentation focuses on client setup and agent features only.", "evidenceIds": [] }, { "productId": "futurehouse", "storyId": "scholarly-sources", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs describe agents (LITERATURE task, Kosmos, Precedent) that explicitly search scholarly literature, trials, and patents, accessing '175M+ papers, trials, and patents' with citation graph and journal quality awareness, and produce cited reports auditable back to specific literature passages. This directly matches the researcher story of searching scholarly/primary sources beyond the open web. Missing for 10: independent/hands-on corroboration of search coverage and source quality beyond vendor docs, and clarity on which specific databases/indices are used.", "evidenceIds": [ "futurehouse-docs-17", "futurehouse-docs-9", "futurehouse-docs-14", "futurehouse-docs-5", "futurehouse-docs-6", "futurehouse-docs-4" ] }, { "productId": "futurehouse", "storyId": "share-collaborate", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack mentions sharing sessions, reports, or collaborative access controls; documentation focuses on API usage, agents, and task execution, not collaboration features. Missing for 10: any mention of sharing/exporting reports, collaborator invites, or team/workspace features.", "evidenceIds": [] }, { "productId": "futurehouse", "storyId": "steer-effort-scope", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence shows job submission with a query and job name, plus a mid-run polling helper (run_tasks_until_done), but no documented parameters or controls for adjusting depth, effort level, or scope of a run before or during execution.", "evidenceIds": [ "futurehouse-docs-4", "futurehouse-docs-12" ] }, { "productId": "futurehouse", "storyId": "structured-reports", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs describe Kosmos/agents producing a 'comprehensive cited report' with statistical results, publication-ready figures, and traceable citations, which implies structured, shareable output, but there's no explicit mention of report sections/tables format or export/sharing mechanisms for stakeholders. missing for 10: explicit description of report structure (sections/tables), export or sharing functionality, and independent confirmation of report format quality.", "evidenceIds": [ "futurehouse-docs-5", "futurehouse-docs-6", "futurehouse-docs-8", "futurehouse-docs-13", "futurehouse-docs-16" ] }, { "productId": "futurehouse", "storyId": "systematic-screening", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The API client supports programmatic task submission (e.g., JobNames.LITERATURE) and batch execution via run_tasks_until_done, and the docs describe a literature agent that can synthesize evidence across hundreds of papers with citations, which could be scripted for repeated, consistent queries. However, there is no explicit documented workflow for structured screening/extraction with defined inclusion/exclusion criteria across a large corpus (e.g., systematic review style tables), and the cookbook page itself returns 404 on the probe, reducing confidence in the full pipeline. missing for 10: explicit systematic-review/extraction workflow docs, example of consistent-criteria batch screening, working cookbook page confirming the API details.", "evidenceIds": [ "futurehouse-docs-4", "futurehouse-docs-12", "futurehouse-docs-19", "futurehouse-docs-7", "futurehouse-probe-2" ] }, { "productId": "futurehouse", "storyId": "transparent-pricing", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Docs confirm a free tier for academics and paid plans for higher rate limits/features, but no concrete pricing figures, tier breakdowns, or specific usage limit numbers are provided anywhere in the pack. missing for 10: actual price points, detailed rate-limit thresholds, feature comparison table across tiers, billing/usage dashboard evidence.", "evidenceIds": [ "futurehouse-docs-15", "futurehouse-docs-20" ] }, { "productId": "notebooklm", "storyId": "agentic-agent-docs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gemini Notebook is an end-user research/synthesis tool, not an agent framework or platform that consumes agent-oriented documentation like llms.txt; nothing in the evidence suggests it acts as an agent client consuming machine-readable docs. This axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "NotebookLM generates summaries, insights, mind maps, briefing documents, AI-suggested report types, and answers with grounded citations directly from uploaded sources, and community feedback confirms real-world use for research insight extraction (e.g., audio overviews, code walkthroughs). Missing for 10: independent benchmarking of insight quality/accuracy and evidence of more proactive/agentic suggestion behavior beyond user-initiated Q&A.", "evidenceIds": [ "notebooklm-docs-2", "notebooklm-docs-5", "notebooklm-docs-27", "notebooklm-docs-28", "notebooklm-docs-14", "notebooklm-comm-1", "notebooklm-comm-5" ] }, { "productId": "notebooklm", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of scheduling, triggers, or autonomous background automations in NotebookLM's documented features; all capabilities (chat, source summarization, audio/video overviews, notes) are user-initiated within a session.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "NotebookLM lets users chat with sources and generate transformed outputs (summaries, audio/video overviews, study guides, slide decks), which is a form of task delegation, but there is no evidence of an agentic assistant that can take autonomous multi-step actions, use tools, browse/search independently, or execute tasks beyond source-grounded Q&A and content transformation; community feedback even notes it can't search for docs on its own (notebooklm-comm-4). missing for 10: autonomous multi-step task execution, tool/agent orchestration, independent web search/retrieval, evidence of true delegation beyond content generation.", "evidenceIds": [ "notebooklm-docs-2", "notebooklm-docs-3", "notebooklm-docs-23", "notebooklm-docs-19", "notebooklm-comm-4" ] }, { "productId": "notebooklm", "storyId": "agentic-headless", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an API, CLI, or headless/automation mode for NotebookLM/Gemini Notebook; all documented usage is through the interactive web/app UI, and community comments even complain about lacking programmatic source ingestion.", "evidenceIds": [ "notebooklm-docs-1", "notebooklm-comm-4" ] }, { "productId": "notebooklm", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence that Gemini Notebook supports MCP server integration or any external tool-plugging mechanism; the product only supports fixed source uploads and generated outputs (audio/video/study guides). missing for 10: any mention of MCP support, tool/plugin ecosystem, or extensibility API.", "evidenceIds": [ "notebooklm-docs-1", "notebooklm-docs-4" ] }, { "productId": "notebooklm", "storyId": "agentic-mcp-server", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gemini Notebook is a research/document-synthesis product, not an agent framework or platform exposing an official MCP server for external agents; no evidence pack item mentions MCP or agent connectivity, and the product's role is a consumer-facing knowledge tool rather than an integration surface for agents.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "NotebookLM lets users interact via natural-language chat to query sources, request custom conversational styles/roles, and ask questions (notebooklm-docs-2, notebooklm-docs-7, notebooklm-docs-28), which is a genuine natural-language interface. However, many core actions (uploading sources, converting notes, generating mind maps/slide decks/audio overviews) are triggered via UI menus/buttons rather than free-form commands, and community feedback notes limits like being unable to have it search for and incorporate new sources on its own (notebooklm-comm-4). Missing for 10: evidence of natural-language commands driving broader agentic actions (e.g., source discovery, format generation, sharing) rather than just chat Q&A and style prompts, and independent confirmation of command robustness.", "evidenceIds": [ "notebooklm-docs-2", "notebooklm-docs-7", "notebooklm-docs-28", "notebooklm-comm-4" ] }, { "productId": "notebooklm", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an official CLI for Gemini Notebook; all documentation describes a web/app UI with no command-line interface mentioned.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "agentic-public-api", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions a documented public API, SDK, or programmatic access method for Gemini Notebook; all evidence describes UI-driven features (sources, chat, audio/video overviews, sharing) and community commentary about workflow limitations, not API access.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "agentic-scoped-keys", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gemini Notebook (NotebookLM) is a research/knowledge-synthesis tool for end users, not a platform that issues API credentials for agents; no evidence pack content relates to credential scoping or agent access control, and this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an official SDK, API, or developer library for building against Gemini Notebook; all evidence describes end-user features (uploading sources, chat, audio/video overviews, notes) rather than programmatic/developer access.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any webhook subscription, API event, or outbound notification capability in Gemini Notebook's docs or community feedback; the product is a research/notebook tool with no mentioned event-driven integration surface.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "agreement-disagreement", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes grounded chat with citations, summaries, mind maps, and study aids, but nothing about explicitly surfacing agreement or contradiction across multiple sources. This is a fair capability to expect from a multi-source research tool, but no docs or community evidence mention comparative or conflict-detection analysis between sources.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "api-interactive-docs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NotebookLM is a research/notebook assistant for document synthesis, not a developer tool or platform with an API to document; the story about an interactive API reference with runnable examples is a category error for this product type.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack of an OpenAPI spec, API documentation, or any machine-readable API for Gemini Notebook; all evidence covers UI features (sources, notes, audio/video overviews, sharing). missing for 10: any mention of a public API, OpenAPI/Swagger spec, or SDK.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "api-sandbox", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NotebookLM is a research/document-synthesis product with no execution environment or production data pipeline to sandbox against; the concept of sandbox vs production testing is a category error for this product type.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a versioned public API for NotebookLM/Gemini Notebook, let alone a documented deprecation policy; all evidence covers end-user features (uploads, summaries, audio/video overviews) not developer API access. Missing for 10: any mention of an API, versioning scheme, or deprecation/support-lifecycle policy.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence only shows per-source actions (uploading individual sources, converting a single note to a source, generating one summary or audio overview at a time) with no mention of bulk/batch operations across many sources, notebooks, or items simultaneously. There is no documented bulk-select, bulk-delete, bulk-export, or batch-processing capability anywhere in the docs or community feedback.", "evidenceIds": [ "notebooklm-docs-1", "notebooklm-docs-12", "notebooklm-docs-4" ] }, { "productId": "notebooklm", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any rule-based or event-triggered automation (e.g., 'when X happens, do Y') in NotebookLM; the product is a research/synthesis tool that responds to manual user prompts and generates static outputs like summaries, audio/video overviews, and study guides, not automated event-driven actions.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack of scheduling, recurring jobs, triggers, or automated workflow execution; NotebookLM is a research/notebook tool driven by manual source uploads and on-demand queries.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NotebookLM is a research/notes synthesis tool with no automation/workflow-building capability, so 'version, review, and roll back automations' is a category error — there is no automation feature to version or roll back.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "autonomous-research-run", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All evidence describes NotebookLM as a source-grounded chat/summarization tool: users upload sources, ask questions, and get single-pass answers, summaries, or generated formats (audio overviews, mind maps, slide decks) — there is no evidence of an autonomous multi-step investigation loop that iteratively searches, plans, or expands beyond the provided sources. A community report explicitly notes it cannot autonomously search for and incorporate new documents into research, reinforcing that it is single-pass/source-bound rather than agentic.", "evidenceIds": [ "notebooklm-docs-2", "notebooklm-docs-5", "notebooklm-docs-27", "notebooklm-comm-4" ] }, { "productId": "notebooklm", "storyId": "export-reference-formats", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "NotebookLM supports exporting notes to Google Docs or Google Sheets and can generate a Slide Deck exportable as PDF, covering document/spreadsheet-style outputs, but there is no evidence of export to reference-manager formats (e.g., BibTeX, RIS, EndNote) which the story explicitly requires. missing for 10: reference-manager file export, independent corroboration of export fidelity.", "evidenceIds": [ "notebooklm-docs-13", "notebooklm-docs-25" ] }, { "productId": "notebooklm", "storyId": "free-tier-trial", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence confirms a paid subscription tier exists (analytics for paid users) implying a separate free tier, and community comments describe hands-on use (audio overviews, code walkthroughs) without mention of payment, suggesting meaningful free access. However, no doc explicitly states free-tier quotas, source limits, or trial duration for researchers. Missing for 10: explicit free-tier usage caps/limits documentation, direct confirmation of no-cost access, and independent confirmation of research-scale usability without upgrading.", "evidenceIds": [ "notebooklm-docs-9", "notebooklm-comm-5", "notebooklm-comm-6" ] }, { "productId": "notebooklm", "storyId": "inline-citations", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs confirm grounded chat responses come with clear in-line citations, hovering shows the exact quoted text, and clicking a citation navigates directly to the source location for verification — directly matching the story's requirement to verify claims against underlying sources. Community evidence corroborates general usefulness but doesn't specifically contest citation accuracy. Missing for 10: independent hands-on verification of citation accuracy/completeness across diverse source types.", "evidenceIds": [ "notebooklm-docs-2", "notebooklm-docs-6", "notebooklm-docs-21" ] }, { "productId": "notebooklm", "storyId": "long-horizon-runs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack describes an asynchronous/background 'long research job' mode with completion notifications; all cited capabilities (chat, summaries, audio/video overviews, flashcards, etc.) are generated within an active session, not unattended long-running jobs with alerts. Missing for 10: any documented async job queue, background processing status, or notification/alert mechanism when a task completes.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "monitoring-alerts", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of any standing-search/alert capability that surfaces new relevant sources automatically; sources must be manually uploaded or discovered by the user. Community feedback explicitly confirms this gap: 'you had to give it URLs or docs, you couldn't just have it search for docs and then incorporate them into your research' ([notebooklm-comm-4]).", "evidenceIds": [ "notebooklm-comm-4", "notebooklm-docs-1" ] }, { "productId": "notebooklm", "storyId": "openness-api-parity", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items mention any public API, developer SDK, or programmatic access to Gemini Notebook features; all evidence describes UI-only workflows (upload, chat, generate audio/video/slides, notes, sharing). Community comments only discuss UI limitations, not API availability. missing for 10: any mention of a public API, SDK, or programmatic endpoints; documentation of API parity with UI features; developer access/authentication details.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "openness-full-export", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs show only piecemeal export options—notes to Google Docs/Sheets, audio overview downloads, and slide decks as PDF—but no evidence of a full data export (sources, chat history, citations, mind maps) or a genuine 'leave the platform' workflow in open, portable formats. Missing for 10: bulk/full-notebook export, open-format guarantees (vs Google Docs/Sheets lock-in), and any documentation or community confirmation of a complete account data export/exit path.", "evidenceIds": [ "notebooklm-docs-13", "notebooklm-docs-10", "notebooklm-docs-25" ] }, { "productId": "notebooklm", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "NotebookLM is a closed, proprietary Google SaaS product with no evidence of an open-license source release; open-source licensing is not a plausible axis for this kind of hosted commercial AI product beyond the general absence-implies-none rule, but here Google's entire distribution model (Workspace core service, paid tiers) signals a closed product category, so this is a category mismatch rather than a missing feature.", "evidenceIds": [ "notebooklm-docs-20" ] }, { "productId": "notebooklm", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gemini Notebook is a fully-hosted Google SaaS product with no self-hosted or open-source deployment option; self-hosting is not a plausible axis for this category of closed cloud service.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "own-corpus-upload", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs confirm PDF/document upload (up to 500k words/200MB per source), grounded Q&A with inline citations navigable to source location, source guide summaries, and note-taking over uploaded corpora—directly matching the story. Community evidence corroborates real-world use for structured research and studying, though one comment notes it can't proactively search for and pull in new docs beyond what's uploaded. Missing for 10: independent benchmarking of research quality/accuracy across large multi-PDF corpora, and confirmation of robust handling of very large or messy academic corpora at scale.", "evidenceIds": [ "notebooklm-docs-1", "notebooklm-docs-2", "notebooklm-docs-4", "notebooklm-docs-5", "notebooklm-docs-6", "notebooklm-docs-21", "notebooklm-docs-26", "notebooklm-comm-1", "notebooklm-comm-4" ] }, { "productId": "notebooklm", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, region selection, or storage location controls for Gemini Notebook; the docs cover features like sources, audio/video overviews, and sharing but say nothing about where data is stored or user control over it.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data-training opt-out, privacy policy, or data usage controls for Gemini Notebook; all citations concern features (sources, summaries, audio/video overviews) rather than training-data privacy posture.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses data retention controls, deletion policies, or user ability to manage/export/delete stored data and sources beyond basic note export; the axis clearly applies to an AI-native privacy-conscious user but no documentation or community mention covers retention/deletion controls.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions telemetry, usage-tracking opt-out, privacy controls, or data-collection settings; only feature docs and general community commentary are present.", "evidenceIds": [] }, { "productId": "notebooklm", "storyId": "scholarly-sources", "verdict": "disputed", "quality": 3, "confidence": "medium", "rationale": "Docs vaguely mention users can 'discover new sources' (notebooklm-docs-1), but there is no evidence of integration with scholarly databases (e.g., PubMed, JSTOR, Google Scholar) for searching primary literature, and a hands-on community report explicitly states you cannot have NotebookLM search for and incorporate sources itself—you must supply URLs or docs yourself (notebooklm-comm-4). This directly contradicts the implied 'discover' capability, making it a disputed claim rather than a confirmed feature. missing for 10: evidence of any scholarly/primary-source search integration, first-party documentation describing how 'discover sources' actually searches literature, and independent confirmation that discovery works as claimed.", "evidenceIds": [ "notebooklm-docs-1", "notebooklm-comm-4" ] }, { "productId": "notebooklm", "storyId": "share-collaborate", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs confirm notebooks can be made public via shareable link, that Workspace admins now treat Gemini Notebook as a core service (implying org-level sharing), and that notes/slide decks/reports can be exported to Google Docs/Sheets or PDF so collaborators can view and continue working on them. However, evidence only describes public link sharing and analytics, not granular collaborator invite/edit permissions typical of collaborative docs, and there's no independent/hands-on account of real-world collaboration workflows. Missing for 10: explicit collaborator role/permission controls (view vs edit), independent corroboration of multi-user collaboration in practice.", "evidenceIds": [ "notebooklm-docs-8", "notebooklm-docs-9", "notebooklm-docs-13", "notebooklm-docs-20", "notebooklm-docs-25" ] }, { "productId": "notebooklm", "storyId": "steer-effort-scope", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "NotebookLM lets users pick conversational style/persona (Default, Custom \"PhD student\" etc.) and choose report/output formats (FAQ, briefing, study guide), which offers some steering of scope and tone, but there is no evidence of controlling depth/effort (e.g., reasoning length, source breadth, iteration count) or adjusting a run mid-execution — outputs like Audio/Video Overviews are generated as fixed artifacts rather than steerable in-progress runs. missing for 10: mid-run steering controls, explicit depth/effort sliders or parameters, ability to expand/narrow scope during execution, independent evidence of these controls working as intended.", "evidenceIds": [ "notebooklm-docs-7", "notebooklm-docs-22", "notebooklm-docs-27", "notebooklm-docs-15" ] }, { "productId": "notebooklm", "storyId": "structured-reports", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "NotebookLM supports Reports with FAQ/study guide/briefing document/custom formats and auto-generated source summaries, plus export of notes to Google Docs/Sheets and Slide Deck generation for sharing, which together cover structured, shareable outputs. However, evidence doesn't confirm native table generation within reports or a single unified 'report' combining sections+tables+summary in one export artifact. missing for 10: explicit table support in generated reports, a single cohesive exportable report format (vs separate docs/sheets/slides), independent/hands-on validation of report quality for stakeholder sharing.", "evidenceIds": [ "notebooklm-docs-27", "notebooklm-docs-5", "notebooklm-docs-13", "notebooklm-docs-19", "notebooklm-docs-25" ] }, { "productId": "notebooklm", "storyId": "systematic-screening", "verdict": "partial", "quality": 3, "confidence": "medium", "rationale": "NotebookLM supports uploading many sources into one notebook and chatting with grounded, cited answers, which could be used ad hoc to ask the same question across sources, but there is no evidence of a systematic screening/extraction feature (e.g., structured criteria templates, per-paper data tables, PRISMA-style tracking, or bulk batch processing) needed for a repeatable literature review workflow. Community feedback also notes the product cannot auto-search/pull in new sources itself, requiring manual curation, which undercuts a large-scale systematic workflow. Missing for 10: structured extraction templates/tables applied consistently across many papers, evidence of handling large paper sets with defined screening criteria, and confirmation this scales beyond source-count limits.", "evidenceIds": [ "notebooklm-docs-2", "notebooklm-docs-6", "notebooklm-docs-28", "notebooklm-comm-4" ] }, { "productId": "notebooklm", "storyId": "transparent-pricing", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs mention some usage limits (500,000 words/200MB per source) and note that usage analytics require a paid subscription and Workspace inclusion, but there is no comprehensive pricing page, tier comparison, or explicit usage caps (e.g., number of notebooks, audio overviews per day) that a researcher would need to fully evaluate cost before committing. Missing for 10: full pricing tier breakdown, free vs paid feature/usage limits, and independent confirmation of these limits.", "evidenceIds": [ "notebooklm-docs-4", "notebooklm-docs-9", "notebooklm-docs-20" ] }, { "productId": "sakana-marlin", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The llms.txt probe returned a 404, and no other evidence shows agent-oriented docs (like an API spec or agent-readable documentation) for Marlin; the evidence pack is entirely marketing copy about the product's research capabilities, not machine-readable docs.", "evidenceIds": [ "sakana-marlin-probe-1", "sakana-marlin-probe-2" ] }, { "productId": "sakana-marlin", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Marlin autonomously researches user-provided topics, mapping causal relationships, comparing hypotheses, and generating structured strategic insights and reports with citations, going beyond simple summarization. Missing for 10: independent/hands-on third-party validation beyond vendor-curated testimonials, and no visibility into underlying data/insight quality benchmarks.", "evidenceIds": [ "sakana-marlin-docs-2", "sakana-marlin-docs-11", "sakana-marlin-docs-22", "sakana-marlin-docs-16", "sakana-marlin-docs-7", "sakana-marlin-docs-9" ] }, { "productId": "sakana-marlin", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Marlin's docs clearly describe a single research task running autonomously for up to ~8 hours without further human input once a topic is set (docs-9, docs-10, docs-16), which matches the 'runs in background autonomously' idea. However, this is a one-shot session, not a recurring/scheduled automation you configure and forget — there's no evidence of triggers, schedules, or multi-run automation management typical of 'set up automations.' Missing for 10: scheduled/recurring automation setup, background job management UI, independent hands-on corroboration of unattended runtime.", "evidenceIds": [ "sakana-marlin-docs-9", "sakana-marlin-docs-10", "sakana-marlin-docs-16", "sakana-marlin-docs-21" ] }, { "productId": "sakana-marlin", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Marlin is explicitly designed as a built-in AI agent that users delegate entire research/strategy tasks to, working autonomously for hours with minimal human input beyond initial framing (docs-1, docs-9, docs-10, docs-16, docs-21). This directly matches the story of delegating tasks to a built-in assistant within the product.\nMissing for 10: independent/hands-on verification beyond vendor testimonials, and detail on interactive control while a task is delegated.", "evidenceIds": [ "sakana-marlin-docs-1", "sakana-marlin-docs-9", "sakana-marlin-docs-10", "sakana-marlin-docs-16", "sakana-marlin-docs-21" ] }, { "productId": "sakana-marlin", "storyId": "agentic-headless", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Marlin is presented as a research-report generation web product with a UI and pay-per-use credits, with no CLI, API, SDK, or webhook documentation for headless/CI usage; probes for llms.txt and OpenAPI specs both returned 404, indicating no programmatic interface is exposed.", "evidenceIds": [ "sakana-marlin-probe-1", "sakana-marlin-probe-2", "sakana-marlin-docs-24" ] }, { "productId": "sakana-marlin", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Sakana Marlin is an autonomous research/report-generation agent, not a platform with an MCP client integration surface; no evidence discusses plugging in external MCP servers or tool ecosystems, and this is not the kind of capability such a research product would expose.", "evidenceIds": [] }, { "productId": "sakana-marlin", "storyId": "agentic-mcp-server", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Sakana Marlin is an autonomous research agent product, not a platform serving tools to other agents; evidence shows no MCP server capability and none is expected for this product type.", "evidenceIds": [] }, { "productId": "sakana-marlin", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Marlin is initiated by giving it a research topic and a brief natural-language exchange to set direction (docs-10), suggesting natural-language input drives its operation, but there is no evidence of a broader natural-language command interface (e.g., chat-style control, follow-up instructions, or command syntax) beyond initial topic-setting. Missing for 10: documentation of ongoing NL command/control during execution, examples of varied NL commands, independent/hands-on confirmation of NL interaction quality.", "evidenceIds": [ "sakana-marlin-docs-10", "sakana-marlin-docs-9", "sakana-marlin-docs-21" ] }, { "productId": "sakana-marlin", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an official CLI; Marlin appears to be a web-based research tool with a pay-per-credit UI, and probes for API/llms.txt endpoints returned 404s, suggesting no developer-facing interface is exposed.", "evidenceIds": [ "sakana-marlin-probe-1", "sakana-marlin-probe-2" ] }, { "productId": "sakana-marlin", "storyId": "agentic-public-api", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No documented public API is evidenced; probes for llms.txt and OpenAPI/swagger endpoints all returned 404s, and all other evidence describes the product's research capabilities, not a programmatic interface.", "evidenceIds": [ "sakana-marlin-probe-1", "sakana-marlin-probe-2" ] }, { "productId": "sakana-marlin", "storyId": "agentic-scoped-keys", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Sakana Marlin is an autonomous research report generation product, not an API/platform that issues credentials to agents; scoped API credential issuance is a category mismatch for this product type.", "evidenceIds": [] }, { "productId": "sakana-marlin", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any official SDK, API, or developer library for Marlin; probes for llms.txt and OpenAPI specs both returned 404, and all docs describe an end-user research product with no mention of programmatic/SDK access.", "evidenceIds": [ "sakana-marlin-probe-1", "sakana-marlin-probe-2" ] }, { "productId": "sakana-marlin", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a webhook or event subscription mechanism; probes for API/OpenAPI specs returned 404s and docs focus only on research report generation. Missing for 10: any webhook documentation, event subscription API, or callback mechanism.", "evidenceIds": [ "sakana-marlin-probe-1", "sakana-marlin-probe-2" ] }, { "productId": "sakana-marlin", "storyId": "agreement-disagreement", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Marketing copy claims Marlin 'resolves contradictions' and 'compares and evaluates multiple hypotheses' rather than merely aggregating, implying some handling of conflicting sources, but there is no evidence of a UI feature or report section that explicitly surfaces where sources agree/disagree to the analyst. Missing for 10: concrete example of a report showing conflicting source viewpoints, screenshot/description of how disagreement is presented, independent corroboration beyond vendor marketing.", "evidenceIds": [ "sakana-marlin-docs-1", "sakana-marlin-docs-16", "sakana-marlin-docs-22" ] }, { "productId": "sakana-marlin", "storyId": "api-interactive-docs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Sakana Marlin is an autonomous research/report-generation product, not a developer API/platform with a public API surface; an interactive API reference with runnable examples is not a relevant capability for this product category.", "evidenceIds": [] }, { "productId": "sakana-marlin", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Probes for OpenAPI/swagger spec files and llms.txt all returned 404, and no documentation mentions a machine-readable API spec. Missing for 10: any published OpenAPI/Swagger file, API reference docs, or equivalent machine-readable spec.", "evidenceIds": [ "sakana-marlin-probe-1", "sakana-marlin-probe-2" ] }, { "productId": "sakana-marlin", "storyId": "api-sandbox", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Sakana Marlin is an autonomous research/strategy report generator, not a coding or data-manipulation tool with a production environment to isolate from; sandbox-vs-production testing is not a relevant axis for this product category.", "evidenceIds": [] }, { "productId": "sakana-marlin", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any public API, versioning scheme, or deprecation policy; probes for OpenAPI/llms.txt endpoints returned 404s, and all docs describe the research product itself, not a developer API. Missing for 10: any API documentation, versioning scheme, deprecation policy.", "evidenceIds": [ "sakana-marlin-probe-1", "sakana-marlin-probe-2" ] }, { "productId": "sakana-marlin", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All evidence describes Marlin running a single autonomous research task per topic (one report, one investigation) rather than processing many items/topics in bulk; no mention of batch submission, multi-item queues, or parallel bulk execution.", "evidenceIds": [] }, { "productId": "sakana-marlin", "storyId": "automation-rules-engine", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Sakana Marlin is a deep-research/report-generation agent, not a rules/trigger automation platform; no concept of user-defined event-triggered rules applies to this product's category.", "evidenceIds": [] }, { "productId": "sakana-marlin", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Sakana Marlin is a single-run autonomous research/report tool triggered by a user topic; no evidence of scheduling, recurrence, cron-like triggers, or workflow automation for repeated jobs. Probes also show no API/OpenAPI surface that could support scheduled invocation.", "evidenceIds": [ "sakana-marlin-docs-9", "sakana-marlin-docs-10", "sakana-marlin-probe-1", "sakana-marlin-probe-2" ] }, { "productId": "sakana-marlin", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack mentions versioning, review workflows, or rollback for automations; Marlin is presented as an autonomous research report generator with no artifact-management features described.", "evidenceIds": [] }, { "productId": "sakana-marlin", "storyId": "autonomous-research-run", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Vendor docs describe autonomous multi-step research: forming hypotheses, gathering info, resolving contradictions, running for hours across thousands of cycles without further human input, producing a full structured report - directly matching the story. This is corroborated by beta-tester quotes praising depth beyond chat-based research tools, though all evidence is vendor-published/testimonial rather than independent hands-on verification. missing for 10: independent third-party evaluation or benchmark of the autonomous multi-step process, technical detail on how contradictions/hypotheses are actually verified", "evidenceIds": [ "sakana-marlin-docs-1", "sakana-marlin-docs-9", "sakana-marlin-docs-10", "sakana-marlin-docs-16", "sakana-marlin-docs-6", "sakana-marlin-docs-22" ] }, { "productId": "sakana-marlin", "storyId": "export-reference-formats", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Marlin auto-generates full reports with references and PowerPoint slides (docs-3, docs-9, docs-20), covering the 'documents' part of the story, but there is no evidence of spreadsheet export or reference-manager file formats (e.g., BibTeX/RIS/EndNote) for citations. missing for 10: spreadsheet export, reference-manager file export (BibTeX/RIS/EndNote), independent corroboration of export formats.", "evidenceIds": [ "sakana-marlin-docs-3", "sakana-marlin-docs-9", "sakana-marlin-docs-20" ] }, { "productId": "sakana-marlin", "storyId": "free-tier-trial", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Marlin is explicitly pay-per-use available only to paid Pro/Team/Enterprise plans, requires adding a card to start, and cancelling mid-run still consumes credits — there is no free tier or trial for researchers to test it meaningfully.", "evidenceIds": [ "sakana-marlin-docs-8", "sakana-marlin-docs-24", "sakana-marlin-docs-5", "sakana-marlin-docs-23" ] }, { "productId": "sakana-marlin", "storyId": "inline-citations", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Vendor docs claim Marlin generates 'fully referenced reports' grounded in primary sources with appendices and references, and a testimonial praises its higher-quality citations to primary vs secondary sources, suggesting citation support exists. However, there is no independent verification, no example of inline citation format, and no detail on how claims map to sources for auditability. Missing for 10: independent hands-on verification of citation accuracy, example output showing citation linking, and confirmation citations are traceable/clickable to primary sources.", "evidenceIds": [ "sakana-marlin-docs-7", "sakana-marlin-docs-15", "sakana-marlin-docs-20", "sakana-marlin-docs-3" ] }, { "productId": "sakana-marlin", "storyId": "long-horizon-runs", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Vendor docs clearly describe long unattended autonomous research runs (~8 hours) producing full reports, which supports the core of the story, but there is no mention of a notification mechanism when results are ready and no independent/hands-on corroboration beyond marketing copy. missing for 10: evidence of a completion notification/alert feature, independent verification of unattended runtime and reliability, and API/technical docs confirming job-control (start/monitor/cancel) beyond the marketing page.", "evidenceIds": [ "sakana-marlin-docs-9", "sakana-marlin-docs-10", "sakana-marlin-docs-1", "sakana-marlin-docs-5" ] }, { "productId": "sakana-marlin", "storyId": "monitoring-alerts", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Marlin is a one-shot deep-research report generator that runs autonomously for a single session (up to ~8 hours) and produces a report; there is no evidence of standing searches, saved alerts, or recurring monitoring for new sources over time.", "evidenceIds": [] }, { "productId": "sakana-marlin", "storyId": "openness-api-parity", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any public API for Sakana Marlin; probes for llms.txt and OpenAPI spec both returned 404, and all documentation describes only UI/credit-based access.", "evidenceIds": [ "sakana-marlin-probe-1", "sakana-marlin-probe-2", "sakana-marlin-docs-24" ] }, { "productId": "sakana-marlin", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of data export, open-format download, or account portability features; probes for llms.txt and OpenAPI both returned 404, and docs only describe generated reports/slides, not export of underlying user data.", "evidenceIds": [ "sakana-marlin-probe-1", "sakana-marlin-probe-2" ] }, { "productId": "sakana-marlin", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any open-source license or public source code repository for Marlin; it is presented as a paid SaaS research product, and probes for open API/docs artifacts returned 404s.", "evidenceIds": [ "sakana-marlin-docs-8", "sakana-marlin-probe-1", "sakana-marlin-probe-2" ] }, { "productId": "sakana-marlin", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Sakana Marlin is a hosted SaaS research product with pay-per-use/credit pricing, not open-source or self-hostable software; self-hosting is a category mismatch for this type of managed AI service.", "evidenceIds": [ "sakana-marlin-docs-8", "sakana-marlin-docs-23", "sakana-marlin-docs-24" ] }, { "productId": "sakana-marlin", "storyId": "own-corpus-upload", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence describes Marlin as an autonomous web-research agent that gathers information via web navigation and generates reports, but there is no mention of uploading a user's own PDFs or corpus for the agent to research over. Missing for 10: any document/file upload feature, corpus ingestion, or evidence of researching over user-supplied materials rather than open web sources.", "evidenceIds": [] }, { "productId": "sakana-marlin", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, region selection, or storage location controls for Sakana Marlin; the evidence covers only research capabilities and pricing.", "evidenceIds": [] }, { "productId": "sakana-marlin", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses data-training opt-out, privacy controls, or data usage policy for Sakana Marlin; missing for 10: any mention of training opt-out settings, privacy policy, or data retention controls.", "evidenceIds": [] }, { "productId": "sakana-marlin", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses data retention, deletion controls, or privacy settings for Marlin; the evidence pack only covers research capabilities, pricing, and product marketing.", "evidenceIds": [] }, { "productId": "sakana-marlin", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses telemetry, usage tracking, or any privacy/opt-out controls for Sakana Marlin. Missing for 10: any documentation of telemetry practices, an opt-out mechanism, or privacy policy details.", "evidenceIds": [] }, { "productId": "sakana-marlin", "storyId": "scholarly-sources", "verdict": "partial", "quality": 6, "confidence": "low", "rationale": "Vendor docs claim research is 'grounded strictly in primary sources' and a testimonial notes it references primary rather than secondary information compared to other generative AI tools, but there is no evidence of dedicated scholarly database access (e.g., PubMed, arXiv, JSTOR integration) or citation verification—only general web navigation is described. missing for 10: evidence of scholarly/academic database integration, independent verification of primary-source claim, and details on how it distinguishes scholarly vs open-web sources.", "evidenceIds": [ "sakana-marlin-docs-7", "sakana-marlin-docs-15", "sakana-marlin-docs-1", "sakana-marlin-docs-16" ] }, { "productId": "sakana-marlin", "storyId": "share-collaborate", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence describes any collaboration or sharing features—no mention of shared workspaces, links, comments, or multi-user access to reports/sessions; the pack only covers autonomous research generation, pricing tiers, and API probes returning 404s.", "evidenceIds": [ "sakana-marlin-docs-8", "sakana-marlin-probe-1", "sakana-marlin-probe-2" ] }, { "productId": "sakana-marlin", "storyId": "steer-effort-scope", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Marlin only allows a brief initial exchange to set direction before running fully autonomously for up to ~8 hours with no mid-run steering, and there's no documented control over depth/effort/scope parameters (e.g., report length, time budget, source breadth) beyond the initial topic framing; cancellation is possible but not adjustment. missing for 10: mid-execution steering controls, explicit depth/effort/scope parameters or settings, independent evidence of pre-run configurability beyond a 'brief exchange'.", "evidenceIds": [ "sakana-marlin-docs-10", "sakana-marlin-docs-9", "sakana-marlin-docs-5", "sakana-marlin-docs-21" ] }, { "productId": "sakana-marlin", "storyId": "structured-reports", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs describe autonomous generation of a full structured report with main body, appendices, references, and presentation slides/executive summary — directly matching sections, tables (implied by structured strategic options), and summary needs for stakeholder sharing. Missing for 10: no independent/hands-on verification of table formatting or actual sample report shown, and no evidence beyond vendor marketing copy.", "evidenceIds": [ "sakana-marlin-docs-3", "sakana-marlin-docs-9", "sakana-marlin-docs-20", "sakana-marlin-docs-2" ] }, { "productId": "sakana-marlin", "storyId": "systematic-screening", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Marlin is positioned as an autonomous business/market strategy research agent producing single deep-dive reports, not as a tool for systematic multi-paper screening/extraction with consistent criteria (a literature-review workflow). No evidence describes handling many papers, applying consistent inclusion/extraction criteria, or batch processing across a corpus — the described unit of work is one topic producing one report.", "evidenceIds": [ "sakana-marlin-docs-9", "sakana-marlin-docs-21", "sakana-marlin-docs-14" ] }, { "productId": "sakana-marlin", "storyId": "transparent-pricing", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Marlin's docs mention it's offered as a pay-per-use add-on to Pro/Team/Enterprise plans, credits are consumed even if cancelled mid-run, and additional credits cost ¥98 each, giving a researcher some pricing signal. However, there's no concrete breakdown of how many credits a typical run consumes, no explicit usage caps, and no comparison table of plan tiers — missing for 10: full plan pricing table, credit-consumption-per-task estimates, explicit usage limits, independent/hands-on cost verification.", "evidenceIds": [ "sakana-marlin-docs-5", "sakana-marlin-docs-8", "sakana-marlin-docs-23", "sakana-marlin-docs-24" ] }, { "productId": "undermind", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Direct probes show no llms.txt, no docs.md, and no openapi spec (all 404), meaning there is no agent-consumable documentation file for a generic AI agent to fetch. The only agent-oriented artifact is the MCP server page itself, which is a different integration mechanism, not llms.txt-style docs.", "evidenceIds": [ "undermind-probe-1", "undermind-probe-2", "undermind-probe-3" ] }, { "productId": "undermind", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Undermind's core capability is AI-generated synthesis and insight extraction from literature data: it runs deep research plans, produces ranked papers with written synthesis, answers cross-paper questions from PDFs, traces citations, and proactively notifies users of new relevant papers—all generated from the user's research data within the product. Community reviews corroborate that these AI-derived insights are often more useful than manual search (undermind-comm-2, undermind-comm-6, undermind-comm-15, undermind-comm-16), though some found gaps in coverage (undermind-comm-1, undermind-comm-4). Missing for 10: independent verification of insight/synthesis accuracy at scale and clearer support for arbitrary user-uploaded (non-literature) datasets.", "evidenceIds": [ "undermind-docs-4", "undermind-docs-5", "undermind-docs-6", "undermind-docs-9", "undermind-docs-12", "undermind-comm-15", "undermind-comm-16" ] }, { "productId": "undermind", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The only evidence of background automation is a single line about being notified when relevant papers are published, with no detail on how such alerts are configured, scheduled, or run autonomously as multi-step agent workflows. missing for 10: documentation of automation/scheduling setup, evidence of autonomous multi-step background agent tasks, and any hands-on confirmation of the notification feature working.", "evidenceIds": [ "undermind-docs-9" ] }, { "productId": "undermind", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Undermind is positioned as an MCP server/tool that other AI clients (Cursor, Claude, ChatGPT) connect to, not as a product with its own built-in AI assistant that users delegate tasks to within Undermind itself; evidence describes it being added to external agent tools rather than an in-product assistant. missing for 10: any evidence of a native, built-in AI assistant/chat agent inside Undermind's own UI that a user can delegate tasks to.", "evidenceIds": [ "undermind-docs-1", "undermind-docs-3", "undermind-docs-21" ] }, { "productId": "undermind", "storyId": "agentic-headless", "verdict": "partial", "quality": 2, "confidence": "low", "rationale": "Undermind's enterprise page claims 'programmatic queries via API' and integration into 'other tools and workflows,' hinting at automatable/headless use, but there is no CLI, no published API reference, and probes for OpenAPI/API docs all returned 404s. The only concretely documented automation path is the MCP server, which is designed for interactive agent clients (Cursor, VS Code, Claude) rather than headless CI pipelines. Missing for 10: documented API/CLI with auth and examples, evidence of CI/automation usage, and confirmation the enterprise API is publicly accessible rather than bespoke.", "evidenceIds": [ "undermind-docs-2", "undermind-docs-17", "undermind-probe-3" ] }, { "productId": "undermind", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "All MCP-related evidence describes Undermind acting as an MCP *server* that other clients (Cursor, VS Code, Claude, ChatGPT) can plug into to use Undermind's own tools — the reverse of this story, which asks whether a user can plug external MCP servers into Undermind so it can use their tools. No evidence shows Undermind hosting/consuming external MCP servers as a client.", "evidenceIds": [ "undermind-docs-1", "undermind-docs-3", "undermind-docs-13", "undermind-docs-14", "undermind-docs-19", "undermind-docs-21" ] }, { "productId": "undermind", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Undermind publishes an official hosted MCP server (mcp.undermind.ai/mcp) with documented setup instructions for Cursor, VS Code, Claude, and any MCP-compatible client, plus specific tool capabilities (literature review, PDF Q&A, notes, curation). This is first-party documentation with concrete transport/endpoint details, confirmed by probe evidence of the docs page existing. Missing for 10: independent/hands-on community confirmation specifically of MCP connectivity (community evidence only covers the web search product, not MCP usage) and no public API schema (openapi probes 404).", "evidenceIds": [ "undermind-docs-1", "undermind-docs-3", "undermind-docs-13", "undermind-docs-14", "undermind-docs-15", "undermind-docs-19", "undermind-probe-4" ] }, { "productId": "undermind", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Undermind ships an official MCP server (Cursor, VS Code, Claude, ChatGPT app) that lets users issue open-ended natural-language research goals which the tool autonomously plans, searches, and synthesizes into reports, fitting the ai-native/agentic story well. Missing for 10: independent hands-on confirmation of the MCP natural-language workflow specifically (community evidence covers the web search UI, not the MCP NL commands) and any public usage examples/logs.", "evidenceIds": [ "undermind-docs-3", "undermind-docs-4", "undermind-docs-13", "undermind-docs-14", "undermind-docs-21", "undermind-probe-4" ] }, { "productId": "undermind", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows an MCP server, API access, and web/ChatGPT app integrations, but there is no mention of an official CLI tool for Undermind anywhere in the docs or probes; llms.txt, docs-md, and openapi probes all 404, and no CLI is documented.", "evidenceIds": [ "undermind-docs-14", "undermind-probe-1", "undermind-probe-2", "undermind-probe-3" ] }, { "productId": "undermind", "storyId": "agentic-public-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Undermind documents a public MCP server (mcp.undermind.ai) with clear tool definitions for search, PDF Q&A, and workspace notes, which does let an AI agent drive the product programmatically (undermind-docs-3,4,5,13,14,15,19). However, a general documented REST/OpenAPI-style public API is only vaguely alluded to ('Programmatic queries via API' under Enterprise) and probes for llms.txt, docs.md, and openapi/swagger specs all return 404s, indicating no broadly published API reference. Missing for 10: a public OpenAPI/Swagger spec or REST API docs, and confirmation that the enterprise API is self-serve rather than sales-gated.", "evidenceIds": [ "undermind-docs-3", "undermind-docs-4", "undermind-docs-13", "undermind-docs-14", "undermind-docs-15", "undermind-docs-17", "undermind-probe-1", "undermind-probe-2", "undermind-probe-3", "undermind-probe-4" ] }, { "productId": "undermind", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of scoped/least-privilege API credential issuance for agents; the API is only mentioned generically ('Programmatic queries via API') with no docs on credential scoping, permissions, or key management, and OpenAPI probes returned 404s.", "evidenceIds": [ "undermind-docs-17", "undermind-probe-3" ] }, { "productId": "undermind", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only mentions a vague 'Programmatic queries via API' for enterprise customers and an MCP server, but no official SDKs (client libraries, language bindings) are documented; probes for OpenAPI specs and docs (llms.txt, mcp.md, openapi.json) all return 404, indicating no public developer SDK resources exist.", "evidenceIds": [ "undermind-docs-17", "undermind-probe-1", "undermind-probe-2", "undermind-probe-3" ] }, { "productId": "undermind", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence mentions a notification feature for new papers (undermind-docs-9) but nothing indicates this is delivered via webhooks or any programmatic subscription mechanism; no webhook API, endpoint, or docs are present, and probes for API/openapi specs all 404.", "evidenceIds": [ "undermind-docs-9", "undermind-probe-3" ] }, { "productId": "undermind", "storyId": "agreement-disagreement", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack describes literature search, synthesis, citation tracing, and PDF Q&A features, but nowhere does it mention surfacing conflicting findings, agreement/disagreement across sources, or qualifying claims by consensus vs. dispute. Citation tracing (docs-12) only supports tracing a single claim to its source, not comparing multiple sources' stances. Missing for 10: any feature or documentation showing detection/display of cross-source agreement or contradiction, any UI element flagging conflicting conclusions, community evidence of this behavior.", "evidenceIds": [ "undermind-docs-12", "undermind-docs-4" ] }, { "productId": "undermind", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "There is no evidence of an interactive API reference or runnable examples; probes explicitly show no OpenAPI/Swagger spec and no docs.md/llms.txt exist. The 'Programmatic queries via API' mention is a bare feature claim with no interactive reference or runnable examples provided.", "evidenceIds": [ "undermind-probe-3", "undermind-probe-2", "undermind-probe-1", "undermind-docs-17" ] }, { "productId": "undermind", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Undermind mentions 'Programmatic queries via API' for enterprise but there is no evidence of a downloadable OpenAPI/Swagger spec; direct probes for openapi.json, swagger.json, and llms.txt all returned 404. Missing for 10: any published machine-readable API spec, documented API schema, or discoverable spec endpoint.", "evidenceIds": [ "undermind-docs-17", "undermind-probe-3", "undermind-probe-1", "undermind-probe-2" ] }, { "productId": "undermind", "storyId": "api-sandbox", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Undermind is a literature-research/search product, not an infrastructure or agentic runtime tool; sandbox-vs-production data isolation is not an applicable concept for its offering.", "evidenceIds": [] }, { "productId": "undermind", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "There is a mention of a 'Programmatic queries via API' for enterprise, but no evidence of versioning or a documented deprecation policy; probes for OpenAPI/docs all returned 404s. Missing for 10: any API versioning scheme, changelog, or deprecation policy documentation.", "evidenceIds": [ "undermind-docs-17", "undermind-probe-3" ] }, { "productId": "undermind", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Undermind's MCP tools explicitly support bulk-style operations: reading full-text PDFs in parallel and answering questions across many papers at once, and running a deep literature review that autonomously searches and synthesizes across large numbers of papers. Enterprise API access also enables programmatic bulk queries. However, there is no evidence of bulk editing/tagging/exporting or batch management operations across items (e.g., bulk-star, bulk-move to folders) — missing for 10: documented batch update/edit/export APIs, evidence of bulk actions beyond reading/analysis, independent verification of parallel-processing claims.", "evidenceIds": [ "undermind-docs-5", "undermind-docs-4", "undermind-docs-17", "undermind-docs-7", "undermind-docs-8" ] }, { "productId": "undermind", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The only automation-relevant capability is a notification feature that alerts users when relevant papers are published (undermind-docs-9), which is a basic event trigger but not a configurable rule engine with user-defined conditions and multiple downstream actions. Missing for 10: ability to define custom trigger conditions, chain multiple actions, or integrate rules into broader workflows beyond a single notification type.", "evidenceIds": [ "undermind-docs-9" ] }, { "productId": "undermind", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "There's a notification feature for new papers (undermind-docs-9) but no evidence of scheduling recurring jobs/workflows, cron-like automation, or configurable recurring tasks; the product focuses on on-demand deep research via MCP tools rather than persistent scheduled automation.", "evidenceIds": [ "undermind-docs-9" ] }, { "productId": "undermind", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Undermind is a literature-research/search tool; the story asks for versioning, reviewing, and rolling back 'automations' (workflows/agents), which is not a capability class this product's evidence pack addresses—no automation-building feature exists to version or roll back. This is a category mismatch rather than an unmet capability.", "evidenceIds": [] }, { "productId": "undermind", "storyId": "autonomous-research-run", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly describe an autonomous multi-step deep literature review agent that plans its own searches, follows citations and key authors, reads full-text PDFs in parallel, and stops only when new searches stop finding relevant papers — not a single-pass summary. Community reviews corroborate multi-minute, iterative search behavior yielding comprehensive results beyond a simple query-response. Missing for 10: independent technical breakdown of the multi-step planning/agentic loop and more recent hands-on validation of the 'stops when exhausted' claim.", "evidenceIds": [ "undermind-docs-4", "undermind-docs-16", "undermind-docs-5", "undermind-comm-12", "undermind-comm-16", "undermind-comm-15" ] }, { "productId": "undermind", "storyId": "export-reference-formats", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows Undermind creates/edits Markdown notes and reports with citations, and can curate/star papers, but there is no mention of exporting to common document formats (e.g., Word/PDF), spreadsheets (e.g., CSV/Excel), or reference-manager formats (e.g., BibTeX, RIS, EndNote).", "evidenceIds": [] }, { "productId": "undermind", "storyId": "free-tier-trial", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Docs imply a tiered system (e.g. '10x higher usage limits' for paid vs default) suggesting a free/limited tier exists, but community evidence shows a hard institutional/company email requirement blocking sign-up, with an independent researcher explicitly reporting they 'can't get in' and another calling the requirement 'obnoxious' and a 'roadblock' — concretely contradicting the ability for many researchers to try it meaningfully for free. Missing for 10: explicit vendor documentation of a free tier or trial with stated limits, and confirmation the email gate has been removed or has an exception path for independent researchers.", "evidenceIds": [ "undermind-docs-18", "undermind-comm-3", "undermind-comm-13" ] }, { "productId": "undermind", "storyId": "inline-citations", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Undermind's docs explicitly claim in-line citations traceable to source papers ([undermind-docs-12], [undermind-docs-6]) and community reviews corroborate it reliably surfaces cited references (e.g. [undermind-comm-8] notes it 'solves' the reference-provision problem unlike a chatbot). However, no independent hands-on verification of citation accuracy/completeness at the claim level is present, and some reviewers note gaps in coverage (missing gray literature/theses) which could affect verifiability of some claims. Missing for 10: independent audit of citation accuracy per-claim, and confirmation citations withstand scrutiny across all source types.", "evidenceIds": [ "undermind-docs-12", "undermind-docs-6", "undermind-comm-8", "undermind-comm-1", "undermind-comm-4" ] }, { "productId": "undermind", "storyId": "long-horizon-runs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Undermind's deep literature review runs autonomously, planning its own searches and stopping only when exhausted (undermind-docs-4/16), and it has a notification feature for relevant papers (undermind-docs-9), suggesting async, unattended operation. However, the notification feature is documented as an ongoing 'new paper published' alert rather than a 'job complete, come see results' notification, and community reports describe run times of minutes (3-6 min) rather than long unattended background jobs (undermind-comm-14/15/16). Missing for 10: explicit documentation that a single research job can run for extended/unattended periods (hours+) and trigger a completion notification, plus independent confirmation of this exact workflow.", "evidenceIds": [ "undermind-docs-4", "undermind-docs-16", "undermind-docs-9", "undermind-comm-14", "undermind-comm-15", "undermind-comm-16" ] }, { "productId": "undermind", "storyId": "monitoring-alerts", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Undermind explicitly offers a 'Get notified whenever relevant papers are published' alert feature, which directly matches the standing-search/alert story, plus curated folders and starred papers for ongoing tracking. However, there is no independent/hands-on evidence of how the alert system works in practice (frequency, delivery channel, reliability), and community discussion focuses on one-off search quality rather than alerting. Missing for 10: independent corroboration of alert functionality, details on alert configuration/frequency, and hands-on user reports of ongoing alerts working as described.", "evidenceIds": [ "undermind-docs-9", "undermind-docs-7", "undermind-docs-8" ] }, { "productId": "undermind", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Undermind exposes core research capabilities (deep literature search, PDF Q&A, note creation, curation, starring) via an official MCP server, and separately claims 'Programmatic queries via API' for enterprise customers, showing some AI-native parity. However, there is no public API/OpenAPI documentation (all probes 404), no evidence that UI-only features like notifications/alerts or workspace management are exposed programmatically, and the API claim is a single unelaborated enterprise line rather than a documented full-parity API. Missing for 10: public API docs/OpenAPI spec, confirmation that all UI features (alerts, workspace/library management) are API-accessible, and independent verification of API completeness.", "evidenceIds": [ "undermind-docs-4", "undermind-docs-5", "undermind-docs-6", "undermind-docs-7", "undermind-docs-8", "undermind-docs-17", "undermind-docs-9", "undermind-probe-1", "undermind-probe-2", "undermind-probe-3" ] }, { "productId": "undermind", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of any data export feature or open-format export capability for user data/papers/workspaces; probes for docs/API endpoints also 404. The evidence covers MCP integration and search quality, not data portability or export/leave capability.", "evidenceIds": [] }, { "productId": "undermind", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Undermind is a closed, proprietary SaaS research/search product; there is no evidence of any open-source license or public source repository, and this is a commercial hosted service, not an open-source project, so the open-source-license axis doesn't apply.", "evidenceIds": [] }, { "productId": "undermind", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Undermind is a hosted SaaS literature-research/search product accessed via web app, API, or MCP endpoint; there is no evidence of an open-source core or any self-hostable deployment option, and self-hosting is not a plausible axis for this kind of proprietary hosted service.", "evidenceIds": [] }, { "productId": "undermind", "storyId": "own-corpus-upload", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Undermind reads full-text PDFs and lets users curate/star papers discovered via its own search engine, but nothing indicates a feature to upload arbitrary personal PDFs or a private corpus for the agent to research over — all workflows described start from Undermind's own literature search rather than user-supplied documents.", "evidenceIds": [ "undermind-docs-5", "undermind-docs-7", "undermind-docs-8" ] }, { "productId": "undermind", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, regional storage options, or any data-location controls for Undermind. missing for 10: any mention of region selection, data residency policy, or storage location controls.", "evidenceIds": [] }, { "productId": "undermind", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data privacy, opt-out of AI training, or data usage policies for Undermind; nothing here confirms or denies such a control exists.", "evidenceIds": [] }, { "productId": "undermind", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses data retention policies, deletion controls, or privacy/data-handling settings for Undermind; all evidence concerns search quality, MCP integration, and product features. This axis applies to any AI-native product handling user data/queries, but there is no documented capability for controlling or deleting stored data.", "evidenceIds": [] }, { "productId": "undermind", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions telemetry, usage tracking, opt-out settings, or privacy controls for Undermind; all evidence covers product features, MCP integration, and general reception.", "evidenceIds": [] }, { "productId": "undermind", "storyId": "scholarly-sources", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Undermind is purpose-built for scholarly literature search: it runs deep literature reviews over papers, reads full-text PDFs including figures/tables/equations, and traces claims back to source papers via citations, with independent community reports confirming it surfaces relevant academic papers beyond Google Scholar. Some community feedback notes gaps in coverage (dissertations, gray literature), which tempers but doesn't negate the core capability. Missing for 10: no independent benchmark on primary-source/preprint coverage breadth, and some users report missed gray literature/theses.", "evidenceIds": [ "undermind-docs-4", "undermind-docs-5", "undermind-docs-12", "undermind-docs-10", "undermind-comm-2", "undermind-comm-5", "undermind-comm-15", "undermind-comm-1", "undermind-comm-4" ] }, { "productId": "undermind", "storyId": "share-collaborate", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence covers workspace files, folders, and note creation but never mentions sharing sessions/reports with collaborators, multi-user access, or permission controls. missing for 10: any mention of sharing/collaboration features, invite/permission mechanisms, or multi-user workspace access.", "evidenceIds": [] }, { "productId": "undermind", "storyId": "steer-effort-scope", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Undermind's research runs are described as fully autonomous — it 'plans its own searches... and stops only when new searches stop finding relevant papers' (undermind-docs-4, undermind-docs-16), with no documented controls for setting depth, effort, or scope before or during a run. A user comment notes confusion about the 'Discovery Progress and Exhaustiveness' indicator (undermind-comm-16), suggesting only passive visibility, not steering. No evidence of parameters, sliders, or mid-run adjustments exists in the pack.", "evidenceIds": [ "undermind-docs-4", "undermind-docs-16", "undermind-comm-16" ] }, { "productId": "undermind", "storyId": "structured-reports", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Undermind produces a ranked list of papers with a written synthesis and can create/edit Markdown notes, syntheses, and reports with citations linking back to sources, which supports shareable structured output. However, there is no evidence of built-in tables, formal 'sections' structuring, or a dedicated stakeholder-facing report/export format beyond Markdown notes. Missing for 10: explicit table generation, multi-section report templates, and export/sharing formats (PDF/Word) for stakeholders beyond in-workspace Markdown.", "evidenceIds": [ "undermind-docs-4", "undermind-docs-6", "undermind-docs-5" ] }, { "productId": "undermind", "storyId": "systematic-screening", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Undermind's MCP tools support open-ended literature review with ranked results, full-text PDF question-answering across many papers, and note/synthesis creation, which covers much of a screening-and-extraction workflow (undermind-docs-4, undermind-docs-5, undermind-docs-6, undermind-docs-7). However, there is no evidence of formal criteria-based screening (e.g., inclusion/exclusion rules) or structured, consistent extraction fields across a batch of papers, and community feedback notes gaps in coverage (missed gray literature/theses) that could undermine consistency (undermind-comm-1, undermind-comm-4). Missing for 10: explicit support for defined screening criteria, structured/consistent extraction schema, and independent validation of consistency at scale.", "evidenceIds": [ "undermind-docs-4", "undermind-docs-5", "undermind-docs-6", "undermind-docs-7", "undermind-comm-1", "undermind-comm-4" ] }, { "productId": "undermind", "storyId": "transparent-pricing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence includes only a fragmentary marketing snippet ('10x higher usage limits, unlimited workspaces...') with no actual price points, plan names, or explicit usage caps, and no dedicated pricing page is cited. A researcher cannot compare plans or understand limits before committing from this evidence alone.", "evidenceIds": [ "undermind-docs-18" ] } ]