[ { "productId": "browser-use", "storyId": "agent-framework-adapters", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence documents an MCP server for connecting to Claude, Cursor, or Windsurf, and a Python library for custom code, but there is no mention of documented adapters for Claude Agent SDK, Vercel AI SDK, LangChain, or CrewAI specifically.", "evidenceIds": [ "browser-use-docs-12", "browser-use-probe-3", "browser-use-gh-3" ] }, { "productId": "browser-use", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Browser Use publishes a discoverable llms.txt at docs.browser-use.com/llms.txt confirmed live via probe (HTTP 200), which is exactly the agent-oriented docs entry point an AI-native user could point an agent at, and the broader docs site is structured/markdown-friendly for agent consumption. Missing for 10: no evidence of additional structured formats like llms-full.txt or explicit guidance encouraging agents to consume it, and no independent community confirmation of agents successfully using it.", "evidenceIds": [ "browser-use-probe-1" ] }, { "productId": "browser-use", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Browser Use's agent can extract and return structured data/results from web tasks (e.g., extracting follower data to CSV, structured JSON output), which counts as AI-generated output from data it gathers, but there is no evidence of proactive 'insights and suggestions' generated from a user's own stored data inside a product dashboard — it's task-driven extraction, not analytics-style suggestion generation. missing for 10: dedicated insights/suggestions surface, evidence of proactive recommendations, analysis of user's own historical data corpus rather than ad-hoc scraped web data.", "evidenceIds": [ "browser-use-gh-2", "browser-use-docs-3", "browser-use-docs-1" ] }, { "productId": "browser-use", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Browser Use Cloud lets users kick off agent runs via API (client.runs.create) that execute asynchronously in a hosted browser, with session reuse and event polling to monitor progress without keeping a local process open, which supports a form of unattended background execution. However there is no documented scheduling, cron-like triggers, or webhook-based automation setup for recurring/background jobs, so it's unclear whether truly hands-off recurring automations are supported. Missing for 10: explicit scheduling/trigger mechanism, evidence of long-running unattended jobs beyond single API-invoked runs, and independent confirmation of background reliability.", "evidenceIds": [ "browser-use-docs-1", "browser-use-docs-4", "browser-use-docs-5", "browser-use-docs-14" ] }, { "productId": "browser-use", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Browser Use is itself an agent/automation product (the AI acting inside the browser), not a host application that delegates to a separate built-in assistant — this is the agent-role exception where the axis does not apply. It ships as a library/cloud API/MCP server for developers to build agents with, not as an end-user app containing an embedded assistant.", "evidenceIds": [] }, { "productId": "browser-use", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Browser Use ships both an open-source Python library and a cloud API (client.runs.create) that are inherently script/automatable, implying headless/CI use, and gh-3 explicitly pitches automating the web 'from your own code, and with any LLM.' However, there is no explicit documentation of headless mode flags, Docker images, or CI pipeline examples/integration guides. Missing for 10: explicit headless-mode configuration docs, CI/CD pipeline examples (e.g., GitHub Actions), and independent confirmation of running unattended in CI.", "evidenceIds": [ "browser-use-docs-1", "browser-use-gh-3", "browser-use-docs-19", "browser-use-docs-2" ] }, { "productId": "browser-use", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Browser Use is itself an agent/automation product; evidence (browser-use-docs-12) shows it ships as an MCP *server* that other clients (Claude, Cursor, Windsurf) connect to, not as an MCP *client* that consumes external MCP servers' tools. Per the agent-role exception, this client-side 'plug in MCP servers' story is out of scope for a product that is itself an agent unless it explicitly runs as an MCP client, which no evidence shows.", "evidenceIds": [ "browser-use-docs-12", "browser-use-probe-3" ] }, { "productId": "browser-use", "storyId": "agentic-mcp-server", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Docs describe an official MCP server enabling Claude, Cursor, Windsurf or any MCP client to run Browser Use tasks (browser-use-docs-12, browser-use-probe-3), but a hands-on community report says the author had to switch tools because Browser Use 'doesn't support MCP integration' in Cursor (browser-use-comm-4), directly contradicting the documented claim. Missing for 10: independent corroboration that MCP connection actually works end-to-end, and resolution of the conflicting user report.", "evidenceIds": [ "browser-use-docs-12", "browser-use-probe-3", "browser-use-comm-4" ] }, { "productId": "browser-use", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Browser Use is fundamentally natural-language driven: tasks are issued as plain-English strings like \"Find the top Hacker News story\" or \"Fill in this job application with my resume and information\", with the agent interpreting and executing them autonomously, corroborated by GitHub examples and community hands-on use for LinkedIn automation. missing for 10: independent benchmark of instruction-following accuracy, and clearer docs on limits/failure modes of natural-language parsing.", "evidenceIds": [ "browser-use-docs-1", "browser-use-gh-1", "browser-use-gh-2", "browser-use-comm-5" ] }, { "productId": "browser-use", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents a Python SDK, cloud API, MCP server, and web dashboard, but never mentions an official standalone CLI tool for Browser Use. Since a browser-automation product could plausibly ship a CLI, absence of any such evidence means this axis is unmet.", "evidenceIds": [] }, { "productId": "browser-use", "storyId": "agentic-public-api", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Browser Use documents a public Cloud API (client.runs.create, sessions, events polling, structured output, CDP connection) across multiple docs pages, indicating a real programmatic interface beyond the UI. However, probes for a formal OpenAPI/swagger spec returned 404s, so there's no machine-readable API contract, only prose docs and SDK examples. Missing for 10: a published OpenAPI/spec artifact, independent third-party confirmation of API usage beyond vendor docs.", "evidenceIds": [ "browser-use-docs-1", "browser-use-docs-4", "browser-use-docs-5", "browser-use-docs-14", "browser-use-probe-2", "browser-use-probe-1" ] }, { "productId": "browser-use", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of scoped or least-privilege API key/credential issuance (e.g., role-based tokens, permission scopes) for agents; docs mention API keys implicitly via client usage but no mention of scoping, restricted permissions, or credential management features. Community/GitHub evidence also silent on this.", "evidenceIds": [] }, { "productId": "browser-use", "storyId": "agentic-sdks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Browser Use ships an official open-source Python library (github, docs-19) and a cloud client SDK with documented usage patterns (client.runs.create, sessions, events polling) shown in docs-1/4/5/14, giving AI-native devs a concrete first-party SDK to build against. Missing for 10: evidence of SDKs beyond Python (e.g. JS/TS), and no OpenAPI spec was found (probe-2 all 404) or independent third-party corroboration of SDK usage.", "evidenceIds": [ "browser-use-docs-1", "browser-use-docs-4", "browser-use-docs-5", "browser-use-docs-19", "browser-use-gh-3", "browser-use-probe-2" ] }, { "productId": "browser-use", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The docs describe an event system based on polling `runs.events()` with a cursor, not webhook subscriptions; no evidence anywhere in the pack mentions webhooks, callback URLs, or push notifications.", "evidenceIds": [ "browser-use-docs-5", "browser-use-docs-14" ] }, { "productId": "browser-use", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Docs show static code snippets (e.g., client.runs.create examples) but no evidence of an interactive, runnable API console/reference; explicit probes for OpenAPI/Swagger specs at standard paths all returned 404, indicating no interactive API explorer exists.", "evidenceIds": [ "browser-use-probe-2", "browser-use-docs-1", "browser-use-probe-1" ] }, { "productId": "browser-use", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe for OpenAPI/swagger spec files returned 404 on all candidate paths, and no documentation references a downloadable machine-readable API spec despite having a REST/cloud API.", "evidenceIds": [ "browser-use-probe-2" ] }, { "productId": "browser-use", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes cloud browser sessions, live preview, and CDP connections, but nothing indicates a dedicated sandbox/staging mode that isolates test runs from production data or real accounts. Users are shown reusing real logged-in profiles (browser-use-comm-5) rather than isolated test environments, and no docs mention a sandbox distinct from production.", "evidenceIds": [ "browser-use-docs-4", "browser-use-docs-10", "browser-use-comm-5" ] }, { "productId": "browser-use", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows an API version label (\"V4\") in docs, but there is no documented deprecation policy, versioning changelog, or API stability guarantees anywhere in the pack; the OpenAPI spec probe also 404s, indicating no formal API contract is published.", "evidenceIds": [ "browser-use-docs-3", "browser-use-docs-15", "browser-use-probe-2" ] }, { "productId": "browser-use", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The library is pitched for automating the web 'at scale' from custom code (gh-3) and cloud sessions can hold multiple runs, hinting at multi-task orchestration, but there is no explicit documentation of a batch/bulk API, parallel run submission, or looping over many items as a first-class feature. Missing for 10: dedicated bulk/batch endpoint or SDK pattern, concurrency limits/guidance, and hands-on evidence of running many items in one operation.", "evidenceIds": [ "browser-use-gh-3", "browser-use-docs-4", "browser-use-docs-1" ] }, { "productId": "browser-use", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Browser Use's evidence covers on-demand task runs, sessions, observability polling, CAPTCHA/stealth, and MCP integration, but nothing describes user-defined rules or triggers that fire actions automatically on external events (e.g., webhooks, schedules, conditional triggers). The product is presented as an agent you invoke to perform a task, not an event-driven automation/rules engine.", "evidenceIds": [ "browser-use-docs-1", "browser-use-docs-4", "browser-use-docs-5", "browser-use-docs-12", "browser-use-gh-3" ] }, { "productId": "browser-use", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers runs, sessions, observability, stealth/proxy/CAPTCHA handling, auth profiles, and MCP integration, but nowhere mentions cron-like scheduling, recurring triggers, or workflow automation over time. No docs or community evidence describe a scheduler or recurring-job feature.", "evidenceIds": [] }, { "productId": "browser-use", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Browser Use documents runs, sessions, event polling, and observability, but there is no evidence of versioning automation definitions, reviewing changes, or rolling back to prior versions of a task/automation. Missing for 10: version history/diffing of automations, review/approval workflow, rollback mechanism.", "evidenceIds": [ "browser-use-docs-4", "browser-use-docs-5", "browser-use-docs-14" ] }, { "productId": "browser-use", "storyId": "captcha-handling", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Browser Use documents a clear captcha stance: automatic CAPTCHA solving is enabled by default for cloud Agent runs and standalone Cloud Browser sessions, with an explicit human-fallback path via the live preview if the challenge persists. This directly matches the story's requirement of a documented stance rather than silent failure. Missing for 10: independent/hands-on verification of captcha-solving success rates and explicit behavior/limits for the self-hosted open-source library (docs focus on Cloud).", "evidenceIds": [ "browser-use-docs-8", "browser-use-docs-9", "browser-use-docs-15" ] }, { "productId": "browser-use", "storyId": "cdp-connect-existing-stack", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs explicitly describe connecting existing Playwright/Puppeteer/CDP code to Browser Use's browsers via CDP URL, with a documented choice between Browser Use driving or the developer's own code connecting directly over CDP (docs-2, docs-18), and community reports confirm connecting to a real local Chrome profile via CDP for existing automation. missing for 10: independent hands-on validation specifically with Playwright/Puppeteer libraries (not just CDP raw), and more detail on session/auth persistence when using external code.", "evidenceIds": [ "browser-use-docs-2", "browser-use-docs-18", "browser-use-docs-4", "browser-use-comm-5" ] }, { "productId": "browser-use", "storyId": "credential-vault-2fa", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Browser Use supports saved/reused login profiles (docs-10) and a documented 2FA workflow where the run pauses at the challenge and a human enters the code via live view (docs-11/16), but there is no vault/secrets-manager integration for storing credentials and injecting them without model exposure, and TOTP is handled via human-in-the-loop rather than automated secret injection. missing for 10: a credential vault/secrets-manager integration, evidence that passwords/TOTP secrets are injected without ever passing through the model context, fully automated TOTP handling without human intervention, independent confirmation of the 2FA flow working in practice.", "evidenceIds": [ "browser-use-docs-10", "browser-use-docs-11", "browser-use-docs-16" ] }, { "productId": "browser-use", "storyId": "deterministic-replay-caching", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of caching resolved actions or generated code for deterministic, lower-cost replay; the product is LLM-driven agent automation with sessions/runs but nothing about caching or replay without re-invoking the model. missing for 10: any mention of action/code caching, deterministic replay mechanism, or cost/latency savings from skipping re-prompting.", "evidenceIds": [] }, { "productId": "browser-use", "storyId": "dom-action-primitives", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Browser Use's core agent is built around natural-language task instructions (e.g. \"Find the top Hacker News story\", \"Fill in this job application\") executed via an LLM-driven agent that perceives the DOM/page state and decides actions, which is the essence of DOM-understanding, layout-resilient automation — this is corroborated by GitHub task examples and community hands-on use (LinkedIn automation, resume filling). However, the evidence pack lacks explicit documentation of discrete act/click/type primitives with described-element targeting or any stated guarantee/mechanism for surviving selector/layout changes; it's inferred from the agent's general design rather than directly documented. missing for 10: explicit API/primitive-level documentation of click/type/act-on-described-element functions, and direct evidence/testing showing resilience to selector or layout changes rather than just general LLM-driven task completion.", "evidenceIds": [ "browser-use-gh-1", "browser-use-gh-2", "browser-use-comm-5", "browser-use-docs-1" ] }, { "productId": "browser-use", "storyId": "ethics-abuse-guardrails", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents stealth, proxy, and CAPTCHA-solving features in detail, but there is no published acceptable-use policy, terms of service, or anti-abuse statement governing what these stealth/automation features may or may not be used for; community discussion even raises security/abuse concerns without any vendor policy response cited.", "evidenceIds": [] }, { "productId": "browser-use", "storyId": "file-download-upload", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack has no documentation of file upload/download handling or of artifacts being stored and retrievable after a run — GH task examples merely reference a resume being filled in and CSV export, but no confirmation these are handled as retrievable files via any API or session mechanism. Missing for 10: explicit file upload API/tooling, download/save-to-cloud-storage feature, and an artifact retrieval endpoint or docs section.", "evidenceIds": [ "browser-use-gh-1", "browser-use-gh-2" ] }, { "productId": "browser-use", "storyId": "hosted-task-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs show a hosted Cloud API (client.runs.create) that creates runs, supports polling via runs.events() with cursors, and returns structured results without the caller managing browser infrastructure (stealth, proxies, CAPTCHA solving handled server-side). Missing for 10: no explicit webhook callback mechanism is documented (only polling is shown), and no public OpenAPI spec was found to confirm full REST surface.", "evidenceIds": [ "browser-use-docs-1", "browser-use-docs-3", "browser-use-docs-5", "browser-use-docs-14", "browser-use-docs-6", "browser-use-docs-7", "browser-use-docs-8", "browser-use-probe-2" ] }, { "productId": "browser-use", "storyId": "live-view-human-takeover", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs describe a live_view_url/live preview that lets a human take control mid-run for cases like CAPTCHAs or 2FA, and events can be polled to monitor a run, which supports live-watch-and-intervene workflows. However, this is scoped to specific triggers (CAPTCHA/2FA) rather than a general 'agent gets stuck, operator takes over anytime' workflow, and there's no independent/hands-on evidence confirming smooth mid-run handoff in practice. missing for 10: general-purpose stuck-detection/handoff beyond CAPTCHA/2FA scenarios, independent hands-on confirmation of live takeover working reliably, clear UI/replay-debugging tooling details.", "evidenceIds": [ "browser-use-docs-9", "browser-use-docs-11", "browser-use-docs-16", "browser-use-docs-14", "browser-use-docs-5" ] }, { "productId": "browser-use", "storyId": "local-browser-mode", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Browser Use ships an open-source Python library explicitly positioned for local, code-driven automation ('For a local agent, use the open-source library'; 'automate the web at scale, from your own code, and with any LLM'), and community reports confirm running it locally against a real local Chrome browser/profile without a cloud account. missing for 10: no explicit walkthrough showing zero network/account calls during local runs, and no independent benchmark of purely offline/local operation.", "evidenceIds": [ "browser-use-docs-19", "browser-use-gh-3", "browser-use-comm-5", "browser-use-docs-2" ] }, { "productId": "browser-use", "storyId": "login-session-persistence", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly describe saving a login profile once and reusing it to start future browsers already logged in, plus a 2FA guide for handling the initial login flow, and community evidence confirms local profile reuse works for logged-in automation (e.g., LinkedIn). missing for 10: independent/hands-on corroboration specifically of the cloud profile-reuse feature (only local profile reuse is community-validated) and no detail on profile storage/security guarantees.", "evidenceIds": [ "browser-use-docs-10", "browser-use-docs-11", "browser-use-docs-16", "browser-use-comm-5" ] }, { "productId": "browser-use", "storyId": "model-agnostic-llm", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "GitHub docs explicitly market the library as usable with any LLM ('Use the Python library ... with any LLM'), and independent community testing corroborates this by reporting successful use with Gemini models rather than being locked to one vendor. Missing for 10: a dedicated docs page enumerating specific supported providers/configuration examples and broader independent confirmation across multiple providers beyond Gemini.", "evidenceIds": [ "browser-use-gh-3", "browser-use-comm-1" ] }, { "productId": "browser-use", "storyId": "nl-task-to-completion", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs and GitHub examples show natural-language goals (e.g., \"Find the top Hacker News story\", \"Fill in this job application\") driving an agent that navigates, logs in, handles 2FA/CAPTCHA, and completes multi-step flows end-to-end via both cloud API and open-source library; community reports corroborate real-world use (e.g., LinkedIn automation). Missing for 10: independent third-party benchmark of complex multi-step task success rates and more robust evidence of reliability at scale beyond anecdotal community reports.", "evidenceIds": [ "browser-use-docs-1", "browser-use-gh-1", "browser-use-gh-2", "browser-use-docs-10", "browser-use-docs-11", "browser-use-docs-8", "browser-use-comm-5", "browser-use-gh-3" ] }, { "productId": "browser-use", "storyId": "observe-suggest-actions", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows live-preview URLs for human intervention during CAPTCHA/2FA and event polling for observability, but nothing about an explicit preview/plan step where candidate actions are surfaced for developer review before the agent commits to acting.", "evidenceIds": [ "browser-use-docs-9", "browser-use-docs-11", "browser-use-docs-14", "browser-use-docs-16" ] }, { "productId": "browser-use", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show the API (client.runs.create, sessions, events polling, structured output, live_view_url for 2FA/CAPTCHA handoff, stealth/proxy defaults) mirrors most cloud UI capabilities, and there's an official MCP server for coding-agent access, suggesting broad but not explicitly confirmed feature parity with the dashboard/UI. However there's no discoverable OpenAPI/formal API spec (404s on all candidate paths) and no explicit vendor statement that 100% of UI functionality is API-reachable; missing for 10: a canonical API reference/OpenAPI spec, and explicit parity documentation confirming every UI action (e.g., live preview manual takeover) is independently scriptable via API rather than requiring the UI.", "evidenceIds": [ "browser-use-docs-1", "browser-use-docs-4", "browser-use-docs-5", "browser-use-docs-9", "browser-use-docs-11", "browser-use-docs-12", "browser-use-probe-2", "browser-use-probe-3" ] }, { "productId": "browser-use", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack describes agent runs, sessions, CAPTCHA handling, and pricing, but nothing documents an account/data export feature (e.g., downloading all run history, sessions, or stored data in an open format) that would let a user leave the platform with their data intact; the open-source library allows self-hosting but that's a separate capability from exporting existing cloud account data.", "evidenceIds": [] }, { "productId": "browser-use", "storyId": "openness-open-license", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The GitHub repo (browser-use/browser-use) and docs reference an 'open-source library' with a public quickstart, indicating the core Python library's source is publicly viewable, but no evidence explicitly states an open-source license (e.g., MIT/Apache) or shows license text. missing for 10: explicit license file/declaration, confirmation of license type, evidence of full source (vs. cloud API) being open.", "evidenceIds": [ "browser-use-gh-3", "browser-use-docs-19", "browser-use-probe-1" ] }, { "productId": "browser-use", "storyId": "openness-self-host", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Browser Use ships an open-source Python library (github.com/browser-use/browser-use) that runs locally and independently of the Cloud API, explicitly positioned as the option for self-hosted/local agents (\"For a local agent, use the open-source library\"), and community reports confirm running it locally connected to a real browser/profile. missing for 10: no first-party self-hosting guide covering infra/deployment (e.g. Docker, scaling), and no independent audit of parity between self-hosted and cloud feature sets (stealth, CAPTCHA solving, proxies are cloud-only per docs).", "evidenceIds": [ "browser-use-docs-19", "browser-use-gh-3", "browser-use-comm-5", "browser-use-probe-1" ] }, { "productId": "browser-use", "storyId": "parallel-fleet-scale", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs show programmatic session/run creation (client.runs.create, session IDs holding multiple runs) and event polling for observability, implying some ability to manage sessions programmatically, but there is no documented concurrency limit, no fleet/parallel-session guidance, and no scaling architecture described. missing for 10: documented concurrency limits, guidance/examples for running multiple concurrent sessions at scale, rate-limit or quota specs, and independent evidence of parallel session management working in practice.", "evidenceIds": [ "browser-use-docs-1", "browser-use-docs-4", "browser-use-docs-5", "browser-use-docs-14", "browser-use-gh-3" ] }, { "productId": "browser-use", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack material addresses data residency, region selection, or storage location controls for cloud runs/sessions; the open-source library option avoids the cloud entirely but that's not the same as choosable region/residency within the product. missing for 10: any mention of data center regions, residency options, or storage location controls.", "evidenceIds": [] }, { "productId": "browser-use", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data usage for AI model training, opt-out policies, or privacy/data-retention commitments for Browser Use Cloud or the open-source library.", "evidenceIds": [] }, { "productId": "browser-use", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of data retention controls, deletion APIs, or privacy/data-lifecycle policy documentation anywhere in the pack; only session/profile reuse and credit pricing are mentioned. missing for 10: retention policy documentation, data deletion API/UI, export/erasure controls, any privacy compliance statement.", "evidenceIds": [] }, { "productId": "browser-use", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions telemetry, usage data collection, or any opt-out/privacy setting for Browser Use; the documentation and community items cover unrelated features like stealth browsing, CAPTCHA solving, and MCP integration.", "evidenceIds": [] }, { "productId": "browser-use", "storyId": "schema-structured-extraction", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Docs mention structured output but V4 returns `run.result` as a plain string with a recommendation to 'ask for JSON only, then validate it client-side' — there's no native Zod/Pydantic schema binding or first-party typed-schema extraction feature shown. This is a workaround rather than a built-in schema-validated extraction pipeline. missing for 10: no evidence of a documented schema/type-binding API (e.g., passing a Pydantic/Zod schema directly to the agent), no SDK-level validation helpers, no independent/hands-on confirmation that structured JSON output reliably conforms to a given schema.", "evidenceIds": [ "browser-use-docs-3", "browser-use-gh-2" ] }, { "productId": "browser-use", "storyId": "session-replay-recordings", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs describe an observability/events stream (runs.events()) for monitoring a run and a live_view_url for real-time human intervention, which could support building a step timeline, but there is no explicit mention of recorded video or screenshot capture for post-hoc replay debugging of failed runs. Missing for 10: documented video recording of sessions, screenshot capture per action, and a dedicated replay/timeline UI for past runs.", "evidenceIds": [ "browser-use-docs-5", "browser-use-docs-14", "browser-use-docs-11", "browser-use-docs-16" ] }, { "productId": "browser-use", "storyId": "stealth-fingerprint-proxies", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs explicitly state cloud browsers run in a hardened Chromium fork with stealth enabled by default and residential proxies across 195+ countries, directly matching the story's stealth+proxy ask, with automatic CAPTCHA solving as a complementary layer. Missing for 10: explicit control/documentation for selecting a specific geo-target rather than automatic 195+ country rotation, and independent/hands-on evidence confirming bot-detection evasion actually works in practice.", "evidenceIds": [ "browser-use-docs-6", "browser-use-docs-7", "browser-use-docs-8", "browser-use-docs-15" ] }, { "productId": "browser-use", "storyId": "task-completion-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Docs explicitly describe polling patterns for run status (runs.events() with cursor, polling ordered V4 events) but no webhook or callback-based notification mechanism is mentioned anywhere in the evidence pack.", "evidenceIds": [ "browser-use-docs-5", "browser-use-docs-14" ] }, { "productId": "browser-use", "storyId": "usage-pricing-transparency", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Pricing page shows credits-based model ($5+ credits, no subscription, one-time $15 signup credit) but there is no documented per-task or per-browser-hour cost breakdown, and no documented rate/concurrency limits anywhere in the evidence. missing for 10: explicit per-task/per-browser-hour cost figures, documented rate limits, documented concurrency limits, any independent confirmation of pricing transparency.", "evidenceIds": [ "browser-use-docs-13", "browser-use-docs-17" ] }, { "productId": "browser-use", "storyId": "vision-computer-use", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Browser Use's docs describe DOM-based agent actions, live preview/human handoff for CAPTCHA/2FA, and CDP-based control, but there is no evidence of a dedicated vision or computer-use mode that acts directly on screenshots for canvas/UI elements the DOM can't reach.", "evidenceIds": [] }, { "productId": "browser-use", "storyId": "workflow-loops-conditionals", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Browser Use runs are essentially single natural-language task strings within a session/run model (create run, poll events, reuse session) with no documented constructs for loops, conditionals, or parameterized workflow templates; the Python library is described as scriptable but no workflow-composition API (branching, iteration, variables) is shown.", "evidenceIds": [ "browser-use-docs-1", "browser-use-docs-4", "browser-use-docs-5", "browser-use-gh-3" ] }, { "productId": "hyperbrowser", "storyId": "agent-framework-adapters", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows generic SDKs, an MCP server, and built-in support for AI models (Claude, OpenAI, Gemini, Grok, BrowserUse) plus Playwright/Puppeteer/CDP session control, but there is no documentation of adapters specifically for Claude Agent SDK, Vercel AI SDK, LangChain, or CrewAI framework integration as the story requires.", "evidenceIds": [ "hyperbrowser-docs-18", "hyperbrowser-docs-21", "hyperbrowser-docs-8", "hyperbrowser-docs-10" ] }, { "productId": "hyperbrowser", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Probe confirms a live, well-formed llms.txt at hyperbrowser.ai/llms.txt (HTTP 200) describing the product for agent consumption, and docs additionally expose an official MCP server that agents can connect to for structured access. missing for 10: no docs.md/full agent-oriented docs mirror (docs.md 404) and no OpenAPI spec discoverable, so agent-friendly docs coverage beyond llms.txt is incomplete.", "evidenceIds": [ "hyperbrowser-probe-1", "hyperbrowser-probe-2", "hyperbrowser-probe-3", "hyperbrowser-docs-8", "hyperbrowser-probe-4" ] }, { "productId": "hyperbrowser", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "medium", "rationale": "Hyperbrowser is a browser-automation/scraping infrastructure API (cloud sessions, agents, extraction, sandboxes) — it has no user-facing dashboard or analytics surface for a user's own data; its AI features (extract, page.ai) operate on scraped web content, not on the user's in-product data, so the 'insights from my data inside the product' axis is a category mismatch for this kind of tool.", "evidenceIds": [] }, { "productId": "hyperbrowser", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Hyperbrowser supports triggering agent tasks via API and cloud sessions, which could underlie automations, but there is no evidence of scheduling, triggers, or persistent background jobs that run autonomously without a caller invoking them each time. missing for 10: scheduled/triggered automation, background job orchestration, evidence of persistent autonomous runs, independent confirmation of unattended operation.", "evidenceIds": [ "hyperbrowser-docs-1", "hyperbrowser-docs-11", "hyperbrowser-docs-19", "hyperbrowser-docs-9" ] }, { "productId": "hyperbrowser", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Hyperbrowser ships built-in AI agents (HyperAgent, page.ai(), browser agents) that let users delegate natural-language tasks like 'search for flights and select the cheapest option' with a single API call, and the agent executes multi-step browser actions autonomously. This is a first-party, well-documented delegation capability, not just a passive tool.\nmissing for 10: independent hands-on evidence validating task delegation reliability/quality beyond vendor docs, and no in-product chat UI shown (delegation is via API/SDK only).", "evidenceIds": [ "hyperbrowser-docs-1", "hyperbrowser-docs-2", "hyperbrowser-docs-11", "hyperbrowser-docs-17", "hyperbrowser-docs-21" ] }, { "productId": "hyperbrowser", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Hyperbrowser is fundamentally a headless, API/SDK/CLI-driven cloud browser service — sessions run in the cloud with no local browser needed, exposing WebSocket/CDP endpoints for Playwright/Puppeteer plus official SDKs and a CLI (`hx`), which is exactly the shape needed for CI/automation pipelines. Missing for 10: explicit CI pipeline examples (e.g., GitHub Actions docs) and independent hands-on confirmation of CI usage.", "evidenceIds": [ "hyperbrowser-docs-10", "hyperbrowser-docs-19", "hyperbrowser-docs-18", "hyperbrowser-docs-29", "hyperbrowser-docs-9", "hyperbrowser-docs-15", "hyperbrowser-docs-16" ] }, { "productId": "hyperbrowser", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Hyperbrowser documents itself as an MCP *server* that exposes its own scraping/automation tools to AI models (hyperbrowser-docs-8, hyperbrowser-gh-1, hyperbrowser-probe-4), but this is the opposite of the story, which asks whether Hyperbrowser (or its HyperAgent/browser-agent features) can act as an MCP *client* and plug in external MCP servers to use their tools. No evidence describes Hyperbrowser consuming or connecting to third-party MCP servers.", "evidenceIds": [ "hyperbrowser-docs-8", "hyperbrowser-gh-1", "hyperbrowser-probe-4" ] }, { "productId": "hyperbrowser", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Hyperbrowser ships an official MCP server (hyperbrowser.ai/docs/integrations/model-context-protocol) exposing tools like scrape_webpage, extraction, and crawling for AI agents to connect to, backed by a public GitHub repo listing the tool set. Missing for 10: independent/hands-on confirmation of the MCP server working with a real agent client and more detail on the full tool list/setup.", "evidenceIds": [ "hyperbrowser-docs-8", "hyperbrowser-gh-1", "hyperbrowser-probe-4" ] }, { "productId": "hyperbrowser", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Hyperbrowser explicitly supports natural-language commands via page.ai() and HyperAgent ('Write natural language commands instead of complex selectors'), plus natural-language driven extraction (page.extract()/Extract API) and an MCP server exposing these to AI models. missing for 10: independent hands-on verification that NL commands reliably succeed across varied tasks, and broader third-party corroboration beyond vendor docs.", "evidenceIds": [ "hyperbrowser-docs-2", "hyperbrowser-docs-23", "hyperbrowser-docs-3", "hyperbrowser-docs-8" ] }, { "productId": "hyperbrowser", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs mention an official CLI called `hx` for setting up and working with sandboxes, confirming a first-party CLI exists, but the evidence pack gives no further detail on its command set, installation, or broader agentic workflow support beyond sandboxes. missing for 10: fuller CLI documentation (commands, usage examples), independent/hands-on confirmation, and evidence it covers more than just sandbox setup.", "evidenceIds": [ "hyperbrowser-docs-29" ] }, { "productId": "hyperbrowser", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Hyperbrowser exposes a fully documented REST API (Scrape, Crawl, Extract, Sessions, Agents) plus official SDKs and a CLI, all covered in detailed first-party docs, and community comments confirm real developers have used the API/docs (even correcting a code sample). Missing for 10: a discoverable OpenAPI/swagger spec (probe found only 404s) and independent third-party validation of full API completeness.", "evidenceIds": [ "hyperbrowser-docs-15", "hyperbrowser-docs-16", "hyperbrowser-docs-22", "hyperbrowser-docs-18", "hyperbrowser-docs-19", "hyperbrowser-docs-10", "hyperbrowser-probe-1", "hyperbrowser-probe-3", "hyperbrowser-comm-6" ] }, { "productId": "hyperbrowser", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of scoped or least-privilege API key/credential management for agents — only mentions of a generic 'credits' usage-tracking system, not permission-scoped API keys or role-based access controls. missing for 10: any API key scoping/permission model, docs on creating restricted-scope credentials, or agent-specific credential issuance.", "evidenceIds": [ "hyperbrowser-docs-27" ] }, { "productId": "hyperbrowser", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Hyperbrowser documents official SDKs (hyperbrowser-docs-18) alongside a dedicated SDK introduction page, plus consistent references throughout docs to using SDKs (or Puppeteer/Playwright) to integrate programmatically (hyperbrowser-docs-19, hyperbrowser-docs-9, hyperbrowser-docs-29). Missing for 10: independent/hands-on developer confirmation of SDK quality or completeness across languages, and no direct OpenAPI spec found (probe-3) which would reinforce SDK-generation credibility.", "evidenceIds": [ "hyperbrowser-docs-18", "hyperbrowser-docs-19", "hyperbrowser-docs-9", "hyperbrowser-docs-29" ] }, { "productId": "hyperbrowser", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions webhooks or event subscription mechanisms; the product docs cover sessions, agents, scraping, extraction, MCP, and sandboxes but nothing about webhook notifications for events like task completion or crawl status. missing for 10: any documentation of webhook subscription/callback endpoints, event types, or webhook configuration.", "evidenceIds": [] }, { "productId": "hyperbrowser", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Hyperbrowser's docs include an api-reference section with code snippets (e.g., fetch-a-web-page.md) and quickstart examples like page.ai()/page.extract(), showing static example code, but there's no evidence of an interactive, runnable API console — probes for openapi/swagger specs all returned 404s, indicating no auto-generated interactive reference. missing for 10: evidence of an actual interactive/try-it API explorer, runnable code sandboxes, or OpenAPI-based interactive docs.", "evidenceIds": [ "hyperbrowser-docs-28", "hyperbrowser-docs-2", "hyperbrowser-docs-3", "hyperbrowser-probe-3" ] }, { "productId": "hyperbrowser", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe for an OpenAPI/Swagger spec at all standard locations (openapi.json, swagger.json, etc.) returned 404s, and no docs page claims a downloadable machine-readable API spec exists — only human-readable API reference pages are present.", "evidenceIds": [ "hyperbrowser-probe-3", "hyperbrowser-probe-2", "hyperbrowser-docs-28" ] }, { "productId": "hyperbrowser", "storyId": "api-sandbox", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Hyperbrowser ships dedicated 'Sandboxes' (isolated exec environments with sub-50ms startup) and isolated cloud browser 'Sessions' that are separate from any user production environment, which aligns with testing in isolation. However, the docs never explicitly frame this as protecting 'production data' or describe production/staging separation guarantees, and there is no independent/hands-on confirmation of isolation guarantees. Missing for 10: explicit production-vs-sandbox data isolation claims, independent verification of sandbox isolation, and security/compliance detail on data leakage prevention.", "evidenceIds": [ "hyperbrowser-docs-25", "hyperbrowser-docs-26", "hyperbrowser-docs-29", "hyperbrowser-docs-10" ] }, { "productId": "hyperbrowser", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of API versioning scheme or documented deprecation policy; OpenAPI/spec probes returned 404 and no changelog or versioning docs appear in the pack. missing for 10: versioned API endpoints/headers, changelog, explicit deprecation policy documentation.", "evidenceIds": [ "hyperbrowser-probe-3", "hyperbrowser-probe-2" ] }, { "productId": "hyperbrowser", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Hyperbrowser's Crawl API can traverse and extract from many pages in one request, and the Extract/Scrape APIs support structured data pulls, with community evidence of higher concurrency limits enabling parallel bulk sessions. However there's no explicit documented 'batch job over an arbitrary list of items/URLs' endpoint, and no first-party proof of large-scale bulk-run examples or rate/queue management for bulk workloads.\nmissing for 10: explicit bulk/batch API for arbitrary item lists, documented large-scale throughput examples, independent benchmarks of bulk operation reliability.", "evidenceIds": [ "hyperbrowser-docs-16", "hyperbrowser-docs-15", "hyperbrowser-docs-22", "hyperbrowser-comm-1" ] }, { "productId": "hyperbrowser", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Hyperbrowser provides on-demand browser automation, scraping, extraction, and agent APIs, but there is no evidence of an event-driven rules/trigger engine (e.g., webhooks on page changes, scheduled triggers, condition-based automation) that would let a user define 'if X happens, do Y' automation. All capabilities described are invoked synchronously via API calls, not autonomously triggered by events. Missing for 10: rule/trigger definition mechanism, event listeners or webhooks, scheduling/condition-based automation execution.", "evidenceIds": [] }, { "productId": "hyperbrowser", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack of scheduling, cron-like recurring jobs, or persistent workflow automation triggers; Hyperbrowser's docs cover on-demand sessions, agents, scraping, extraction, sandboxes and MCP integration but nothing about recurring/scheduled execution. Missing for 10: any scheduler/cron feature, recurring job API, or workflow trigger documentation.", "evidenceIds": [] }, { "productId": "hyperbrowser", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Hyperbrowser is a cloud browser/automation infrastructure product with sessions, recordings, and agents, but there is no evidence of version control, review workflows, or rollback capabilities for automations themselves. Session recordings enable debugging replay but not versioning/review/rollback of automation scripts or flows. Missing for 10: any versioning system, diff/review interface, or rollback mechanism for automations.", "evidenceIds": [] }, { "productId": "hyperbrowser", "storyId": "captcha-handling", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Hyperbrowser explicitly documents an automatic CAPTCHA-solving stance (enabled at session creation) alongside stealth mode for anti-bot detection, giving automation engineers a documented policy rather than silent failures. However, a hands-on community report describes being 'instantly blocked by cloudflare' in a real automation use case, contradicting the claimed automatic bypass/solving reliability in practice. missing for 10: independent verification that CAPTCHA solving works reliably across real sites, clarification of behavior when solving fails (human fallback vs silent failure), and resolution of the Cloudflare-blocking report.", "evidenceIds": [ "hyperbrowser-docs-5", "hyperbrowser-docs-4", "hyperbrowser-comm-3" ] }, { "productId": "hyperbrowser", "storyId": "cdp-connect-existing-stack", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly state sessions expose a WebSocket endpoint compatible with Playwright, Puppeteer, or any CDP-compatible tool, and the intro reiterates 'Control Chrome browsers in the cloud using Puppeteer, Playwright, or our SDKs.' This directly supports connecting existing automation code without rewriting it, plus profiles for persisting cookies/session state across sessions. Missing for 10: independent/hands-on developer confirmation of drop-in compatibility with existing large codebases and any noted friction points.", "evidenceIds": [ "hyperbrowser-docs-10", "hyperbrowser-docs-19", "hyperbrowser-docs-9", "hyperbrowser-docs-6" ] }, { "productId": "hyperbrowser", "storyId": "credential-vault-2fa", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows session/profile persistence (cookies, storage) and stealth/captcha features, but nothing about a credentials vault, secret injection into login forms, or TOTP/2FA handling without exposing secrets to the model.", "evidenceIds": [ "hyperbrowser-docs-6" ] }, { "productId": "hyperbrowser", "storyId": "deterministic-replay-caching", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any caching mechanism for resolved actions or generated code; Hyperbrowser's docs cover agents, extraction, sessions, stealth, proxies, and sandboxes but nothing about replaying deterministic cached actions to avoid re-prompting the LLM.", "evidenceIds": [] }, { "productId": "hyperbrowser", "storyId": "dom-action-primitives", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "HyperAgent's page.ai() lets developers issue natural-language commands (e.g., 'search for flights... select the cheapest option') instead of writing selectors, and docs explicitly say it lets you 'write natural language commands instead of complex selectors,' matching the DOM-understanding action-primitive story. However, evidence shows a single high-level task-execution call rather than discrete act/click/type primitives, and there's no independent/hands-on evidence confirming resilience to selector or layout changes. Missing for 10: granular primitive-level API docs (separate click/type/act calls), independent verification of survival across DOM/layout changes, and community confirmation of reliability.", "evidenceIds": [ "hyperbrowser-docs-2", "hyperbrowser-docs-17", "hyperbrowser-docs-23" ] }, { "productId": "hyperbrowser", "storyId": "ethics-abuse-guardrails", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Hyperbrowser doesn't publish a clear acceptable-use/anti-abuse policy in its docs, and when community members pressed on robots.txt/rate-limit compliance, the company added an 'ethical scraping' page reactively — but a hands-on tester found it still doesn't actually support respecting robots.txt or limiting request rate, and Hyperbrowser's own docs promote circumventing bot-detection countermeasures (stealth/CAPTCHA-solving) rather than governing their limits. Missing for 10: an actual published acceptable-use policy document, terms defining permitted use of stealth/CAPTCHA features, and evidence of enforcement matching any stated ethical posture.", "evidenceIds": [ "hyperbrowser-comm-7", "hyperbrowser-comm-8", "hyperbrowser-comm-9", "hyperbrowser-docs-4", "hyperbrowser-docs-5", "hyperbrowser-docs-20" ] }, { "productId": "hyperbrowser", "storyId": "file-download-upload", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack details scraping, extraction, session recording, and sandbox exec, but nowhere mentions file download/upload handling on target sites or artifact retrieval mechanisms for such files. Since Hyperbrowser exposes raw Playwright/Puppeteer CDP endpoints, this axis is plausible for the product category, but no documentation or community evidence confirms the capability.", "evidenceIds": [ "hyperbrowser-docs-10", "hyperbrowser-docs-19", "hyperbrowser-docs-7" ] }, { "productId": "hyperbrowser", "storyId": "hosted-task-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Hyperbrowser's docs confirm a hosted HTTP API where AI agents can start browser tasks with a single API call and run them in managed cloud sessions (agents, scrape, crawl, extract), removing the need to manage a browser directly. However, the evidence pack never explicitly documents a polling endpoint or webhook callback mechanism for retrieving task results—only 'watch them execute live' and SDK/CDP connection details are mentioned. Missing for 10: explicit API documentation of a status/poll endpoint, explicit webhook/callback support, and independent confirmation of async result retrieval flow.", "evidenceIds": [ "hyperbrowser-docs-1", "hyperbrowser-docs-11", "hyperbrowser-docs-19", "hyperbrowser-docs-9", "hyperbrowser-probe-1" ] }, { "productId": "hyperbrowser", "storyId": "live-view-human-takeover", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Hyperbrowser docs confirm you can 'watch tasks execute live' and supports session recordings (rrweb/video) for later replay/debugging, but there is no evidence of a mechanism for a human to take control mid-run and intervene when the agent gets stuck — no live handoff, pause/resume, or manual takeover feature is documented. Missing for 10: documented live human takeover/intervention controls during an active agent run, and independent confirmation this works in practice.", "evidenceIds": [ "hyperbrowser-docs-1", "hyperbrowser-docs-11", "hyperbrowser-docs-7", "hyperbrowser-docs-14" ] }, { "productId": "hyperbrowser", "storyId": "local-browser-mode", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "All evidence describes Hyperbrowser as a managed cloud-browser service — sessions are explicitly cloud-hosted with WebSocket/CDP endpoints, and control is via cloud sessions requiring credits/subscription, not local execution without a cloud account. No evidence of a local-browser dev mode or offline/no-account option exists.", "evidenceIds": [ "hyperbrowser-docs-10", "hyperbrowser-docs-19", "hyperbrowser-docs-27" ] }, { "productId": "hyperbrowser", "storyId": "login-session-persistence", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs explicitly describe Profiles that persist cookies, local storage, session storage, and cache across multiple sessions, directly enabling agents to skip login walls on repeat runs. Missing for 10: no independent/hands-on confirmation that profile reuse actually bypasses login in practice, and no detail on profile lifecycle/management limits.", "evidenceIds": [ "hyperbrowser-docs-6" ] }, { "productId": "hyperbrowser", "storyId": "model-agnostic-llm", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Hyperbrowser's docs explicitly state 'Built in support for Claude, OpenAI, Gemini, Grok, and BrowserUse agents,' indicating the agent framework is not locked to a single LLM vendor and can work with multiple providers. Missing for 10: independent/community confirmation of multi-provider use, and detailed docs on how to configure/swap providers or bring custom API keys.", "evidenceIds": [ "hyperbrowser-docs-21", "hyperbrowser-docs-17", "hyperbrowser-docs-23" ] }, { "productId": "hyperbrowser", "storyId": "nl-task-to-completion", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs show HyperAgent's page.ai() executing natural-language multi-step goals (e.g., searching flights and selecting cheapest option) and a separate managed Agents API that starts autonomous browser-agent tasks with a single call, supporting navigation, clicking, and form interactions end-to-end. Missing for 10: independent/hands-on verification of complex multi-step task completion and success-rate data beyond first-party docs.", "evidenceIds": [ "hyperbrowser-docs-2", "hyperbrowser-docs-1", "hyperbrowser-docs-11", "hyperbrowser-docs-17", "hyperbrowser-docs-23", "hyperbrowser-docs-21" ] }, { "productId": "hyperbrowser", "storyId": "observe-suggest-actions", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Hyperbrowser's agent APIs (page.ai(), HyperAgent, MCP tools) execute actions directly via natural-language commands, and extract()/scrape() only pull data — there is no documented observe/plan primitive that lets a developer preview candidate actions before the agent commits to acting. missing for 10: an explicit plan/observe API or dry-run mode, evidence of returning candidate actions for approval, any UI or SDK method to inspect proposed steps prior to execution.", "evidenceIds": [ "hyperbrowser-docs-2", "hyperbrowser-docs-17", "hyperbrowser-docs-23", "hyperbrowser-docs-3" ] }, { "productId": "hyperbrowser", "storyId": "openness-api-parity", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Hyperbrowser is API-first: sessions, scraping, crawling, extraction, agents, sandboxes, profiles, recordings, stealth/CAPTCHA solving are all exposed via API/SDK/CLI, and the dashboard is largely a viewer over the same session/agent primitives rather than a separate feature surface. missing for 10: no public OpenAPI spec was found (404s on probes), and no explicit statement confirming every dashboard-only setting (e.g., billing/team management) is API-controllable.", "evidenceIds": [ "hyperbrowser-docs-1", "hyperbrowser-docs-10", "hyperbrowser-docs-15", "hyperbrowser-docs-16", "hyperbrowser-docs-22", "hyperbrowser-docs-25", "hyperbrowser-docs-29", "hyperbrowser-probe-3" ] }, { "productId": "hyperbrowser", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of a data export/portability feature for user data (sessions, profiles, recordings, extracted data) in open formats, nor any account-deletion/data-takeout mechanism; profiles and recordings are stored server-side with no documented export path.", "evidenceIds": [] }, { "productId": "hyperbrowser", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Hyperbrowser's core cloud service (sessions, scraping, sandboxes, MCP server) is presented purely as a closed commercial SaaS API/product with no license or public repo for the platform itself. The only open-source item mentioned, HyperAgent, is a separate client-side automation framework, not the Hyperbrowser product's own source code, so it doesn't satisfy this story.", "evidenceIds": [ "hyperbrowser-docs-17", "hyperbrowser-docs-23", "hyperbrowser-docs-27" ] }, { "productId": "hyperbrowser", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Hyperbrowser is presented entirely as a managed cloud service (cloud browser sessions, sandboxes, MCP server) with no evidence of a self-hostable core product, open-source repo of the main platform, or on-prem deployment option; only HyperAgent (a client-side automation library) is open source, not the underlying cloud infrastructure. missing for 10: any docs or repo for self-hosting the core browser/session infrastructure, deployment/Docker instructions, or license terms permitting self-hosting.", "evidenceIds": [ "hyperbrowser-docs-19", "hyperbrowser-docs-25", "hyperbrowser-docs-17" ] }, { "productId": "hyperbrowser", "storyId": "parallel-fleet-scale", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Hyperbrowser clearly supports programmatic, isolated cloud browser sessions controllable via Playwright/Puppeteer/SDKs (hyperbrowser-docs-10, hyperbrowser-docs-19), and community feedback indicates it offers more concurrency than competitors like Browserbase (hyperbrowser-comm-1), implying tiered concurrency limits exist. However, the evidence pack contains no first-party documentation page explicitly listing numeric concurrency limits per plan or fleet-scale orchestration guidance. Missing for 10: explicit documented concurrency-limit numbers/tiers, guidance on running many sessions in parallel, and independent verification of limits in practice.", "evidenceIds": [ "hyperbrowser-docs-10", "hyperbrowser-docs-19", "hyperbrowser-comm-1", "hyperbrowser-docs-27" ] }, { "productId": "hyperbrowser", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, region selection, or geographic storage location for Hyperbrowser sessions/data; proxy/geo-routing docs relate to traffic egress, not data storage location. Missing for 10: any mention of data residency options, region selection controls, or storage location guarantees.", "evidenceIds": [] }, { "productId": "hyperbrowser", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Hyperbrowser is a browser automation/scraping infrastructure product; no evidence anywhere in the pack addresses a data-training opt-out or any AI-model training data policy. Even the community threads about ethical scraping/robots.txt concern outbound scraping behavior, not whether Hyperbrowser uses customer data to train models. Absence of evidence for this applicable privacy-posture axis means none.", "evidenceIds": [] }, { "productId": "hyperbrowser", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack content addresses data retention policies, deletion controls, or privacy settings for stored session data, recordings, or profiles; community evidence even suggests scraping is done without regard to opt-out mechanisms like robots.txt, but this doesn't speak to retention/deletion controls directly. This is a fair axis for a cloud browser automation service handling user data (recordings, profiles, scraped data), so absence of evidence yields 'none' rather than 'na'.", "evidenceIds": [ "hyperbrowser-docs-6", "hyperbrowser-docs-7", "hyperbrowser-comm-8", "hyperbrowser-comm-9" ] }, { "productId": "hyperbrowser", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of any telemetry opt-out setting or privacy control; the only related mention is credit-based usage tracking for billing, which is a business metric, not telemetry, and no docs describe disabling analytics/tracking.", "evidenceIds": [ "hyperbrowser-docs-27" ] }, { "productId": "hyperbrowser", "storyId": "schema-structured-extraction", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Hyperbrowser documents both an Extract API and page.extract() method that support schema-based extraction with Zod, explicitly for type-safe structured results from visited pages, and this is available both in standalone scrape/crawl calls and agent workflows (HyperAgent). Missing for 10: no independent/hands-on verification of schema-validation correctness or Pydantic-specific support (docs only mention Zod), and no example showing enforcement failure/error handling.", "evidenceIds": [ "hyperbrowser-docs-3", "hyperbrowser-docs-22", "hyperbrowser-docs-24", "hyperbrowser-docs-17" ] }, { "productId": "hyperbrowser", "storyId": "session-replay-recordings", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Hyperbrowser documents session recordings (rrweb DOM-level recordings and MP4 video) which support visual replay of a browser session, and MCP/scrape tools mention screenshot capture, but there is no evidence of a dedicated step-by-step action/timeline log for agent runs or a debugging UI tied to failed runs specifically. missing for 10: explicit step-by-step action timeline artifact, agent-run failure/debug UI, independent hands-on confirmation of replay-based debugging workflow.", "evidenceIds": [ "hyperbrowser-docs-7", "hyperbrowser-docs-1", "hyperbrowser-gh-1" ] }, { "productId": "hyperbrowser", "storyId": "stealth-fingerprint-proxies", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Hyperbrowser explicitly documents Stealth Mode, an 'Ultra Stealth Mode', CAPTCHA solving, and proxy routing for geo-targeting/IP rotation, directly matching the story's anti-bot and geo-targeted proxy needs (hyperbrowser-docs-4, -12, -20, -5, -13). However, a first-hand community report describes being 'instantly blocked by cloudflare' despite using the service for an automation use case, and another user documents the scraper still exposing bot-like UA/IP patterns detectable by target sites, concretely contradicting the anti-detection claims in practice (hyperbrowser-comm-3, hyperbrowser-comm-8). missing for 10: independent benchmark showing stealth/proxy reliably evades modern bot defenses, explicit confirmation of 'residential' proxy type versus generic rotating proxies, and resolution of the Cloudflare-block report.", "evidenceIds": [ "hyperbrowser-docs-4", "hyperbrowser-docs-12", "hyperbrowser-docs-20", "hyperbrowser-docs-5", "hyperbrowser-docs-13", "hyperbrowser-comm-3", "hyperbrowser-comm-8" ] }, { "productId": "hyperbrowser", "storyId": "task-completion-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack mentions webhooks, callbacks, or push notifications for task/session completion; all documented status mechanisms appear to be live/polling-based (watch execution live, sessions, recordings). Absence of evidence for this applicable capability yields none.", "evidenceIds": [] }, { "productId": "hyperbrowser", "storyId": "usage-pricing-transparency", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Hyperbrowser docs mention a credit-based pricing model (hyperbrowser-docs-27) but provide no explicit per-task/per-browser-hour cost table or concurrency/rate-limit documentation in the evidence pack. Community feedback directly contradicts the transparency claim: users report confusion over credit costs versus competitors (hyperbrowser-comm-2) and one user states it appears 'not possible to follow robots.txt or limit RPS as a user of this service' (hyperbrowser-comm-9), indicating undocumented or absent rate-limiting behavior. missing for 10: a concrete pricing/rate-limit page with per-hour or per-task cost breakdown, explicit concurrency caps, and independent confirmation that documented limits match real usage.", "evidenceIds": [ "hyperbrowser-docs-27", "hyperbrowser-comm-2", "hyperbrowser-comm-9", "hyperbrowser-comm-10" ] }, { "productId": "hyperbrowser", "storyId": "vision-computer-use", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Hyperbrowser documents 'built-in support for Claude, OpenAI, Gemini, Grok, and BrowserUse agents' (model-native agents) and can return screenshots via fetch/scrape APIs, hinting at vision-capable agent modes, but there is no explicit documentation of a dedicated 'vision' or 'computer-use' action mode that operates purely on screenshots for canvases/UIs the DOM can't reach. Missing for 10: explicit computer-use/vision mode docs, screenshot-based click/type action primitives, and any example of handling canvas/non-DOM UI elements.", "evidenceIds": [ "hyperbrowser-docs-21", "hyperbrowser-docs-28", "hyperbrowser-gh-1" ] }, { "productId": "hyperbrowser", "storyId": "workflow-loops-conditionals", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Hyperbrowser's docs describe single-call agent tasks, scraping/extraction, and Playwright/Puppeteer session control, but nothing describes a workflow builder or orchestration layer with loops, conditionals, or parameterized multi-step flows—automation is either one-shot AI prompts (page.ai, Extract API) or raw code you write yourself against Playwright, not a documented workflow composition feature.", "evidenceIds": [ "hyperbrowser-docs-2", "hyperbrowser-docs-17", "hyperbrowser-docs-23", "hyperbrowser-docs-10" ] }, { "productId": "notte", "storyId": "agent-framework-adapters", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Notte documents explicit integration guides for CrewAI (notte-docs-32), Vercel AI SDK (notte-docs-33), OpenAI CUA (notte-docs-34), and Claude-based agents via CLI/MCP (notte-docs-9, notte-docs-31), all pointing at its MCP server or CLI as the browser layer plug-in point. However, there is no documented LangChain adapter and no mention of 'Claude Agent SDK' specifically (only Claude Code/managed agents), so the story's exact framework list is only partially covered. Missing for 10: a LangChain-specific integration doc, explicit Claude Agent SDK adapter naming, and independent confirmation these adapters work hands-on.", "evidenceIds": [ "notte-docs-32", "notte-docs-33", "notte-docs-34", "notte-docs-9", "notte-docs-31", "notte-docs-7", "notte-probe-3" ] }, { "productId": "notte", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Notte serves a verified llms.txt at docs.notte.cc/llms.txt with explicit 'Critical Instructions For AI Agents' directing agents to further docs, plus extensive agent-oriented docs (MCP server, CLI, integration guides for CrewAI, Vercel AI SDK, Claude Code) confirming a mature agentic-docs ecosystem. Missing for 10: independent third-party confirmation that agents actually consume llms.txt successfully in practice.", "evidenceIds": [ "notte-probe-1", "notte-docs-7", "notte-docs-29", "notte-docs-32", "notte-docs-33", "notte-docs-9" ] }, { "productId": "notte", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Notte's evidence covers AI-driven data extraction, browser agents, and automation infrastructure, but nothing shows the product itself analyzing a user's own data to proactively surface insights or suggestions inside a dashboard/UI — extraction is task-driven ('extract the last 10 messages'), not autonomous insight generation. Missing for 10: any documented insights/analytics dashboard, proactive suggestion feature, or evidence of the product surfacing patterns/recommendations from a user's stored data.", "evidenceIds": [ "notte-docs-3", "notte-docs-15", "notte-docs-25", "notte-docs-16" ] }, { "productId": "notte", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Notte supports deploying scripts/agents as serverless 'Functions' invocable via API and schedulable, plus persistent sessions, credential vaults, and cloud browser infrastructure that let automations run unattended in the background (notte-docs-2, notte-docs-26, notte-docs-24, notte-docs-4). This directly matches autonomous background automation for an AI-native user. Missing for 10: no independent/hands-on confirmation of scheduling reliability in production, and community feedback includes at least one report of a failed extraction task (notte-comm-1), so real-world robustness is unverified.", "evidenceIds": [ "notte-docs-2", "notte-docs-26", "notte-docs-24", "notte-docs-4", "notte-docs-5", "notte-comm-1" ] }, { "productId": "notte", "storyId": "agentic-builtin-assistant", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Notte's core offering is an AI web agent that accepts natural-language task descriptions and executes them autonomously on websites (notte-gh-1, notte-docs-18, notte-docs-37), which functions as a built-in AI assistant a user delegates tasks to. However, a hands-on community report describes a concrete failure ('tried to use it to extract data from hyatt.com completely failed... another hype but actually doesn't work browser agent' — notte-comm-1), and the founder himself admits only ~60% reliability on captcha-gated sites (notte-comm-7), directly contradicting the polished 'describe a task, watch it happen' framing. Missing for 10: independent verification of consistent task success, a true conversational/chat-based assistant UI (rather than API/CLI-driven task submission), and resolution of the documented failure case.", "evidenceIds": [ "notte-gh-1", "notte-docs-18", "notte-docs-37", "notte-comm-1", "notte-comm-7" ] }, { "productId": "notte", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Notte offers cloud/remote browser sessions, an API with Bearer token auth, serverless 'Functions' deployments that can be scheduled, and a CLI/SDK — all designed for headless, programmatic, CI-friendly automation without a local browser or UI. Community evidence corroborates real usage of the API/agent stack though with mixed reliability reports on task success. Missing for 10: no explicit CI pipeline example (e.g., GitHub Actions), no independent benchmark confirming headless stability at scale.", "evidenceIds": [ "notte-docs-2", "notte-docs-17", "notte-docs-24", "notte-docs-26", "notte-docs-30", "notte-probe-4", "notte-comm-1" ] }, { "productId": "notte", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All MCP-related evidence describes Notte exposing its own MCP server for external agents (Claude, CrewAI, Vercel AI SDK) to plug into and control Notte's browser tools — the reverse direction of this story. There is no evidence that Notte's own agents can consume or plug in external MCP servers to gain new tools.", "evidenceIds": [ "notte-docs-7", "notte-docs-29", "notte-docs-32", "notte-docs-33", "notte-probe-3" ] }, { "productId": "notte", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Notte publishes a dedicated official MCP server (docs.notte.cc/mcp-server) that gives agents access to cloud browser sessions, page interaction, and data extraction, and this is corroborated across multiple integration guides (CrewAI, Vercel AI SDK) showing agents pointed at the Notte MCP server to get a real browser. Missing for 10: independent/hands-on third-party confirmation that the MCP server works reliably in practice (community evidence only covers the general product, not MCP specifically).", "evidenceIds": [ "notte-docs-7", "notte-docs-29", "notte-docs-32", "notte-docs-33", "notte-probe-3" ] }, { "productId": "notte", "storyId": "agentic-nl-commands", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Notte's core value prop is natural-language task execution ('Describe a task. Watch it happen. One prompt', 'Give AI agents natural language tasks to complete on websites') backed by agent-start API and CLI/MCP integrations, so the capability is clearly built and documented. However, a hands-on community report describes a complete failure when trying to extract data from hyatt.com via the agent, and the founder himself admits captcha/anti-bot handling only works ~60% of the time, concretely undercutting reliability of the NL-driven approach. Missing for 10: independent reproducible success cases beyond vendor demos, and resolution of the documented hyatt.com failure.", "evidenceIds": [ "notte-docs-18", "notte-gh-1", "notte-docs-37", "notte-comm-1", "notte-comm-7" ] }, { "productId": "notte", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Notte documents an official CLI used to generate config, drive browsers, and produce observe/page output (notte-docs-30, notte-docs-31, notte-docs-9, notte-docs-8, notte-probe-4), explicitly positioned for AI agents to control browsers via shell commands. missing for 10: no independent/community hands-on validation of the CLI specifically (only vendor docs), and no detailed CLI command reference beyond scattered mentions.", "evidenceIds": [ "notte-docs-9", "notte-docs-30", "notte-docs-31", "notte-docs-8", "notte-probe-4" ] }, { "productId": "notte", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Notte documents a full public API with Bearer-token authentication, API reference pages (e.g., agent-start endpoint), SDKs, CLI, and MCP server for programmatic/agentic control of sessions, scraping, and functions, matching the ai-native/agenticness story well. missing for 10: no discoverable OpenAPI/swagger spec (404s on probe) and no independent third-party confirmation of API robustness beyond vendor docs.", "evidenceIds": [ "notte-docs-17", "notte-docs-37", "notte-docs-24", "notte-docs-26", "notte-docs-7", "notte-probe-3", "notte-probe-2", "notte-docs-30" ] }, { "productId": "notte", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only a single Bearer-token API authentication scheme (notte-docs-17) and a vault feature that hides stored web credentials from the LLM (notte-docs-14, notte-docs-27), but nothing about issuing scoped, role-limited, or least-privilege API keys specifically for agents. No mention of API key scopes, permission tiers, or per-agent credential restrictions.", "evidenceIds": [ "notte-docs-17", "notte-docs-14", "notte-docs-27" ] }, { "productId": "notte", "storyId": "agentic-sdks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Notte provides a documented REST API with Bearer-token auth, an official CLI, and an MCP server plus integration guides for frameworks like CrewAI, Vercel AI SDK, and OpenAI CUA, all of which support building AI-native automations programmatically. However, no evidence explicitly names or documents a first-party 'SDK' package (e.g., Python/TypeScript client library) and an OpenAPI spec probe returned 404s, suggesting the API surface may not be as formally packaged as a dedicated SDK. missing for 10: explicit official SDK package docs (Python/JS), a working OpenAPI/schema reference, independent developer confirmation of SDK usage.", "evidenceIds": [ "notte-docs-17", "notte-docs-30", "notte-probe-4", "notte-docs-7", "notte-probe-3", "notte-docs-32", "notte-docs-33", "notte-docs-34", "notte-probe-2" ] }, { "productId": "notte", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions webhooks or event subscription mechanisms; the product offers sessions, MCP, CLI, and API endpoints but nothing about push notifications or webhook callbacks. Missing for 10: any documentation of webhook subscription, event types, or delivery mechanism.", "evidenceIds": [] }, { "productId": "notte", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only static API-reference pages (authentication, agent-start) and no OpenAPI/Swagger spec was found at any candidate path (probe-2 returned 404s), and nothing in the pack describes an interactive console or runnable code examples in the API docs.", "evidenceIds": [ "notte-docs-17", "notte-docs-37", "notte-probe-2" ] }, { "productId": "notte", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence shows Notte has human-readable API reference docs (auth, agent-start) but a direct probe for machine-readable spec files (openapi.json, swagger.json, etc.) returned 404 on all candidate paths, indicating no downloadable OpenAPI or equivalent spec is published.", "evidenceIds": [ "notte-probe-2", "notte-docs-17", "notte-docs-37" ] }, { "productId": "notte", "storyId": "api-sandbox", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Notte documents 'isolated browser instances running in the cloud' for each session and a free trial ('Try the full platform without a card'), which implies some session-level isolation from a user's own systems, but there is no explicit sandbox/production-data separation mode, staging environment, or test-data guarantee described anywhere in the docs. missing for 10: explicit sandbox vs production distinction, test-data isolation guarantees, hands-on confirmation that sandbox sessions never touch real production data.", "evidenceIds": [ "notte-docs-24", "notte-docs-22", "notte-docs-35" ] }, { "productId": "notte", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "There is evidence of an API with bearer token auth, but nothing about API versioning or a documented deprecation policy; OpenAPI probes even 404, suggesting no formal spec surfaced. missing for 10: versioning scheme, deprecation policy documentation, changelog entries about breaking changes.", "evidenceIds": [ "notte-docs-17", "notte-probe-2" ] }, { "productId": "notte", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Notte offers scalable cloud browser sessions and serverless 'functions'/API endpoints that can be invoked and scheduled programmatically, which could in principle be used to run automation across many items, but there is no explicit documentation of a batch/bulk-processing feature (e.g., iterating over a list of URLs/records, parallel job queues, or bulk extraction outputs). missing for 10: explicit bulk/batch API or CLI examples, evidence of parallel multi-item execution, and any hands-on confirmation of running the same task across many inputs.", "evidenceIds": [ "notte-docs-1", "notte-docs-2", "notte-docs-26", "notte-docs-3" ] }, { "productId": "notte", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Notte supports scheduling serverless 'functions' (deploy scripts as API endpoints and 'schedule anywhere'), which gives some automation-trigger capability, but there is no evidence of a rules engine, webhooks, or event-based triggers (e.g., 'on page change, do X') as opposed to simple time-based scheduling/API invocation. Missing for 10: explicit event-trigger/webhook support, conditional rule definitions, and any UI/API for defining 'if event then action' automations.", "evidenceIds": [ "notte-docs-2", "notte-docs-26" ] }, { "productId": "notte", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Notte's 'Functions' feature explicitly advertises serverless automations that can be 'invoked and scheduled anywhere' (notte-docs-2, notte-docs-26), directly supporting recurring job scheduling, and workflows can be deployed as API endpoints for automation pipelines. However, there is no documentation of a scheduling UI, cron syntax, or interval/trigger configuration, and no independent confirmation that scheduled jobs work reliably in practice. Missing for 10: concrete scheduling mechanism/API docs (cron expressions, triggers), example of a recurring job configured end-to-end, and independent verification that scheduled runs execute reliably.", "evidenceIds": [ "notte-docs-2", "notte-docs-26", "notte-docs-35" ] }, { "productId": "notte", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of versioning, review workflows, diffs, or rollback capability for automations/workflows/functions; docs mention deploying, generating and editing scripts but nothing about version history or reverting changes.", "evidenceIds": [] }, { "productId": "notte", "storyId": "captcha-handling", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Notte's docs advertise 'captcha solving' as a built-in feature of its browser infrastructure (notte-docs-1), suggesting automatic handling, but there is no documented policy for what happens when solving fails (no human fallback or explicit non-support statement). Concrete contradicting evidence comes from the founder himself in community discussion, admitting captcha solving only works for ~60% of providers and some are 'still work in progress' (notte-comm-7), directly undercutting the blanket 'captcha solving' claim and leaving automation engineers without clarity on failure behavior. missing for 10: documented success-rate/coverage table, explicit fallback or escalation behavior on captcha failure, and independent verification of solve rates beyond the founder's informal comment.", "evidenceIds": [ "notte-docs-1", "notte-comm-7" ] }, { "productId": "notte", "storyId": "cdp-connect-existing-stack", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Notte sessions explicitly expose a CDP endpoint documented to work with Playwright (notte-docs-11), which directly supports connecting existing Playwright/CDP automation code to Notte's cloud browsers rather than rewriting it. Puppeteer isn't explicitly named but CDP is the shared protocol underlying it, and sessions are described as programmatically controllable cloud instances (notte-docs-24). missing for 10: explicit Puppeteer example/docs, independent hands-on confirmation of the CDP/Playwright connection working at scale.", "evidenceIds": [ "notte-docs-11", "notte-docs-24", "notte-docs-10" ] }, { "productId": "notte", "storyId": "credential-vault-2fa", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Notte's docs describe a credential vault where secrets are injected into the browser session but never exposed to the LLM, plus persistent cookies/login state and 'verified identities' (emails/phones) for sign-up and 2FA flows, directly matching the core of the story. However, there's no explicit walkthrough of a TOTP code being generated/entered by the agent, and no independent/hands-on confirmation that 2FA login flows work end-to-end in practice. Missing for 10: concrete TOTP-specific workflow documentation, independent verification of vault+2FA login success.", "evidenceIds": [ "notte-docs-4", "notte-docs-14", "notte-docs-27", "notte-docs-5", "notte-docs-6", "notte-docs-21" ] }, { "productId": "notte", "storyId": "deterministic-replay-caching", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Notte lets you script deterministic parts and generate/edit workflow code (notte-docs-2, notte-docs-26, notte-docs-30, notte-gh-2), which implies some cached/generated code can be replayed without re-prompting the LLM, but there is no explicit documentation of an action/result cache, deterministic replay guarantee, or cost/latency comparison for repeat runs. missing for 10: explicit caching mechanism for resolved actions, documentation of deterministic replay across runs, and cost/latency benchmarks comparing cached vs re-prompted execution.", "evidenceIds": [ "notte-docs-2", "notte-docs-26", "notte-docs-30", "notte-gh-2" ] }, { "productId": "notte", "storyId": "dom-action-primitives", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs describe a genuine action-space abstraction (observe()/act() calls, 'no selectors, no maintenance', natural-language task execution) that maps directly to the described act/click/type primitives, and the CLI/MCP integrations reinforce this as a core product concept. However, there's no first-party benchmark or independent hands-on confirmation that these primitives specifically survive selector/layout changes, and community commentary raises skepticism about action-space reliability versus screenshot+HTML approaches without being a concrete contradiction. Missing for 10: independent reproducibility evidence of resilience across DOM changes, and a documented before/after example showing selector survival.", "evidenceIds": [ "notte-docs-8", "notte-docs-18", "notte-gh-1", "notte-gh-2", "notte-comm-11" ] }, { "productId": "notte", "storyId": "ethics-abuse-guardrails", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item shows Notte publishing an acceptable-use policy, terms governing stealth/captcha-bypass usage, or an anti-abuse stance; docs only describe stealth/proxy/captcha features themselves. A community comment even accuses Notte of 'disrespecting robots.txt' and enabling spam, but this is criticism, not a vendor-published policy to compare against.", "evidenceIds": [ "notte-docs-12", "notte-docs-13", "notte-docs-28", "notte-comm-5" ] }, { "productId": "notte", "storyId": "file-download-upload", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers browser sessions, scraping/extraction, credentials, and CDP/Playwright access, but no documentation mentions file upload/download handling or artifact retrieval from agent-operated sites. This is a plausible capability for a browser-automation platform, so absence of evidence yields 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "notte", "storyId": "hosted-task-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Notte offers a hosted HTTP API (Bearer-token auth, agent-start endpoint, cloud sessions, serverless 'functions' you can invoke and schedule) that let an agent submit a task without managing a browser itself, and sessions expose CDP/live-view for status. However there is no direct documentation of a polling endpoint or webhook callback mechanism for retrieving results, and no OpenAPI/swagger spec was found (404s), so completion-notification patterns are unclear. Missing for 10: explicit polling/webhook result-retrieval documentation, published OpenAPI schema, independent confirmation of end-to-end async task completion.", "evidenceIds": [ "notte-docs-17", "notte-docs-37", "notte-docs-26", "notte-docs-2", "notte-docs-24", "notte-probe-2" ] }, { "productId": "notte", "storyId": "live-view-human-takeover", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Notte clearly supports live session viewing via ViewerUrl and a 'Live View & Replays' screenshare feature, and sessions expose a CDP endpoint connectable via Playwright which could in principle allow manual intervention. However, there is no explicit documentation of a human-takeover/pause-and-control workflow mid-run when an agent gets stuck. Missing for 10: explicit human-in-the-loop takeover mechanism, documentation of pausing agent execution to hand control to a human, and independent confirmation this works in practice.", "evidenceIds": [ "notte-docs-10", "notte-docs-19", "notte-docs-11", "notte-docs-24" ] }, { "productId": "notte", "storyId": "local-browser-mode", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "All evidence describes Notte as a cloud-hosted service — sessions are explicitly 'isolated browser instances running in the cloud,' access requires a Bearer API token, and pricing/credits are core to usage — with no documented option to run the agent against a local browser without a cloud account.", "evidenceIds": [ "notte-docs-24", "notte-docs-17", "notte-docs-22", "notte-docs-1" ] }, { "productId": "notte", "storyId": "login-session-persistence", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Notte docs explicitly document persisting cookies/login state across sessions and 'Authenticated Profiles' as browser profiles per agent, alongside secure credential vaults so agents can skip re-authentication on subsequent runs. missing for 10: no independent/hands-on confirmation that persisted profiles actually skip login walls in practice, and no detail on profile reuse limits/expiry.", "evidenceIds": [ "notte-docs-5", "notte-docs-21", "notte-docs-4", "notte-docs-14", "notte-docs-27" ] }, { "productId": "notte", "storyId": "model-agnostic-llm", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "The pricing page lists 'Bring your own keys' as a feature (Yes for higher tiers), indicating some BYO-LLM-key support, and OpenAI CUA integration doc shows a specific model provider integration, but there is no documentation of broad model-agnostic architecture, no list of supported providers, and no explicit statement that any LLM can be swapped in across the framework. missing for 10: explicit multi-provider support documentation, list of supported LLM vendors, guidance on configuring custom/local models, independent confirmation of model-agnosticism.", "evidenceIds": [ "notte-docs-23", "notte-docs-34" ] }, { "productId": "notte", "storyId": "nl-task-to-completion", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Notte's docs and README explicitly market natural-language, multi-step web task execution (e.g. 'Give AI agents natural language tasks to complete on websites', 'Describe a task. Watch it happen. One prompt', and a worked example of logging into LinkedIn and extracting messages), backed by session/vault/proxy infrastructure. However, a hands-on community report describes a real attempt to use the agent to extract data from hyatt.com that 'completely failed,' directly contradicting the end-to-end reliability claim, and the founder himself admits captcha/anti-bot handling only works for ~60% of providers. Missing for 10: independent successful third-party demonstrations of complex multi-step flows, and resolution of the reported failure case.", "evidenceIds": [ "notte-gh-1", "notte-docs-18", "notte-docs-16", "notte-docs-37", "notte-comm-1", "notte-comm-7" ] }, { "productId": "notte", "storyId": "observe-suggest-actions", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Notte's docs explicitly describe an `observe()` call and CLI `notte page observe` command that returns element IDs/selectors before actions are executed, which is direct evidence of a preview/plan-before-act primitive. However, the evidence pack lacks a full worked example showing the observe→plan→act workflow end-to-end, independent confirmation of its reliability, or details on how proposed actions are presented/reviewed by a developer. missing for 10: a complete observe/plan-then-act workflow example, independent/hands-on verification that observe output is accurate and usable for gating actions, and documentation of any 'plan' abstraction distinct from observe.", "evidenceIds": [ "notte-docs-8", "notte-probe-4", "notte-docs-9" ] }, { "productId": "notte", "storyId": "openness-api-parity", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Notte is built API/CLI/MCP-first: sessions, scraping, agents, functions, vaults, and even live session viewing are all exposed via API/CLI/MCP endpoints (notte-docs-2, -9, -17, -24, -37, -7/-29), suggesting broad UI/API parity for an API-native product. However there's no explicit vendor statement guaranteeing full feature parity, and a probe found no discoverable OpenAPI/swagger spec (404s across common paths), which weakens confidence that every UI capability (e.g. templates, dashboard-only settings) is fully API-exposed. Missing for 10: explicit parity documentation, a public OpenAPI spec, and independent confirmation that all UI-only features (templates, dashboard views) have API equivalents.", "evidenceIds": [ "notte-docs-2", "notte-docs-9", "notte-docs-17", "notte-docs-24", "notte-docs-37", "notte-docs-7", "notte-docs-29", "notte-probe-2" ] }, { "productId": "notte", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of data export/portability features, open-format export of user data, or account deletion/data takeout mechanisms; the docs focus on browser automation, scraping outputs, and credential storage but nothing about exporting one's own account data in open formats. missing for 10: data export feature docs, open-format (e.g. JSON/CSV) account export, data portability/account deletion process, any independent confirmation of exportability.", "evidenceIds": [] }, { "productId": "notte", "storyId": "openness-open-license", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "A public GitHub repository (nottelabs/notte) exists and is referenced in evidence, indicating the source code is at least publicly readable, but no evidence pack item specifies a license type (e.g., MIT/Apache) or confirms open-source licensing terms. A community comment even questions whether Notte is open source at all, suggesting ambiguity rather than confirmed openness. Missing for 10: explicit license file/badge evidence, confirmation of license terms, independent verification that the full product (not just parts) is open-licensed.", "evidenceIds": [ "notte-gh-1", "notte-gh-2", "notte-comm-9" ] }, { "productId": "notte", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All evidence describes Notte as a cloud/SaaS platform (remote browsers, cloud sessions, serverless functions, hosted API with Bearer auth) with no documented self-hosting or on-prem deployment path; a community comment even asks whether an open-source alternative exists, implying Notte itself isn't self-hostable.", "evidenceIds": [ "notte-docs-1", "notte-docs-24", "notte-docs-17", "notte-comm-9" ] }, { "productId": "notte", "storyId": "parallel-fleet-scale", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Notte documents cloud-based, isolated, programmatically controllable sessions (notte-docs-24), API/CLI/SDK control (notte-docs-17, notte-docs-37, notte-probe-4), and serverless scaling claims (notte-docs-1, notte-docs-26), supporting the 'programmatic session management' half of the story. However, no evidence anywhere specifies actual concurrency limits, quotas, or fleet-scale numbers for running many sessions in parallel — pricing/credit pages are mentioned only vaguely by community members (notte-comm-2) without concurrency specifics. missing for 10: documented concurrency/rate limits per plan, explicit multi-session fleet management API/dashboard evidence, independent benchmarks of parallel session throughput.", "evidenceIds": [ "notte-docs-1", "notte-docs-24", "notte-docs-17", "notte-docs-26", "notte-docs-37", "notte-comm-2" ] }, { "productId": "notte", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, region selection, or storage location choices for Notte's cloud infrastructure; only general security/credential features are documented.", "evidenceIds": [] }, { "productId": "notte", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses opting out of AI training data usage, data retention policies for model training, or any privacy commitment regarding customer data not being used to train models. Notte's docs cover security of credentials, sessions, stealth mode, and infrastructure, but nothing about AI-training data usage policy.", "evidenceIds": [] }, { "productId": "notte", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data retention policies, deletion controls, or user-facing options to purge stored data/sessions/credentials; docs mention credential encryption and cookie persistence but nothing about retention windows or deletion mechanisms.", "evidenceIds": [] }, { "productId": "notte", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions telemetry, analytics opt-out, or usage-tracking controls for Notte; missing for 10: any privacy policy or docs section on telemetry collection, opt-out mechanism/flag, or data retention settings.", "evidenceIds": [] }, { "productId": "notte", "storyId": "schema-structured-extraction", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Docs show Pydantic-style schema extraction (BaseModel classes) and structured/markdown output via 'fetch' and scraping concepts (notte-docs-15, notte-docs-3, notte-docs-25), which directly matches the story. However, a hands-on community report states extraction 'completely failed' on a real site (hyatt.com), directly contradicting the reliability of the extraction pipeline in practice. Missing for 10: Zod/TypeScript schema examples (only Python/Pydantic shown), independent corroboration of successful schema-validated extraction, and resolution of the reported failure case.", "evidenceIds": [ "notte-docs-15", "notte-docs-3", "notte-docs-25", "notte-comm-1" ] }, { "productId": "notte", "storyId": "session-replay-recordings", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Notte docs confirm 'Live View & Replays' with screenshare and session playback, plus a live ViewerUrl to watch sessions and CDP/Playwright hooks for programmatic inspection, giving some replay-debugging capability. However, there is no explicit documentation of step-by-step action timelines or a dedicated debugging UI for failed runs, and no independent/hands-on confirmation this replay feature works reliably. missing for 10: documented step-by-step action timeline/debugging tool, independent verification of replay/video debugging in practice.", "evidenceIds": [ "notte-docs-19", "notte-docs-10", "notte-docs-11", "notte-docs-35" ] }, { "productId": "notte", "storyId": "stealth-fingerprint-proxies", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Notte's docs explicitly describe stealth mode with clean/realistic browser fingerprints, built-in anti-bot detection avoidance, and residential proxies with a global network including fixed IPs and BYO options, directly matching the story's ask for fingerprinting and geo/residential proxy control. missing for 10: independent hands-on verification that stealth+proxy combo actually evades sophisticated bot detection in practice, and finer detail on geo-targeting granularity beyond 'global network'.", "evidenceIds": [ "notte-docs-12", "notte-docs-13", "notte-docs-20", "notte-docs-28", "notte-docs-1" ] }, { "productId": "notte", "storyId": "task-completion-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions webhooks, callback URLs, or event-driven notifications for task/session completion; the docs describe sessions, viewer URLs, CDP endpoints and API auth but nothing about push notifications, forcing polling-based status checks instead.", "evidenceIds": [] }, { "productId": "notte", "storyId": "usage-pricing-transparency", "verdict": "disputed", "quality": 3, "confidence": "medium", "rationale": "Notte does have a public pricing page and free-trial claim (notte-docs-22, notte-docs-23), but there is no documentation of concrete per-task/per-browser-hour rates or rate/concurrency limits, and community feedback directly contradicts the transparency claim: users on HN explicitly ask what a 'credit' actually buys and report the pricing page doesn't explain it, calling the credit-based scheme 'broken' and unpredictable (notte-comm-2, notte-comm-3, notte-comm-4). missing for 10: explicit per-task/per-hour rate tables, documented concurrency/rate limits, and resolution of the community complaints about opaque credit meaning.", "evidenceIds": [ "notte-docs-22", "notte-docs-23", "notte-comm-2", "notte-comm-3", "notte-comm-4" ] }, { "productId": "notte", "storyId": "vision-computer-use", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Notte's core action space is DOM/observe-based, but the docs include a dedicated integration guide for OpenAI's Computer Use Agent (CUA) that operates on screenshots atop Notte's browser infrastructure, showing a vision/computer-use path exists. However this is presented as an external integration rather than a first-class 'switch mode' toggle within Notte's own API, and community commentary explicitly questions the reliability of Notte's action-space approach versus screenshot-based methods. Missing for 10: native documented API/flag to toggle into vision mode, first-party examples of vision-based action execution, and independent hands-on confirmation that the CUA integration works reliably.", "evidenceIds": [ "notte-docs-34", "notte-docs-19", "notte-comm-11" ] }, { "productId": "notte", "storyId": "workflow-loops-conditionals", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Notte supports scripting deterministic automations and deploying them as serverless 'Functions'/API endpoints, and lets users generate workflow code via CLI then edit it, going beyond a single one-shot prompt (notte-docs-26, notte-docs-2, notte-docs-30, notte-gh-2). However there is no explicit documentation of workflow-level constructs like loops, conditionals, or parameterized templates. missing for 10: explicit docs on loop/conditional syntax in workflows, parameter binding across runs, independent confirmation of repeatable multi-step workflow composition.", "evidenceIds": [ "notte-docs-26", "notte-docs-2", "notte-docs-30", "notte-gh-2", "notte-docs-36" ] }, { "productId": "skyvern", "storyId": "agent-framework-adapters", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Skyvern documents a Python/TypeScript/REST SDK and an MCP server that plugs into Claude Desktop, Claude Code, Codex, Cursor, and Windsurf, but there is no evidence of documented adapters for Claude Agent SDK, Vercel AI SDK, LangChain, or CrewAI. missing for 10: any documented integration guide or adapter for LangChain, CrewAI, Vercel AI SDK, or Claude Agent SDK specifically.", "evidenceIds": [ "skyvern-docs-1", "skyvern-docs-5", "skyvern-docs-7" ] }, { "productId": "skyvern", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A probe confirms llms.txt is live and returns a structured summary of Skyvern for agent consumption, and skyvern.com/llms provides an agent-oriented docs page listing features in a scannable format. This directly satisfies pointing an agent at llms.txt or agent-oriented docs. Missing for 10: a docs.md/markdown-mirrored docs endpoint (404) and an accessible OpenAPI spec, which would round out machine-readable documentation.", "evidenceIds": [ "skyvern-probe-1", "skyvern-docs-22", "skyvern-docs-23", "skyvern-docs-24", "skyvern-docs-25", "skyvern-probe-2", "skyvern-probe-3" ] }, { "productId": "skyvern", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "medium", "rationale": "Skyvern is a browser-automation/agent platform for executing web tasks and extracting data per user-specified schemas, not a product that analyzes a user's own data corpus to surface proactive insights or suggestions; this axis is a category mismatch for its purpose.", "evidenceIds": [] }, { "productId": "skyvern", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Skyvern's docs show multi-step, code-first and no-code workflows that run via API or cloud UI, persist browser state, and can pause for human approval while capturing recordings/artifacts — all indicative of autonomous background execution (skyvern-docs-5,6,10,11,21). Zapier integration and API-driven triggering (skyvern-docs-16, skyvern-docs-28) supports running without manual intervention, but there's no explicit documentation of a scheduler, cron-like triggers, or continuous monitoring dashboard for unattended runs. Missing for 10: explicit scheduling/trigger docs, evidence of long-running unattended background jobs, and independent confirmation of reliability at scale (community notes some brittleness, e.g. skyvern-comm-2).", "evidenceIds": [ "skyvern-docs-5", "skyvern-docs-6", "skyvern-docs-10", "skyvern-docs-11", "skyvern-docs-21", "skyvern-docs-16", "skyvern-docs-28", "skyvern-comm-2" ] }, { "productId": "skyvern", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Skyvern's core product IS an AI agent you delegate to via natural-language prompts to complete multi-step browser tasks (skyvern-docs-17, skyvern-docs-18), and it also ships a 'Copilot chat for building and debugging workflows interactively' inside the platform (skyvern-docs-25), plus SOP-to-workflow generation from plain English (skyvern-docs-24). This matches an AI-native user delegating tasks to a built-in assistant. Missing for 10: independent/hands-on validation of the copilot chat feature specifically (community evidence focuses on task execution quality, not the assistant/copilot UX), and no detail on assistant's conversational scope beyond workflow authoring.", "evidenceIds": [ "skyvern-docs-17", "skyvern-docs-18", "skyvern-docs-24", "skyvern-docs-25", "skyvern-gh-1" ] }, { "productId": "skyvern", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Skyvern ships a code-first SDK/REST API (Python/TypeScript) that connects to a cloud or self-hosted Chromium instance, explicitly positioned as replacing brittle Selenium scripts and integrating browser automation via API into other products, and can run entirely on your own infrastructure with your own LLM keys, supporting headless/scriptable use suitable for CI. missing for 10: explicit CI/CD pipeline documentation or example (e.g. GitHub Actions integration), and independent confirmation of headless execution in automated environments", "evidenceIds": [ "skyvern-docs-1", "skyvern-docs-2", "skyvern-docs-5", "skyvern-docs-12", "skyvern-docs-13" ] }, { "productId": "skyvern", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows Skyvern exposing an MCP *server* so external AI assistants (Claude, Cursor, etc.) can control Skyvern's browser — the reverse of the story, which asks whether Skyvern can consume external MCP servers' tools as a client. No documentation or community evidence shows Skyvern importing or connecting to third-party MCP servers to extend its own toolset.", "evidenceIds": [ "skyvern-docs-7", "skyvern-probe-4" ] }, { "productId": "skyvern", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Skyvern documents an official MCP server (skyvern-docs-7, skyvern-probe-4) that lets AI assistants like Claude Desktop, Claude Code, Codex, Cursor, and Windsurf control a browser via Skyvern, directly matching the story. Missing for 10: independent/hands-on community verification of the MCP server specifically (community evidence covers other features, not MCP usage) and a clear setup/config example beyond the single doc page.", "evidenceIds": [ "skyvern-docs-7", "skyvern-probe-4" ] }, { "productId": "skyvern", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Skyvern's core interaction model is natural-language: users provide a prompt describing the goal (docs-18), SOPs in plain English are converted to workflows (docs-24), and a Copilot chat and MCP server let AI assistants/users direct browser actions in natural language (docs-25, docs-7, docs-19). This is corroborated by community reports of using it via prompts on real sites, though with mixed reliability on complex flows.\nmissing for 10: independent benchmarking or hands-on confirmation that natural-language commands reliably handle complex multi-step tasks, and clearer evidence of NL-driven success rates beyond anecdotal HN reports.", "evidenceIds": [ "skyvern-docs-18", "skyvern-docs-24", "skyvern-docs-25", "skyvern-docs-7", "skyvern-docs-19", "skyvern-comm-2" ] }, { "productId": "skyvern", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers Skyvern's Python/TypeScript SDKs, REST API, MCP server, and visual dashboard, but nowhere mentions an official CLI tool for AI-native workflows. missing for 10: any documented CLI command, npm/pip CLI package, or terminal-based interface.", "evidenceIds": [ "skyvern-docs-1", "skyvern-docs-5", "skyvern-docs-7" ] }, { "productId": "skyvern", "storyId": "agentic-public-api", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Skyvern documents a public API/SDK surface (Python, TypeScript, REST) for creating tasks, running multi-step browser automations, and extracting structured data via JSON schema, matching the ai-native 'drive via documented API' story; it also ships an MCP server for agent control. Missing for 10: a discoverable OpenAPI/swagger spec (probe found 404s for all candidate paths) and independent/hands-on confirmation of API robustness beyond first-party docs.", "evidenceIds": [ "skyvern-docs-1", "skyvern-docs-5", "skyvern-docs-17", "skyvern-docs-18", "skyvern-docs-4", "skyvern-docs-7", "skyvern-probe-3", "skyvern-probe-4" ] }, { "productId": "skyvern", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of scoped or least-privilege API credential issuance for agents; docs mention API keys and self-hosted LLM keys but nothing about credential scoping, permissions, or restricting agent access levels. Community comments even raise concerns about handling sensitive credentials in plain text with no mitigation shown. Missing for 10: any documentation of scoped API tokens, role-based access control, or least-privilege credential management for agents.", "evidenceIds": [ "skyvern-docs-2", "skyvern-docs-12", "skyvern-comm-7" ] }, { "productId": "skyvern", "storyId": "agentic-sdks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Skyvern explicitly documents official Python and TypeScript SDKs plus a REST API for integrating browser automation, with SDK-level primitives like page.extract and data_extraction_schema shown in docs (skyvern-docs-1, skyvern-docs-5, skyvern-docs-19, skyvern-docs-4). Missing for 10: independent/hands-on developer corroboration of SDK usage and a public API reference (OpenAPI spec probe returned 404s, skyvern-probe-3), so quality is capped below full confidence in completeness.", "evidenceIds": [ "skyvern-docs-1", "skyvern-docs-5", "skyvern-docs-19", "skyvern-docs-4", "skyvern-probe-3" ] }, { "productId": "skyvern", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions webhooks or event subscriptions of any kind; Skyvern's documented integration surfaces are REST/SDK APIs, Zapier, and an MCP server, none of which constitute a webhook subscription mechanism.", "evidenceIds": [] }, { "productId": "skyvern", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only static docs describing SDKs and REST usage, with no interactive API reference or runnable-example explorer; probes explicitly found no OpenAPI/Swagger spec at any candidate path (404s), indicating no interactive reference exists.", "evidenceIds": [ "skyvern-probe-3", "skyvern-probe-2", "skyvern-docs-1" ] }, { "productId": "skyvern", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Skyvern offers a REST API (skyvern-docs-1) but a direct probe for OpenAPI/swagger specs at standard paths returned 404 across all candidates, and no docs mention a downloadable machine-readable spec.", "evidenceIds": [ "skyvern-probe-3", "skyvern-probe-2", "skyvern-docs-1" ] }, { "productId": "skyvern", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Skyvern's docs describe cloud or self-hosted execution, credential handling, and observability, but nothing describes a dedicated sandbox/staging mode isolated from production data or systems — missing for 10: any mention of a sandbox environment, test/staging mode, or data isolation guarantees.", "evidenceIds": [] }, { "productId": "skyvern", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of API versioning scheme or a documented deprecation policy; OpenAPI/spec probes all returned 404 and no changelog or versioning docs appear in the pack. Missing for 10: versioned API endpoints (e.g., /v1/), a published deprecation/support policy, and changelog documentation.", "evidenceIds": [ "skyvern-probe-3", "skyvern-probe-2" ] }, { "productId": "skyvern", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence describes a bulk-operation feature (e.g., running the same task across a list/CSV of items, batch triggering, or concurrent multi-item processing) — the docs focus on single-task API calls, visual workflows, and MCP integration rather than batch/bulk execution.", "evidenceIds": [] }, { "productId": "skyvern", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Skyvern documents a Zapier integration, which could allow external events to trigger Skyvern workflows, but there is no evidence of a native rule/trigger engine, webhooks, or scheduled/event-based automation within Skyvern itself. Missing for 10: documented native event triggers or webhook listeners, schedule-based triggers, and any conditional rule engine inside Skyvern's workflow builder.", "evidenceIds": [ "skyvern-docs-16", "skyvern-docs-6" ] }, { "productId": "skyvern", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes Skyvern's workflow builder, API/SDK, MCP integration, and automation features extensively, but contains no mention of scheduling, cron triggers, or recurring job execution anywhere in the docs, GitHub description, or community discussion. Since Skyvern is a workflow/automation platform, scheduling recurring runs is a fair capability to expect, but it's simply absent from the provided evidence.", "evidenceIds": [] }, { "productId": "skyvern", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack describes version history, change review, or rollback capabilities for Skyvern workflows/automations — only building, running, sharing templates, and artifact capture (recordings/screenshots) are documented. Missing for 10: workflow version history, diff/review UI, rollback-to-previous-version mechanism, any changelog or audit trail for automation edits.", "evidenceIds": [] }, { "productId": "skyvern", "storyId": "captcha-handling", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Skyvern's docs give an explicit, detailed captcha stance: automatic detection and solving via its vision model for reCAPTCHA v2/v3, hCaptcha, Cloudflare Turnstile, FunCaptcha, MTCaptcha, and text/image captchas, avoiding silent failure ambiguity. Missing for 10: independent/hands-on confirmation that captcha solving works reliably in practice (community evidence discusses pricing, mobile UX, and credential handling but not captcha outcomes specifically), and no documented fallback/human-in-the-loop behavior specifically tied to captcha failures.", "evidenceIds": [ "skyvern-docs-9", "skyvern-docs-27" ] }, { "productId": "skyvern", "storyId": "cdp-connect-existing-stack", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs show Skyvern's own SDK connects to a cloud Chromium instance over CDP and layers Playwright on top, and describe 'drop-in AI commands on top of Playwright' with fallback to raw selectors, implying some interoperability with existing Playwright code. However there is no explicit guidance or example showing a developer pointing an existing Playwright/Puppeteer/CDP script at Skyvern's managed browser instead of rewriting into Skyvern's task/workflow API, and no independent confirmation of this specific reuse pattern. Missing for 10: explicit BYO-script CDP endpoint docs, Puppeteer-specific support, and hands-on/community verification of dropping in existing automation code unchanged.", "evidenceIds": [ "skyvern-docs-5", "skyvern-docs-19", "skyvern-docs-4" ] }, { "productId": "skyvern", "storyId": "credential-vault-2fa", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Skyvern's docs explicitly describe storing credentials via password-manager vault integrations (Bitwarden, 1Password, Azure Key Vault) and automatically handling TOTP/2FA, email, and SMS verification during login flows, matching the story closely [skyvern-docs-8][skyvern-docs-26]. However, there's no independent/hands-on verification that secrets are never exposed to the LLM, and a community comment raises concern about credentials being handled in plain text, so missing for 10: independent security audit or hands-on confirmation of secret-masking from the model, and clarification addressing the community's plaintext-handling concern.", "evidenceIds": [ "skyvern-docs-8", "skyvern-docs-26", "skyvern-comm-7" ] }, { "productId": "skyvern", "storyId": "deterministic-replay-caching", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of caching resolved actions or generated code for deterministic, cheaper replay; Skyvern's model is per-run AI-driven navigation via LLM+vision, and community feedback even complains about cost/latency of repeated LLM calls with no mention of a caching mechanism to mitigate this.", "evidenceIds": [ "skyvern-comm-1", "skyvern-comm-9", "skyvern-docs-17" ] }, { "productId": "skyvern", "storyId": "dom-action-primitives", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Skyvern's docs describe exactly this: natural-language act/click/type primitives with vision+DOM understanding that operate on sites 'never seen before' and fall back to selectors only if needed (skyvern-docs-19, skyvern-gh-1, skyvern-docs-17), positioned explicitly as a replacement for brittle Selenium scripts (skyvern-docs-13). However, a hands-on community test found it worked on the happy path but concretely failed to interact with a layout element (a popup) and struggled to hit a tab on a real site (skyvern-comm-2), contradicting the claim that it robustly survives arbitrary layout changes. Missing for 10: independent benchmark data on selector/layout-change robustness, broader corroboration beyond one hands-on report, and resolution of the observed failure mode.", "evidenceIds": [ "skyvern-docs-19", "skyvern-gh-1", "skyvern-docs-17", "skyvern-docs-13", "skyvern-comm-2" ] }, { "productId": "skyvern", "storyId": "ethics-abuse-guardrails", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack of any published acceptable-use policy, terms of service, or anti-abuse statement covering CAPTCHA-solving/stealth automation features; docs describe capabilities (CAPTCHA bypass, bot bypass) but no governance/AUP language is cited. missing for 10: a published acceptable-use policy, anti-abuse terms, or statement on permitted use of stealth/CAPTCHA-bypass features.", "evidenceIds": [] }, { "productId": "skyvern", "storyId": "file-download-upload", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs show Skyvern can log into vendor portals and download PDFs (skyvern-docs-15) and captures per-run artifacts like recordings, screenshots, and network traffic retrievable afterward (skyvern-docs-11), implying file download support, but there is no explicit documentation of file upload capability to sites, nor of a dedicated API/UI for retrieving downloaded artifacts as opposed to just run/debug artifacts. missing for 10: explicit upload-to-site capability documentation, a documented file-download/artifact storage API distinct from debugging screenshots, and independent/hands-on confirmation of file transfer working in practice.", "evidenceIds": [ "skyvern-docs-15", "skyvern-docs-11", "skyvern-docs-14" ] }, { "productId": "skyvern", "storyId": "hosted-task-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm a hosted REST/SDK API where you submit a prompt+URL (optionally a JSON schema) and Skyvern runs the task on cloud Chromium without the caller managing a browser (skyvern-docs-1, skyvern-docs-5, skyvern-docs-18, skyvern-docs-17). However, there is no explicit documentation of a polling endpoint or webhook callback mechanism, and probes found no discoverable OpenAPI spec, so the exact result-retrieval mechanism described in the story is unconfirmed. Missing for 10: explicit webhook/callback docs, explicit polling endpoint docs, and an accessible API reference confirming these mechanics.", "evidenceIds": [ "skyvern-docs-1", "skyvern-docs-5", "skyvern-docs-18", "skyvern-docs-17", "skyvern-probe-3" ] }, { "productId": "skyvern", "storyId": "live-view-human-takeover", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly state a VNC stream lets you watch a live session and take control at any point, plus pause-for-approval human-in-the-loop flows that preserve browser state — directly matching the story. missing for 10: independent/hands-on confirmation of the live takeover UX and details on how control handoff works mid-run beyond the docs description.", "evidenceIds": [ "skyvern-docs-10", "skyvern-docs-21" ] }, { "productId": "skyvern", "storyId": "local-browser-mode", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Skyvern is open-source and its self-hosted docs explicitly state it 'runs entirely on your infrastructure: your servers, your browsers, your LLM API keys' (skyvern-docs-12, skyvern-docs-2), which supports running without a cloud account. However, the core SDK/browser-automation flow described elsewhere connects to a 'cloud Chromium instance over CDP' (skyvern-docs-5), suggesting the default path is cloud-based, and no local-machine dev setup details (docker/local browser config, install steps) are shown. Missing for 10: explicit local-browser dev walkthrough, confirmation that the local-first SDK path bypasses cloud Chromium, and independent hands-on confirmation of local-only operation.", "evidenceIds": [ "skyvern-docs-12", "skyvern-docs-2", "skyvern-docs-5" ] }, { "productId": "skyvern", "storyId": "login-session-persistence", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Skyvern's browser-sessions feature explicitly persists cookies, local storage, and open tabs across operations so 'later operations pick up exactly where the previous one stopped,' and pauses preserve browser state — this directly supports skipping repeated logins. However, the docs don't clearly describe a named 'profile' abstraction, how long sessions persist across truly separate future runs, or how these persisted sessions are managed/reused across different agents or teams. missing for 10: explicit reusable-profile management docs, long-term persistence guarantees across independent runs, independent/hands-on confirmation of skip-login behavior.", "evidenceIds": [ "skyvern-docs-21", "skyvern-docs-10", "skyvern-docs-8" ] }, { "productId": "skyvern", "storyId": "model-agnostic-llm", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs state self-hosted Skyvern runs with 'your own LLM API keys' on your own infrastructure, implying model-agnosticism rather than lock-in to a single vendor, but there is no explicit list of supported providers/models or first-party guide on swapping LLM backends, and no independent confirmation of multi-provider support. Missing for 10: an explicit supported-providers list/config docs, and community/hands-on evidence of using non-default LLMs.", "evidenceIds": [ "skyvern-docs-2", "skyvern-docs-12" ] }, { "productId": "skyvern", "storyId": "nl-task-to-completion", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Skyvern's docs and GitHub strongly claim natural-language goal execution across novel, multi-step web flows (forms, logins, CAPTCHAs) via a single prompt/API (skyvern-docs-17, skyvern-docs-18, skyvern-gh-1), but a hands-on community test found it succeeded only on the 'happy path' and concretely failed on a real multi-step flow (costcotravel.com), struggling to hit a tab and failing to click a popup (skyvern-comm-2). This is a specific documented counter-example contradicting the 'completes multi-step task end to end' claim, not just general skepticism. Missing for 10: independent benchmark results, more hands-on trials showing consistent success on complex/unseen sites, and resolution of the reported failure case.", "evidenceIds": [ "skyvern-docs-17", "skyvern-docs-18", "skyvern-gh-1", "skyvern-comm-2" ] }, { "productId": "skyvern", "storyId": "observe-suggest-actions", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Skyvern's docs mention human-in-the-loop pausing for approval between steps and a VNC stream to watch/take control, which offers some ability to intervene before the agent proceeds, but there is no documented explicit 'plan/preview candidate actions' step (e.g., a dry-run or action list shown before execution). A community comment even notes the absence of assertion/verification-style controls compared to Playwright, suggesting no built-in preview mechanism for validating steps before they run. Missing for 10: an explicit plan/preview UI or API that lists candidate actions before execution, and independent confirmation that the pause-for-approval flow shows planned actions rather than just pausing mid-run.", "evidenceIds": [ "skyvern-docs-10", "skyvern-comm-8" ] }, { "productId": "skyvern", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Skyvern's docs show a strong code-first path (Python/TS/REST SDKs, page.extract, workflow creation via API) that covers most core automation tasks also available in the dashboard, and MCP/REST access is documented. However, several UI-only tooling features (drag-and-drop visual builder, browser recorder, SOP upload, copilot chat) are described only as dashboard capabilities with no documented API equivalent, and no public OpenAPI/swagger spec was discoverable to confirm full API-UI parity. Missing for 10: documented API equivalents for recorder/SOP-upload/copilot-chat features, and a discoverable OpenAPI reference confirming complete parity.", "evidenceIds": [ "skyvern-docs-1", "skyvern-docs-3", "skyvern-docs-5", "skyvern-docs-6", "skyvern-docs-22", "skyvern-docs-23", "skyvern-docs-24", "skyvern-docs-25", "skyvern-probe-3" ] }, { "productId": "skyvern", "storyId": "openness-full-export", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Skyvern is open-source and self-hostable, meaning your data (artifacts, recordings, screenshots, network traffic) stays on your own infrastructure rather than being locked in a vendor's cloud, which implicitly supports data portability. However, there is no explicit documentation of a data export feature, standard open-format export (e.g., JSON/CSV bulk export of run history), or a stated 'leave with your data' workflow. Missing for 10: explicit export functionality/documentation, named open data formats, and any independent confirmation of successful data migration out of the platform.", "evidenceIds": [ "skyvern-docs-12", "skyvern-docs-11", "skyvern-docs-2" ] }, { "productId": "skyvern", "storyId": "openness-open-license", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Skyvern is described as open-source with a GitHub repo, and probe evidence confirms it self-identifies as 'open-source' (skyvern-probe-1), but community evidence directly contradicts full open-license access, noting the project is AGPL3 licensed, which is a legally open license but is called out as a practical non-starter/restrictive for many users (skyvern-comm-3). missing for 10: explicit statement of license terms in docs, confirmation of what percentage of the product (cloud vs self-hosted) is actually open-sourced, and independent corroboration that the full source is readable without restriction.", "evidenceIds": [ "skyvern-gh-1", "skyvern-gh-2", "skyvern-probe-1", "skyvern-comm-3" ] }, { "productId": "skyvern", "storyId": "openness-self-host", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Skyvern has a dedicated self-hosted docs page stating it 'runs entirely on your infrastructure: your servers, your browsers, your LLM API keys' (skyvern-docs-12, skyvern-docs-2), and community evidence confirms it is genuinely open-source (AGPL3) rather than just marketing language (skyvern-comm-3). Missing for 10: independent hands-on confirmation of a successful self-host deployment and clarity on how AGPL licensing affects commercial self-hosting use.", "evidenceIds": [ "skyvern-docs-12", "skyvern-docs-2", "skyvern-comm-3", "skyvern-probe-1" ] }, { "productId": "skyvern", "storyId": "parallel-fleet-scale", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers session persistence, VNC control, self-hosting, and SDK/API access, but nowhere documents concurrency limits, fleet-level session orchestration, or programmatic management of multiple simultaneous browser sessions. Missing for 10: documented concurrency limits, APIs for spinning up/managing many parallel sessions, and any scaling/throughput guidance.", "evidenceIds": [ "skyvern-docs-10", "skyvern-docs-21", "skyvern-docs-5", "skyvern-docs-12" ] }, { "productId": "skyvern", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Skyvern offers a self-hosted deployment mode where 'your servers, your browsers, your LLM API keys' run entirely on the user's own infrastructure, which lets an AI-native user control where data resides by choosing their hosting region themselves — but this is achieved only by self-hosting, not via an explicit region/residency selector in the managed cloud product. Missing for 10: documented data residency/region options in the hosted Skyvern Cloud offering, compliance certifications, or explicit multi-region storage controls.", "evidenceIds": [ "skyvern-docs-2", "skyvern-docs-12" ] }, { "productId": "skyvern", "storyId": "privacy-no-training", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Skyvern offers self-hosted deployment using your own infrastructure and your own LLM API keys, which implicitly lets users avoid sending data to Skyvern-controlled models/training pipelines, but there is no explicit privacy policy, data-retention statement, or 'we do not train on your data' commitment in the evidence for the hosted/cloud offering. Missing for 10: explicit no-training/data-use policy documentation, opt-out mechanism for the cloud product, and independent confirmation of data handling practices.", "evidenceIds": [ "skyvern-docs-2", "skyvern-docs-12" ] }, { "productId": "skyvern", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Skyvern offers self-hosting (docs-2, docs-12) which gives infrastructure-level control over where data lives, and it captures artifacts (recordings, screenshots, network traffic) per run (docs-11), implying some data exists to manage, but there is no documented retention policy, data deletion API/UI, or export/purge controls for the cloud/hosted product. missing for 10: explicit data retention policy, user-facing deletion/export controls, documentation on how long artifacts/credentials are stored in cloud mode, and independent confirmation that self-hosting actually eliminates vendor-side data retention.", "evidenceIds": [ "skyvern-docs-2", "skyvern-docs-12", "skyvern-docs-11", "skyvern-comm-7" ] }, { "productId": "skyvern", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item discusses telemetry, usage tracking, or an opt-out mechanism; while self-hosting exists, there is no explicit statement about data collection or opt-out controls for the cloud/hosted product. missing for 10: any mention of telemetry collection, privacy policy on usage data, or an opt-out setting/flag.", "evidenceIds": [] }, { "productId": "skyvern", "storyId": "schema-structured-extraction", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Skyvern's docs explicitly support structured, schema-based extraction via `page.extract` with a JSON schema or `data_extraction_schema` param, matching the developer's need for typed output rather than raw text (skyvern-docs-4, skyvern-docs-20, skyvern-docs-18). However, evidence only shows JSON-schema validation, not native Zod/Pydantic model binding, and there's no independent/hands-on confirmation of this specific feature. missing for 10: explicit Zod/Pydantic model integration examples, independent verification of extraction accuracy/schema enforcement.", "evidenceIds": [ "skyvern-docs-4", "skyvern-docs-20", "skyvern-docs-18", "skyvern-docs-5" ] }, { "productId": "skyvern", "storyId": "session-replay-recordings", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Skyvern docs explicitly state every run captures session recordings, per-step screenshots, AI reasoning traces, and network traffic for debugging, and community feedback corroborates strong observability into workflow steps. missing for 10: independent hands-on verification of the video/timeline UI itself, and no mention of a true step-by-step interactive timeline scrubber beyond artifact capture.", "evidenceIds": [ "skyvern-docs-11", "skyvern-comm-9" ] }, { "productId": "skyvern", "storyId": "stealth-fingerprint-proxies", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers CAPTCHA solving and authentication/2FA handling, but there is no mention anywhere of stealth fingerprinting, browser fingerprint spoofing, or residential/geo-targeted proxy support. Missing for 10: any documentation of proxy configuration, geo-targeting, or anti-fingerprinting/stealth mode features.", "evidenceIds": [ "skyvern-docs-9", "skyvern-docs-27" ] }, { "productId": "skyvern", "storyId": "task-completion-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item mentions webhooks, callback URLs, or push notifications for task/session completion; the docs discuss artifacts, VNC streaming, and human-in-the-loop review but nothing about event-driven notification instead of polling. missing for 10: any documentation of webhook/callback support, event subscription API, or notification configuration.", "evidenceIds": [] }, { "productId": "skyvern", "storyId": "usage-pricing-transparency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The pricing page is referenced only for its target-audience blurb (skyvern-docs-13); no evidence pack item shows actual per-task or per-browser-hour rates, tiers, or documented rate/concurrency limits. Community comments only express general cost concerns ('pretty pricey', wanting cost down 'at scale') without citing concrete published pricing or limits.", "evidenceIds": [ "skyvern-docs-13", "skyvern-comm-1", "skyvern-comm-9" ] }, { "productId": "skyvern", "storyId": "vision-computer-use", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Skyvern's core action loop is vision-based: it maps visual elements to actions on pages it has never seen, without custom DOM-specific code (skyvern-gh-1), and separately uses its vision model to detect and solve CAPTCHAs, which are canvas-like elements the DOM can't parse (skyvern-docs-9, skyvern-docs-27). Docs also mention falling back to selectors when useful (skyvern-docs-19), implying vision-first with DOM as a secondary path rather than a purely DOM-based tool needing a special switch.\nmissing for 10: explicit documentation of a discrete 'vision/computer-use mode' toggle, dedicated canvas/non-DOM UI examples (e.g., canvas-drawn widgets, non-HTML apps), and independent benchmarking confirming success on such UIs.", "evidenceIds": [ "skyvern-gh-1", "skyvern-docs-9", "skyvern-docs-27", "skyvern-docs-19" ] }, { "productId": "skyvern", "storyId": "workflow-loops-conditionals", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Skyvern clearly supports multi-step, repeatable workflows via both a code-first SDK and a visual no-code drag-and-drop builder (skyvern-docs-5, skyvern-docs-6, skyvern-docs-22), plus SOP-to-workflow generation and a browser recorder for building reusable automations (skyvern-docs-23, skyvern-docs-24). However, the evidence never explicitly documents loop constructs, conditional branching, or parameterized workflow inputs as first-class workflow-builder features. Missing for 10: explicit documentation of loop/iteration blocks, conditional/branching logic, and named/typed workflow parameters in the workflow builder.", "evidenceIds": [ "skyvern-docs-5", "skyvern-docs-6", "skyvern-docs-22", "skyvern-docs-23", "skyvern-docs-24", "skyvern-gh-2" ] }, { "productId": "smooth", "storyId": "agent-framework-adapters", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Smooth positions itself as a browser tool usable by agents like Claude Code, but there is no documentation of adapters for Claude Agent SDK, Vercel AI SDK, LangChain, or CrewAI specifically. Missing for 10: any documented integration guide or adapter package for these named frameworks.", "evidenceIds": [ "smooth-gh-1", "smooth-docs-6" ] }, { "productId": "smooth", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A direct probe confirms docs.smooth.sh/llms.txt returns HTTP 200 with a structured index of docs pages, and the docs themselves are mirrored as .md files (e.g. live-share.md, proxies.md) enabling agent-friendly consumption. One community comment notes the docs pages aren't fully token-efficient, a minor caveat. Missing for 10: independent verification that agents actually consume llms.txt effectively, and no evidence of additional agent-specific doc formats beyond the single llms.txt file.", "evidenceIds": [ "smooth-probe-1", "smooth-docs-2", "smooth-docs-3", "smooth-comm-9" ] }, { "productId": "smooth", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Smooth is a browser-automation SDK/CLI that lets AI agents navigate the web and extract structured data from pages — it is not a data platform or analytics product with a UI that surfaces AI-generated insights/suggestions from a user's own data. This story targets a different product category (BI/analytics-style in-product insights), so it does not apply to Smooth's browser-agent tooling.", "evidenceIds": [ "smooth-gh-1", "smooth-docs-7", "smooth-docs-8" ] }, { "productId": "smooth", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Smooth lets users kick off browser-agent tasks programmatically that run autonomously (navigating, extracting data, solving CAPTCHAs) and provides a live_url to monitor progress, which supports hands-off execution once started. However there's no evidence of scheduling, triggers, webhooks, or persistent 'set it and forget it' background jobs that run without an explicit API call — the model shown is synchronous task invocation, not autonomous background automation setup. Missing for 10: scheduling/cron or event-trigger support, evidence of long-running unattended jobs, and independent confirmation of background execution beyond a single task call.", "evidenceIds": [ "smooth-docs-1", "smooth-docs-2", "smooth-docs-7", "smooth-docs-9" ] }, { "productId": "smooth", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Smooth's core product is a task-delegation interface: users hand off a task description (e.g., navigate, extract, multi-step session workflow) to Smooth's built-in AI/browser agent, which executes autonomously and returns live_url and structured outputs (smooth-docs-1, smooth-docs-7, smooth-docs-8, smooth-docs-5). Community hands-on feedback corroborates it executing complex prompts well (smooth-comm-2, smooth-comm-1), though it is agent-facing (tool for other agents like Claude Code) as well as human-facing. Missing for 10: independent/reproducible benchmarks of task success (raised unanswered in smooth-comm-14) and clearer human-only assistant UX beyond API/CLI task calls.", "evidenceIds": [ "smooth-docs-1", "smooth-docs-7", "smooth-docs-8", "smooth-docs-5", "smooth-docs-2", "smooth-comm-1", "smooth-comm-2" ] }, { "productId": "smooth", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Smooth is explicitly designed as an SDK/API and CLI for programmatic browser automation, with 'plug-and-play' 4-line-of-code task execution and custom tools/session workflows suited to unattended automation, and a documented CLI (smooth-gh-1, smooth-docs-1, smooth-probe-3). However there is no explicit documentation or example of running it inside a CI pipeline (e.g., GitHub Actions), headless flags, or exit-code/automation-specific guidance. Missing for 10: explicit CI/CD integration docs or examples, confirmation of non-interactive/headless auth flow for pipelines, independent confirmation of CI usage.", "evidenceIds": [ "smooth-docs-1", "smooth-gh-1", "smooth-probe-3", "smooth-docs-6", "smooth-docs-7" ] }, { "productId": "smooth", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Smooth documents a 'custom tools' feature for arbitrary functions but there is no mention anywhere in docs, GitHub, or community discussion of MCP server support or integration, so this applicable capability is unevidenced.", "evidenceIds": [ "smooth-docs-6", "smooth-probe-1" ] }, { "productId": "smooth", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Smooth is a browser-automation SDK/CLI used by agents like Claude Code, but the evidence pack contains no mention of an official MCP server or MCP protocol integration anywhere in its docs, GitHub repo, or community discussion — only generic SDK/CLI methods and custom tool functions are documented.", "evidenceIds": [ "smooth-gh-1", "smooth-docs-6", "smooth-probe-1", "smooth-probe-3" ] }, { "productId": "smooth", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Smooth's core interaction model is task-based: you give it a task description that an agent executes in a browser (session workflow, extract, navigate), and it is explicitly positioned as \"a browser for AI agents\" usable by tools like Claude Code, implying natural-language task instructions. However, no evidence shows an explicit example of a natural-language prompt/command syntax or confirms this is exposed to end-users beyond agent-to-agent orchestration. Missing for 10: explicit example of a natural-language task string/command, confirmation of human-facing NL command interface, independent corroboration of NL usability.", "evidenceIds": [ "smooth-docs-1", "smooth-docs-7", "smooth-gh-1", "smooth-docs-8" ] }, { "productId": "smooth", "storyId": "agentic-official-cli", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "GitHub repo describes Smooth CLI explicitly as 'a browser for AI agents, enabling tools like Claude Code to navigate the web' and a docs probe confirms an official CLI overview page exists, showing a first-party CLI built for AI-agent workflows. Missing for 10: independent hands-on confirmation of the CLI's usage/reliability and more detailed CLI documentation content beyond the overview link.", "evidenceIds": [ "smooth-gh-1", "smooth-probe-3" ] }, { "productId": "smooth", "storyId": "agentic-public-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Smooth provides documented SDK/API methods (task execution, session workflows, structured outputs, custom tools, proxies) and an llms.txt docs index plus a CLI, showing a documented programmatic interface for AI-native use. However, no formal OpenAPI/REST spec was found (404s on all standard paths), and there is no independent corroboration of API robustness beyond vendor docs. missing for 10: a discoverable OpenAPI/REST spec, independent/hands-on verification of API completeness and stability.", "evidenceIds": [ "smooth-docs-1", "smooth-docs-6", "smooth-docs-7", "smooth-probe-1", "smooth-probe-2", "smooth-probe-3" ] }, { "productId": "smooth", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of scoped or least-privilege API credential issuance, API key scoping, or permission management for agents; docs cover task execution, sessions, proxies, and privacy features but nothing about credential scoping. missing for 10: scoped API key/token generation, permission/role controls, credential revocation or least-privilege access management.", "evidenceIds": [] }, { "productId": "smooth", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Smooth documents SDK-style integration (4-line task execution, custom tools, structured outputs, session workflows) and a CLI positioned as \"a browser for AI agents\" usable with tools like Claude Code, indicating official first-party SDK/CLI support for AI-native workflows. However, there is no OpenAPI spec, no evidence of multi-language SDKs, and no independent/hands-on confirmation of SDK reliability beyond docs and a GitHub repo. missing for 10: OpenAPI/API spec availability, multi-language SDK coverage, independent developer corroboration of SDK usage/quality.", "evidenceIds": [ "smooth-docs-1", "smooth-docs-6", "smooth-docs-7", "smooth-gh-1", "smooth-probe-3", "smooth-probe-2" ] }, { "productId": "smooth", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions webhooks or event subscriptions; Smooth's documented features (live_url, structured outputs, custom tools, sessions) do not include a webhook/event notification mechanism.", "evidenceIds": [] }, { "productId": "smooth", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows only static markdown docs (llms.txt, feature pages) and explicitly shows the openapi.json/swagger endpoints returning 404, indicating no interactive API reference or runnable-example playground exists. No mention of a Swagger UI, Postman collection, or in-browser code runner is present anywhere in docs or community discussion.", "evidenceIds": [ "smooth-probe-1", "smooth-probe-2" ] }, { "productId": "smooth", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe for OpenAPI/Swagger spec files at all standard paths returned 404, and no docs page references a downloadable machine-readable API spec; only an llms.txt (docs index) is available.", "evidenceIds": [ "smooth-probe-2", "smooth-probe-1" ] }, { "productId": "smooth", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Smooth's docs describe browser-automation features (live sessions, proxies, persistent auth, structured outputs) but nowhere mention a sandbox/staging mode or any mechanism to isolate test runs from production data or accounts. Community feedback even flags unresolved concerns about data handling and security, but no concrete sandbox capability is described or corroborated.", "evidenceIds": [] }, { "productId": "smooth", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of API versioning scheme or a documented deprecation policy; OpenAPI spec probe returned 404s and docs show no changelog/versioning references. missing for 10: versioned API scheme, deprecation policy documentation, changelog/migration guides.", "evidenceIds": [ "smooth-probe-2" ] }, { "productId": "smooth", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Smooth's docs describe single-task execution, session workflows, and structured extraction, but nothing about running/orchestrating bulk operations across many items (e.g., batch task queues, parallel task fan-out) is documented or mentioned by users.", "evidenceIds": [ "smooth-docs-1", "smooth-docs-7", "smooth-docs-8" ] }, { "productId": "smooth", "storyId": "automation-rules-engine", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Smooth is a browser-automation/agent-tool product for running tasks on demand, not an event-driven rules/automation-trigger platform; there is no mention of defining rules or triggers that fire actions on events, so this automation-depth axis (workflow/event triggers) is a category mismatch for this product type.", "evidenceIds": [] }, { "productId": "smooth", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of scheduling, cron-like triggers, or recurring workflow orchestration; Smooth is documented as a task-execution/browser-automation tool (session workflows, extraction, structured output) with no mention of recurring/scheduled jobs. Missing for 10: any scheduling API, cron/trigger mechanism, or recurring workflow docs.", "evidenceIds": [ "smooth-docs-7", "smooth-docs-1" ] }, { "productId": "smooth", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Smooth is a browser-automation/AI-agent-browsing tool (task execution, sessions, structured extraction) rather than an automation-authoring platform with version history or workflow rollback semantics; versioning/review/rollback of 'automations' is not an applicable axis for this product category.", "evidenceIds": [] }, { "productId": "smooth", "storyId": "captcha-handling", "verdict": "full", "quality": 6, "confidence": "medium", "rationale": "Smooth explicitly documents an automatic captcha-solving stance ('Auto-CAPTCHA solvers: Bypass CAPTCHA challenges automatically, allowing for uninterrupted task execution'), giving automation engineers a clear documented behavior rather than silent failure. Community reaction (smooth-comm-3) criticizes the ethics/marketing of this feature but does not present a hands-on failure showing the solver doesn't work, so this remains a documented claim rather than a disputed one. Missing for 10: independent/hands-on verification that auto-solving actually succeeds in practice, and no documentation of fallback behavior (e.g., what happens if a captcha can't be auto-solved).", "evidenceIds": [ "smooth-docs-9", "smooth-comm-3" ] }, { "productId": "smooth", "storyId": "cdp-connect-existing-stack", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Smooth's docs describe its own SDK/task API (structured outputs, sessions, custom tools) but there is no mention of a CDP endpoint, Playwright/Puppeteer connect() compatibility, or any way to point existing automation code at Smooth's browsers; one commenter even notes 'agent-browser helped a lot over playwright but doesn't completely close the gap,' underscoring the absence of such interoperability. Missing for 10: any CDP/WebSocket endpoint, official Playwright/Puppeteer connect examples, or documented browser-endpoint compatibility.", "evidenceIds": [ "smooth-docs-7", "smooth-docs-12", "smooth-comm-8" ] }, { "productId": "smooth", "storyId": "credential-vault-2fa", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The docs describe persistent sessions (log in once, reuse authentication) but there is no mention of a credential vault, secret injection to avoid model exposure, or TOTP/2FA handling anywhere in the evidence pack. This axis clearly applies to a browser-automation agent product, but no capability matching the story is documented.", "evidenceIds": [ "smooth-docs-3" ] }, { "productId": "smooth", "storyId": "deterministic-replay-caching", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of caching resolved actions/generated code for deterministic, cheaper replay; docs mention persistent sessions (auth reuse), structured outputs, and cost efficiency via small models, but nothing about caching or replay of prior task executions to skip re-prompting the LLM. Missing for 10: any mention of action/result caching, replay mechanism, or cost/latency comparison for repeat runs.", "evidenceIds": [ "smooth-docs-3", "smooth-docs-5", "smooth-docs-11" ] }, { "productId": "smooth", "storyId": "dom-action-primitives", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Smooth's docs describe a 'Session Workflow' that lets you orchestrate smaller tasks, navigate to URLs, and extract data within a persistent session, and the whole product is framed as an AI browser agent that understands pages rather than relying on brittle selectors (smooth-docs-7, smooth-gh-1). However, there is no explicit documentation of discrete act/click/type primitives on described elements, nor any evidence/testing showing these survive selector or layout changes — community comments even note it doesn't fully close the gap versus Playwright-style tools (smooth-comm-8). Missing for 10: explicit act/click/type API reference, documented resilience testing against DOM/selector changes, independent verification of robustness claims.", "evidenceIds": [ "smooth-docs-7", "smooth-gh-1", "smooth-comm-8" ] }, { "productId": "smooth", "storyId": "ethics-abuse-guardrails", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack of a published acceptable-use policy, anti-abuse terms, or guidance on permissible use of the stealth/CAPTCHA-bypass and automation features; docs only describe how to use auto-CAPTCHA and proxy features, not what usage is disallowed. Community commentary even calls out the lack of any such framing (e.g., accusing the marketing of enabling spambots), reinforcing the absence rather than disputing a claim.", "evidenceIds": [ "smooth-docs-9", "smooth-comm-3" ] }, { "productId": "smooth", "storyId": "file-download-upload", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes Smooth's session workflows, structured extraction, live-view, proxies, and persistent auth, but nothing addresses file download/upload during a browser session or persisting artifacts for later retrieval. Since browser automation tools plausibly support file transfer (e.g., downloading a report from a site or uploading a document to a form), this axis applies but is unaddressed.", "evidenceIds": [ "smooth-docs-7", "smooth-docs-8", "smooth-docs-2" ] }, { "productId": "smooth", "storyId": "hosted-task-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm a hosted task-submission model (4-line integration, live_url for tracking, persistent sessions, structured outputs) consistent with an agent submitting tasks without managing a browser, but no evidence pack item explicitly documents a polling endpoint or webhook delivery mechanism, and probes found no public OpenAPI/REST spec. missing for 10: explicit polling endpoint docs, explicit webhook/callback docs, confirmed REST API schema (openapi probe 404s).", "evidenceIds": [ "smooth-docs-1", "smooth-docs-2", "smooth-docs-7", "smooth-probe-2", "smooth-probe-3" ] }, { "productId": "smooth", "storyId": "live-view-human-takeover", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs confirm a live_url to watch agent actions in real time (smooth-docs-2), satisfying the 'watch a session live' half of the story, but there is no evidence of any mechanism for a human to intervene or take control mid-run when the agent gets stuck. missing for 10: documented human-takeover/control API or UI, evidence of pausing/resuming agent execution, hands-on confirmation of mid-run intervention.", "evidenceIds": [ "smooth-docs-2", "smooth-docs-7" ] }, { "productId": "smooth", "storyId": "local-browser-mode", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Smooth is documented as a cloud-hosted browser agent service (task execution via live_url, proxies, zero-data-retention as an 'enterprise' add-on), with no docs describing a local-browser/offline mode; a P2P tunnel feature only lets the cloud agent reach your localhost, not run without an account. Community feedback explicitly asks for self-hosting ('Make it self-hostable, the conversation can change'), confirming no local/no-account mode exists.", "evidenceIds": [ "smooth-docs-10", "smooth-docs-12", "smooth-comm-10", "smooth-comm-12" ] }, { "productId": "smooth", "storyId": "login-session-persistence", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Smooth's docs explicitly describe a persistent-sessions feature ('Log in once, then reuse that authentication for future tasks') and a session workflow that maintains a persistent browser session across multi-step tasks, directly matching the story. However, there is no independent/hands-on corroboration of this specific feature working reliably, and no detail on profile management (multiple reusable profiles, storage/export). Missing for 10: independent verification of session persistence in practice, documentation on managing multiple reusable profiles.", "evidenceIds": [ "smooth-docs-3", "smooth-docs-7" ] }, { "productId": "smooth", "storyId": "model-agnostic-llm", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Smooth's docs describe it as using its own internal 'small and efficient AI models' for browser-agent tasks, and none of the documentation, CLI, or API references mention any mechanism for developers to configure or swap in their own LLM provider (e.g., OpenAI, Anthropic, custom model endpoints). This suggests the opposite of model-agnosticism — the models are an implementation detail of the service, not a pluggable choice.", "evidenceIds": [ "smooth-docs-11", "smooth-gh-1", "smooth-probe-1" ] }, { "productId": "smooth", "storyId": "nl-task-to-completion", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs describe exactly this capability: multi-step 'Session Workflow' that navigates URLs, orchestrates sub-tasks, and extracts data, plus a live_url to watch the agent act, and a community commenter confirms 'I just wrote a complex prompt and it did a good job.' This matches the natural-language, end-to-end web task story well. Missing for 10: independently reproducible benchmarks/evals (a commenter explicitly asks for third-party reproducible comparisons and gets no clear answer), and broader hands-on validation beyond a single anecdote.", "evidenceIds": [ "smooth-docs-7", "smooth-docs-8", "smooth-docs-2", "smooth-comm-2", "smooth-comm-14" ] }, { "productId": "smooth", "storyId": "observe-suggest-actions", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Smooth's docs describe live viewing of actions as they execute (live_url) and data extraction, but there is no evidence of a distinct observe/plan step that lets a developer preview candidate actions before committing the agent to act.", "evidenceIds": [] }, { "productId": "smooth", "storyId": "openness-api-parity", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack shows Smooth as an API/CLI/SDK-first browser-automation tool with docs for tasks, sessions, proxies, custom tools, and a live_url for viewing agent actions, but there is no mention of a separate web dashboard/UI or any comparison of UI-only vs API-only capabilities. Without evidence of what a UI offers (or that all UI features are mirrored in the API), the parity claim can't be substantiated. Missing for 10: any documented web UI/dashboard, and an explicit statement or demonstration that all UI actions are also achievable via API.", "evidenceIds": [ "smooth-docs-1", "smooth-docs-7", "smooth-probe-3" ] }, { "productId": "smooth", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Smooth's docs mention Zero Data Retention (deletion of task data) but there is no evidence of a bulk data export feature or open-format export for users to take their data and leave — the closest related item is deletion, not portability. missing for 10: any documented export mechanism, open format specification, or user data portability tooling.", "evidenceIds": [ "smooth-docs-10" ] }, { "productId": "smooth", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Smooth ships a GitHub repo for its SDK/CLI, but there is no evidence of an open-source license for the core product, and community comments explicitly request self-hosting/open-source alternatives ('Make it self-hostable, the conversation can change', 'I'll wait for a free/open source browser'), implying the core service is closed.", "evidenceIds": [ "smooth-gh-1", "smooth-comm-10", "smooth-comm-12" ] }, { "productId": "smooth", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Smooth is offered only as a hosted cloud API/SaaS with no documented self-host option, and community feedback explicitly requests self-hosting as a missing capability ('Make it self-hostable, the conversation can change').", "evidenceIds": [ "smooth-comm-10", "smooth-comm-11" ] }, { "productId": "smooth", "storyId": "parallel-fleet-scale", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers single-session features (persistent sessions, live URL, proxies, structured output) but contains no documentation of concurrency limits, fleet/pool management, or APIs for running many sessions in parallel. Missing for 10: documented concurrency limits, fleet/pool orchestration APIs, rate-limit or scaling guidance, evidence of parallel session usage.", "evidenceIds": [ "smooth-docs-7", "smooth-docs-3", "smooth-docs-2" ] }, { "productId": "smooth", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of data residency/region selection options; only Zero Data Retention (deletion) is mentioned, which is a different capability. Missing for 10: any mention of region choice, data center locations, or residency controls.", "evidenceIds": [ "smooth-docs-10" ] }, { "productId": "smooth", "storyId": "privacy-no-training", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Smooth documents a 'Zero Data Retention' enterprise feature that lets customers delete all data tied to completed tasks, which is adjacent to preventing data reuse, but there is no explicit statement that data is excluded from model training, and this feature is gated to enterprise tier. Community feedback also notes an absence of any detailed security/privacy documentation despite marketing claims of 'enterprise-grade security', raising trust concerns without disputing the ZDR feature itself. missing for 10: explicit AI-training opt-out policy, default (non-enterprise) privacy guarantees, independent verification of data handling.", "evidenceIds": [ "smooth-docs-10", "smooth-comm-11" ] }, { "productId": "smooth", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm a 'Zero Data Retention' feature letting customers delete all data tied to completed tasks, directly addressing retention/deletion control, but it's explicitly gated as an 'enterprise feature' rather than a universal capability, and no detail is given on default retention periods, deletion APIs/CLI commands, or granular controls for non-enterprise users. Community feedback (e.g., concerns about sending data to a third party, no security details found) shows some skepticism but doesn't concretely contradict the ZDR claim itself. missing for 10: default/non-enterprise retention policy, self-serve deletion mechanism (API/CLI), independent verification of ZDR working in practice.", "evidenceIds": [ "smooth-docs-10", "smooth-comm-11" ] }, { "productId": "smooth", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence describes a telemetry/usage-tracking opt-out control; the only related privacy feature is 'Zero Data Retention' for enterprise customers, which addresses data deletion after tasks rather than disabling telemetry/tracking. Community comments raise general privacy/security concerns but do not confirm or deny an opt-out mechanism.", "evidenceIds": [ "smooth-docs-10", "smooth-comm-11" ] }, { "productId": "smooth", "storyId": "schema-structured-extraction", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly describe structured outputs via `response_model` for deterministic typed data and a dedicated `session-extract` method to extract structured data from a page by providing a schema, directly matching the story. Missing for 10: explicit Zod/Pydantic code examples and independent/hands-on confirmation that extraction validation works as documented.", "evidenceIds": [ "smooth-docs-5", "smooth-docs-8", "smooth-docs-7" ] }, { "productId": "smooth", "storyId": "session-replay-recordings", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs mention a `live_url` for viewing agent actions live during a run, but there is no evidence of persisted video recordings, screenshots, or a step-by-step action timeline that can be replayed after a run has finished and failed. Missing for 10: recorded video/screenshot artifacts, post-hoc replay viewer, structured action timeline, and any independent confirmation of replay-based debugging.", "evidenceIds": [ "smooth-docs-2" ] }, { "productId": "smooth", "storyId": "stealth-fingerprint-proxies", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm auto-CAPTCHA solving and configurable proxy server parameters plus persistent authenticated sessions, which support anti-bot automation goals, but there is no explicit mention of residential/geo-targeted proxy pools or stealth browser fingerprinting techniques. Missing for 10: explicit residential/geo-targeted proxy options, stealth fingerprinting details, and independent verification that bot-detection evasion actually works in practice.", "evidenceIds": [ "smooth-docs-4", "smooth-docs-9", "smooth-docs-3", "smooth-comm-3" ] }, { "productId": "smooth", "storyId": "task-completion-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions webhooks, callback URLs, or event-driven notifications for task/session completion; the docs describe live_url viewing, structured outputs, and session workflows but nothing about push notifications versus polling.", "evidenceIds": [] }, { "productId": "smooth", "storyId": "usage-pricing-transparency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of documented per-task/per-browser-hour pricing tiers or rate/concurrency limits; only a vague claim of being '7x more affordable' with no actual pricing page or limits documented, and community comments call it 'too expensive' without citing specifics.", "evidenceIds": [ "smooth-docs-11", "smooth-comm-12", "smooth-comm-15" ] }, { "productId": "smooth", "storyId": "vision-computer-use", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers Smooth's session workflows, extraction, custom tools, proxies, and CAPTCHA solving, but nowhere mentions a vision/computer-use mode operating on screenshots for canvases or non-DOM UI elements. This axis is plausible for a browser-automation agent, but no documentation or community report confirms such a capability exists.", "evidenceIds": [ "smooth-docs-7", "smooth-docs-8", "smooth-docs-6", "smooth-gh-1" ] }, { "productId": "smooth", "storyId": "workflow-loops-conditionals", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Smooth documents a 'Session Workflow' method for multi-step execution—orchestrating smaller tasks, navigating URLs, and extracting data within a persistent browser session—plus structured outputs and custom tools that let developers build deterministic logic around agent calls. However, there is no explicit documentation of native loop/conditional constructs or parameterized workflow templates; any control flow would rely on the surrounding SDK code rather than a built-in workflow engine. Missing for 10: explicit loop/conditional primitives, parameterization/templating of workflows, and independent evidence of repeatable multi-step automations beyond simple session chaining.", "evidenceIds": [ "smooth-docs-7", "smooth-docs-5", "smooth-docs-6" ] }, { "productId": "stagehand", "storyId": "agent-framework-adapters", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Docs and community evidence confirm a documented MCP/stdio integration for Claude Code agents (and a Browserbase MCP server actually powered by Stagehand), but there is no evidence of documented adapters for Vercel AI SDK, LangChain, or CrewAI as the story specifically requires. missing for 10: Vercel AI SDK adapter docs, LangChain adapter docs, CrewAI adapter docs, and any first-party guide unifying these framework integrations.", "evidenceIds": [ "stagehand-docs-11", "stagehand-comm-9", "stagehand-probe-rt-1" ] }, { "productId": "stagehand", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "The probe confirms Stagehand's docs site serves a valid llms.txt (HTTP 200) with structured links to docs pages, directly enabling an agent to be pointed at agent-oriented documentation. missing for 10: no evidence of independent/community confirmation that agents actually consume this llms.txt successfully in practice, and no detail on completeness/coverage of the llms.txt beyond the introduction page.", "evidenceIds": [ "stagehand-probe-1" ] }, { "productId": "stagehand", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Stagehand is a browser-automation/data-extraction library (act/observe/extract on web pages), not a product that stores 'my data' and surfaces AI-generated insights/dashboards from it — this consumer-analytics axis doesn't apply to its category.", "evidenceIds": [] }, { "productId": "stagehand", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Stagehand supports cloud-hosted Browserbase sessions with persistent user data/contexts, caching to speed repeat runs, and a real-time observability dashboard, all of which enable automations to run without a local machine and be monitored asynchronously. However there's no documented scheduler, trigger system, or explicit 'run unattended over time' capability distinguishing this from any scripted automation run on-demand. missing for 10: explicit scheduling/cron or trigger mechanism, evidence of long-running unattended multi-step autonomous sessions, independent confirmation of background execution reliability.", "evidenceIds": [ "stagehand-docs-7", "stagehand-docs-8", "stagehand-docs-9", "stagehand-docs-10", "stagehand-docs-6" ] }, { "productId": "stagehand", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Stagehand's core capability is letting an AI-native user delegate actions to a built-in AI (act/extract/observe) via natural language, with self-healing and DOM handling reducing manual work — this is the product's central value proposition and is corroborated by community praise for its natural-language automation and self-healing behavior. missing for 10: independent quantitative benchmarks of task delegation success/reliability and more explicit UI/chat-assistant framing beyond programmatic act()/extract() calls.", "evidenceIds": [ "stagehand-docs-1", "stagehand-docs-2", "stagehand-docs-3", "stagehand-docs-5", "stagehand-docs-13", "stagehand-comm-8", "stagehand-comm-1" ] }, { "productId": "stagehand", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Stagehand is a Node/TS library built on Playwright with a scriptable API (act/observe/extract), headless browser support via Browserbase or CDP-attached Chromium, and npm-installable package confirmed via runtime probe, making CI/headless automation straightforward. missing for 10: explicit first-party CI pipeline examples (GitHub Actions/Docker recipes) and independent hands-on confirmation of a full unattended CI run.", "evidenceIds": [ "stagehand-docs-4", "stagehand-docs-9", "stagehand-docs-15", "stagehand-probe-rt-1", "stagehand-comm-4" ] }, { "productId": "stagehand", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All evidence shows Stagehand exposing its own browser-automation tools via MCP (server role) to other agents like Claude Code, not Stagehand acting as an MCP client that consumes external MCP servers' tools. No evidence describes configuring Stagehand to plug into third-party MCP servers so it can call their tools.", "evidenceIds": [ "stagehand-docs-11", "stagehand-probe-rt-1" ] }, { "productId": "stagehand", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Stagehand ships an official MCP server ('Give a Claude Code agent persistent Stagehand browser tools over MCP/stdio') and a runtime probe confirms `npx -y @browserbasehq/mcp` actually boots and identifies as 'Browserbase MCP Server ... powered by Browserbase and Stagehand', responding to JSON-RPC initialize. Community evidence corroborates real-world use (the Stagehand MCP server won Anthropic's Claude MCP hackathon). Missing for 10: independent third-party review of the full tool surface/robustness of the MCP server beyond the hackathon anecdote and the first-party docs/probe.", "evidenceIds": [ "stagehand-docs-11", "stagehand-probe-rt-1", "stagehand-comm-9" ] }, { "productId": "stagehand", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Stagehand's core API (act/extract/observe) is explicitly designed around natural-language instructions instead of hardcoded selectors, with docs describing instruction-driven execution and self-healing when selectors break. Community commentary independently corroborates that this NL-driven approach dynamically generates Playwright actions and is more resilient than traditional automation frameworks. Missing for 10: independent quantitative benchmarks on NL command accuracy/reliability across diverse sites.", "evidenceIds": [ "stagehand-docs-1", "stagehand-docs-3", "stagehand-docs-5", "stagehand-docs-14", "stagehand-comm-8" ] }, { "productId": "stagehand", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Stagehand is distributed as an npm SDK/library plus an MCP server; the evidence pack shows npm install and MCP server invocation via npx, but no dedicated official CLI tool for direct AI-native command-line interaction is documented anywhere.", "evidenceIds": [ "stagehand-probe-rt-1", "stagehand-docs-11" ] }, { "productId": "stagehand", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Stagehand ships a well-documented public SDK API (act(), extract(), observe(), page methods, selfHeal, caching) with a dedicated docs site and llms.txt, and an npm package that a runtime probe confirms actually installs and exports a working class plus a functioning MCP server exposing the same tools. Community posts corroborate real developer usage of this API for browser automation. Missing for 10: an OpenAPI/REST spec (404s) and deeper independent (non-community) validation of API stability/versioning.", "evidenceIds": [ "stagehand-docs-1", "stagehand-docs-2", "stagehand-docs-3", "stagehand-docs-4", "stagehand-docs-5", "stagehand-probe-1", "stagehand-probe-2", "stagehand-probe-rt-1", "stagehand-comm-8" ] }, { "productId": "stagehand", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Stagehand is a browser-automation library; evidence covers API keys for model/Browserbase access but no scoped/least-privilege credential issuance, role-based permissions, or token-scoping features for agents are documented anywhere in docs, community, or probes. Missing for 10: any mention of scoped API key issuance, permission scoping, or least-privilege credential management for agents.", "evidenceIds": [ "stagehand-docs-9", "stagehand-probe-rt-1" ] }, { "productId": "stagehand", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Stagehand ships an official TypeScript/JS SDK (@browserbasehq/stagehand npm package) with documented APIs (act, observe, extract) and confirmed runtime install/import success, making it directly usable by AI-native developers building agentic browser automation. missing for 10: no evidence of official SDKs in other languages (e.g., Python) or independent third-party corroboration of SDK API stability beyond docs and one runtime probe.", "evidenceIds": [ "stagehand-probe-rt-1", "stagehand-docs-2", "stagehand-docs-3", "stagehand-docs-1", "stagehand-probe-1" ] }, { "productId": "stagehand", "storyId": "agentic-webhooks", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Stagehand is a browser automation SDK for agents/scripts to control web pages, not an event-driven platform; no evidence of any webhook subscription mechanism, and this is a category error for the product type rather than a missing feature.", "evidenceIds": [] }, { "productId": "stagehand", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows standard prose documentation pages (docs.stagehand.dev) and confirms no OpenAPI/swagger spec exists (404s on all candidate paths), with no mention anywhere of an interactive, runnable-example API reference (e.g., live code sandbox or Swagger-style explorer). The llms.txt probe only shows a docs index for LLM ingestion, not an interactive reference.", "evidenceIds": [ "stagehand-probe-2", "stagehand-probe-1", "stagehand-docs-1" ] }, { "productId": "stagehand", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Direct probes for OpenAPI/swagger specs at all standard paths returned 404, and no documentation mentions a downloadable machine-readable API spec; only an llms.txt exists which is not an API spec.", "evidenceIds": [ "stagehand-probe-2", "stagehand-probe-1" ] }, { "productId": "stagehand", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Stagehand's docs cover browser session persistence, regions, and observability but nothing about a dedicated sandbox/staging mode or safeguards to prevent hitting production data; the evidence pack shows only live browser automation against real or Browserbase-hosted sites.", "evidenceIds": [] }, { "productId": "stagehand", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Docs reference a 'v4' version path, but there is no evidence of a documented API versioning scheme or deprecation policy for Stagehand's SDK/API; no openapi spec or changelog/deprecation docs were found (openapi probe 404s).", "evidenceIds": [ "stagehand-probe-2" ] }, { "productId": "stagehand", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers single-page act/extract/observe primitives, caching, and session persistence, but nothing describes batch/bulk operations across many items or parallelized runs at once. A browser-automation framework could plausibly support this (e.g., running many sessions in parallel), so the axis applies, but no such capability is documented.", "evidenceIds": [] }, { "productId": "stagehand", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence describes Stagehand's act/observe/extract primitives for executing AI-driven browser actions, caching, and self-healing selectors, but nothing about defining persistent rules that automatically trigger on events (e.g., webhooks, schedules, DOM-change listeners) outside of an explicit script invocation.", "evidenceIds": [] }, { "productId": "stagehand", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Stagehand is a browser-automation SDK/library for scripting and controlling browser sessions via natural language, not a scheduling/orchestration platform; the evidence pack contains no concept of cron-like recurring job scheduling, and this capability is outside the product's category (scheduling would be handled by an external orchestrator invoking Stagehand scripts).", "evidenceIds": [] }, { "productId": "stagehand", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Stagehand is a browser automation library/SDK, not a workflow/automation builder with saved automation artifacts; versioning, review, and rollback of 'automations' is not a fair axis for a code-driven browser scripting tool — this is a category error, not a missing feature.", "evidenceIds": [] }, { "productId": "stagehand", "storyId": "captcha-handling", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Only a single community anecdote claims Stagehand/Browserbase have 'built in proxies and captcha' handling, but no first-party documentation in the pack states an explicit captcha policy (auto-solve, human fallback, or non-support). missing for 10: official docs describing captcha handling behavior, guidance on fallback/human-in-the-loop when captchas are hit, and independent verification beyond one forum comment.", "evidenceIds": [ "stagehand-comm-1", "stagehand-comm-3" ] }, { "productId": "stagehand", "storyId": "cdp-connect-existing-stack", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Stagehand explicitly supports attaching over CDP to any existing Chromium browser and lets you keep using familiar page-level (Playwright) methods when you don't need AI inference, which lets developers reuse existing Playwright/CDP automation rather than rewriting from scratch. However there is no evidence of native Puppeteer script compatibility, and community comments note Stagehand is positioned as a replacement/wrapper rather than a drop-in for arbitrary existing Playwright suites. Missing for 10: explicit Puppeteer interop, first-party guide on migrating existing Playwright test suites unmodified, and independent confirmation that CDP-attach preserves auth/session state seamlessly.", "evidenceIds": [ "stagehand-docs-15", "stagehand-docs-4", "stagehand-comm-8", "stagehand-comm-2" ] }, { "productId": "stagehand", "storyId": "credential-vault-2fa", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers session/state persistence (cookies, local storage, Browserbase contexts) but there is no mention of vault-based credential storage, secret injection to avoid model exposure, or TOTP/2FA handling anywhere in the docs or community evidence.", "evidenceIds": [ "stagehand-docs-7", "stagehand-docs-8", "stagehand-docs-9" ] }, { "productId": "stagehand", "storyId": "deterministic-replay-caching", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Stagehand's docs explicitly state that act(), observe(), and extract() results are cached server-side to reduce LLM costs and speed up automations, directly matching the story's request for cached deterministic replay at lower cost/latency. Additional support for persistent user data (cookies/local storage, Browserbase contexts) reinforces cross-run determinism, though there's no independent/hands-on corroboration of cache-hit behavior or cache invalidation details. Missing for 10: independent verification of caching effectiveness/determinism claims, details on cache invalidation when page structure changes, and quantified latency/cost savings.", "evidenceIds": [ "stagehand-docs-6", "stagehand-docs-7", "stagehand-docs-8" ] }, { "productId": "stagehand", "storyId": "dom-action-primitives", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs describe act()/observe()/extract() as natural-language, DOM-understanding primitives with selfHeal to re-infer broken selectors and automatic iframe/shadow-DOM handling, directly matching the story; independent HN commentary corroborates that Stagehand dynamically regenerates Playwright actions making it resilient to DOM changes. Missing for 10: rigorous independent benchmark of resilience across real layout changes and more than a single third-party account.", "evidenceIds": [ "stagehand-docs-1", "stagehand-docs-3", "stagehand-docs-5", "stagehand-docs-13", "stagehand-comm-8" ] }, { "productId": "stagehand", "storyId": "ethics-abuse-guardrails", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item references a published acceptable-use policy, anti-abuse terms, or governance statement about how Stagehand's automation/anti-detection features may or may not be used; docs focus entirely on features (act/observe/extract, caching, proxies) with no mention of usage policy or abuse prevention stance.", "evidenceIds": [] }, { "productId": "stagehand", "storyId": "file-download-upload", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Stagehand's act/observe/extract primitives, session persistence, and MCP integration, but nowhere mentions file download/upload handling or artifact retrieval after a session ends. Since browser automation tools plausibly support file transfer, this is an applicable axis with no supporting evidence.", "evidenceIds": [] }, { "productId": "stagehand", "storyId": "hosted-task-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Stagehand is presented as an SDK/library (with an MCP server for agent tool-use) rather than a hosted HTTP task-submission API; the probe explicitly found no OpenAPI/REST endpoint (openapi paths all 404), and no docs describe submitting a task and polling or receiving a webhook result. Evidence shows Browserbase-hosted browser sessions and MCP/stdio integration, but nothing matching the described async HTTP task API.", "evidenceIds": [ "stagehand-probe-2", "stagehand-docs-11", "stagehand-probe-rt-1" ] }, { "productId": "stagehand", "storyId": "live-view-human-takeover", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Stagehand's docs mention Browserbase's session dashboard offering real-time browser screen recording and replay, which covers 'watching a session live,' but there is no evidence of a mechanism to hand control back to a human mid-run when the agent stalls. missing for 10: explicit live take-over/human-in-the-loop control feature, evidence of pausing agent execution for manual intervention, and independent confirmation of this workflow.", "evidenceIds": [ "stagehand-docs-10", "stagehand-docs-9" ] }, { "productId": "stagehand", "storyId": "local-browser-mode", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs confirm Stagehand can attach to a local Chromium browser over CDP (localBrowser.connect) and persist local user data on disk, meaning it can run fully locally without Browserbase/cloud accounts; Browserbase is presented as optional rather than required. missing for 10: independent hands-on confirmation of a pure-local run with no cloud account, and clearer docs distinguishing local-only setup from Browserbase defaults.", "evidenceIds": [ "stagehand-docs-15", "stagehand-docs-7", "stagehand-docs-4" ] }, { "productId": "stagehand", "storyId": "login-session-persistence", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly describe persisting local browser user-data directories (cookies/local storage survive across runs) and Browserbase contexts for persisting session/browser data across cloud runs, directly matching the reusable-profile/login-skip story. Missing for 10: independent hands-on confirmation that a persisted profile actually skips a real login wall, and more detail on context lifecycle/expiry limits.", "evidenceIds": [ "stagehand-docs-7", "stagehand-docs-8", "stagehand-docs-9" ] }, { "productId": "stagehand", "storyId": "model-agnostic-llm", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack never documents which LLM providers Stagehand supports or describes any provider-swapping configuration; it only notes that a 'model key' is required to run (stagehand-probe-rt-1), without specifying that multiple vendors' models are interchangeable. Missing for 10: any docs or examples showing configuration for OpenAI/Anthropic/other providers, a provider-agnostic API surface, or community confirmation of using non-default models.", "evidenceIds": [ "stagehand-probe-rt-1" ] }, { "productId": "stagehand", "storyId": "nl-task-to-completion", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Stagehand's docs confirm natural-language act()/observe()/extract() primitives that can navigate, click, and fill forms, plus self-healing and iframe/shadow-DOM handling that support robustness across steps, and community posts confirm real-world scraping/automation use (stagehand-comm-1, stagehand-comm-9). However, the evidence shows these as composable single-action primitives rather than a documented single-call 'give it a goal and it autonomously completes a multi-step flow' agent mode — multi-step chains appear to require the developer to sequence act/observe/extract calls or wire it via MCP into another agent (stagehand-docs-11). Missing for 10: first-party or hands-on evidence of a single natural-language goal driving full end-to-end multi-step task completion without developer-orchestrated step chaining.", "evidenceIds": [ "stagehand-docs-1", "stagehand-docs-3", "stagehand-docs-5", "stagehand-docs-13", "stagehand-docs-11", "stagehand-comm-1", "stagehand-comm-8" ] }, { "productId": "stagehand", "storyId": "observe-suggest-actions", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "observe() is a first-class primitive documented explicitly for discovering actionable elements and returning structured actions 'you can execute or validate before acting,' directly matching the preview/plan-before-act story, and it composes with act()/extract() as separate committed steps (stagehand-docs-3). Runtime probe confirms the package installs and exposes the Stagehand class in practice. missing for 10: independent hands-on developer confirmation specifically of observe()'s preview behavior (community evidence discusses self-healing/act() but not observe() directly).", "evidenceIds": [ "stagehand-docs-3", "stagehand-docs-1", "stagehand-probe-rt-1" ] }, { "productId": "stagehand", "storyId": "openness-api-parity", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Stagehand is fundamentally a code-first SDK/API for browser automation (act/observe/extract calls, MCP integration); it has no separate primary UI whose feature set the API would need to match, aside from the auxiliary Browserbase session dashboard which is a different product's observability layer. The API-vs-UI parity framing is a category error for this kind of dev tool.", "evidenceIds": [] }, { "productId": "stagehand", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "stagehand", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains only docs, community discussion, and runtime probes about Stagehand's automation features; none of it references a public source repository, license, or open-source status. Missing for 10: any mention of a GitHub repo, license file, or open-source claim.", "evidenceIds": [] }, { "productId": "stagehand", "storyId": "openness-self-host", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Stagehand is installable via npm and can run against a locally-controlled Chromium instance via CDP (localBrowser.connect), and the runtime probe confirms local install/execution without requiring Browserbase credentials, indicating the core library can be self-hosted. However, several advanced features (proxies, captcha handling, session recording, multi-region hosting) are documented as Browserbase-cloud-only, and there is no explicit self-hosting/Docker deployment guide or licensing statement. Missing for 10: dedicated self-host deployment docs, confirmation that captcha/proxy features work without Browserbase, and independent hands-on confirmation of a fully self-hosted setup.", "evidenceIds": [ "stagehand-docs-15", "stagehand-probe-rt-1", "stagehand-docs-9", "stagehand-docs-8" ] }, { "productId": "stagehand", "storyId": "parallel-fleet-scale", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows Stagehand/Browserbase supports single-session configuration (regions, contexts, persistence, observability) but nowhere documents fleet-level concurrency limits or APIs for managing many concurrent sessions programmatically. Missing for 10: documented concurrency caps, fleet/session-pool management API, and any evidence of running many sessions in parallel.", "evidenceIds": [ "stagehand-docs-9", "stagehand-docs-10", "stagehand-docs-8" ] }, { "productId": "stagehand", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs state Browserbase-hosted sessions can be started in one of four regions and that this can keep session data in a required jurisdiction, giving some data-residency control (stagehand-docs-9). However this only covers Browserbase-run browser sessions, not other data flows like server-side act/observe/extract caching (stagehand-docs-6) or local user-data directories (stagehand-docs-7), and there is no independent/hands-on confirmation of residency guarantees. Missing for 10: region control over cached LLM/action results and other stored artifacts, explicit compliance/residency guarantees, and third-party verification.", "evidenceIds": [ "stagehand-docs-9", "stagehand-docs-6", "stagehand-docs-7" ] }, { "productId": "stagehand", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "stagehand", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Docs describe persisting user data (cookies/local storage, Browserbase contexts) and choosing a data region for jurisdiction, but there is no evidence of any deletion controls, retention limits, or user-facing settings to purge stored session/browser data. missing for 10: explicit retention policy, deletion/purge mechanism, and user controls over how long session data or extracted data is kept.", "evidenceIds": [ "stagehand-docs-7", "stagehand-docs-8", "stagehand-docs-9" ] }, { "productId": "stagehand", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item discusses telemetry, usage tracking, or opt-out settings for Stagehand; documentation covers automation features, caching, and Browserbase infrastructure but is silent on telemetry practices.", "evidenceIds": [] }, { "productId": "stagehand", "storyId": "schema-structured-extraction", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Stagehand's documented extract() API takes an instruction plus an output shape (schema), directly matching typed/schema-validated extraction rather than raw text scraping. Missing for 10: explicit mention of Zod/Pydantic naming in the evidence pack, independent/hands-on confirmation of schema validation behavior beyond docs.", "evidenceIds": [ "stagehand-docs-2", "stagehand-docs-3" ] }, { "productId": "stagehand", "storyId": "session-replay-recordings", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Stagehand relies on Browserbase's session dashboard for real-time screen recording and replay, giving some visibility into runs, but there is no documented step-by-step action timeline correlated with agent decisions, no screenshot-per-step artifact, and no dedicated debugging/replay tooling built into Stagehand itself. missing for 10: native step-by-step action timeline tied to LLM decisions, per-step screenshots, first-party replay/debug UI (beyond Browserbase's generic session recording), independent hands-on confirmation of replay-based debugging workflows.", "evidenceIds": [ "stagehand-docs-10" ] }, { "productId": "stagehand", "storyId": "stealth-fingerprint-proxies", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "A community hands-on comment praises Stagehand/Browserbase's 'built in proxies and captcha' handling for adversarial sites, and docs confirm Browserbase supports multi-region session placement (geo-targeting) — but there is no first-party documentation of stealth fingerprinting configuration or explicit residential-proxy selection controls. missing for 10: dedicated docs on fingerprint spoofing/stealth mode, explicit residential proxy configuration options, and independent verification that bot-block evasion works reliably.", "evidenceIds": [ "stagehand-comm-1", "stagehand-comm-3", "stagehand-docs-9" ] }, { "productId": "stagehand", "storyId": "task-completion-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any webhook/callback mechanism for task or session completion; Stagehand's docs focus on act/observe/extract, caching, and session dashboards but nothing about push notifications replacing polling.", "evidenceIds": [] }, { "productId": "stagehand", "storyId": "usage-pricing-transparency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack items mention pricing tiers, per-task/per-browser-hour costs, or documented rate/concurrency limits — docs cover only technical features (caching, contexts, regions) and community comments are unrelated to pricing transparency.", "evidenceIds": [] }, { "productId": "stagehand", "storyId": "vision-computer-use", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No documentation describes a vision/computer-use mode operating on screenshots; the only related evidence is a community comment stating the team is 'working on a better computer use integration' — implying it isn't shipped yet.", "evidenceIds": [ "stagehand-comm-9" ] }, { "productId": "stagehand", "storyId": "workflow-loops-conditionals", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Stagehand is a code-first SDK (act/observe/extract primitives) that developers embed in regular TypeScript/Python code, which implicitly allows loops, conditionals, and parameters since it's just function calls in a host language — but no docs explicitly describe workflow composition, control-flow patterns, or parameterized repeatable workflows as a first-class feature. missing for 10: explicit workflow/orchestration documentation, examples of loops or conditionals chaining act/observe/extract calls, parameterized workflow templates, and any community evidence of engineers building such multi-step conditional workflows.", "evidenceIds": [ "stagehand-docs-2", "stagehand-docs-3", "stagehand-docs-4", "stagehand-docs-6" ] }, { "productId": "steel", "storyId": "agent-framework-adapters", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Steel documents a concrete integration with the Claude Agent SDK exposing its browser as in-process MCP tools (steel-docs-9), showing at least one first-party framework adapter exists. However, the evidence pack contains no documented adapters or integration guides for Vercel AI SDK, LangChain, or CrewAI, so the broader multi-framework claim is only partially substantiated. missing for 10: documented adapters for Vercel AI SDK, LangChain, and CrewAI, plus independent corroboration of any of these integrations working in practice.", "evidenceIds": [ "steel-docs-9" ] }, { "productId": "steel", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Probe confirms llms.txt is live at docs.steel.dev/llms.txt (HTTP 200) with agent-oriented framing, and the docs also expose an OpenAPI spec, making the docs machine/agent consumable. missing for 10: no independent third-party confirmation that agents actually consume the llms.txt file successfully in practice.", "evidenceIds": [ "steel-probe-1", "steel-probe-2" ] }, { "productId": "steel", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Steel is browser automation/session infrastructure for AI agents, not a product holding a user's own dataset to analyze; its agent-traces feature is a raw activity timeline/export, not AI-generated insights or suggestions over user data. This axis is a category mismatch for an infra API rather than a data/analytics product.", "evidenceIds": [ "steel-docs-6", "steel-docs-7" ] }, { "productId": "steel", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Steel provides on-demand browser sessions, CLI, SDK, and agent-trace tooling for agents to control browsers, but nothing in the evidence describes a scheduling/trigger mechanism or persistent background job runner that lets a user set up automations to run autonomously without invocation — sessions are explicitly spun up 'on demand' by an agent/script, not scheduled by Steel itself.", "evidenceIds": [ "steel-docs-1", "steel-docs-8", "steel-docs-10", "steel-probe-rt-1" ] }, { "productId": "steel", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Steel is browser infrastructure/API tooling for AI agents (session management, stealth, proxies, CLI, MCP tool exposure) — it is consumed by external AI agents, not itself a product with a built-in AI assistant a user delegates tasks to. This story is a category error for an infrastructure/API product like Steel.", "evidenceIds": [] }, { "productId": "steel", "storyId": "agentic-headless", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Steel provides a documented CLI for end-to-end headless browser workflows from the terminal, a REST/SDK API for programmatic session creation, an open-source Docker image for self-hosting, and a verified probe confirming a keyless self-host roundtrip (docker run, health check, session creation via API, CLI install, SDK install) — all strongly supporting CI/headless automation use. missing for 10: no explicit first-party CI pipeline example (e.g., GitHub Actions template) or independent hands-on report of running Steel inside an actual CI system.", "evidenceIds": [ "steel-docs-8", "steel-gh-4", "steel-gh-1", "steel-probe-rt-1", "steel-probe-3", "steel-docs-1" ] }, { "productId": "steel", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Steel is a cloud browser infrastructure/API product, not an agent or orchestrator that would consume external MCP servers' tools. The evidence shows the opposite direction — Steel itself is exposed as MCP tools to other agent frameworks (e.g., Claude Agent SDK) — meaning Steel plays the tool-provider role, not the MCP-client role this story describes.", "evidenceIds": [ "steel-docs-9" ] }, { "productId": "steel", "storyId": "agentic-mcp-server", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Steel's docs mention an integration with the Claude Agent SDK that 'exposes a cloud browser as in-process MCP tools,' showing some official MCP tool exposure for agents, but there's no evidence of a standalone, general-purpose official MCP server endpoint independent of this one SDK integration. missing for 10: a dedicated/standalone MCP server doc or endpoint usable by any agent framework, independent corroboration or hands-on proof of MCP connectivity.", "evidenceIds": [ "steel-docs-9" ] }, { "productId": "steel", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Steel exposes its browser control as MCP tools within agent SDKs (e.g., Claude Agent SDK) so an AI agent can translate natural-language user requests into Steel API calls, and the CLI/SDK/API allow full programmatic control — but there's no evidence of a native natural-language interface to Steel itself (e.g., a chat command layer); control still requires structured API/CLI calls or a separate agent framework. Missing for 10: a first-party NL command interface or chat-driven control surface, and independent confirmation that NL-driven agent use works end-to-end in production.", "evidenceIds": [ "steel-docs-9", "steel-docs-8", "steel-gh-1" ] }, { "productId": "steel", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Steel ships a documented official CLI (steel-docs-8) that supports end-to-end browser workflows from the terminal, and probe evidence confirms real installation via setup.steel.dev installing 'steel CLI 0.4.4' into a fresh environment (steel-probe-rt-1), corroborating the docs. Missing for 10: independent third-party reviews of the CLI's UX/reliability beyond the vendor-run probe, and more detail on advanced CLI subcommands/scripting capabilities.", "evidenceIds": [ "steel-docs-8", "steel-probe-3", "steel-probe-rt-1" ] }, { "productId": "steel", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Steel publishes a full OpenAPI spec (steel-probe-2), documented Sessions API with SDKs, and a CLI, all confirmed hands-on by a runtime probe showing session creation, health checks, and SDK usage working end-to-end. This is strong first-party documentation plus independent verification of a working public API. Missing for 10: no third-party community deep-dive validating API completeness beyond the probe.", "evidenceIds": [ "steel-probe-2", "steel-docs-1", "steel-docs-8", "steel-probe-3", "steel-probe-rt-1" ] }, { "productId": "steel", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack of scoped or least-privilege API key/credential issuance (e.g., role-based keys, permission scopes, restricted tokens) for agents — only session isolation, proxies, and CLI features are documented, none of which address credential scoping.", "evidenceIds": [] }, { "productId": "steel", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Steel ships an official steel-sdk npm package (verified working via probe: exports a Steel client class), a documented OpenAPI spec, and an official CLI for full browser workflows, all covered in first-party docs and confirmed by a hands-on runtime probe. missing for 10: explicit multi-language SDK coverage (e.g., Python/other languages) and independent community confirmation of SDK usage beyond the CLI/API.", "evidenceIds": [ "steel-probe-rt-1", "steel-probe-2", "steel-probe-3", "steel-docs-8", "steel-docs-9" ] }, { "productId": "steel", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack mentions webhooks or event subscriptions; Steel's documented surfaces are Sessions API, CLI, MCP tools, and agent traces, none of which describe a webhook/event notification mechanism.", "evidenceIds": [] }, { "productId": "steel", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Steel publishes a full OpenAPI spec (docs.steel.dev/openapi.json) and an llms.txt, and a community commenter independently praised the docs/API reference quality, suggesting an interactive, well-documented API surface. However, there's no explicit evidence of an in-browser 'try it' / runnable-example console distinct from static docs. Missing for 10: direct confirmation of an interactive try-it console with live runnable code snippets, and independent hands-on verification of that specific feature.", "evidenceIds": [ "steel-probe-2", "steel-probe-1", "steel-comm-3" ] }, { "productId": "steel", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A probe confirms a live, valid OpenAPI JSON spec served at docs.steel.dev/openapi.json (HTTP 200 with 'openapi' key), directly satisfying the machine-readable spec requirement, complemented by an llms.txt index for discoverability. missing for 10: independent third-party corroboration beyond the automated probe.", "evidenceIds": [ "steel-probe-2", "steel-probe-1" ] }, { "productId": "steel", "storyId": "api-sandbox", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Steel's core Sessions API spins up isolated, on-demand browser instances each with their own state, cookies, and storage, and this isolation was independently verified via a self-hosted runtime probe that created a live, separate browser session from a throwaway Docker instance — effectively a sandbox with no shared production state. Missing for 10: explicit documentation framing sessions as a 'test vs production' environment, and no first-party guidance on staging/production data separation policies beyond session isolation.", "evidenceIds": [ "steel-docs-1", "steel-docs-10", "steel-probe-rt-1", "steel-gh-4" ] }, { "productId": "steel", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions API versioning scheme or a documented deprecation policy; only an OpenAPI spec and general docs are shown, neither of which discuss version lifecycle or deprecation commitments.", "evidenceIds": [] }, { "productId": "steel", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Steel's Sessions API allows spinning up isolated browser sessions on demand and reusing auth context across multiple sessions, which implies you could programmatically launch many sessions for parallel/bulk tasks, but there is no explicit documentation of a batch/bulk API, concurrency limits, or guidance for orchestrating many items at once. missing for 10: explicit bulk/batch API or documented pattern for running many operations concurrently, concurrency/rate limits, and independent evidence of large-scale parallel session usage.", "evidenceIds": [ "steel-docs-1", "steel-docs-4", "steel-docs-10" ] }, { "productId": "steel", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Steel's evidence covers sessions, proxies, captcha solving, CLI, and agent traces, but nothing describes a rules engine or event-triggered automation (e.g., 'if X happens then do Y') that a user could configure declaratively.", "evidenceIds": [] }, { "productId": "steel", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Steel is a browser automation/session API for AI agents (sessions, stealth, proxies, CLI, MCP tools), but nothing in the evidence describes a scheduler, cron-like trigger, or recurring job/workflow orchestration feature; usage is invoked on-demand per session, not on a schedule.", "evidenceIds": [] }, { "productId": "steel", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Steel is a cloud browser/session API for AI agents (sessions, stealth, proxies, agent traces, CLI) — it is not an automation-authoring platform with versioned workflows to review or roll back; that concept doesn't map onto its product category.", "evidenceIds": [] }, { "productId": "steel", "storyId": "captcha-handling", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Steel documents a single explicit flag covering captcha detection, solving, and verification for the whole session, plus a separate human-in-the-loop debug URL feature for manual takeover when needed — giving automation engineers a documented stance rather than silent failures. Missing for 10: independent/hands-on verification that automatic captcha solving actually succeeds on real-world captchas, and clearer documentation of failure/fallback behavior when auto-solve fails.", "evidenceIds": [ "steel-docs-2", "steel-docs-5" ] }, { "productId": "steel", "storyId": "cdp-connect-existing-stack", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Steel explicitly supports connecting existing Puppeteer, Playwright, or Selenium code via CDP to control its browser instances, and a runtime probe confirms live sessions expose websocket/debugger URLs consistent with CDP connectivity. Docs and SDK further corroborate first-class session management compatible with standard automation libraries. Missing for 10: independent third-party hands-on confirmation specifically of a rewritten Playwright/Puppeteer script running unmodified against Steel.", "evidenceIds": [ "steel-gh-1", "steel-probe-rt-1", "steel-docs-1" ] }, { "productId": "steel", "storyId": "credential-vault-2fa", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Steel's docs show session-level auth persistence (reusing cookies/storage across sessions) but no evidence of a credentials vault, secret injection without model exposure, or TOTP/2FA challenge automation — the core asks of this story are unaddressed.", "evidenceIds": [ "steel-docs-4" ] }, { "productId": "steel", "storyId": "deterministic-replay-caching", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Steel's evidence covers session management, stealth, proxies, human-in-the-loop debugging, and agent traces, but nothing about caching resolved actions or generated code to enable deterministic, lower-cost replay without re-invoking the LLM. Agent traces (steel-docs-6/7) provide observability/export, not action-cache replay for cost savings.", "evidenceIds": [] }, { "productId": "steel", "storyId": "dom-action-primitives", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Steel's evidence shows only traditional CDP/Puppeteer/Selenium-based control and CLI commands like click/fill/type (steel-docs-8, steel-gh-1), which are selector-based automation primitives, not AI/DOM-understanding 'act on described element' primitives that resolve targets semantically and survive selector/layout changes. No documentation or hands-on evidence describes a Stagehand-like natural-language action resolver.", "evidenceIds": [ "steel-docs-8", "steel-gh-1" ] }, { "productId": "steel", "storyId": "ethics-abuse-guardrails", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack documents Steel's stealth/captcha-solving, proxy, and CAPTCHA features extensively, but contains no published acceptable-use policy, terms of service, or anti-abuse statement governing what these stealth capabilities may be used for. No AUP, ToS, or anti-abuse page is cited or referenced anywhere in the docs, GitHub repo, or community discussion. Missing for 10: a published acceptable-use policy, anti-abuse/misuse guidelines, or ToS language specifically addressing stealth/captcha feature usage.", "evidenceIds": [] }, { "productId": "steel", "storyId": "file-download-upload", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence covers session artifacts (traces, screenshots, page-to-markdown conversion) and CLI-driven browser control, but nothing documents actual file upload to web forms or downloading files from a site with persistent artifact retrieval — a distinct capability from trace export.", "evidenceIds": [] }, { "productId": "steel", "storyId": "hosted-task-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Steel's docs describe a Sessions API that hands agents a raw, controllable browser (via CDP/Puppeteer/Playwright) plus a CLI for scripted step-by-step actions, but there is no evidence of a higher-level 'submit a task, poll or get a webhook for the result' abstraction — the agent still must drive the browser session itself rather than delegate a task and retrieve a finished output.", "evidenceIds": [ "steel-docs-1", "steel-docs-8", "steel-gh-1", "steel-probe-rt-1" ] }, { "productId": "steel", "storyId": "live-view-human-takeover", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Steel's debug URL feature is explicitly documented for human-in-the-loop workflows enabling users to directly interact with and control a live browser session, and runtime proof confirms sessions expose a live debugger/websocket URL for real-time viewing/control. The agent-traces timeline feature complements this by letting engineers review what happened, though it's more post-hoc than live takeover. Missing for 10: explicit documentation of mid-run handoff back to the agent after human control, and independent/community corroboration of the human-in-the-loop debug feature specifically (vs. general product commentary).", "evidenceIds": [ "steel-docs-5", "steel-docs-6", "steel-probe-rt-1" ] }, { "productId": "steel", "storyId": "local-browser-mode", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Steel Browser is open-source and can be self-hosted via Docker with no cloud account, confirmed by a runtime probe showing a local Docker container booting the browser API, creating live sessions, and working with the official CLI and SDK entirely locally. missing for 10: independent third-party (non-vendor) confirmation of long-term local dev workflow beyond the single recorded probe.", "evidenceIds": [ "steel-probe-rt-1", "steel-gh-4", "steel-gh-1", "steel-probe-3" ] }, { "productId": "steel", "storyId": "login-session-persistence", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Steel docs explicitly document reusing auth context across sessions to let agents skip repeated logins, backed by session isolation, custom session IDs, and a real API/CLI/SDK confirmed via runtime probe. Missing for 10: independent third-party hands-on confirmation of the reuse-auth-context feature specifically (only vendor docs cite it) and no explicit profile-export/import UX details beyond the docs description.", "evidenceIds": [ "steel-docs-4", "steel-docs-1", "steel-docs-10", "steel-probe-rt-1" ] }, { "productId": "steel", "storyId": "model-agnostic-llm", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "steel", "storyId": "nl-task-to-completion", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Steel provides the browser primitives (sessions, navigate/click/fill/extract via CLI or API, CDP control) that a multi-step web task requires, and its CLI/MCP integrations let external agent frameworks drive those actions from natural-language goals. However, Steel's own docs state the agent loop and NL reasoning are handled by the paired SDK (e.g., Claude Agent SDK), not by Steel itself — Steel 'handles the browser' while the SDK runs the reasoning loop, so Steel alone does not accept a raw NL goal and autonomously plan/execute it end to end. missing for 10: evidence of Steel natively parsing/planning from a raw NL instruction without an external agent/LLM orchestrating the steps, and independent hands-on proof of a full NL-driven multi-step flow completed unattended.", "evidenceIds": [ "steel-docs-8", "steel-docs-9", "steel-gh-1", "steel-probe-rt-1" ] }, { "productId": "steel", "storyId": "observe-suggest-actions", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Steel exposes browser observation primitives (page-to-markdown/readability/screenshot extraction, agent traces/timeline, and a debug URL for human-in-the-loop review) that let a developer inspect page state or a human intervene mid-session, but there is no documented 'plan' or dry-run API that lets an agent preview a set of candidate actions before committing to execute them. missing for 10: an explicit plan/observe-then-act primitive or dry-run action preview API, evidence of independent developers using it specifically for pre-commit action review.", "evidenceIds": [ "steel-gh-3", "steel-docs-5", "steel-docs-6" ] }, { "productId": "steel", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "low", "rationale": "Steel is fundamentally API-first: the Sessions API, CAPTCHA/proxy/auth-reuse, human-in-the-loop debug URLs, agent traces, and even the CLI are all documented as thin wrappers over the same REST/OpenAPI surface (steel-docs-1..10, steel-probe-2, steel-probe-3), and the runtime probe confirms the API alone (no UI) can create/list live sessions end-to-end. However, there's no explicit evidence enumerating the dashboard/UI feature set and confirming 1:1 parity with the API — it's inferred rather than directly stated. Missing for 10: an explicit doc/comparison confirming every UI-only feature (e.g. dashboard analytics, trace viewer) is also exposed via API, and independent confirmation of full parity.", "evidenceIds": [ "steel-docs-1", "steel-docs-2", "steel-docs-4", "steel-docs-5", "steel-docs-6", "steel-docs-8", "steel-probe-2", "steel-probe-rt-1" ] }, { "productId": "steel", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Steel's agent-traces feature explicitly supports exporting run data as markdown, JSON, or a ZIP with markdown+screenshots (steel-docs-7), and the product itself is open-source and self-hostable (steel-gh-4, steel-probe-rt-1), meaning users are never locked into a proprietary cloud and can run/keep everything themselves. However, there's no documented comprehensive 'export all my data' capability covering sessions, auth contexts, or account-level data beyond traces. Missing for 10: a documented full-account data export/portability feature covering sessions, auth states, and configs, not just trace recordings.", "evidenceIds": [ "steel-docs-7", "steel-gh-4", "steel-probe-rt-1" ] }, { "productId": "steel", "storyId": "openness-open-license", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "The product's llms.txt explicitly states 'Steel is the open-source browser API for AI agents,' there's a public GitHub repo (steel-dev/steel-browser) with feature docs, and a runtime probe confirms the self-hostable OSS image can be pulled and run via Docker — all consistent with source availability. However, no evidence pack item names or shows a specific OSS license (e.g., MIT/AGPL) or a LICENSE file, so the exact openness terms are unconfirmed. Missing for 10: explicit license file/type citation, independent confirmation of license terms.", "evidenceIds": [ "steel-gh-1", "steel-gh-4", "steel-probe-1", "steel-probe-rt-1", "steel-comm-3" ] }, { "productId": "steel", "storyId": "openness-self-host", "verdict": "full", "quality": 10, "confidence": "high", "rationale": "Steel is explicitly open-source with a pre-built Docker image (combined API + UI), and a runtime probe confirms a full keyless self-host roundtrip: docker-running the OSS image, health check succeeding, and creating/listing live browser sessions, corroborating vendor docs and GitHub claims.", "evidenceIds": [ "steel-gh-4", "steel-gh-1", "steel-probe-rt-1", "steel-probe-1" ] }, { "productId": "steel", "storyId": "parallel-fleet-scale", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Steel's Sessions API and CLI clearly support spinning up isolated on-demand browser sessions with custom session IDs, and a runtime probe confirms working session creation/listing via self-hosted API — solid programmatic session management. However, the evidence pack contains no documented concurrency limits, quotas, or fleet-scale guidance for running many sessions in parallel. Missing for 10: explicit documented concurrency/session limits, guidance or examples for orchestrating many simultaneous sessions, and independent corroboration of scale behavior.", "evidenceIds": [ "steel-docs-1", "steel-docs-10", "steel-docs-8", "steel-probe-rt-1" ] }, { "productId": "steel", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Steel offers a self-hostable open-source Docker image (steel-gh-4, steel-probe-rt-1) which lets users control where their data physically resides by hosting it themselves, but there is no explicit region/residency selection feature documented for the managed cloud offering. missing for 10: explicit region-selection UI/API for the managed cloud service, documentation on data residency guarantees or compliance certifications (e.g., GDPR/SOC2 region controls).", "evidenceIds": [ "steel-gh-4", "steel-probe-rt-1" ] }, { "productId": "steel", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "steel", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Steel is open-source and self-hostable (steel-gh-4, steel-probe-rt-1), which gives users full control over where session data lives and how long it's retained, and session lifecycle docs show sessions can be created/stopped with custom IDs (steel-docs-10). However, there is no explicit documentation of a retention policy, data-deletion API, or GDPR-style controls for the managed cloud offering. Missing for 10: documented retention/deletion controls or policy for the hosted cloud service, explicit data-purge API, independent confirmation of retention behavior.", "evidenceIds": [ "steel-gh-4", "steel-probe-rt-1", "steel-docs-10" ] }, { "productId": "steel", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions telemetry, analytics, usage tracking, or an opt-out mechanism; the self-hostable OSS nature is noted (steel-probe-rt-1) but that alone does not confirm any telemetry opt-out control exists or is documented.", "evidenceIds": [] }, { "productId": "steel", "storyId": "schema-structured-extraction", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Steel's docs/GitHub only show raw content extraction utilities (markdown, readability, screenshots, PDF conversion) and generic 'extract content' CLI commands, with no mention of Zod/Pydantic-style schema validation or typed structured outputs. The axis is fair for a browser-automation API since competitors offer schema-based extraction, but no evidence shows Steel provides this.", "evidenceIds": [ "steel-gh-3", "steel-docs-8" ] }, { "productId": "steel", "storyId": "session-replay-recordings", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Steel's Agent Traces feature explicitly turns a run into a timeline of agent activity with screenshots, and lets you export as markdown, JSON, or a ZIP with markdown+screenshots, directly matching the replay-debugging story for automation engineers. Missing for 10: explicit video recording/playback evidence and independent/hands-on corroboration of the traces UI beyond first-party docs.", "evidenceIds": [ "steel-docs-6", "steel-docs-7", "steel-docs-5" ] }, { "productId": "steel", "storyId": "stealth-fingerprint-proxies", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Steel's docs explicitly document stealth mode (single flag covering detection evasion, captcha solving, and verification) plus both Managed Residential Proxies and Bring-Your-Own-Proxy (BYOP) for geo-targeting, directly matching the story, and community comment confirms custom proxy support works in practice. Missing for 10: no independent/hands-on evidence quantifying bot-detection bypass success rates or geo-targeting granularity, and no first-party benchmark showing reduced block rates.", "evidenceIds": [ "steel-docs-2", "steel-docs-3", "steel-comm-4" ] }, { "productId": "steel", "storyId": "task-completion-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions webhooks, event callbacks, or push notifications for session/task completion; Steel's docs cover sessions API, CLI, proxies, captcha, and traces but nothing about async status notifications, implying developers must poll session status endpoints.", "evidenceIds": [] }, { "productId": "steel", "storyId": "usage-pricing-transparency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No first-party documentation in the evidence pack lays out per-task or per-browser-hour pricing or concurrency/rate limits; the only pricing-related evidence is a community report of inconsistency between the pricing page and docs pricing ($59 vs $99), which itself signals the opposite of transparent, dependable pricing rather than confirming it.", "evidenceIds": [ "steel-comm-1" ] }, { "productId": "steel", "storyId": "vision-computer-use", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Steel can capture screenshots, convert pages to markdown/PDF, and offers a debug URL for human-in-the-loop control, but there is no mention of a vision/computer-use action mode where an agent issues click/type actions based on screenshot coordinates instead of DOM selectors.", "evidenceIds": [] }, { "productId": "steel", "storyId": "workflow-loops-conditionals", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Steel's docs describe browser session management, stealth, CLI scripting of sequential steps (create session → navigate → click/fill → extract → stop), and MCP tool exposure, but nothing in the evidence pack shows a workflow-composition layer with loops, conditionals, or parameterized branching — the CLI and SDK are linear step sequences, not a control-flow DSL. missing for 10: any documented loop/conditional constructs, parameterized workflow templates, or reusable multi-branch automation definitions.", "evidenceIds": [ "steel-docs-8", "steel-docs-10", "steel-gh-1" ] } ]