[ { "productId": "auth0", "storyId": "act-as-oauth-provider", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Auth0's core value proposition includes acting as an OAuth 2.0 authorization server that lets you 'secure your own API' and issue access tokens via flows like Client Credentials (M2M) and Authorization Code, which is the mechanism for turning an app into a token issuer for other clients (auth0-docs-2, auth0-docs-8/25, auth0-docs-54). Organizations also let you configure federated login for external business partners (auth0-docs-6/44). However, the evidence pack lacks explicit documentation of a self-service 'third-party app' registration/consent flow (dynamic client registration, developer marketplace, per-client consent screens) that fully mirrors 'Sign in with my app' for arbitrary third-party developers, missing for 10: dedicated third-party client registration/consent UI docs, dynamic client registration support, and independent hands-on confirmation of building a public OAuth provider on Auth0.", "evidenceIds": [ "auth0-docs-2", "auth0-docs-8", "auth0-docs-25", "auth0-docs-54", "auth0-docs-6", "auth0-docs-44" ] }, { "productId": "auth0", "storyId": "active-session-visibility", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack describes session/device visibility or revocation features (active session lists, per-device revocation, or 'log out all devices' controls) for end users or admins; the pack covers login flows, MFA, SSO, RBAC, migration, and AI agent integrations but never session management.", "evidenceIds": [] }, { "productId": "auth0", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Auth0 has a live llms.txt file (HTTP 200) that serves as a documentation index, and individual docs pages expose .md variants pointing agents back to the llms.txt index, directly matching the agent-oriented docs pattern. Missing for 10: no independent/community confirmation that agents actually consume this successfully in practice.", "evidenceIds": [ "auth0-probe-1", "auth0-probe-2", "auth0-docs-63" ] }, { "productId": "auth0", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack's AI material (auth0-docs-15 to 20, 29-31, 41-42) is entirely about Auth0 securing AI agents and enabling agentic authentication flows, not about Auth0 itself surfacing AI-generated insights or suggestions from the customer's own tenant/usage data (e.g., no AI-powered anomaly analysis, dashboard copilot, or suggested configuration insights). No evidence shows an in-product AI assistant analyzing logs/user data and proactively suggesting actions.", "evidenceIds": [] }, { "productId": "auth0", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Auth0 is an identity platform, not a workflow/automation builder, but it does support agentic use cases where an AI agent can act on a user's behalf asynchronously — e.g. Actions run automatically as tenant-defined logic on login events, and CIBA lets agents request approval for sensitive operations 'even when the user is not actively using the application,' enabling some background/autonomous agent behavior. This is far short of a general-purpose 'set up automations that run autonomously' capability. Missing for 10: a scheduler or workflow orchestration engine, evidence of user-configured recurring/background jobs, and any hands-on/independent confirmation that agents actually run unattended via these primitives.", "evidenceIds": [ "auth0-docs-18", "auth0-docs-31", "auth0-docs-52", "auth0-docs-59" ] }, { "productId": "auth0", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Auth0 documents an MCP server that lets external AI tools (Claude, Cursor, Windsurf) manage the tenant via natural language, and features for building AI agents into customer apps — but this is the reverse of the story: Auth0 itself does not ship a built-in AI assistant inside its own product/dashboard that users delegate tasks to. No evidence of an in-product Auth0 assistant exists in the pack.", "evidenceIds": [ "auth0-docs-13", "auth0-docs-41", "auth0-gh-1" ] }, { "productId": "auth0", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Auth0 supports headless/CI-friendly automation via the Client Credentials (M2M) flow for machine-to-machine auth, an official CLI (auth0-cli) for managing tenant config and automating tasks, and Management/Authentication APIs suitable for scripted use. However, evidence doesn't show CI-specific guides, service-account key rotation patterns, or non-interactive testing/staging workflows tailored for CI pipelines. missing for 10: CI-specific documentation/examples, headless testing guidance, evidence of CI pipeline integration patterns.", "evidenceIds": [ "auth0-docs-8", "auth0-docs-25", "auth0-probe-5", "auth0-docs-61" ] }, { "productId": "auth0", "storyId": "agentic-mcp-client", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Auth0's AI-agent docs explicitly state that its identity platform lets AI agents 'connect to MCP servers' securely (unique agent identities, Token Vault for token management) rather than Auth0 itself being the MCP client consuming tools — this is infrastructure for developers' agents, not Auth0 acting as the tool-consuming agent. Evidence is limited to marketing-style doc snippets with no technical walkthrough or hands-on confirmation of actual MCP tool invocation. Missing for 10: a concrete technical guide/example showing an agent built on Auth0 actually calling tools from a connected MCP server, and independent/hands-on corroboration of this specific flow.", "evidenceIds": [ "auth0-docs-20", "auth0-docs-68", "auth0-docs-17", "auth0-docs-30" ] }, { "productId": "auth0", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Auth0 ships an official MCP Server (auth0-mcp-server) documented on their docs site and GitHub, letting AI agents like Claude Desktop, Cursor, or Windsurf connect via OAuth 2.0 Device Authorization to manage the tenant (create apps, deploy Actions, manage users, debug logs). This is a first-party, well-documented server-side MCP integration with both docs and GitHub repo corroboration. Missing for 10: independent/hands-on third-party review of the MCP server's reliability in production use.", "evidenceIds": [ "auth0-docs-13", "auth0-docs-41", "auth0-docs-60", "auth0-gh-1", "auth0-probe-4" ] }, { "productId": "auth0", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Auth0 ships an official MCP Server (documented and open-sourced on GitHub) that lets AI tools like Claude Desktop, Cursor, or Windsurf manage the Auth0 tenant (creating apps, deploying Actions, debugging logs, managing users) via natural language, using secure OAuth 2.0 Device Authorization for auth. missing for 10: independent hands-on user reports validating the natural-language UX, and detail on which operations are not yet supported via MCP.", "evidenceIds": [ "auth0-docs-13", "auth0-docs-41", "auth0-docs-60", "auth0-gh-1", "auth0-probe-4" ] }, { "productId": "auth0", "storyId": "agentic-official-cli", "verdict": "full", "quality": 7, "confidence": "high", "rationale": "Auth0 ships an official CLI (github.com/auth0/auth0-cli, documented at auth0.com/docs/deploy-monitor/auth0-cli) for managing tenant configuration and automating tasks, which fits the 'official CLI' story for AI-native/agentic workflows. Missing for 10: no independent/hands-on review of the CLI's scripting or automation ergonomics, and no explicit detail on machine-readable output formats for agent integration.", "evidenceIds": [ "auth0-docs-61", "auth0-probe-5" ] }, { "productId": "auth0", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Auth0 exposes a well-documented Management/Authentication API, CLI (auth0-cli), and even an MCP server for AI-native/agentic access, all backed by first-party docs and a public GitHub repo. missing for 10: a discoverable OpenAPI/swagger spec (probe found all candidate paths 404) and independent hands-on developer corroboration specifically of the API/CLI (not just general product sentiment).", "evidenceIds": [ "auth0-docs-61", "auth0-probe-5", "auth0-docs-13", "auth0-docs-41", "auth0-gh-1", "auth0-probe-3", "auth0-probe-1" ] }, { "productId": "auth0", "storyId": "agentic-scoped-keys", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Auth0 explicitly documents scoped, least-privilege credential issuance for AI agents: M2M client-credentials flow with defined scopes, Token Vault for scoped third-party API tokens, and Auth0 for AI Agents messaging that agents avoid 'broad, unrestricted access' via OAuth 2.0 scoping and unique per-agent identities. This directly matches the story of issuing scoped/least-privilege API credentials to an agent. missing for 10: independent/hands-on verification of scope granularity in practice, and no example showing a concrete least-privilege scope configuration for an agent use case.", "evidenceIds": [ "auth0-docs-8", "auth0-docs-16", "auth0-docs-17", "auth0-docs-20", "auth0-docs-30", "auth0-docs-59", "auth0-docs-68" ] }, { "productId": "auth0", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Auth0 publishes a dedicated AI-agent developer surface (auth0.com/ai/docs) covering agent identity, Token Vault for API access, CIBA async approvals, and FGA for RAG pipelines, described as implementable with 'just a few lines of code,' implying official SDK-level integration for AI-native builders. However, the evidence pack never explicitly names or links a concrete 'Auth0 AI SDK' package, code sample, or language-specific SDK reference for AI agent workflows. Missing for 10: explicit SDK package/repo names for AI agent integration, code snippets/quickstart demonstrating SDK usage, and independent developer corroboration of building against these SDKs.", "evidenceIds": [ "auth0-docs-15", "auth0-docs-16", "auth0-docs-17", "auth0-docs-18", "auth0-docs-19", "auth0-docs-20", "auth0-docs-68" ] }, { "productId": "auth0", "storyId": "agentic-webhooks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Auth0 has log streaming (export tenant logs to analysis services) which is an event-export mechanism, but this is not a true webhook subscription API for arbitrary event types—it's log streaming to specific integrations (Datadog, Splunk, etc.), not a generic webhook subscription model. There's no documented dedicated webhook subscription API/endpoint for AI-native agents to subscribe to auth events. Missing for 10: a dedicated webhooks/event-subscription API, documentation of webhook payload formats and subscription management, and any AI-agent-specific webhook integration.", "evidenceIds": [ "auth0-docs-12", "auth0-docs-27", "auth0-docs-47", "auth0-docs-58" ] }, { "productId": "auth0", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of an interactive API reference with runnable examples; the OpenAPI probe explicitly returned 404 on all candidate paths, and docs are static markdown/prose rather than an interactive try-it-out console.", "evidenceIds": [ "auth0-probe-3", "auth0-probe-2" ] }, { "productId": "auth0", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows an explicit probe for OpenAPI/swagger spec files at Auth0's common paths, all returning 404, and no other citation mentions a downloadable machine-readable API spec (only llms.txt discovery files and MCP/CLI tooling are documented).", "evidenceIds": [ "auth0-probe-3" ] }, { "productId": "auth0", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of sandbox/test tenants, staging environments, or any mechanism to test integrations against non-production data — all citations focus on production auth flows, MFA, RBAC, migration, logging, and AI-agent identity features. This is a fair axis for an auth platform (tenants/environments are common), but nothing in the pack shows Auth0 providing an explicit sandbox separate from production.", "evidenceIds": [] }, { "productId": "auth0", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items reference API versioning, version numbers, or a documented deprecation policy for Auth0's Management/Authentication APIs; the openapi.json probe even returned 404s. Missing for 10: any documentation of API versioning scheme, deprecation notices/changelog policy, or sunset timelines for breaking changes.", "evidenceIds": [ "auth0-probe-3" ] }, { "productId": "auth0", "storyId": "audit-logs-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Auth0 documents a log streaming service that exports tenant logs (including auth/admin activity) to log analysis services in near real-time, which covers the SIEM export/streaming part of the story. However, no evidence addresses tamper-evidence (immutability, cryptographic integrity, WORM storage) of the audit logs, nor lists specific SIEM integrations or retention guarantees. Missing for 10: explicit tamper-evident/immutable log guarantees, named SIEM integrations (e.g., Splunk, Datadog, Sumo Logic), and independent verification of log integrity.", "evidenceIds": [ "auth0-docs-12", "auth0-docs-27", "auth0-docs-47", "auth0-docs-58" ] }, { "productId": "auth0", "storyId": "auth-event-webhooks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Auth0's log streaming service (auth0-docs-12/27/47/58) lets developers export tenant logs — including auth events — in near real-time to external log-analysis/webhook-style destinations, which can be used to keep other systems in sync, and Actions can trigger custom logic on login/signup events. However, the evidence never explicitly documents a webhook subscription model scoped to specific event types (sign-up, sign-in, user changes) as a first-class event-stream API. Missing for 10: explicit webhook endpoint documentation, granular event-type subscription docs, and independent/hands-on confirmation of reliable real-time sync use.", "evidenceIds": [ "auth0-docs-12", "auth0-docs-27", "auth0-docs-47", "auth0-docs-58", "auth0-docs-52" ] }, { "productId": "auth0", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Auth0 documents bulk user import/export for migrations (auth0-docs-11, auth0-docs-46, auth0-docs-57) and an MCP server that lets AI agents perform 'complex, multi-step operations' like creating apps or managing users via natural language (auth0-docs-41, auth0-docs-13, auth0-gh-1), which gives some automation-depth support. However, there is no evidence of a genuine bulk/batch API (e.g., batch create/update/delete many resources in one call) exposed to AI-native tooling beyond the narrow bulk-import use case. Missing for 10: explicit bulk operation endpoints or MCP tool calls that act on many items at once (not just user migration), and any hands-on confirmation that the MCP server or CLI supports batch actions across large item sets.", "evidenceIds": [ "auth0-docs-11", "auth0-docs-46", "auth0-docs-57", "auth0-docs-41", "auth0-docs-13", "auth0-gh-1" ] }, { "productId": "auth0", "storyId": "automation-rules-engine", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Auth0 Actions are versioned, event-triggered Node.js functions that let users define custom logic firing automatically on login/identity events (e.g., post-login, M2M token issuance), which directly matches the 'rules that trigger actions on events' story, and log streaming further extends automated event export to external systems. Missing for 10: independent/hands-on validation of Actions' event-trigger breadth, and no explicit AI-native framing of this specific automation feature (AI-agent evidence in the pack focuses on auth flows, not rule/event automation).", "evidenceIds": [ "auth0-docs-52", "auth0-docs-12", "auth0-docs-47", "auth0-docs-58" ] }, { "productId": "auth0", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Auth0 is an identity/authentication platform; scheduling recurring jobs or workflows is not a capability within its product category, and nothing in the evidence suggests it offers job/workflow scheduling.", "evidenceIds": [] }, { "productId": "auth0", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Auth0 Actions are described as 'versioned' Node.js functions for customizing login flows, which offers a thin sliver of versioning for one type of automation, but there is no evidence of review workflows, diffing, approval processes, or rollback mechanisms for Actions or other automations like Flows/Forms. missing for 10: rollback/restore functionality, review/approval workflow, version history UI or API, coverage beyond Actions (e.g., Forms, Rules, tenant config).", "evidenceIds": [ "auth0-docs-52" ] }, { "productId": "auth0", "storyId": "bot-attack-protection", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Auth0 docs explicitly describe proactive IP-blocking against consecutive failed login attempts to prevent brute-force/DDoS, breached-password detection using a compromised-credential database to block credential stuffing, and bot-detection-adjacent adaptive MFA that only challenges risky logins. These directly address bot/credential-stuffing/brute-force protection on auth flows. missing for 10: no explicit CAPTCHA/bot-detection documentation detail, no independent/hands-on validation of attack-blocking efficacy, and no security-engineer testimonial confirming real-world effectiveness.", "evidenceIds": [ "auth0-docs-36", "auth0-docs-55", "auth0-docs-50", "auth0-docs-35" ] }, { "productId": "auth0", "storyId": "bulk-user-import-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Auth0 docs clearly describe bulk import and export of user data ('Migrate your user data into and out of Auth0 with automatic migration and bulk imports and exports' — auth0-docs-46/57/66) and gradual migration via import mode (auth0-docs-10/11). However, the evidence never specifically confirms password-hash import support or details the export format/API, and one community comment even flags Auth0 as having 'strong vendor lock-in' (auth0-comm-15), which cuts against the 'never locked in' framing. Missing for 10: explicit documentation of password-hash import fields, concrete export API/schema details, and independent confirmation that migration avoids lock-in.", "evidenceIds": [ "auth0-docs-10", "auth0-docs-11", "auth0-docs-46", "auth0-docs-57", "auth0-docs-66", "auth0-comm-15" ] }, { "productId": "auth0", "storyId": "deployment-data-control", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Auth0 offers a 'Private Cloud' managed deployment on AWS/Azure for data-residency and isolation needs, which gives some deployment control, but this is still an Auth0-managed service, not a self-hosted/on-prem deployment, and there's no evidence of bringing your own database or fully self-managed instances outside Auth0's control plane. Missing for 10: self-managed/on-prem deployment option, ability to use own database/user store as system of record (only migration import/export is documented, not BYO-DB), and independent confirmation of Private Cloud's isolation guarantees.", "evidenceIds": [ "auth0-docs-14", "auth0-docs-28", "auth0-docs-48", "auth0-docs-64", "auth0-docs-46" ] }, { "productId": "auth0", "storyId": "device-flow-for-cli-agents", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Auth0 documents the OAuth 2.0 Device Authorization Flow explicitly for input-constrained devices/CLIs (docs-9, docs-26, docs-40), and its own MCP Server product uses this exact flow to let AI tools like Claude Desktop authenticate against a tenant without pasting long-lived secrets (docs-13, docs-41, auth0-gh-1). This is a first-party, standards-based implementation directly matching the CLI/headless-agent use case in the story. missing for 10: independent/hands-on third-party verification of the device flow specifically in CLI/agent contexts beyond Auth0's own docs and MCP server example.", "evidenceIds": [ "auth0-docs-9", "auth0-docs-26", "auth0-docs-40", "auth0-docs-13", "auth0-docs-41", "auth0-gh-1", "auth0-probe-4" ] }, { "productId": "auth0", "storyId": "edge-middleware-route-protection", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers Auth0's general auth features, MFA, SSO, MCP server, and AI agent tooling, but contains no mention of Next.js SDK, edge middleware, or session verification at the edge — no evidence of first-party framework SDK/middleware integration for route protection.", "evidenceIds": [] }, { "productId": "auth0", "storyId": "enterprise-saml-oidc-sso", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Auth0 explicitly documents SAML-based enterprise authentication (auth0-docs-33), federating an existing enterprise directory service so employees can log in with existing credentials (auth0-docs-37), and pricing/tenant features listing 'Enterprise Connection,' 'Self-Service SSO,' and 'SCIM' as core capabilities (auth0-docs-21), which is Auth0's well-known Enterprise Connections feature covering SAML/OIDC federation with providers like Okta, Entra, and Google Workspace. Missing for 10: explicit named documentation citing Okta/Entra ID/Google Workspace as specific supported enterprise connections, and independent/hands-on corroboration of these specific IdP integrations working in practice.", "evidenceIds": [ "auth0-docs-33", "auth0-docs-37", "auth0-docs-21", "auth0-docs-4" ] }, { "productId": "auth0", "storyId": "fine-grained-authorization", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Auth0 explicitly offers Auth0 FGA, described as 'a unified, relationship-based platform that helps secure multi-tenant B2B APIs and apps, and decouples complex authorization logic from your code,' going beyond basic RBAC, plus document-level access control for RAG pipelines and 'advanced authorization mechanics for more flexible access control.' This directly matches the relationship/policy-based fine-grained authorization ask, distinct from the separate RBAC docs. Missing for 10: deeper technical documentation on FGA's authorization model/policy language, and independent/hands-on validation of FGA in production beyond marketing copy.", "evidenceIds": [ "auth0-docs-62", "auth0-docs-19", "auth0-docs-42", "auth0-docs-56" ] }, { "productId": "auth0", "storyId": "guided-competitor-migration", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Auth0 documents vendor-maintained tooling for exactly this need: gradual migration via import mode that migrates users transparently as they log in (no forced password reset) plus bulk import/export for one-time migrations from an external user store or database. Missing for 10: named step-by-step guides for migrating from specific competing providers (e.g., Okta, Cognito, Firebase) rather than generic import-mode docs, and independent/hands-on evidence confirming migrations complete without any password resets in practice.", "evidenceIds": [ "auth0-docs-10", "auth0-docs-11", "auth0-docs-46", "auth0-docs-57", "auth0-docs-66" ] }, { "productId": "auth0", "storyId": "human-approval-for-agent-actions", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Auth0 explicitly documents CIBA support for agent workflows: 'Using standards like CIBA (Client-Initiated Backchannel Authentication), agents can request approval for sensitive operations via push notifications, SMS, or email, even when the user is not actively using the application,' directly matching the async human-approval-for-sensitive-agent-transaction story. This is first-party vendor documentation specifically targeted at AI agent use cases (auth0.com/ai/docs), though there is no independent/hands-on corroboration or deeper implementation detail (e.g., code samples, latency/timeout behavior). Missing for 10: independent/hands-on verification of CIBA-for-agents in practice, and detailed implementation/config documentation beyond the marketing-level description.", "evidenceIds": [ "auth0-docs-18", "auth0-docs-31", "auth0-docs-16" ] }, { "productId": "auth0", "storyId": "m2m-client-credentials", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Auth0 has dedicated first-party documentation on the Client Credentials flow explicitly for M2M applications like daemons and backend services (auth0-docs-8/25), plus a dedicated Machine-to-Machine feature page describing Client ID/Secret exchange for access tokens without human interaction (auth0-docs-54). This directly matches the story's requirement of credential issuance for backend/agent authentication without a human in the loop. missing for 10: independent/hands-on third-party corroboration specifically confirming the M2M client-credentials flow works as documented (community evidence covers general Auth0 sentiment, not this specific flow).", "evidenceIds": [ "auth0-docs-8", "auth0-docs-25", "auth0-docs-54" ] }, { "productId": "auth0", "storyId": "mfa-second-factors", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Auth0 docs explicitly list MFA factors including one-time passwords (TOTP) and recovery codes (backup codes) [auth0-docs-38, auth0-docs-23], and Adaptive MFA documentation shows risk-based, step-up-like enforcement that only prompts for MFA when a login looks risky [auth0-docs-50]. These are all vendor-first-party docs with no independent corroboration or explicit 'step-up authentication' API walkthrough in the pack. Missing for 10: independent/hands-on confirmation of step-up enforcement behavior, and explicit docs using the term 'step-up' tied to specific high-risk actions rather than just adaptive login risk scoring.", "evidenceIds": [ "auth0-docs-38", "auth0-docs-23", "auth0-docs-50" ] }, { "productId": "auth0", "storyId": "oauth-oidc-standard-flows", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Auth0 documents core OAuth2/OIDC flows including client credentials (M2M), device authorization flow (explicitly noted as ratified OAuth 2.0), and general authentication/authorization via Universal Login and SDKs, sparing developers from implementing protocol details themselves. However, the evidence pack lacks explicit first-party documentation or quickstart specifically calling out 'authorization code with PKCE' or refresh token rotation/handling details, and there is no independent hands-on confirmation of PKCE support quality. missing for 10: explicit PKCE flow documentation, refresh token rotation/handling details, independent developer corroboration of authorization-code+PKCE implementation ease.", "evidenceIds": [ "auth0-docs-8", "auth0-docs-9", "auth0-docs-40", "auth0-docs-25", "auth0-docs-2", "auth0-comm-7", "auth0-comm-8" ] }, { "productId": "auth0", "storyId": "openness-api-parity", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Auth0 exposes a Management API surfaced through both an official CLI (auth0-docs-61, probe-5) and an MCP server that explicitly lets AI agents 'create apps, deploy Actions, debug logs, and manage users — all without touching the dashboard' (auth0-gh-1, auth0-docs-41), showing strong API/automation parity with UI tasks. However, there is no explicit documentation or independent verification claiming 100% feature parity between the Management API and every UI/dashboard capability, and a direct OpenAPI spec probe returned 404s (auth0-probe-3). Missing for 10: explicit 'full API parity' documentation, independent hands-on confirmation of parity, and a public OpenAPI/swagger spec.", "evidenceIds": [ "auth0-gh-1", "auth0-docs-41", "auth0-docs-61", "auth0-probe-3", "auth0-probe-5" ] }, { "productId": "auth0", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Auth0 documents bulk export/import of user data and log streaming export (auth0-docs-46, auth0-docs-66, auth0-docs-12), showing some data portability, but there is no explicit claim of exporting *all* tenant/user data in standardized open formats, and community commentary notes 'strong vendor lock-in' (auth0-comm-15) as a caveat. missing for 10: explicit open-format (e.g., JSON/CSV standard) full-tenant export guarantee, independent confirmation that a customer successfully exported and migrated away, and no evidence of a comprehensive one-click 'leave with all data' feature.", "evidenceIds": [ "auth0-docs-46", "auth0-docs-66", "auth0-docs-12", "auth0-docs-27", "auth0-comm-15" ] }, { "productId": "auth0", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Auth0 is a proprietary, closed-source SaaS identity platform; there is no evidence of an open-license source release for the core product (only an MCP server and CLI are on GitHub, which are tooling, not the product itself).", "evidenceIds": [] }, { "productId": "auth0", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Auth0 is offered exclusively as a managed SaaS (with a Private Cloud option deployed by Auth0 on AWS/Azure, not self-hosted by the customer); there is no evidence of an open-source or self-hostable core product that an AI-native user could run on their own infrastructure.", "evidenceIds": [ "auth0-docs-14", "auth0-docs-28", "auth0-docs-48", "auth0-docs-64" ] }, { "productId": "auth0", "storyId": "org-level-sso-domains", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Auth0 Organizations lets security engineers represent each B2B customer/partner and configure branded, federated login flows per business (auth0-docs-6, auth0-docs-24, auth0-docs-44), and pricing page shows Enterprise Connection, Self-Service SSO, and SCIM as organization-related add-ons (auth0-docs-21). However, the evidence pack lacks explicit documentation of verified/claimed domains for automatic org routing or of just-in-time provisioning behavior tied to organizations. missing for 10: explicit docs on domain verification/home-realm-discovery per organization, explicit JIT provisioning documentation scoped to Organizations, and independent/hands-on confirmation of this multi-tenant SSO setup working end-to-end.", "evidenceIds": [ "auth0-docs-6", "auth0-docs-24", "auth0-docs-44", "auth0-docs-21" ] }, { "productId": "auth0", "storyId": "organizations-multitenant", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Auth0 docs explicitly describe an Organizations feature for representing B2B customers/partners, managing their membership, and configuring branded federated login flows per business (auth0-docs-6/24/44/65), plus RBAC for role-based permissions (auth0-docs-7/39/45). However, the evidence pack contains no mention of invitation flows or invitation APIs for organization members, which is a core part of the story. Missing for 10: explicit documentation of organization invitation/onboarding flows, and independent/hands-on corroboration of multi-tenant org setup working end-to-end.", "evidenceIds": [ "auth0-docs-6", "auth0-docs-24", "auth0-docs-44", "auth0-docs-65", "auth0-docs-7", "auth0-docs-39" ] }, { "productId": "auth0", "storyId": "passkeys-passwordless-login", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Auth0 documents WebAuthn (security keys and device biometrics) as an MFA factor, a dedicated Passwordless feature (email/SMS OTP, magic links), and email/SMS one-time code login as first-class sign-in options alongside password/social login. This covers passkeys/WebAuthn and passwordless magic link/OTP as first-class methods per official docs and feature pages. Missing for 10: independent hands-on developer confirmation specifically of passkey/WebAuthn setup ease, and clearer documentation of magic link (vs OTP) as a distinct flow.", "evidenceIds": [ "auth0-docs-38", "auth0-docs-51", "auth0-docs-34", "auth0-docs-5", "auth0-docs-23" ] }, { "productId": "auth0", "storyId": "prebuilt-auth-ui", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Auth0 documents hosted Universal Login for sign-in/sign-up with password/social/passwordless options, plus no-code Forms editor and Actions for customizing identity flows, and community evidence (HN) corroborates that Universal Login works well and doesn't require JavaScript. However, evidence lacks explicit detail on prebuilt embeddable UI component libraries (e.g., Lock widget) or a dedicated self-service profile-management UI, and one comment notes the offering can be 'rough around the edges.' Missing for 10: explicit documentation of embeddable/prebuilt UI SDK components, dedicated user profile self-management UI, and independent hands-on confirmation of profile management specifically.", "evidenceIds": [ "auth0-docs-49", "auth0-docs-53", "auth0-docs-52", "auth0-docs-1", "auth0-docs-22", "auth0-comm-1", "auth0-comm-4" ] }, { "productId": "auth0", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Auth0 Private Cloud deployment docs mention meeting 'data-residency' requirements via dedicated deployment on AWS or Azure, implying some regional choice, but this is only for the enterprise-tier Private Cloud offering, not the standard multi-tenant product. missing for 10: explicit list of supported regions/data centers, region-selection mechanism in the standard product/dashboard, and any documentation on residency guarantees for the mainstream SaaS tier.", "evidenceIds": [ "auth0-docs-14", "auth0-docs-28", "auth0-docs-48", "auth0-docs-64" ] }, { "productId": "auth0", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Auth0 is an identity/authentication platform, not a data-holding AI model provider or content platform where 'training data opt-out' would be a relevant axis; nothing in the evidence pack relates to AI model training data usage at all.", "evidenceIds": [] }, { "productId": "auth0", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers user migration/export/import, log streaming, and SSO/MFA, but nothing addresses configurable data retention periods or user data deletion controls (e.g., right-to-be-forgotten, account deletion APIs, retention policy settings). Absence of evidence for this applicable privacy-posture axis means it is unverified.", "evidenceIds": [ "auth0-docs-46", "auth0-docs-57", "auth0-docs-66", "auth0-docs-12" ] }, { "productId": "auth0", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry/usage-tracking opt-out settings for Auth0 or its AI/MCP tooling; all documentation covers authentication, MFA, SSO, RBAC, migration, and AI-agent identity features unrelated to telemetry controls.", "evidenceIds": [] }, { "productId": "auth0", "storyId": "rbac-roles-in-tokens", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Auth0 docs clearly describe RBAC where roles are assigned to users and permissions to roles (auth0-docs-7, auth0-docs-39, auth0-docs-45), and the broader access-management platform page confirms role-based access control for apps and APIs (auth0-docs-56). Auth0's RBAC is standardly surfaced via permissions claims in access tokens for authorization checks, consistent with the OAuth2-based API security flows documented (auth0-docs-2, auth0-docs-8). Missing for 10: explicit first-party documentation snippet showing the permissions/roles claim appearing inside issued session/access tokens, and independent hands-on confirmation of token enforcement behavior.", "evidenceIds": [ "auth0-docs-7", "auth0-docs-39", "auth0-docs-45", "auth0-docs-56", "auth0-docs-2" ] }, { "productId": "auth0", "storyId": "scim-directory-sync", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "SCIM appears only as a bullet item ('SCIM New') on the pricing page listing enterprise features, with no documentation describing how SCIM provisioning/deprovisioning actually works, how groups sync, or how IdP-triggered deprovisioning revokes app access. Missing for 10: dedicated SCIM configuration docs, deprovisioning behavior details, group sync specifics, and any independent/hands-on confirmation it works as described.", "evidenceIds": [ "auth0-docs-21" ] }, { "productId": "auth0", "storyId": "scoped-agent-identities", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Auth0 explicitly markets giving each AI agent a unique machine identity (auth0-docs-20, auth0-docs-68) via M2M Client Credentials flow with distinct Client ID/Secret and scoped tokens (auth0-docs-8, auth0-docs-25, auth0-docs-54), plus RBAC to scope permissions narrowly (auth0-docs-7, auth0-docs-39, auth0-docs-45). Missing for 10: explicit documentation of instant/one-click revocation for agent credentials, and independent hands-on validation of this specific agent-identity workflow beyond vendor docs.", "evidenceIds": [ "auth0-docs-20", "auth0-docs-68", "auth0-docs-8", "auth0-docs-25", "auth0-docs-54", "auth0-docs-7", "auth0-docs-39" ] }, { "productId": "auth0", "storyId": "secure-mcp-server-auth", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Auth0's 'AI for Agents' marketing explicitly claims agents can be given unique identities and 'connect to MCP servers more securely' via OAuth 2.0, and mentions Token Vault/CIBA for scoped, verifiable access, but there is no technical documentation showing MCP-spec-specific features (dynamic client registration, resource indicators/RFC8707, PKCE enforcement for MCP resource servers) or a hands-on example of fronting a self-built MCP server with Auth0 as authorization server — the only concrete MCP server doc (auth0-docs-13/41) is Auth0's own tenant-management MCP server, not a customer's protected MCP endpoint. missing for 10: concrete technical walkthrough of protecting a custom MCP server with Auth0-issued scoped tokens, evidence of MCP-spec conformance (DCR, resource metadata), independent/hands-on verification.", "evidenceIds": [ "auth0-docs-20", "auth0-docs-68", "auth0-docs-16", "auth0-docs-17", "auth0-docs-18", "auth0-docs-13" ] }, { "productId": "auth0", "storyId": "session-lifecycle-revocation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers login flows, SSO, MFA, RBAC, Organizations, and AI-agent security features, but contains no documentation of session lifetime configuration, refresh-token rotation/expiry controls, or an API/dashboard action for immediately revoking a live session — a core, expected capability for an identity platform. Since this is a fair and applicable axis for Auth0 but no supporting evidence is present, it cannot be credited as full/partial.", "evidenceIds": [] }, { "productId": "auth0", "storyId": "social-login-breadth", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Auth0's docs explicitly describe adding social login (Facebook, X, etc.) alongside enterprise federation and SAML as configurable connection options rather than custom code, which is a core, well-documented Auth0 capability (auth0-docs-1/22/32/37/33). Missing for 10: an explicit enumerated list of all supported social/OAuth providers and independent hands-on verification that adding a new provider truly requires zero code.", "evidenceIds": [ "auth0-docs-1", "auth0-docs-22", "auth0-docs-32", "auth0-docs-37", "auth0-docs-33", "auth0-docs-43" ] }, { "productId": "auth0", "storyId": "token-vending-for-agent-tools", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Auth0 documents a dedicated Token Vault that obtains, stores, and refreshes short-lived tokens for third-party APIs (Google, Slack, GitHub, Gmail, Spotify, etc.) so agents can act on the user's behalf without handling raw credentials, plus CIBA-based user consent/approval flows for sensitive operations. This directly matches the story's core ask of delegated, user-consented, short-lived token exchange for third-party tool calls. Missing for 10: independent/hands-on verification of the Token Vault in production agent workflows and more detail on token lifetime/scoping guarantees beyond vendor docs.", "evidenceIds": [ "auth0-docs-17", "auth0-docs-30", "auth0-docs-59", "auth0-docs-18", "auth0-docs-31", "auth0-docs-16" ] }, { "productId": "better-auth", "storyId": "act-as-oauth-provider", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "The docs explicitly state the MCP plugin lets an app 'act as an OAuth authorization server and protected resource for... clients' (issuing tokens), and the Device Authorization plugin implements RFC 8628 for third-party/limited-input clients — both directly support turning the app into an OAuth provider. However, evidence is scoped mainly to MCP-client and device-flow use cases rather than a generic, broadly-documented 'OAuth provider for any third-party app' plugin with client registration/management. Missing for 10: explicit generic OAuth Provider plugin docs (client registration, scopes, consent screen) for arbitrary third-party apps beyond MCP/device flows, and independent/hands-on confirmation of this specific capability.", "evidenceIds": [ "better-auth-docs-12", "better-auth-docs-60", "better-auth-docs-17", "better-auth-docs-56", "better-auth-probe-4" ] }, { "productId": "better-auth", "storyId": "active-session-visibility", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm users can revoke their own sessions to log out of other devices/browsers, and session config (expiry, updateAge) is documented, but there's no evidence of a UI/API for listing all active sessions with device metadata, nor for admin-side viewing/revoking of a specific user's sessions (admin plugin covers banning/impersonation, not session listing/revocation). missing for 10: documented endpoint/UI for listing sessions with device info, admin-initiated session revocation for other users, independent/hands-on confirmation of multi-device session management.", "evidenceIds": [ "better-auth-docs-10", "better-auth-docs-37", "better-auth-docs-47", "better-auth-docs-15" ] }, { "productId": "better-auth", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Direct probe confirms llms.txt is live and returns 200 with structured docs content, plus .md-suffixed doc pages are directly fetchable (docs-1 through docs-76 all reference .md doc URLs), and Better Auth also hosts a dedicated remote MCP server exposing documentation search/setup help to MCP-capable clients like Cursor and Claude Code. Missing for 10: no independent/third-party report of an agent actually consuming llms.txt or the MCP docs server successfully.", "evidenceIds": [ "better-auth-probe-1", "better-auth-docs-18", "better-auth-docs-59", "better-auth-probe-2" ] }, { "productId": "better-auth", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Better Auth is an authentication framework/library, not a data product with dashboards or analytics; 'AI-generated insights from data' is a category error for its role — it provides auth infrastructure (MCP server for docs, agent auth tooling) rather than surfacing insights from user data.", "evidenceIds": [] }, { "productId": "better-auth", "storyId": "agentic-autonomous-automation", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Better Auth is an authentication framework/library, not an automation or workflow-orchestration platform; there is no concept of scheduling or running autonomous background automations. This story is a category error for this product type — the axis does not apply.", "evidenceIds": [] }, { "productId": "better-auth", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "better-auth", "storyId": "agentic-headless", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Better Auth is a code-first, server-side library configured entirely in code (not a dashboard), which supports scriptable/CI-friendly setup, and it ships a CLI (used for generating schemas, testing device auth flow, etc.) that can run non-interactively. However, there is no direct evidence of automated/headless test suites, CI pipeline examples, or documentation explicitly addressing running Better Auth in CI/automation contexts, and one community report notes admin scripts/tests were 'hacky' due to header-based request design. missing for 10: explicit CI/automation documentation or examples, evidence of non-interactive CLI scripting for auth setup, and confirmation that the header-based request model doesn't hinder headless script/test usage.", "evidenceIds": [ "better-auth-docs-45", "better-auth-docs-28", "better-auth-docs-49", "better-auth-comm-5", "better-auth-probe-5" ] }, { "productId": "better-auth", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Better Auth is an authentication library/framework for developers, not an AI agent or assistant with a runtime that consumes external MCP servers' tools. Its MCP-related features are about exposing itself as an MCP server/OAuth resource (docs-12, docs-59) or hosting a docs MCP server for other clients (docs-18, docs-32), not about the product itself plugging in and using tools from external MCP servers — so this client-side agenticness axis is a category error for this product.", "evidenceIds": [] }, { "productId": "better-auth", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Better Auth documents a hosted remote MCP server that exposes documentation search/setup help to MCP-capable clients like Cursor, Codex, and Claude Code, and separately ships an MCP plugin letting apps act as an OAuth authorization server for MCP clients — both confirmed by an independent probe of the docs page. Missing for 10: independent/hands-on community confirmation that agents successfully connect via this MCP server (only vendor docs/probe evidence, no user reports).", "evidenceIds": [ "better-auth-docs-18", "better-auth-docs-59", "better-auth-docs-60", "better-auth-probe-4" ] }, { "productId": "better-auth", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Better Auth hosts a remote MCP server (better-auth-docs-18/32/43/59, probe-4) that lets MCP-capable AI clients (Cursor, Claude Code, etc.) query documentation, examples, and setup help in natural language, giving AI-native users a way to 'operate' parts of the developer workflow conversationally. However, this only covers doc search/setup guidance, not actual operation of the auth product itself (e.g., managing users, configuring providers, running admin actions) via natural-language commands — that still requires writing code/CLI commands. Missing for 10: evidence of NL-driven execution of real auth operations (user management, config changes) rather than just documentation assistance, and independent confirmation of the MCP server's usefulness beyond vendor docs.", "evidenceIds": [ "better-auth-docs-18", "better-auth-docs-32", "better-auth-docs-43", "better-auth-docs-59", "better-auth-probe-4" ] }, { "productId": "better-auth", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Better Auth documents an official CLI (better-auth/docs/concepts/cli) used for migrations, schema generation, and even testing the device authorization flow, confirming it's a real, functional tool rather than a stub. Missing for 10: independent hands-on community verification specifically of CLI usage/quality, and more detail on the full range of CLI subcommands beyond migration/device-auth testing.", "evidenceIds": [ "better-auth-probe-5", "better-auth-docs-28", "better-auth-docs-49" ] }, { "productId": "better-auth", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Better Auth exposes a well-documented server API (auth.api methods), client SDK (authClient), and machine-readable docs formats (llms.txt, docs.md) explicitly aimed at AI-native consumption, plus a documented CLI and MCP plugin for programmatic auth flows. missing for 10: a discoverable OpenAPI/Swagger schema (probe found all standard OpenAPI paths 404) and independent hands-on confirmation of API robustness by third parties.", "evidenceIds": [ "better-auth-docs-23", "better-auth-docs-4", "better-auth-docs-61", "better-auth-probe-1", "better-auth-probe-2", "better-auth-probe-5", "better-auth-probe-3" ] }, { "productId": "better-auth", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Better Auth's API Key plugin lets developers create and manage API keys with custom expiration, rate limiting, and refill systems, and the MCP plugin/OAuth support plus 'Auth for AI agents. MCP auth, token exchange, and agent delegation' messaging directly target scoped credentialing for agents. However, there's no explicit documentation of fine-grained scope/permission definitions per API key or agent-specific least-privilege templates, and no independent/hands-on verification of this specific agent-credential workflow. missing for 10: documented scope/permission granularity for API keys or agent tokens, worked example of issuing a least-privilege credential to an agent, independent corroboration of the agent-delegation feature working as claimed.", "evidenceIds": [ "better-auth-docs-16", "better-auth-docs-27", "better-auth-docs-42", "better-auth-docs-73", "better-auth-docs-76", "better-auth-docs-60", "better-auth-docs-12", "better-auth-probe-4" ] }, { "productId": "better-auth", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Better Auth ships an official TypeScript SDK (the `better-auth` package and `authClient`), well documented with framework integrations across 20+ frameworks, plus a hosted MCP server for AI-agent-assisted setup/doc search — all of which support AI-native/agentic building. However, community feedback notes the client library is JS-only with no documented path for building SDKs in other languages, limiting broader AI-native/cross-platform SDK usage. Missing for 10: multi-language official SDKs, first-party API reference (OpenAPI probe 404), independent verification of agent-driven SDK usage beyond docs.", "evidenceIds": [ "better-auth-docs-20", "better-auth-docs-21", "better-auth-docs-45", "better-auth-docs-59", "better-auth-probe-3", "better-auth-comm-15" ] }, { "productId": "better-auth", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any webhook subscription feature in Better Auth's docs (rate limiting, sessions, MCP, plugins, etc. are covered, but no webhook/event subscription mechanism is mentioned anywhere in the pack).", "evidenceIds": [] }, { "productId": "better-auth", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows only static markdown documentation with code snippets, and a direct probe for an OpenAPI/interactive API reference (Swagger, openapi.json, etc.) found all candidate paths returning 404, indicating no interactive, runnable API reference is exposed.", "evidenceIds": [ "better-auth-probe-3" ] }, { "productId": "better-auth", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows an explicit probe for OpenAPI/swagger spec endpoints that all returned 404, and no documentation in the pack references an OpenAPI/machine-readable spec output for Better Auth's API surface. While Better Auth exposes many auth endpoints, there's no evidence of a downloadable machine-readable spec.", "evidenceIds": [ "better-auth-probe-3" ] }, { "productId": "better-auth", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Better Auth's docs show it is self-hosted with your own database (SQLite/Postgres/MySQL) and no explicit sandbox/test-mode environment, test API keys, or documented way to isolate test traffic from production data is mentioned anywhere in the evidence pack. While a developer could manually stand up a separate database, there is no first-party 'sandbox environment' feature, staging mode, or test-data isolation guidance cited. Missing for 10: documented sandbox/staging environment, test API keys or test mode, guidance on isolating test data from production.", "evidenceIds": [ "better-auth-docs-36", "better-auth-docs-2" ] }, { "productId": "better-auth", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of semantic versioning, API stability guarantees, or a documented deprecation policy for Better Auth's SDK/API; the only related probe (openapi.json/swagger.json) returned 404s, and no changelog or versioning docs are cited. missing for 10: versioning policy documentation, deprecation/changelog process, evidence of API stability guarantees for AI/agent consumers.", "evidenceIds": [ "better-auth-probe-3" ] }, { "productId": "better-auth", "storyId": "audit-logs-export", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack shows only a marketing-page activity feed mockup listing generic events (session created, token revoked, 2FA enabled) with no documentation of tamper-evident logging, hashing/integrity guarantees, or any SIEM export/streaming/webhook mechanism for audit or admin activity. No dedicated audit-log feature, webhook events reference, or SIEM integration docs appear anywhere in the pack.", "evidenceIds": [ "better-auth-docs-63" ] }, { "productId": "better-auth", "storyId": "auth-event-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item documents a webhook system or event-stream API for auth events; the only related item (docs-63) shows a UI activity-feed mockup, not a subscribable webhook/event mechanism for external systems to sync with.", "evidenceIds": [ "better-auth-docs-63" ] }, { "productId": "better-auth", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes admin, organization, and API-key plugins that manage individual users/keys/roles, but nowhere documents batch/bulk endpoints (e.g., bulk user import, bulk revoke, bulk key issuance) that an AI-native user could invoke in one call. Bulk operations are a plausible axis for an auth admin API, but no evidence supports it.", "evidenceIds": [ "better-auth-docs-15", "better-auth-docs-16", "better-auth-docs-27", "better-auth-docs-72" ] }, { "productId": "better-auth", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows no documentation of an event-hook or rules-trigger system (e.g., 'on sign-up run X', webhooks, or automation rules) — only static configuration options like rate limits and session expiration. Auth frameworks commonly support such hooks, so the axis applies, but no evidence of it exists here.", "evidenceIds": [] }, { "productId": "better-auth", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Better Auth is an authentication framework/library, not a workflow/job-scheduling or automation orchestration tool; scheduling recurring jobs or workflows is entirely outside its category and product scope.", "evidenceIds": [] }, { "productId": "better-auth", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Better Auth is an authentication library/framework, not an automation-building or workflow-orchestration product; there is no concept of 'automations' to version, review, or roll back. The one loosely related item (auth config living in version-controlled code, reviewable in PRs) refers to developer-authored config files, not an AI-native automation feature with rollback semantics, so this axis is a category mismatch for this product.", "evidenceIds": [] }, { "productId": "better-auth", "storyId": "bot-attack-protection", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Better Auth documents a built-in rate limiter with customizable rules, including strict per-path limits like 3 requests/10s on /sign-in/email, which directly mitigates brute-force and credential-stuffing attacks, plus secure password hashing (scrypt) and 2FA/backup codes as additional layers. However, there is no evidence of dedicated bot/CAPTCHA detection, IP reputation, breached-password checks, or anomaly-based bot mitigation beyond simple rate limiting. missing for 10: explicit bot detection/CAPTCHA integration, credential-stuffing-specific defenses (e.g., breached password screening), independent security audit or hands-on validation of these protections.", "evidenceIds": [ "better-auth-docs-11", "better-auth-docs-25", "better-auth-docs-38", "better-auth-docs-57", "better-auth-docs-70", "better-auth-docs-24", "better-auth-docs-40" ] }, { "productId": "better-auth", "storyId": "bulk-user-import-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Better Auth documents a migration guide specifically for moving from Auth0 that explicitly covers importing email/password users with proper password hash handling, plus social accounts and 2FA data (docs-8, docs-30, docs-41). However, there is no evidence of a generic bulk-import tool for arbitrary user stores, nor any documented export functionality to avoid lock-in — the 'own your data in your own DB' claim is community sentiment (better-auth-comm-1) rather than a documented export feature. missing for 10: a generic (non-Auth0-specific) bulk import mechanism, explicit export/dump tooling, and independent confirmation that migrated password hashes work end-to-end.", "evidenceIds": [ "better-auth-docs-8", "better-auth-docs-30", "better-auth-docs-41", "better-auth-comm-1" ] }, { "productId": "better-auth", "storyId": "deployment-data-control", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Better Auth is a self-hosted, open-source TypeScript library installed directly into your app (npm install), storing all user data in a database you own and control (SQLite/PostgreSQL/MySQL/etc.), with no vendor dashboard or hosted service dependency — confirmed by docs and community reports of full data ownership ('You get all of your auth data in your own db', 'I own all the data', 'adding an extra user column is just a Postgres column'). Missing for 10: no independent security-audit/compliance documentation confirming enterprise self-hosting patterns beyond community anecdotes, and comm-17 flags speculative concern about future closed-source cloud dependency (not a current contradiction).", "evidenceIds": [ "better-auth-docs-1", "better-auth-docs-2", "better-auth-docs-36", "better-auth-comm-1", "better-auth-comm-3", "better-auth-comm-13", "better-auth-comm-17", "better-auth-docs-45" ] }, { "productId": "better-auth", "storyId": "device-flow-for-cli-agents", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Better Auth ships a dedicated Device Authorization plugin implementing RFC 8628 explicitly for CLI/headless clients (smart TVs, CLI apps, IoT devices), and docs show it can be tested via the Better Auth CLI, directly matching the story. missing for 10: independent/hands-on third-party verification of the device flow in production and explicit agent-specific integration examples beyond docs.", "evidenceIds": [ "better-auth-docs-17", "better-auth-docs-49", "better-auth-docs-56", "better-auth-docs-28", "better-auth-docs-74" ] }, { "productId": "better-auth", "storyId": "edge-middleware-route-protection", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Better Auth is confirmed framework-agnostic with support for Next.js, Nuxt, SvelteKit, Astro, Hono and 20+ frameworks, and server-side session verification via auth.api methods, but the evidence pack lacks any explicit mention of edge-runtime middleware, Next.js middleware helpers, or edge-verified session checks. missing for 10: explicit Next.js middleware/edge-runtime route-protection docs, evidence of edge-compatible session verification, independent hands-on confirmation of edge middleware usage.", "evidenceIds": [ "better-auth-docs-20", "better-auth-docs-23", "better-auth-docs-21", "better-auth-probe-2" ] }, { "productId": "better-auth", "storyId": "enterprise-saml-oidc-sso", "verdict": "full", "quality": 6, "confidence": "medium", "rationale": "Better Auth's official SSO plugin explicitly supports OIDC, OAuth2, and SAML 2.0, which are the protocols used by Okta, Entra, and Google Workspace for workforce SSO (better-auth-docs-14). However, there is no documentation or community evidence naming specific IdP integrations (Okta/Entra/Google Workspace) or showing a hands-on enterprise SSO setup. missing for 10: named provider configuration guides (Okta/Entra/Google Workspace specifics), admin/dashboard SSO management, and independent/hands-on validation of the SSO plugin in production.", "evidenceIds": [ "better-auth-docs-14" ] }, { "productId": "better-auth", "storyId": "fine-grained-authorization", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Better Auth's organization plugin lets teams assign roles and permissions per-org/team, and the admin plugin manages user roles, which provides some resource-scoped (team/org) access control beyond a single global role — but this is still fundamentally RBAC, not a documented policy- or relationship-based (ReBAC/ABAC) authorization engine with custom resource-level rules or relationship graphs. missing for 10: no evidence of a policy engine, attribute-based rules, or relationship-based (Zanzibar-style) permission model; no docs describing per-resource ACLs beyond org/team roles.", "evidenceIds": [ "better-auth-docs-13", "better-auth-docs-54", "better-auth-docs-71", "better-auth-docs-15", "better-auth-docs-72", "better-auth-docs-75" ] }, { "productId": "better-auth", "storyId": "guided-competitor-migration", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Better Auth publishes a vendor-maintained Auth0 migration guide covering email/password migration 'with proper hashing,' social accounts, and 2FA, which implies preserving existing password hashes rather than forcing resets [better-auth-docs-8, better-auth-docs-30, better-auth-docs-41]. However, this guide is limited to Auth0 specifically — there's no evidence of similar guides/tooling for other major competitors (Clerk, Cognito, Firebase, Okta), and no explicit statement or independent confirmation that end users won't be forced to reset passwords during migration. Missing for 10: migration guides for other competing providers, explicit no-password-reset guarantee, and independent/hands-on confirmation of a smooth Auth0 migration.", "evidenceIds": [ "better-auth-docs-8", "better-auth-docs-30", "better-auth-docs-41" ] }, { "productId": "better-auth", "storyId": "human-approval-for-agent-actions", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Better Auth documents a generic Device Authorization plugin (RFC 8628 approval flow) and a vague marketing line about 'agent delegation' for AI agents, but there is no documented CIBA-style or step-up approval mechanism that pauses an autonomous agent's transaction pending asynchronous human confirmation. No evidence ties any plugin to sensitive-transaction gating for agents.", "evidenceIds": [ "better-auth-docs-17", "better-auth-docs-49", "better-auth-docs-76" ] }, { "productId": "better-auth", "storyId": "m2m-client-credentials", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Better Auth offers building blocks for service-to-service auth — an API Key plugin for authenticating requests without user login, a JWT plugin for token issuance/verification, and an MCP plugin that lets the app act as an OAuth authorization server for AI agent clients (docs-76: 'Auth for AI agents. MCP auth, token exchange, and agent delegation.') — but none of the evidence explicitly documents an OAuth2 client-credentials grant flow for machine-to-machine authentication. Device Authorization plugin explicitly requires human approval, so it doesn't satisfy the 'no human in the loop' requirement. missing for 10: explicit client-credentials grant documentation, example of service-to-service token issuance without any user context, independent/hands-on validation of M2M flows.", "evidenceIds": [ "better-auth-docs-16", "better-auth-docs-7", "better-auth-docs-60", "better-auth-docs-76", "better-auth-docs-17" ] }, { "productId": "better-auth", "storyId": "mfa-second-factors", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Better Auth's 2FA plugin documents TOTP authenticator app setup, backup code generation, and enabling/disabling 2FA with trusted device management (better-auth-docs-39, -40, -48, -53, -69), directly satisfying the core MFA requirement. However, the evidence pack contains no explicit documentation of step-up enforcement (e.g., requiring re-verification for sensitive actions or conditional MFA triggers) beyond basic enable/disable. Missing for 10: explicit step-up/conditional enforcement policy documentation, independent hands-on confirmation of TOTP+backup-code flow working end-to-end.", "evidenceIds": [ "better-auth-docs-6", "better-auth-docs-39", "better-auth-docs-40", "better-auth-docs-48", "better-auth-docs-53", "better-auth-docs-69" ] }, { "productId": "better-auth", "storyId": "oauth-oidc-standard-flows", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Better Auth documents built-in social OAuth (Google, GitHub, Apple, Discord), an SSO plugin supporting OIDC/OAuth2/SAML, a JWT plugin with JWKS, and even the Device Authorization Grant (RFC 8628) — all abstracting protocol details behind config/plugins rather than requiring manual implementation. However, the evidence pack never explicitly calls out PKCE handling or refresh-token rotation mechanics for the authorization code flow, so full protocol-detail coverage isn't directly confirmed. missing for 10: explicit PKCE flow documentation, explicit refresh-token lifecycle docs, independent hands-on test of OAuth flow correctness.", "evidenceIds": [ "better-auth-docs-5", "better-auth-docs-14", "better-auth-docs-7", "better-auth-docs-17", "better-auth-docs-49" ] }, { "productId": "better-auth", "storyId": "openness-api-parity", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Better Auth is explicitly code/API-first — 'No dashboard clicks. Your auth lives in code' — and virtually every capability (signup, 2FA, sessions, admin operations like banning/impersonating, organization management, API keys) is exposed via auth.api server methods or the authClient SDK rather than a separate GUI, meaning there is no UI-only feature gap by design. missing for 10: no explicit UI product to compare against for full parity claims, no OpenAPI spec discovered (probe found 404s) to formally enumerate API surface, and no independent hands-on confirmation that admin/org UI-equivalent actions are 100% API-reachable.", "evidenceIds": [ "better-auth-docs-45", "better-auth-docs-23", "better-auth-docs-15", "better-auth-docs-13", "better-auth-docs-16", "better-auth-probe-3" ] }, { "productId": "better-auth", "storyId": "openness-full-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Better Auth stores all auth data in your own standard SQL database (SQLite/Postgres/MySQL) rather than a vendor silo, and community reviewers explicitly praise that 'you get all of your auth data in your own db' and 'you are not tied to any db provider,' which inherently satisfies data-ownership/exit concerns. However there is no dedicated documented export feature, standard open export format (e.g., CSV/JSON dump tool), or migration-out guide beyond the reverse (Auth0-to-BetterAuth) migration guide. Missing for 10: explicit data-export tooling/documentation, a formal 'leave the platform' guide, and independent verification of export completeness.", "evidenceIds": [ "better-auth-docs-36", "better-auth-comm-1", "better-auth-comm-3", "better-auth-comm-13" ] }, { "productId": "better-auth", "storyId": "openness-open-license", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Community evidence repeatedly confirms Better Auth is open source and self-hostable (own DB, no vendor lock-in), and users treat it as a codebase they can read/extend (e.g., adding custom plugins). However, no direct citation of the specific license (e.g., MIT) or a link to the source repository/license file is provided. Missing for 10: explicit license name confirmation, direct link to GitHub repo/license file.", "evidenceIds": [ "better-auth-comm-1", "better-auth-comm-3", "better-auth-comm-10", "better-auth-comm-13", "better-auth-docs-45" ] }, { "productId": "better-auth", "storyId": "openness-self-host", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Better Auth is an open-source, code-first library installed via npm into your own codebase, storing all data in your own SQLite/PostgreSQL/MySQL database with no vendor lock-in ('You are not tied to any db provider', 'Your auth lives in code version controlled'), which community reviews confirm as a self-hosting advantage. missing for 10: no explicit dedicated 'self-hosting guide' or deployment/infra doc, and no independent audit of production self-hosted deployments beyond community testimonials.", "evidenceIds": [ "better-auth-docs-1", "better-auth-docs-2", "better-auth-docs-36", "better-auth-docs-45", "better-auth-comm-1", "better-auth-comm-3", "better-auth-comm-13" ] }, { "productId": "better-auth", "storyId": "org-level-sso-domains", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Better Auth ships an organization plugin for multi-tenant orgs (teams/roles) and a separate SSO plugin supporting OIDC/OAuth2/SAML 2.0, plus a marketing claim of 'SSO, SAML 2.0, SCIM, and directory sync' for enterprise readiness — together implying per-org SSO with provisioning is achievable. However, the evidence never explicitly documents binding an SSO connection to a specific organization, domain verification/ownership checks, or just-in-time (JIT) user provisioning on SSO login. Missing for 10: explicit docs on per-organization SSO connection scoping, domain verification workflow, and JIT provisioning behavior on first SSO login.", "evidenceIds": [ "better-auth-docs-13", "better-auth-docs-14", "better-auth-docs-55", "better-auth-docs-62", "better-auth-docs-75" ] }, { "productId": "better-auth", "storyId": "organizations-multitenant", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "The organization plugin explicitly supports members, teams, roles/permissions, and multi-tenancy (docs-13, docs-26, docs-54, docs-71, docs-75), and community reports confirm real-world use of organizations/multi-tenant setups (better-auth-comm-3). However, invitation flows are asserted only implicitly via 'members and teams' management and marketing copy ('Multi-tenancy built in... invitations') without dedicated docs excerpts detailing the invitation API/flow itself. missing for 10: explicit documentation of invitation creation/acceptance endpoints, role-based permission examples, and independent hands-on verification of the invitation flow specifically.", "evidenceIds": [ "better-auth-docs-13", "better-auth-docs-26", "better-auth-docs-54", "better-auth-docs-71", "better-auth-docs-75", "better-auth-comm-3" ] }, { "productId": "better-auth", "storyId": "passkeys-passwordless-login", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Better Auth ships a passkey plugin (WebAuthn) referenced in docs/site (better-auth-docs-19, better-auth-docs-44) and independently corroborated by a hands-on community comparison noting the passkey plugin requires just 7 lines of server code (better-auth-comm-14). Magic links and passwordless are also listed as first-class supported methods (better-auth-docs-44 'Passkeys, magic links, API keys, JWTs, and more'). Missing for 10: dedicated docs page excerpt for magic-link/email-OTP plugin configuration details and independent hands-on verification of magic link/email OTP specifically (only passkeys got hands-on confirmation).", "evidenceIds": [ "better-auth-docs-19", "better-auth-docs-44", "better-auth-comm-14" ] }, { "productId": "better-auth", "storyId": "prebuilt-auth-ui", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Better Auth's evidence shows only client SDK methods (authClient.signUp.email, etc.) and API-level primitives (session, 2FA, organization plugins) — there is no mention of prebuilt UI components or hosted sign-in/sign-up/profile pages anywhere in the docs pack. Community commentary explicitly contrasts it with Clerk as 'more hackable,' implying it is headless rather than shipping ready-made UI.", "evidenceIds": [ "better-auth-docs-4", "better-auth-docs-35", "better-auth-docs-67", "better-auth-comm-4" ] }, { "productId": "better-auth", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Better Auth is a self-hosted library where you bring your own database (SQLite/PostgreSQL/MySQL/etc.), and community comments confirm 'you own all the data' and it's 'not tied to any db provider,' which implicitly lets a developer choose the physical location/region of their data store. However, there is no explicit region/residency feature, configuration flag, or documentation addressing data-residency requirements directly. missing for 10: explicit residency/region controls or docs, compliance statements (GDPR/data-locality), multi-region deployment guidance.", "evidenceIds": [ "better-auth-docs-36", "better-auth-comm-1", "better-auth-comm-3", "better-auth-comm-13" ] }, { "productId": "better-auth", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Better Auth is an authentication framework/library, not an AI model provider or data-processing service that trains AI models on user data; the question of preventing data from being used for AI training is a category error for this product type.", "evidenceIds": [] }, { "productId": "better-auth", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Better Auth stores auth data in the developer's own database (SQLite/Postgres/MySQL) rather than a vendor-hosted store, and community comments emphasize 'I own all the data' and full schema control, plus docs show users can revoke their own sessions and rotate secrets — giving some data-deletion control. However there is no explicit documentation of a data retention policy, account/user deletion API, or GDPR-style erasure workflow in the evidence pack. Missing for 10: explicit user/account deletion API docs, data retention policy statements, and compliance (e.g. GDPR/CCPA erasure) documentation.", "evidenceIds": [ "better-auth-docs-10", "better-auth-docs-9", "better-auth-comm-1", "better-auth-comm-3", "better-auth-comm-13" ] }, { "productId": "better-auth", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of telemetry, analytics collection, or any opt-out mechanism for Better Auth's CLI or library; while telemetry opt-out is a fair question for a dev tool/CLI like this, no documentation or community evidence addresses it.", "evidenceIds": [] }, { "productId": "better-auth", "storyId": "rbac-roles-in-tokens", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Better Auth's organization and admin plugins support assigning roles/permissions and managing access control (better-auth-docs-13, better-auth-docs-54, better-auth-docs-71, better-auth-docs-15, better-auth-docs-75), and the JWT plugin can issue tokens for verification (better-auth-docs-7). However, there is no explicit documentation showing that custom roles/permissions are automatically embedded into session/JWT claims for authorization checks — the JWT and role/permission systems appear documented separately rather than as an integrated enforcement pipeline. missing for 10: explicit docs on custom permission schemas beyond built-in roles, evidence that role/permission data is serialized into JWT/session claims, and independent/hands-on confirmation of authorization checks using token claims.", "evidenceIds": [ "better-auth-docs-13", "better-auth-docs-54", "better-auth-docs-71", "better-auth-docs-15", "better-auth-docs-29", "better-auth-docs-7", "better-auth-docs-75" ] }, { "productId": "better-auth", "storyId": "scim-directory-sync", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows SSO/SAML plugin, organization plugin, and admin plugin, but no mention of SCIM protocol support for syncing users/groups from IdPs or automated deprovisioning propagation. No SCIM endpoint, provisioning API, or directory sync capability is documented anywhere in the pack.", "evidenceIds": [] }, { "productId": "better-auth", "storyId": "scoped-agent-identities", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Better Auth explicitly markets 'Auth for AI agents: MCP auth, token exchange, and agent delegation' and ships an API Key plugin (create/manage/verify keys, custom expiration, rate limiting, remaining-count/refill) plus an MCP OAuth plugin and session/token revocation, which together give the building blocks for distinct, scoped, revocable machine identities separate from human credentials. However the docs don't show a concrete worked example of assigning narrowly-scoped per-agent permissions (vs. organization-level roles) or an 'instant revocation' guarantee specifically for agent identities, and there's no independent/hands-on validation of this agent-identity workflow. missing for 10: explicit per-agent scoped-permission example, documented instant-revocation SLA for API keys/agent tokens, independent corroboration of agent-identity isolation.", "evidenceIds": [ "better-auth-docs-16", "better-auth-docs-42", "better-auth-docs-60", "better-auth-docs-76", "better-auth-docs-63", "better-auth-docs-10" ] }, { "productId": "better-auth", "storyId": "secure-mcp-server-auth", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Better Auth ships a first-party MCP plugin explicitly described as letting an app 'act as an OAuth authorization server and protected resource for Model Context Protocol clients,' directly matching the story of fronting an MCP server with spec-compliant OAuth for scoped tokens; it also supports JWT/JWKS verification and API keys that align with scoped, verifiable tokens. Missing for 10: independent hands-on validation of the MCP OAuth flow specifically (community evidence covers other features, not this plugin) and no explicit mention of granular scope definitions within the MCP plugin docs excerpted here.", "evidenceIds": [ "better-auth-docs-12", "better-auth-docs-60", "better-auth-docs-76", "better-auth-docs-7", "better-auth-probe-4" ] }, { "productId": "better-auth", "storyId": "session-lifecycle-revocation", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs show configurable session expiry and rolling refresh via `session: { expiresIn, updateAge }`, and explicit server-side revocation of user sessions (self-revocation for logging out compromised sessions on other devices), plus admin plugin capable of banning/impersonating users tied to session control. Missing for 10: no explicit documentation of an admin-initiated forced revocation of another user's specific session (only self-revocation and broader ban/impersonate flows are shown), and no independent hands-on test confirming revocation propagation latency.", "evidenceIds": [ "better-auth-docs-37", "better-auth-docs-47", "better-auth-docs-10", "better-auth-docs-15", "better-auth-docs-29" ] }, { "productId": "better-auth", "storyId": "social-login-breadth", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs and marketing pages explicitly state built-in support for social providers (Google, GitHub, Apple, Discord, and more) configured via a `socialProviders` config block, plus a dedicated SSO plugin supporting OIDC, OAuth2, and SAML 2.0 for enterprise IdPs, all achievable through configuration rather than custom code. Community testimonials corroborate that OAuth providers (Google, Apple, GitHub) work out-of-the-box with minimal code. Missing for 10: an exhaustive named list of all supported providers, and independent hands-on verification of configuring a less-common/niche provider beyond the commonly cited ones.", "evidenceIds": [ "better-auth-docs-5", "better-auth-docs-52", "better-auth-docs-68", "better-auth-docs-14", "better-auth-docs-55", "better-auth-docs-62", "better-auth-comm-1" ] }, { "productId": "better-auth", "storyId": "token-vending-for-agent-tools", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Better Auth explicitly markets 'Auth for AI agents. MCP auth, token exchange, and agent delegation' and documents an MCP plugin that lets an app act as an OAuth authorization server/protected resource for MCP clients, plus a Device Authorization plugin for consent-based token issuance on limited-input clients — these are directly relevant primitives for an agent obtaining delegated, short-lived tokens. However, there is no detailed documentation of an actual token-vault/exchange mechanism for arbitrary third-party APIs (only a marketing tagline plus generic OAuth-server framing), and no independent/hands-on evidence confirming this flow works as a user-consented delegation vault. Missing for 10: concrete docs/example of token exchange for third-party API scopes, evidence of short-lived token issuance tied to explicit user consent for agent tool calls, and independent verification of the 'token exchange'/'agent delegation' claims.", "evidenceIds": [ "better-auth-docs-76", "better-auth-docs-12", "better-auth-docs-60", "better-auth-docs-17", "better-auth-docs-56" ] }, { "productId": "clerk", "storyId": "act-as-oauth-provider", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Clerk's docs describe 'OAuth scoped access' letting third-party applications request limited access to a user's data through Clerk's API, and changelog notes 'Custom OAuth scopes' for MCP clients — both indicating Clerk can act as an OAuth provider issuing tokens to third-party clients. However, the evidence never details the full provider mechanics (client app registration, consent screens, token/refresh endpoints, discovery metadata) expected from a genuine 'Sign in with my app' OAuth provider setup. Missing for 10: detailed docs/tutorial on registering third-party OAuth client apps against a Clerk-backed app, consent-screen/token-issuance flow documentation, and independent/hands-on confirmation that third parties can integrate 'Sign in with [customer's app]'.", "evidenceIds": [ "clerk-docs-10", "clerk-docs-29", "clerk-docs-44", "clerk-docs-55" ] }, { "productId": "clerk", "storyId": "active-session-visibility", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Clerk explicitly advertises active device monitoring and per-device session revocation in the user profile UI, directly matching the user-facing part of the story. However, the evidence pack does not explicitly document admin-side dashboard controls to view/revoke all users' sessions or a one-click 'revoke all devices' bulk action. missing for 10: explicit admin dashboard session/device management UI, bulk 'revoke all sessions' action, independent/hands-on confirmation of revocation behavior.", "evidenceIds": [ "clerk-supp-session-devices", "clerk-docs-22", "clerk-docs-6" ] }, { "productId": "clerk", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Clerk hosts a working llms.txt (HTTP 200, confirmed by probe) and docs.md machine-readable docs, plus a changelog llms-full.txt/llms.txt, giving agents direct access to agent-oriented documentation; it also ships an official MCP server so agents can pull SDK snippets directly. missing for 10: independent third-party confirmation that agents successfully consume llms.txt in practice beyond the probe check.", "evidenceIds": [ "clerk-probe-1", "clerk-probe-2", "clerk-docs-52", "clerk-docs-53", "clerk-docs-54", "clerk-docs-55", "clerk-docs-2", "clerk-probe-4" ] }, { "productId": "clerk", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Clerk is an authentication/user-management infrastructure product, not a data/analytics product; generating AI insights or suggestions from a user's own data is outside its category — the evidence pack only covers auth, orgs, sessions, MCP for SDK snippets, and agent tool authorization, none of which constitute AI-generated insights on user data.", "evidenceIds": [] }, { "productId": "clerk", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Clerk's evidence covers auth/identity features (webhooks, M2M tokens, eve agent authorization, MCP server for coding assistants) but none of this describes a capability for an AI-native user to configure workflows or automations that run autonomously in the background — Clerk enables agents to authenticate, not to be orchestrated or scheduled. Missing for 10: any documented automation/workflow builder, scheduling, or background task runner within Clerk itself.", "evidenceIds": [ "clerk-docs-5", "clerk-docs-11", "clerk-docs-16", "clerk-docs-43" ] }, { "productId": "clerk", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Clerk's AI-related evidence is about enabling external AI coding agents (via an MCP server) to consume Clerk's docs/snippets, and about infrastructure ('eve') for authenticating and authorizing AI agents built by developers — not a built-in assistant inside Clerk's own product that an end-user could delegate tasks to. No evidence of an embedded assistant in the Clerk dashboard or admin console.", "evidenceIds": [ "clerk-docs-2", "clerk-docs-42", "clerk-docs-11", "clerk-docs-16" ] }, { "productId": "clerk", "storyId": "agentic-headless", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Clerk exposes a Backend API, OpenAPI spec, API keys and M2M tokens for machine-to-machine/programmatic auth, and a scriptable CLI plus an open-source migration tool that respects rate limits — all usable in automated/non-interactive contexts. It also supports the OAuth device-code grant for browser-less clients (CLIs, TVs). However there is no explicit documentation of CI pipelines, headless test-mode, or automated end-to-end testing workflows for the auth flows themselves.\nmissing for 10: explicit CI/test-automation guides, headless browser/session testing support, first-party CI examples.", "evidenceIds": [ "clerk-docs-11", "clerk-docs-43", "clerk-docs-7", "clerk-supp-migration-tool", "clerk-docs-52", "clerk-probe-3", "clerk-docs-37" ] }, { "productId": "clerk", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Clerk's evidence only shows it publishing its own MCP server for other AI agents to consume Clerk's SDK docs (clerk-docs-2/26/-4), which is the opposite direction of the story — Clerk acting as an MCP client that plugs in external MCP servers to use their tools. The 'eve' agent-authorization features (clerk-docs-11, -16, -27) discuss gating tool calls and OAuth brokering for agent tools, but there is no evidence Clerk itself connects to or consumes third-party MCP servers.", "evidenceIds": [ "clerk-docs-2", "clerk-docs-26", "clerk-docs-11", "clerk-docs-16", "clerk-docs-27" ] }, { "productId": "clerk", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Clerk documents an official remote MCP server enabling AI agents (Claude, Cursor, GitHub Copilot) to access Clerk SDK snippets and implementation patterns, and its CLI offers a one-command connector to wire this MCP server into AI clients. This is exactly the story's requirement and is corroborated by both docs and a live probe confirming the page exists. Missing for 10: independent/hands-on community confirmation that the MCP connection works reliably in practice.", "evidenceIds": [ "clerk-docs-2", "clerk-docs-26", "clerk-docs-42", "clerk-docs-41", "clerk-probe-4", "clerk-probe-5" ] }, { "productId": "clerk", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Clerk exposes a remote MCP server so AI coding agents (Claude, Cursor, Copilot, Codex) can pull SDK snippets and scaffold auth/session/org code via natural-language prompts, and the CLI can connect this MCP server in one command; clerk.com/agents explicitly shows 'Hand Codex the Clerk docs and it scaffolds sign-in, sessions, and orgs.' However this is developer/setup-time tooling (agent-assisted integration), not an end-user natural-language interface for operating the live Clerk product (e.g., managing users/orgs via chat) — missing for 10: evidence of natural-language control over runtime product operations (user/org management, billing, settings) rather than just code scaffolding, and independent hands-on confirmation of the MCP server's reliability.", "evidenceIds": [ "clerk-docs-2", "clerk-docs-41", "clerk-docs-51", "clerk-probe-4", "clerk-probe-5" ] }, { "productId": "clerk", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Clerk ships an official CLI (clerk.com/cli, llms.txt) that scaffolds auth into a repo, manages webhooks locally, ships to production, and connects the Clerk MCP server to AI clients in one command — directly serving AI-native/agentic workflows. missing for 10: independent/hands-on community verification of the CLI's reliability and no deeper detail on full command coverage beyond the marketing/docs pages.", "evidenceIds": [ "clerk-docs-37", "clerk-docs-38", "clerk-docs-39", "clerk-docs-40", "clerk-docs-41", "clerk-probe-1", "clerk-probe-5" ] }, { "productId": "clerk", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Clerk publishes a documented, discoverable Backend API with an OpenAPI spec (clerk-probe-3), a CLI and llms.txt entry points for programmatic/AI-native access (clerk-probe-1, clerk-probe-2, clerk-docs-37/40), session tokens and API keys/M2M tokens for programmatic auth (clerk-docs-6, clerk-docs-43), and a dedicated remote MCP server plus docs guiding AI agents to use it (clerk-docs-2, clerk-docs-42, clerk-probe-4). This is strong, well-documented, first-party evidence of API-driven and agent-friendly access. missing for 10: independent/hands-on developer confirmation that the public API/OpenAPI spec is fully complete and stable in practice (community evidence is silent on the API itself, only on general product reliability), and no third-party audit of API completeness.", "evidenceIds": [ "clerk-probe-3", "clerk-probe-1", "clerk-probe-2", "clerk-docs-37", "clerk-docs-40", "clerk-docs-6", "clerk-docs-43", "clerk-docs-2", "clerk-docs-42", "clerk-probe-4" ] }, { "productId": "clerk", "storyId": "agentic-scoped-keys", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Clerk explicitly supports machine-to-machine tokens and API keys for agent-to-agent auth, custom OAuth scopes for MCP clients, gating individual tool calls against caller's permissions/scopes, and OAuth scoped access limiting third-party data access — directly enabling scoped, least-privilege credentials for agents. missing for 10: independent/hands-on verification of the scoping granularity in production and no community corroboration of this specific agentic credentialing workflow.", "evidenceIds": [ "clerk-docs-11", "clerk-docs-43", "clerk-docs-16", "clerk-docs-27", "clerk-docs-55", "clerk-docs-10", "clerk-docs-44", "clerk-supp-m2m-pricing" ] }, { "productId": "clerk", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Clerk ships official SDKs (e.g., the clerk/javascript GitHub repo, framework-specific guides like Next.js) plus a CLI and a first-party remote MCP server that lets AI agents like Claude, Cursor, and GitHub Copilot pull up-to-date SDK snippets and implementation patterns, directly serving AI-native builders (clerk-gh-1, clerk-docs-2/26/42, clerk-docs-41, clerk-probe-4/5). Community feedback (clerk-comm-1) raises quality concerns about the JS SDK being bloated/obfuscated, tempering confidence without disputing that official SDKs exist and are documented. Missing for 10: independent hands-on validation of SDK developer experience and broader multi-language SDK coverage beyond JS/Next.js in the evidence pack.", "evidenceIds": [ "clerk-gh-1", "clerk-docs-2", "clerk-docs-26", "clerk-docs-41", "clerk-docs-42", "clerk-probe-4", "clerk-probe-5", "clerk-comm-1" ] }, { "productId": "clerk", "storyId": "agentic-webhooks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Clerk documents native webhook support for event notifications (user created/updated, billing events) plus CLI tooling to relay and verify webhook deliveries locally with signature verification, giving a clear event-subscription mechanism for automation/agentic consumers. Missing for 10: independent/hands-on corroboration of webhook reliability and a full enumerated event catalog.", "evidenceIds": [ "clerk-docs-5", "clerk-docs-25", "clerk-docs-38" ] }, { "productId": "clerk", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Clerk publishes an OpenAPI JSON spec (clerk-probe-3) and various docs pages, but there is no evidence of an interactive API reference UI (e.g., a 'try it out' console or runnable code sandbox) that lets users execute API calls directly from the docs.", "evidenceIds": [ "clerk-probe-3" ] }, { "productId": "clerk", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A probe confirms Clerk publishes a machine-readable OpenAPI spec at a stable public URL (clerk.com/openapi.json) that returns HTTP 200 and contains a valid 'openapi' key, directly satisfying the story. Missing for 10: no independent third-party validation of the spec's completeness/accuracy beyond the probe check.", "evidenceIds": [ "clerk-probe-3" ] }, { "productId": "clerk", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no mention of a sandbox/test environment separate from production (no dev-instance vs prod-instance distinction, no test mode, no synthetic data environment) for AI agents or otherwise to safely test against.", "evidenceIds": [] }, { "productId": "clerk", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows an OpenAPI spec exists and extensive feature docs, but nowhere is there mention of API versioning scheme, version headers, or a documented deprecation policy for Clerk's APIs/SDKs.", "evidenceIds": [] }, { "productId": "clerk", "storyId": "audit-logs-export", "verdict": "disputed", "quality": 3, "confidence": "low", "rationale": "Clerk documents an 'Admin Logs' audit trail for dashboard/admin actions and general webhook event notifications, but there's no evidence of tamper-evidence, SIEM streaming/export integrations, or full auth-event logging — and a hands-on community report explicitly states Clerk is 'lacking audit logs/versioning,' directly contradicting the audit-log claim. missing for 10: SIEM/webhook-to-SIEM streaming or export integration, tamper-evidence/immutability guarantees, comprehensive auth-event (not just admin-action) audit coverage, and resolution of the community-reported lack of audit logs.", "evidenceIds": [ "clerk-docs-54", "clerk-docs-5", "clerk-comm-1" ] }, { "productId": "clerk", "storyId": "auth-event-webhooks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Clerk documents first-party webhooks for auth events like user creation/updates, plus a CLI feature to relay webhook deliveries locally for testing signatures offline, directly supporting sync-to-external-systems use cases. Missing for 10: a full enumerated list of supported event types (sign-in specifically), independent/hands-on developer confirmation of webhook reliability, and details on delivery guarantees/retries.", "evidenceIds": [ "clerk-docs-5", "clerk-docs-38", "clerk-docs-25" ] }, { "productId": "clerk", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Clerk documents an open-source migration tool that bulk-creates users from a JSON/CSV file via the Backend API (respecting rate limits), which is a genuine bulk operation capability, but there is no evidence of broader bulk operations (e.g., bulk role/permission updates, bulk org membership changes, bulk deletions) or of an AI agent invoking such bulk actions through the MCP server (which only exposes SDK snippets, not execution). missing for 10: evidence of bulk operations beyond user import, AI-agent-driven bulk actions via API/MCP, and bulk management of organizations/roles/permissions.", "evidenceIds": [ "clerk-docs-7", "clerk-supp-migration-tool", "clerk-docs-42" ] }, { "productId": "clerk", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Clerk offers webhooks that fire on events like user creation/update, giving a basic event-trigger mechanism, but there's no evidence of a built-in rules engine, conditional logic, or automated actions beyond delivering the webhook payload — developers must build the 'action' side themselves outside Clerk. Missing for 10: native rules/automation engine, conditional trigger logic, built-in actions (not just event notifications), and any documentation of automation workflows beyond webhook delivery.", "evidenceIds": [ "clerk-docs-5" ] }, { "productId": "clerk", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Clerk is an authentication/user-management platform; scheduling recurring jobs or workflows is outside its product category (no cron/workflow-orchestration capability is offered or implied anywhere in the evidence).", "evidenceIds": [] }, { "productId": "clerk", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Clerk is an authentication/user-management platform, not an automation-building tool; there is no concept of user-created 'automations' to version, review, or roll back. This axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "clerk", "storyId": "bot-attack-protection", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Clerk documents built-in bot detection ('advanced bot and multi-account protection...machine learning', 'Bot Detection...continually updated machine learning'), brute-force prevention on OTPs ('built-in brute force prevention'), disposable-email blocking against fraudulent sign-ups, and MFA (SMS, authenticator app, backup codes) to harden auth flows against credential stuffing. These are first-party vendor claims without independent penetration-test corroboration. missing for 10: independent/hands-on verification of bot/brute-force protection efficacy, and explicit mention of credential-stuffing-specific defenses (e.g., breached-password detection) beyond general bot ML claims.", "evidenceIds": [ "clerk-supp-bot-detection", "clerk-docs-49", "clerk-docs-50", "clerk-docs-9", "clerk-docs-20" ] }, { "productId": "clerk", "storyId": "bulk-user-import-export", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Clerk documents an open-source migration tool that imports users from JSON/CSV via the Backend API (clerk-docs-7, clerk-supp-migration-tool), covering the bulk-import half of the story, but the evidence never mentions preserving/importing password hashes specifically, nor any official export tool or process for getting users back out of Clerk. Missing for 10: explicit password-hash import support, an official export/data-portability tool, and confirmation of round-trip migration to avoid lock-in.", "evidenceIds": [ "clerk-docs-7", "clerk-supp-migration-tool" ] }, { "productId": "clerk", "storyId": "deployment-data-control", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Clerk is presented throughout the evidence as a hosted SaaS (managed dashboard, CLI, remote MCP server, cloud APIs) with no mention of self-hosting, on-premise/private instance deployment, or bringing your own user database. No docs, changelog, or community evidence describe data residency controls or self-managed deployment options. missing for 10: any self-hosted/on-prem deployment option, BYO-database support, or documented data residency/private-instance controls.", "evidenceIds": [] }, { "productId": "clerk", "storyId": "device-flow-for-cli-agents", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Clerk's changelog explicitly describes an OAuth device authorization grant ('CLIs, TVs, game consoles... can ask a user to approve access from a browser-capable device without entering their credentials') matching the story's core need, and a related changelog entry mentions custom OAuth scopes for MCP clients. However, this is only surfaced as a brief changelog note rather than a dedicated implementation guide, and Clerk's more prominently documented machine-auth pattern is long-lived API keys/M2M tokens, which is the opposite pattern the story wants to avoid. Missing for 10: a full first-party guide/tutorial for implementing the device flow for CLIs/agents, SDK examples, and independent/hands-on confirmation it works as described.", "evidenceIds": [ "clerk-docs-52", "clerk-docs-55", "clerk-docs-11", "clerk-docs-43" ] }, { "productId": "clerk", "storyId": "edge-middleware-route-protection", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Clerk's docs confirm first-party Next.js support and JWT-based short-lived session tokens for backend/API authentication (clerk-docs-19, clerk-docs-6/22/32), and community feedback praises the Next.js integration story (clerk-comm-11), supporting the framework-SDK claim. However, the evidence pack never explicitly documents the clerkMiddleware()/edge-runtime session verification mechanism or peer-framework (Remix, SvelteKit, etc.) middleware parity. Missing for 10: explicit middleware API docs, edge-runtime verification details, and coverage of non-Next.js framework middleware equivalents.", "evidenceIds": [ "clerk-docs-19", "clerk-docs-6", "clerk-docs-22", "clerk-docs-32", "clerk-comm-11", "clerk-gh-1" ] }, { "productId": "clerk", "storyId": "enterprise-saml-oidc-sso", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Clerk's docs explicitly state Enterprise SSO support for SAML and OIDC protocols with named IdPs Azure AD, Okta, and Google Workspace, plus user data sync — directly matching the story. Missing for 10: independent/hands-on verification of enterprise SSO setup (e.g., a third-party case study or admin walkthrough) and specifics on workforce-directory sync/SCIM depth.", "evidenceIds": [ "clerk-docs-8", "clerk-docs-28" ] }, { "productId": "clerk", "storyId": "fine-grained-authorization", "verdict": "partial", "quality": 3, "confidence": "medium", "rationale": "Clerk documents custom Roles and fine-grained Permissions scoped to Organizations, which goes beyond the two default roles, but this is still a classic RBAC model (roles+permission strings tied to org membership) — there is no documented relationship-based (ReBAC/Zanzibar-style) or policy-based (ABAC/OPA-style) resource-level authorization engine. A community report explicitly flags the RBAC as limited ('roles tied to organizations, not stored on user itself'), reinforcing that Clerk does not offer authorization beyond role/permission checks. missing for 10: relationship-graph or ABAC/policy engine, resource-level (object-instance) permission checks, independent confirmation that custom Permissions support conditions/attributes rather than static strings.", "evidenceIds": [ "clerk-docs-4", "clerk-docs-17", "clerk-docs-21", "clerk-docs-46", "clerk-comm-3" ] }, { "productId": "clerk", "storyId": "guided-competitor-migration", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Clerk documents a vendor-maintained, open-source migration tool that ingests a JSON/CSV list of users and creates them via the Backend API, explicitly for migrating off other providers (clerk-docs-7, clerk-supp-migration-tool). However, the evidence never confirms the tool imports existing password hashes or otherwise avoids forcing users to reset passwords — a key part of this story. Missing for 10: explicit documentation of password/credential hash migration support, guidance on preserving existing sessions, and independent/hands-on confirmation that migrated users are not forced to reset passwords.", "evidenceIds": [ "clerk-docs-7", "clerk-supp-migration-tool" ] }, { "productId": "clerk", "storyId": "human-approval-for-agent-actions", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Clerk documents a device-authorization-grant-style flow (clerk-docs-52) that lets a user approve access from a separate browser-capable device — conceptually adjacent to CIBA-style out-of-band approval — and a 'reverification window' for sensitive actions (clerk-docs-53), but neither is presented as a mechanism for an autonomous agent to pause a specific transaction and await asynchronous human approval. Missing for 10: explicit CIBA/agent-transaction-approval workflow, agent-side APIs to request and await approval mid-task, and any hands-on or documented example of this exact use case.", "evidenceIds": [ "clerk-docs-52", "clerk-docs-53" ] }, { "productId": "clerk", "storyId": "m2m-client-credentials", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Clerk documents dedicated Machine-to-Machine (M2M) tokens distinct from user session tokens, explicitly described as authenticating 'calls between your agents' (vs. API keys for programmatic callers), with tool-call authorization gated by permissions/scopes — directly matching the client-credentials/agent-auth use case. This is a metered, real platform feature (present in pricing tiers), not just a docs mention.\n\nmissing for 10: explicit naming of the OAuth2 'client_credentials' grant type, independent/hands-on verification of the M2M flow, and details on token lifetime/rotation for service accounts.", "evidenceIds": [ "clerk-docs-11", "clerk-docs-43", "clerk-docs-16", "clerk-docs-27", "clerk-supp-m2m-pricing" ] }, { "productId": "clerk", "storyId": "mfa-second-factors", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Clerk docs explicitly list authenticator app and backup codes as second-factor MFA strategies, alongside SMS, and document the needs_second_factor sign-in status for enforcement. Step-up/reverification is also documented (customizable reverification window for sensitive actions), covering the step-up enforcement requirement. missing for 10: independent hands-on verification of TOTP/backup-code flows and step-up reverification working end-to-end, and more detail on configuring per-action step-up policies.", "evidenceIds": [ "clerk-docs-9", "clerk-docs-20", "clerk-docs-30", "clerk-docs-45", "clerk-docs-53" ] }, { "productId": "clerk", "storyId": "oauth-oidc-standard-flows", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Clerk's SDKs/components abstract social-login OAuth, and docs show OIDC support for Enterprise SSO, OAuth scoped access, a device-authorization grant for CLIs/TVs, custom OAuth scopes for MCP clients, and short-lived JWT session tokens — all handled without developers touching raw protocol code. However, the evidence never explicitly confirms an authorization-code+PKCE flow or a standard OAuth refresh-token grant/rotation mechanism (session tokens are proprietary short-lived JWTs, not documented OAuth refresh tokens). Missing for 10: explicit PKCE flow documentation, explicit refresh-token grant/rotation details, and independent/hands-on confirmation that the abstraction fully hides protocol nuances in production use.", "evidenceIds": [ "clerk-docs-6", "clerk-docs-10", "clerk-docs-28", "clerk-docs-44", "clerk-docs-52", "clerk-docs-55", "clerk-docs-22" ] }, { "productId": "clerk", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Clerk exposes a documented OpenAPI/Backend API (clerk-probe-3), session-token backend auth (clerk-docs-6), webhooks, migration via Backend API (clerk-docs-7), and org/role/permission management that mirrors the UI (clerk-docs-3,4,24), suggesting broad API parity with the dashboard/UI experience. However, there is no explicit vendor claim of full UI-to-API parity, and community feedback flags gaps like RBAC limitations and reliability issues that suggest not every UI capability is cleanly API-accessible (clerk-comm-3).\nmissing for 10: explicit parity documentation/statement, evidence that billing/impersonation/admin-dashboard actions are fully API-driven, independent confirmation of complete feature parity.", "evidenceIds": [ "clerk-probe-3", "clerk-docs-6", "clerk-docs-7", "clerk-docs-3", "clerk-docs-24", "clerk-comm-3" ] }, { "productId": "clerk", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows a migration tool for importing user data INTO Clerk (JSON/CSV via Backend API) and a general Backend/OpenAPI surface, but nothing documents a bulk data-export feature or open-format export path for users wanting to leave the platform.", "evidenceIds": [ "clerk-docs-7", "clerk-supp-migration-tool", "clerk-probe-3" ] }, { "productId": "clerk", "storyId": "openness-open-license", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Clerk's client SDKs are published on GitHub (clerk/javascript) and it explicitly ships an open-source migration tool, showing some source is publicly readable, but the core Clerk platform/backend is a closed SaaS with no evidence of an open license covering the full product. Missing for 10: explicit license text (e.g., MIT) for the SDK repo, confirmation that the core auth/backend service is open source, and independent verification of license terms.", "evidenceIds": [ "clerk-gh-1", "clerk-supp-migration-tool" ] }, { "productId": "clerk", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Clerk is a hosted, closed-source authentication/user-management SaaS; there is no evidence of a self-hostable core product, on-prem deployment option, or open-source server. Only a small migration CLI tool is open-source, not the core product itself.", "evidenceIds": [ "clerk-docs-7", "clerk-supp-migration-tool" ] }, { "productId": "clerk", "storyId": "org-level-sso-domains", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Clerk documents Organizations for multi-tenant B2B apps and Enterprise SSO (SAML/OIDC) with IdP sync, which together support giving customer orgs their own SSO connections. However, there is no explicit documentation of per-organization SSO connection scoping, verified-domain-based auto-join/JIT provisioning tied to a specific org, or an admin flow for org-level domain verification — these are typically distinct enterprise features not evidenced here. Missing for 10: explicit per-org SSO connection binding, domain verification workflow, and JIT provisioning-on-domain-match documentation, plus independent confirmation these work together as described.", "evidenceIds": [ "clerk-docs-3", "clerk-docs-8", "clerk-docs-28", "clerk-docs-24", "clerk-docs-31" ] }, { "productId": "clerk", "storyId": "organizations-multitenant", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Clerk's Organizations feature is well documented: multi-tenant B2B support with grouping users, default admin/member roles plus custom roles and permissions, multiple org membership with active-organization session context, and built-in UI for creating orgs, switching, and viewing memberships/invitations (clerk-docs-3, clerk-docs-4/17/46, clerk-docs-24, clerk-docs-34, clerk-docs-12). This covers the core story out of the box via prebuilt components and APIs. Missing for 10: independent/hands-on corroboration of the invitation flow specifically, and one community comment (clerk-comm-3) notes RBAC is organization-scoped rather than user-scoped, a design caveat rather than a failure.", "evidenceIds": [ "clerk-docs-3", "clerk-docs-4", "clerk-docs-12", "clerk-docs-24", "clerk-docs-34", "clerk-docs-46", "clerk-comm-3" ] }, { "productId": "clerk", "storyId": "passkeys-passwordless-login", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Clerk documents passkey and biometric sign-in support and email/SMS one-time passcodes with brute-force prevention, showing first-class passwordless/WebAuthn options, but passkeys and biometric sign-in are explicitly gated behind paid Pro+ plans rather than being universally free/first-class, and no evidence explicitly confirms magic links as a distinct sign-in strategy. missing for 10: explicit magic-link documentation, confirmation that passkeys are available on all plans (not just Pro+), and independent/hands-on validation of the passwordless UX.", "evidenceIds": [ "clerk-supp-passkeys", "clerk-supp-bot-detection", "clerk-docs-9" ] }, { "productId": "clerk", "storyId": "prebuilt-auth-ui", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Clerk's docs strongly evidence prebuilt drop-in UI components (SignIn/SignUp, profile management), customization of branding/CSS, Next.js and other framework integration, and CLI-driven production deployment (domains, OAuth providers, DNS) supporting shipping to production. Community threads raise reliability/complexity concerns but do not contradict the core UI-component/profile-management capability itself. Missing for 10: explicit first-party documentation of hosted/account-portal pages (as opposed to embedded components) and independent hands-on confirmation of the full sign-up/sign-in/profile flow in production.", "evidenceIds": [ "clerk-docs-1", "clerk-docs-15", "clerk-docs-18", "clerk-docs-19", "clerk-docs-23", "clerk-docs-33", "clerk-docs-37", "clerk-docs-40", "clerk-gh-1" ] }, { "productId": "clerk", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, regional data storage, or the ability to choose where user data is hosted; Clerk's docs cover auth, orgs, MFA, MCP, CLI, etc., but nothing about region/residency controls.", "evidenceIds": [] }, { "productId": "clerk", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack addresses AI training data usage or opt-out policies for Clerk; the evidence covers auth, RBAC, sessions, MCP tooling, and reliability complaints but nothing about data being used for AI training or a mechanism to prevent it. Since Clerk stores user PII, this is a plausible privacy question for the product category, but it's unaddressed.", "evidenceIds": [] }, { "productId": "clerk", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of data retention policies, user data deletion APIs/webhooks, GDPR/CCPA compliance controls, or configurable retention windows — nothing addresses this privacy-posture story despite Clerk being a user-data platform where such controls would be expected.", "evidenceIds": [] }, { "productId": "clerk", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions telemetry, usage tracking, analytics data collection, or an opt-out mechanism for Clerk's own product telemetry; this is an applicable axis (Clerk is a SaaS/dev-tool product that could plausibly collect usage telemetry) but nothing addresses it.", "evidenceIds": [] }, { "productId": "clerk", "storyId": "rbac-roles-in-tokens", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Clerk's docs clearly show role/permission definition (default admin/member plus custom roles and fine-grained permissions) scoped to Organizations, and confirm that session tokens carry Organization context including memberships and Roles for authorization checks. However, a hands-on community report explicitly criticizes Clerk for lacking 'proper RBAC' because roles are tied to Organizations rather than being storable directly on the user, which is a real limitation on how broadly the enforcement model applies. Missing for 10: evidence of user-level (non-org) role/permission enforcement, first-party code samples showing permission checks against the session JWT claims, and independent corroboration beyond the single critical community post.", "evidenceIds": [ "clerk-docs-4", "clerk-docs-17", "clerk-docs-21", "clerk-docs-24", "clerk-docs-34", "clerk-docs-46", "clerk-docs-22", "clerk-comm-3" ] }, { "productId": "clerk", "storyId": "scim-directory-sync", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack shows Clerk supports Enterprise SSO via SAML/OIDC and syncs user data on sign-in, but there is no mention anywhere of SCIM provisioning/deprovisioning, group sync, or automated deactivation triggered by IdP changes — the core of this story.", "evidenceIds": [ "clerk-docs-8", "clerk-docs-28" ] }, { "productId": "clerk", "storyId": "scoped-agent-identities", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Clerk's 'eve' agent-auth product provides M2M tokens as a distinct credential type from user API keys/sessions, explicitly for 'calls between your agents,' and lets you gate individual tool calls against the caller's scoped Clerk permissions, plus custom OAuth scopes for MCP clients — directly supporting least-privilege, non-human machine identities for agents. Missing for 10: explicit documentation of instant/granular revocation specifically for M2M tokens (only session/device revocation is documented), and independent/hands-on corroboration beyond first-party docs.", "evidenceIds": [ "clerk-docs-11", "clerk-docs-43", "clerk-docs-16", "clerk-docs-27", "clerk-docs-55", "clerk-supp-m2m-pricing", "clerk-supp-session-devices" ] }, { "productId": "clerk", "storyId": "secure-mcp-server-auth", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Clerk shows real building blocks for this story — OAuth scoped access, M2M tokens for agent-to-agent calls, an 'eve' feature to gate tool calls against caller permissions/scopes and broker OAuth on the caller's behalf, and a changelog entry specifically about 'Custom OAuth scopes: give MCP clients precise, discoverable' scopes. However, evidence stops short of a dedicated guide showing a spec-compliant OAuth authorization flow specifically fronting a self-hosted MCP server (Clerk's own MCP server is for feeding docs to coding agents, a different use case). Missing for 10: a dedicated first-party tutorial/reference for protecting a custom MCP server with Clerk OAuth end-to-end, and independent/hands-on confirmation of spec compliance with the MCP authorization spec.", "evidenceIds": [ "clerk-docs-10", "clerk-docs-16", "clerk-docs-27", "clerk-docs-43", "clerk-docs-55", "clerk-docs-52" ] }, { "productId": "clerk", "storyId": "session-lifecycle-revocation", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Clerk's docs describe short-lived session JWTs with refresh (clerk-docs-6/22/32) and explicit session-lifecycle management including active device monitoring and per-device/session revocation (clerk-supp-session-devices), which covers expiry, refresh, and revocation as claimed. However a hands-on community report describes production token-refresh failures requiring manual weekend fixes (clerk-comm-1), directly contradicting the reliability of the refresh mechanism in practice. Missing for 10: independent verification of instant/reliable server-side revocation propagation and no corroborating success reports to offset the refresh-failure account.", "evidenceIds": [ "clerk-docs-6", "clerk-docs-22", "clerk-supp-session-devices", "clerk-comm-1" ] }, { "productId": "clerk", "storyId": "social-login-breadth", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Evidence confirms Clerk supports configurable OAuth flows (scoped access, docs-10/29/44) and Enterprise SSO via SAML/OIDC with multiple IdPs like Azure AD, Okta, Google Workspace (docs-8/28), implying broad provider support without custom code. However, the pack never explicitly enumerates the standard social-provider list (Google, GitHub, Facebook, etc.) or shows a dashboard toggle UI, and community notes raise reliability concerns (clerk-comm-3/4) though these are about RBAC/uptime, not OAuth setup itself. missing for 10: explicit list/screenshot of supported social providers, first-party doc confirming zero-code toggle configuration, independent hands-on confirmation of ease of adding new providers.", "evidenceIds": [ "clerk-docs-8", "clerk-docs-28", "clerk-docs-10", "clerk-docs-29", "clerk-docs-44" ] }, { "productId": "clerk", "storyId": "token-vending-for-agent-tools", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Clerk's 'eve' agent-auth docs explicitly describe brokering OAuth to obtain a provider token on the caller's behalf, gated by the caller's Clerk permissions/scopes, and Clerk separately supports OAuth scoped access for third-party API delegation — directly matching the token-vault/exchange concept for delegated tool calls. Missing for 10: no detail on token lifetime/refresh mechanics specific to third-party tokens (as opposed to Clerk's own short-lived session tokens), no independent/hands-on corroboration of the eve broker in production use.", "evidenceIds": [ "clerk-docs-16", "clerk-docs-27", "clerk-docs-10", "clerk-docs-29", "clerk-docs-44", "clerk-docs-55", "clerk-docs-6" ] }, { "productId": "keycloak", "storyId": "act-as-oauth-provider", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Keycloak is designed exactly for this: developers register their app as a realm/client, and Keycloak issues OAuth2/OIDC tokens (with discovery, introspection, revocation, dynamic client registration, client credentials for machine clients) so third-party clients can authenticate against the developer's own Keycloak-backed app, effectively 'Sign in with my app'. Missing for 10: independent hands-on case study of a developer specifically building a public-facing 'Sign in with X' third-party integration, and clearer docs on scoping/consent screens for external clients.", "evidenceIds": [ "keycloak-docs-3", "keycloak-docs-4", "keycloak-docs-20", "keycloak-docs-39", "keycloak-docs-52", "keycloak-docs-53", "kc-supp-client-credentials", "keycloak-docs-23" ] }, { "productId": "keycloak", "storyId": "active-session-visibility", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Keycloak's Account Console lets users view and manage their own sessions and history, while the Admin Console/Admin REST API lets admins manage users' sessions and permissions, and single sign-out revokes sessions across all connected apps — covering the core of the story for both personas. Missing for 10: explicit documentation of a per-device 'revoke individually vs revoke all' UI/API call-out and independent/hands-on confirmation of this specific workflow.", "evidenceIds": [ "keycloak-docs-48", "keycloak-docs-21", "keycloak-docs-13", "keycloak-docs-15", "keycloak-docs-51" ] }, { "productId": "keycloak", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Direct probes show no llms.txt (404), no markdown docs endpoint, and no OpenAPI/agent-oriented docs endpoint; only standard human-facing documentation exists.", "evidenceIds": [ "keycloak-probe-1", "keycloak-probe-2", "keycloak-probe-3" ] }, { "productId": "keycloak", "storyId": "agentic-ai-insights", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Keycloak is an identity and access management server (authentication, SSO, authorization); it has no concept of user data analytics or AI-generated insights/suggestions. This axis is a category error for an IAM product, not a missing feature.", "evidenceIds": [] }, { "productId": "keycloak", "storyId": "agentic-autonomous-automation", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Keycloak is an identity and access management server, not an automation/agent platform; setting up autonomous background automations is outside its product category (wrong axis), even though it exposes APIs that other automation tools could call.", "evidenceIds": [] }, { "productId": "keycloak", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "keycloak", "storyId": "agentic-headless", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Keycloak documents strong headless/CI primitives: running from a container image with `start-dev` (keycloak-docs-2, keycloak-docs-45), an Operator for declarative management (keycloak-docs-43, keycloak-docs-54), the kcadm.sh Admin CLI and REST API for scripted administration (kc-supp-kcadm), realm import/export as JSON (keycloak-docs-10), and health REST endpoints for readiness checks (keycloak-docs-17) — all standard building blocks for CI automation. However, hands-on community reports concretely contradict smooth CI/IaC usage: configuration is stored in the database rather than files, making it hard to manage via yaml/configmaps or test changes before production (keycloak-comm-19), the Admin API is described as inconsistent and hard to keep idempotent for Ansible-driven automation (keycloak-comm-13), and one operator explicitly states Keycloak 'doesn't play well with infrastructure-as-code ideas' and is best treated as a 'pet' not 'cattle' (keycloak-comm-3). Missing for 10: first-party CI/CD pipeline examples or GitHub Actions templates, evidence of clean idempotent automation workflows, and resolution of the documented IaC friction.", "evidenceIds": [ "keycloak-docs-2", "keycloak-docs-45", "keycloak-docs-43", "keycloak-docs-54", "kc-supp-kcadm", "keycloak-docs-10", "keycloak-docs-17", "keycloak-comm-19", "keycloak-comm-13", "keycloak-comm-3" ] }, { "productId": "keycloak", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Keycloak is an identity and access management server, not an AI agent or assistant; the story asks about plugging MCP servers into a product so it can use tools, which is an agentic client capability wholly outside Keycloak's category as an IAM/SSO platform.", "evidenceIds": [] }, { "productId": "keycloak", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Keycloak is an identity/access management server; no evidence of an official MCP server for connecting AI agents, and probes for llms.txt/openapi endpoints returned 404. Since Keycloak is not itself an AI agent but a platform/service, this axis applies, and absence of evidence means 'none'.", "evidenceIds": [ "keycloak-probe-1", "keycloak-probe-2", "keycloak-probe-3" ] }, { "productId": "keycloak", "storyId": "agentic-nl-commands", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Keycloak is an identity/access-management server, not an AI agent or natural-language interface product; operating it via natural-language commands is outside its product category (admin console, REST API, CLI are the interfaces). This is a category mismatch, not a missing feature.", "evidenceIds": [] }, { "productId": "keycloak", "storyId": "agentic-official-cli", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Keycloak ships an official Admin CLI (kcadm.sh) documented in the Server Administration Guide, enabling scripted realm/user/role/client management equivalent to the Admin REST API — well suited for automation by AI agents or scripts. Missing for 10: no independent/hands-on corroboration of kcadm reliability, and no explicit AI-agent-oriented tooling or examples beyond generic scripting support.", "evidenceIds": [ "kc-supp-kcadm", "keycloak-docs-6" ] }, { "productId": "keycloak", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Keycloak documents an Admin REST API and a full set of OIDC/OAuth2 protocol endpoints (well-known config, token introspection/revocation, dynamic client registration, device grant), plus the kcadm CLI and SCIM endpoints for scripted/agentic driving of the product. Community reports confirm the API is used in real automation (Ansible) though one report notes inconsistency/idempotency friction, a minor caveat rather than a failure. missing for 10: a discoverable OpenAPI/Swagger spec (probe shows 404s) and stronger independent corroboration of full API completeness.", "evidenceIds": [ "keycloak-docs-6", "keycloak-docs-23", "keycloak-docs-38", "keycloak-docs-39", "keycloak-docs-52", "keycloak-docs-53", "kc-supp-kcadm", "kc-supp-scim", "kc-supp-client-credentials", "keycloak-comm-13" ] }, { "productId": "keycloak", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Keycloak's client credentials grant lets service accounts (which cover automated/agent clients) authenticate via client_id/secret or JWT, and its Authorization Services/fine-grained authorization let admins define exact least-privilege policies and scopes per client, which is the core mechanism needed to issue scoped API credentials to a non-human actor like an AI agent. However, none of the evidence specifically addresses AI-agent use cases, dynamic/short-lived credential issuance workflows for agents, or independent hands-on validation of scoped-credential setups. Missing for 10: agent-specific tooling or examples, dynamic/ephemeral credential issuance patterns, independent/hands-on confirmation that scoped client-credentials setups work as intended for agentic workloads.", "evidenceIds": [ "kc-supp-client-credentials", "kc-supp-authz-services", "keycloak-docs-24", "keycloak-docs-39" ] }, { "productId": "keycloak", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers Keycloak's REST APIs, OIDC endpoints, admin CLI (kcadm), and protocol standards, but nothing documents official client SDKs/libraries for developers (let alone AI-native/agentic use) — and probes confirm no machine-readable API spec (openapi.json 404) or llms.txt is exposed. Missing for 10: any mention of official SDKs (Java, Node, Python, etc.), agent/AI-specific integration libraries, or programmatic SDK documentation.", "evidenceIds": [ "keycloak-docs-6", "keycloak-docs-23", "kc-supp-kcadm", "keycloak-probe-1", "keycloak-probe-2", "keycloak-probe-3" ] }, { "productId": "keycloak", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions webhooks or an event-subscription mechanism for external systems; Keycloak's admin/account consoles, REST APIs, and SPI extensibility are documented, but nothing indicates a webhook subscription feature for events like login, user creation, etc. missing for 10: any documentation of webhook config, event listener SPI exposed as webhooks, or third-party corroboration of webhook support.", "evidenceIds": [] }, { "productId": "keycloak", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only a link to 'Documentation for the Administration RESTful API' (keycloak-docs-6) with no mention of an interactive, runnable API explorer; probes for OpenAPI/Swagger specs and machine-readable docs all returned 404s, indicating no interactive API reference is exposed.", "evidenceIds": [ "keycloak-docs-6", "keycloak-probe-3", "keycloak-probe-1", "keycloak-probe-2" ] }, { "productId": "keycloak", "storyId": "api-machine-spec", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Keycloak exposes machine-readable OIDC discovery metadata (the /.well-known/openid-configuration endpoint documented in the OIDC layers guide), which is a standard machine-readable description of its authentication/authorization endpoints — a partial equivalent to an API spec for that subsystem. However, direct probes for a full OpenAPI/Swagger spec (openapi.json, swagger.json, etc.) all returned 404, and the docs only reference prose documentation for the Admin REST API rather than a downloadable machine-readable spec covering the full REST surface. Missing for 10: a published OpenAPI/Swagger document for the Admin REST API or other REST endpoints, and any docs explicitly stating such a spec is downloadable.", "evidenceIds": [ "keycloak-docs-4", "keycloak-docs-38", "keycloak-docs-6", "keycloak-probe-3" ] }, { "productId": "keycloak", "storyId": "api-sandbox", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Keycloak docs show a local dev-mode Docker instance (start-dev), realm-based tenant isolation, and JSON import/export of realms, which could be used to build a sandbox separate from production. However, a hands-on community report explicitly states that because Keycloak stores its configuration in the database rather than files, 'there's no easy supported way to test some changes before production,' directly contradicting the idea of frictionless sandbox testing. missing for 10: no first-party guidance on a supported staging/sandbox workflow, no evidence of data-masking or safe test-data seeding, and the one concrete community account says testing before production is hard.", "evidenceIds": [ "keycloak-docs-2", "keycloak-docs-20", "keycloak-docs-10", "keycloak-docs-59", "keycloak-comm-19" ] }, { "productId": "keycloak", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence documents Keycloak's Admin REST API and OIDC endpoints exist, but nothing describes API versioning schemes or a documented deprecation policy; one community comment even calls the API 'inconsistent' for automation purposes. Missing for 10: any documentation of API version numbers, backward-compatibility guarantees, or a formal deprecation/sunset policy.", "evidenceIds": [ "keycloak-docs-6", "keycloak-docs-23", "keycloak-comm-13" ] }, { "productId": "keycloak", "storyId": "audit-logs-export", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of audit logging, event listeners, admin event tracking, tamper-evidence, or SIEM export/streaming capabilities—only general docs on protocols, admin console, federation, and community sentiment. Audit-log export to SIEM is a fair expectation for an IAM platform, so this is 'none' rather than 'na', but nothing in the pack substantiates it.", "evidenceIds": [] }, { "productId": "keycloak", "storyId": "auth-event-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of webhooks, event streams, or an event listener SPI for auth events (sign-ups, sign-ins, user changes); it only covers OIDC/SAML protocols, admin console, REST API, and federation topics. Missing for 10: any documentation of webhook/event-stream subscription mechanism, event listener SPI, or third-party event-forwarding examples.", "evidenceIds": [] }, { "productId": "keycloak", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Keycloak exposes an Admin REST API and the kcadm CLI for scripting operations across realms, users, roles, and clients, plus realm import/export as JSON, which together enable programmatic/bulk administration. However, there's no documented dedicated bulk-operation endpoint, and community feedback notes the API is inconsistent and hard to use idempotently for automation (keycloak-comm-13), undercutting reliability for large-scale scripted changes. Missing for 10: explicit bulk/batch API documentation, first-party examples of bulk user/client management, and independent confirmation that automation at scale works smoothly.", "evidenceIds": [ "kc-supp-kcadm", "keycloak-docs-6", "keycloak-docs-10", "keycloak-comm-13" ] }, { "productId": "keycloak", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Keycloak documents built-in automated triggers like brute-force lockout policies that fire actions (temporary/permanent account lockout) based on login-failure events, and its SPI-based extensibility could theoretically support custom event listeners, but there is no documented general-purpose rules/automation engine for user-defined event-triggered actions. missing for 10: a documented rules/automation engine or webhook system letting users define arbitrary triggers/actions, independent evidence of custom event-listener usage, and any AI-native automation tooling.", "evidenceIds": [ "kc-supp-brute-force", "keycloak-docs-5" ] }, { "productId": "keycloak", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Keycloak is an identity and access management server; scheduling recurring jobs or workflows is not part of its product category, which centers on authentication, authorization, and identity federation.", "evidenceIds": [] }, { "productId": "keycloak", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Keycloak is an identity/access-management server, not an automation-building platform; it has no concept of 'automations' to version, review, or roll back (only realm JSON import/export, which is unrelated to automation workflows). This axis is a category mismatch for an IAM product.", "evidenceIds": [] }, { "productId": "keycloak", "storyId": "bot-attack-protection", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Keycloak documents built-in brute-force detection with temporary and permanent account lockout events (kc-supp-brute-force), and CVE-2023-6927 evidence shows security issues (redirect URI validation) are patched, not indicating brute-force protection failure. Standard OIDC/OAuth flows plus WebAuthn/passkey support (kc-supp-passkeys) further harden auth flows against credential-based attacks. Missing for 10: no explicit bot-detection/CAPTCHA feature documentation, no independent third-party testing or benchmark of brute-force protection effectiveness against real attacks.", "evidenceIds": [ "kc-supp-brute-force", "kc-supp-passkeys", "keycloak-docs-13", "keycloak-comm-7" ] }, { "productId": "keycloak", "storyId": "bulk-user-import-export", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Keycloak documents realm import/export as JSON files (keycloak-docs-10), which is the standard mechanism for bulk user migration, and the Admin CLI/REST API (kc-supp-kcadm) can further script user data extraction and loading. However, the evidence pack never explicitly confirms that password hashes are preserved/portable in this export-import flow, nor is there any hands-on or community confirmation of successful password-hash migration to/from Keycloak. Missing for 10: explicit documentation of password hash format/algorithm compatibility during import, worked example of exporting and re-importing credentials, and independent/community verification that this round-trip works without lock-in issues.", "evidenceIds": [ "keycloak-docs-10", "kc-supp-kcadm" ] }, { "productId": "keycloak", "storyId": "deployment-data-control", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Keycloak is explicitly self-hosted (Docker/container image, Operator install) and lets operators run their own production database (e.g., switch to PostgreSQL) and realm data entirely under their control, with no mandatory SaaS dependency; community reports corroborate years of stable self-hosted production use. missing for 10: no explicit first-party statement on data residency/air-gapped deployment guarantees beyond generic self-hosting instructions.", "evidenceIds": [ "keycloak-docs-2", "keycloak-docs-27", "keycloak-docs-45", "keycloak-docs-54", "keycloak-docs-59", "keycloak-comm-5", "keycloak-comm-10", "keycloak-comm-14" ] }, { "productId": "keycloak", "storyId": "device-flow-for-cli-agents", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Keycloak's Server Administration Guide explicitly documents the 'Device authorization grant' for clients with limited input/no browser, and the OIDC endpoints reference lists the device authorization endpoint (/protocol/openid-connect/auth/device) used to obtain a device code and user code — exactly the flow needed for CLIs/headless agents to authenticate without long-lived secrets. This is paired with client-credentials/service-account support for other machine flows, giving a complete non-interactive auth story. Missing for 10: independent/hands-on community evidence specifically validating CLI or agent use of the device flow (community citations cover other topics, not this flow).", "evidenceIds": [ "kc-supp-device-grant", "keycloak-docs-38", "keycloak-docs-23" ] }, { "productId": "keycloak", "storyId": "edge-middleware-route-protection", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows only generic OIDC/SAML protocol support and endpoints (well-known config, introspection, revocation) but no mention of a first-party Next.js SDK, edge middleware, or session verification at the edge; Keycloak's client adapters for specific frameworks are not referenced anywhere in the pack.", "evidenceIds": [ "keycloak-docs-4", "keycloak-docs-23", "keycloak-docs-38", "keycloak-docs-40" ] }, { "productId": "keycloak", "storyId": "enterprise-saml-oidc-sso", "verdict": "full", "quality": 7, "confidence": "high", "rationale": "Keycloak's identity brokering natively supports both SAML 2.0 and OIDC identity providers configured via the admin console, and production users confirm integrating multiple external IdPs for SSO in real deployments. Missing for 10: explicit named walkthroughs/screenshots for Okta, Entra, or Google Workspace specifically, and independent hands-on verification of these exact IdP integrations rather than generic SAML/OIDC brokering docs.", "evidenceIds": [ "keycloak-docs-9", "keycloak-docs-46", "kc-supp-social-brokering", "keycloak-comm-5", "keycloak-comm-14" ] }, { "productId": "keycloak", "storyId": "fine-grained-authorization", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Keycloak's Authorization Services documentation explicitly describes resource-level, policy-based fine-grained authorization beyond RBAC, using resources, scopes, and policies evaluated via UMA permission tickets — going well past simple role checks (kc-supp-authz-services, keycloak-docs-49, keycloak-docs-24, keycloak-docs-37). This is well-documented first-party capability with a dedicated guide, and community evidence corroborates general production stability though doesn't specifically validate this subsystem in depth. Missing for 10: independent/hands-on community validation specifically of the fine-grained authorization services feature (relationship-based policies, custom policy scripts) rather than general product reviews.", "evidenceIds": [ "kc-supp-authz-services", "keycloak-docs-49", "keycloak-docs-24", "keycloak-docs-37", "keycloak-docs-12" ] }, { "productId": "keycloak", "storyId": "guided-competitor-migration", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Keycloak's user federation SPI lets teams implement a custom provider for existing user stores (e.g., a legacy password database), which is the standard technique used to migrate users without forcing resets, and identity brokering/account linking also support gradual migration. However, there is no vendor-maintained migration guide or tooling specifically targeting competing auth providers (Auth0, Okta, Firebase, etc.) in the evidence pack. Missing for 10: explicit vendor migration guides/tooling for specific competitor providers, documented password-hash-preserving import workflow, and case studies of password-reset-free migrations.", "evidenceIds": [ "keycloak-docs-47", "keycloak-docs-8", "keycloak-docs-9", "kc-supp-federation", "kc-supp-social-brokering", "keycloak-docs-10" ] }, { "productId": "keycloak", "storyId": "human-approval-for-agent-actions", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents OAuth2/OIDC flows, device authorization grant, client credentials, and fine-grained authorization services, but nowhere mentions CIBA (Client-Initiated Backchannel Authentication) or any mechanism for out-of-band human approval tied to a specific agent-initiated transaction. Device grant is a different flow (device-initiated, browser-based user authorization) rather than a backend-initiated async approval-of-transaction pattern.", "evidenceIds": [ "kc-supp-device-grant", "kc-supp-client-credentials", "kc-supp-authz-services" ] }, { "productId": "keycloak", "storyId": "m2m-client-credentials", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Keycloak natively documents the OAuth2 client_credentials grant explicitly for service-to-service (non-user) authentication, backed by service accounts, client_id/secret or JWT auth, plus supporting infrastructure (token introspection, revocation, dynamic client registration) that lets machine clients authenticate and manage tokens without human interaction. This is first-party documented and corroborated by broad community usage as a stable OAuth2/OIDC provider in production. Missing for 10: independent hands-on validation specifically of the client-credentials flow (vs. general Keycloak stability reports).", "evidenceIds": [ "kc-supp-client-credentials", "keycloak-docs-39", "keycloak-docs-52", "keycloak-docs-53", "keycloak-docs-38", "kc-supp-authz-services" ] }, { "productId": "keycloak", "storyId": "mfa-second-factors", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Docs confirm Keycloak supports enabling two-factor authentication via the account console and reference WebAuthn/passkey setups, showing MFA capability exists, but no evidence explicitly describes TOTP authenticator app enrollment, backup/recovery codes, or configurable step-up authentication policies. missing for 10: explicit TOTP authenticator app documentation, backup codes mechanism, step-up/conditional MFA enforcement details, and independent confirmation of these specific features.", "evidenceIds": [ "keycloak-docs-13", "keycloak-docs-51", "kc-supp-passkeys", "kc-supp-brute-force" ] }, { "productId": "keycloak", "storyId": "oauth-oidc-standard-flows", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Keycloak's official docs show standard OIDC discovery, client registration with redirect URIs (implying auth code+PKCE flow), token introspection/revocation endpoints, and explicit support for refresh tokens and client credentials grants, all abstracting protocol details behind admin console configuration and standard endpoints. Community evidence corroborates long-term production use for OAuth2/OIDC SSO across many stacks, though some note it requires OAuth/OIDC familiarity to configure well. Missing for 10: explicit first-party documentation/code sample showing PKCE flag toggling or client library code eliminating hand-rolled protocol logic.", "evidenceIds": [ "keycloak-docs-4", "keycloak-docs-23", "keycloak-docs-38", "keycloak-docs-40", "keycloak-docs-52", "keycloak-docs-53", "kc-supp-client-credentials", "keycloak-docs-28", "keycloak-docs-50", "keycloak-comm-17", "keycloak-comm-5" ] }, { "productId": "keycloak", "storyId": "openness-api-parity", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Keycloak documents a comprehensive Admin REST API (kc-docs-6, kc-supp-kcadm) that the kcadm CLI wraps, and the same operations (realms, users, roles, clients, permissions) are said to be available via API as via the Admin Console — strong first-party support for API-as-UI-parity. However, a hands-on report notes the API is 'inconsistent' and hard to use idempotently for automation (keycloak-comm-13), SCIM explicitly excludes protected admin resources ('never through SCIM', kc-supp-scim), and no discoverable OpenAPI/machine-readable spec was found (keycloak-probe-3), undermining full AI-native parity and discoverability. missing for 10: independent corroboration that all UI-only workflows (e.g. theme customization, some fine-grained authz UI flows) have equal API parity, a public/discoverable OpenAPI spec, and resolution of reported API inconsistency for automation.", "evidenceIds": [ "keycloak-docs-6", "kc-supp-kcadm", "keycloak-comm-13", "kc-supp-scim", "keycloak-probe-3" ] }, { "productId": "keycloak", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Keycloak documents realm import/export as JSON, giving admins a way to extract configuration and user data in an open format, and being self-hosted/open-source avoids vendor lock-in. However, there is no documented end-user-facing 'export my data' feature for individual account holders (only admin console/CLI export), and no mention of exporting in other open standards (e.g., CSV, SCIM bulk export) beyond JSON realm dumps. Missing for 10: end-user self-service data export UI, broader open-format support, and independent confirmation of export completeness/fidelity.", "evidenceIds": [ "keycloak-docs-10", "keycloak-docs-11", "kc-supp-kcadm" ] }, { "productId": "keycloak", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains extensive product documentation and community commentary but no citation identifies an open-source license (e.g., Apache 2.0), a public source repository, or any statement about being able to read Keycloak's source code. Missing for 10: explicit license statement, link to source repository (e.g., GitHub), any confirmation of open licensing terms.", "evidenceIds": [] }, { "productId": "keycloak", "storyId": "openness-self-host", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Keycloak ships docker images and a documented self-host quickstart (docker run with KC_BOOTSTRAP_ADMIN_*), operator install, production DB setup, and reverse-proxy/clustering guides, and multiple community users confirm years of stable self-hosted production use. Missing for 10: independent benchmark/security audit beyond anecdotal community reports and no first-party statement addressing high resource usage some users report.", "evidenceIds": [ "keycloak-docs-2", "keycloak-docs-27", "keycloak-docs-45", "keycloak-docs-54", "keycloak-docs-59", "keycloak-docs-18", "keycloak-comm-5", "keycloak-comm-10", "keycloak-comm-11", "keycloak-comm-14" ] }, { "productId": "keycloak", "storyId": "org-level-sso-domains", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Keycloak documents an 'Organizations' feature in the realm admin guide allowing creation of named, realm-scoped organizations, and separately documents identity brokering/federation and first-broker-login style provisioning generally, but the evidence pack never shows organization-specific domain verification or org-linked JIT provisioning end-to-end. Missing for 10: evidence of domain verification per organization, evidence of binding a distinct SSO/IdP connection to each organization, and explicit documentation of JIT user provisioning tied to org membership.", "evidenceIds": [ "kc-supp-orgs", "kc-supp-social-brokering", "kc-supp-federation", "keycloak-docs-20" ] }, { "productId": "keycloak", "storyId": "organizations-multitenant", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Keycloak ships a dedicated 'Organizations' feature (Server Admin Guide: create organizations, set unique names within a realm) that is designed exactly for multi-tenant B2B scenarios, going beyond the older realm-as-tenant model. However, the evidence pack only documents organization creation/naming and does not show the membership management or invitation-flow mechanics in detail (e.g., inviting users by email, org-scoped roles, self-registration into an org). Missing for 10: documented invitation-flow API/UI details, membership assignment mechanics, and independent/hands-on confirmation that these work smoothly in production.", "evidenceIds": [ "kc-supp-orgs", "keycloak-docs-20", "keycloak-docs-32" ] }, { "productId": "keycloak", "storyId": "passkeys-passwordless-login", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Keycloak's admin docs explicitly document WebAuthn and a dedicated Passkeys section (conditional UI/autofill, modal UI, passkey mediation, loginless WebAuthn), showing first-class passwordless/passkey support. However, there is no evidence of built-in magic-link or email OTP sign-in as first-class methods—these are not mentioned anywhere in the pack. Missing for 10: documentation of magic-link authentication, native email OTP flow, and independent/hands-on confirmation of passkey UX.", "evidenceIds": [ "kc-supp-passkeys", "keycloak-docs-13", "keycloak-docs-51" ] }, { "productId": "keycloak", "storyId": "prebuilt-auth-ui", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Keycloak ships hosted, production-ready login/registration/forgot-password/account-management pages out of the box (keycloak-docs-3, -27, -28, -32, -33, -13, -51) and supports full UI customization via its theming system (keycloak-docs-7, -26, -41) and self-registration/profile/2FA management through the Account Console (keycloak-docs-13, -48, -51). This directly covers sign-in, sign-up, and profile management via hosted pages/customizable themes.\n\nMissing for 10: independent/hands-on evidence validating ease of theme customization in practice, and no mention of embeddable prebuilt UI component libraries (e.g., React/Vue widgets) beyond server-rendered hosted pages — the story leans on hosted pages more than 'components' but that gap in evidence keeps quality below top marks.", "evidenceIds": [ "keycloak-docs-3", "keycloak-docs-7", "keycloak-docs-13", "keycloak-docs-26", "keycloak-docs-27", "keycloak-docs-28", "keycloak-docs-32", "keycloak-docs-33", "keycloak-docs-41", "keycloak-docs-48", "keycloak-docs-50", "keycloak-docs-51" ] }, { "productId": "keycloak", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Keycloak is self-hosted software that can be run via Docker/container images against a database the operator chooses (e.g., PostgreSQL), which inherently lets an operator pick the physical region/infrastructure for data storage. However, there is no explicit documentation addressing data residency, region selection, or compliance controls as a named feature. Missing for 10: explicit residency/region documentation, multi-region deployment guidance, data localization/compliance statements.", "evidenceIds": [ "keycloak-docs-2", "keycloak-docs-45", "keycloak-docs-59" ] }, { "productId": "keycloak", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Keycloak is an identity and access management server; it has no relationship to AI model training data usage or opting out of AI training, so this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "keycloak", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Keycloak lets admins manage/delete users via the Admin Console and Admin REST API, and users can view/manage some of their own account data (profile, sessions) via the Account Console, which indirectly supports data control and deletion. However, there is no explicit documentation of data-retention policies, automated data expiry, GDPR-style export/erasure workflows, or user-initiated account deletion. Missing for 10: explicit retention policy configuration, documented self-service account/data deletion, data export tooling, and GDPR/compliance-specific retention controls.", "evidenceIds": [ "keycloak-docs-6", "keycloak-docs-13", "keycloak-docs-48", "kc-supp-kcadm" ] }, { "productId": "keycloak", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry collection or an opt-out mechanism for Keycloak; the only related item (keycloak-docs-44) describes self-hosted operational metrics for monitoring, not vendor usage tracking. Since telemetry opt-out is a plausible question for any self-hosted server software, and no evidence supports or denies it, this is 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "keycloak", "storyId": "rbac-roles-in-tokens", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Keycloak's docs explicitly describe role-based authorization plus fine-grained/UMA-based authorization services managed centrally via the admin console and REST API (keycloak-docs-24, keycloak-docs-49, kc-supp-authz-services), and its OIDC layer issues signed access tokens that carry these claims, validated via introspection endpoints (keycloak-docs-38, keycloak-docs-40, keycloak-docs-52). This matches standard Keycloak behavior where realm/client roles are embedded in JWT access tokens for downstream authorization checks. Missing for 10: an explicit first-party quote showing role claims embedded in the JWT payload, and independent/hands-on confirmation of token-based enforcement in production.", "evidenceIds": [ "keycloak-docs-24", "keycloak-docs-49", "keycloak-docs-38", "keycloak-docs-40", "keycloak-docs-52", "kc-supp-authz-services", "keycloak-docs-4" ] }, { "productId": "keycloak", "storyId": "scim-directory-sync", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Keycloak's Server Administration Guide documents built-in SCIM support for managing users and groups per realm, including enabling SCIM, obtaining the SCIM API base URL, and setting up a service account client — directly matching the story's sync mechanism. However, evidence is first-party docs only with no independent/hands-on corroboration that deprovisioning in the source IdP actually propagates through SCIM to revoke app access, and no detail on directionality (Keycloak as SCIM client vs server) or deprovisioning semantics. Missing for 10: independent corroboration of real-world SCIM deprovisioning flow, clarity on SCIM client vs server mode, and details on how revocation cascades to relying-party access.", "evidenceIds": [ "kc-supp-scim" ] }, { "productId": "keycloak", "storyId": "scoped-agent-identities", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Keycloak supports OAuth2 client-credentials grant for machine-to-machine service accounts (distinct non-human identities), fine-grained authorization services to scope permissions per client, and a token revocation endpoint that can immediately revoke access/refresh tokens — the core building blocks needed to give each agent its own least-privilege, revocable identity. However, this requires assembling generic OAuth2 primitives (service accounts + authz policies + revocation endpoint) rather than a purpose-built 'agent identity' feature, and there is no evidence of agent-specific provisioning at scale or hands-on confirmation that revocation is truly instant in practice. Missing for 10: dedicated agent/non-human-identity lifecycle tooling, evidence of bulk/scaled machine-identity issuance, and independent confirmation of revocation latency/behavior.", "evidenceIds": [ "kc-supp-client-credentials", "kc-supp-authz-services", "keycloak-docs-53", "keycloak-docs-24", "keycloak-docs-20" ] }, { "productId": "keycloak", "storyId": "secure-mcp-server-auth", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Keycloak is a standards-compliant OAuth2/OIDC provider with the core primitives an MCP OAuth flow needs — dynamic client registration, discovery metadata, token introspection/revocation, client-credentials grants for machine clients, and fine-grained authorization services — all documented in first-party docs. However, no evidence anywhere in the pack mentions MCP specifically, any MCP-server integration guide, resource-indicator/audience-restricted tokens for MCP's spec, or a reference implementation fronting an MCP server, so it's inferred generic capability rather than a demonstrated MCP-specific flow. Missing for 10: explicit MCP-server integration documentation or tutorial, evidence of audience/resource-scoped token issuance tailored to MCP's OAuth profile, and independent/hands-on confirmation of using Keycloak to gate an MCP server.", "evidenceIds": [ "keycloak-docs-4", "keycloak-docs-38", "keycloak-docs-39", "keycloak-docs-40", "keycloak-docs-52", "keycloak-docs-53", "kc-supp-client-credentials", "kc-supp-authz-services", "keycloak-docs-1" ] }, { "productId": "keycloak", "storyId": "session-lifecycle-revocation", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Keycloak's docs show explicit server-side session/token revocation (token revocation endpoint, single sign-out) and admin/user session management (admin console 'manage sessions', account console lets users view/manage sessions), plus introspection for validating token state. Refresh is implied via standard OIDC refresh tokens and the introspection/revocation endpoints supporting both access and refresh tokens. missing for 10: explicit documentation of session/token expiry (idle/max lifespan) configuration and refresh-token rotation policy, and independent hands-on confirmation that revocation immediately kills active sessions", "evidenceIds": [ "keycloak-docs-53", "keycloak-docs-52", "keycloak-docs-21", "keycloak-docs-48", "keycloak-docs-15", "keycloak-docs-40", "keycloak-docs-11" ] }, { "productId": "keycloak", "storyId": "social-login-breadth", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Keycloak's docs explicitly state that social login and identity brokering (OIDC/SAML IdPs) are enabled purely through admin console configuration with 'no code or changes to your application required,' and community evidence corroborates real-world use integrating many external IdPs for SSO. Missing for 10: no evidence enumerating the exact list/breadth of pre-built social providers, and no independent hands-on account specifically validating the configuration-only social login flow.", "evidenceIds": [ "keycloak-docs-16", "keycloak-docs-22", "keycloak-docs-46", "kc-supp-social-brokering", "keycloak-docs-25", "keycloak-comm-5" ] }, { "productId": "keycloak", "storyId": "token-vending-for-agent-tools", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Keycloak provides the underlying OAuth2/OIDC building blocks that such a flow would need — identity brokering to link user accounts with third-party IdPs (keycloak-docs-25, keycloak-docs-46, kc-supp-social-brokering), client-credentials/service-account flows for machine access (kc-supp-client-credentials), a device grant suited to limited-input clients (kc-supp-device-grant), and token introspection/revocation for short-lived token lifecycle management (keycloak-docs-52, keycloak-docs-53). However, none of the evidence documents an explicit 'token vault' or RFC 8693 token-exchange feature for retrieving/exchanging stored third-party IdP tokens on behalf of a user for downstream API calls, nor any agent-specific consent UX. Missing for 10: explicit token exchange/RFC8693 support docs, documented retrieval of brokered IdP access tokens for calling third-party APIs, and agent/tool-specific delegated-consent flow documentation.", "evidenceIds": [ "keycloak-docs-25", "keycloak-docs-46", "kc-supp-social-brokering", "kc-supp-client-credentials", "kc-supp-device-grant", "keycloak-docs-52", "keycloak-docs-53" ] }, { "productId": "workos", "storyId": "act-as-oauth-provider", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "WorkOS explicitly supports building 'Sign in with [your application]' OAuth applications via AuthKit Connect, describing it as a spec-compatible OAuth authorization server where third-party clients get a login button and users authenticate as the actor (docs-27, docs-33, docs-45), plus M2M app support for programmatic API access (docs-32). This directly delivers the 'become an OAuth provider' story with concrete docs. missing for 10: independent/hands-on developer corroboration of setting up third-party OAuth client flows, and more detail on token issuance/scopes management beyond the docs excerpts.", "evidenceIds": [ "workos-docs-27", "workos-docs-33", "workos-docs-45", "workos-docs-32" ] }, { "productId": "workos", "storyId": "active-session-visibility", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers SSO, Directory Sync, RBAC, MFA, Audit Logs, Vault, and MCP integration, but contains no mention of a session/device management feature that lets users or admins view active sessions/devices and revoke them individually or all at once. This is a fair axis for an authentication platform, but no documentation, dashboard feature, or API reference in the pack addresses it.", "evidenceIds": [] }, { "productId": "workos", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Direct probe evidence confirms WorkOS serves a working llms.txt (HTTP 200) and markdown-formatted docs.md at the expected agent-oriented URLs, plus an OpenAPI spec, making its documentation machine-readable and agent-consumable exactly as the story describes. Missing for 10: no independent/community confirmation that agents actually consume these successfully in practice.", "evidenceIds": [ "workos-probe-1", "workos-probe-2", "workos-probe-3" ] }, { "productId": "workos", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "WorkOS provides MCP-based agent access to manage dashboard data (create/read/update orgs, users, etc.) and fraud/bot detection (Radar), but there is no evidence of AI-generated insights or suggestions surfaced to the user from their own data — no analytics dashboard, anomaly explanations, or recommendation features are documented. missing for 10: any AI-driven insights/analytics dashboard, evidence of suggestion generation from customer data, or reporting on usage/security patterns via AI.", "evidenceIds": [ "workos-docs-9", "workos-docs-12", "workos-supp-radar" ] }, { "productId": "workos", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "WorkOS's MCP server (workos-docs-9, workos-docs-15, workos-docs-44) lets an agent act on WorkOS dashboard data when invoked, and Directory Sync pushes event updates via webhooks (workos-docs-21), but there is no evidence of a scheduling/automation engine or persistent unattended background jobs — MCP access is invocation-based tool access, not autonomous background automation. Missing for 10: any scheduler/trigger system, evidence of agents running unattended over time, or documented autonomous workflow execution.", "evidenceIds": [ "workos-docs-9", "workos-docs-15", "workos-docs-44", "workos-docs-21" ] }, { "productId": "workos", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "WorkOS's MCP evidence describes external AI agents connecting to and operating WorkOS via its API/dashboard data (workos-docs-9, workos-docs-44) — this is WorkOS acting as a tool controlled by outside agents, not a built-in assistant inside WorkOS that users delegate tasks to. No evidence of any native AI assistant embedded in the WorkOS dashboard or product itself.", "evidenceIds": [ "workos-docs-9", "workos-docs-44", "workos-docs-15" ] }, { "productId": "workos", "storyId": "agentic-headless", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "WorkOS exposes a full REST API (openapi.json), Node SDK with API-key-only initialization and automatic retry/backoff logic, and M2M application support for programmatic, non-interactive API access — all of which support scripted or CI-driven automation of identity/auth workflows. However, there is no explicit CI/CD documentation, GitHub Actions examples, or first-party guidance on running WorkOS-integrated tests/workflows headlessly, and CLI Auth (device flow) still requires a human completing a web login step. missing for 10: explicit CI/headless usage docs or examples, confirmation that M2M/API-key flows are fully non-interactive end-to-end, independent evidence of real-world CI usage.", "evidenceIds": [ "workos-docs-32", "workos-gh-2", "workos-gh-3", "workos-gh-4", "workos-probe-3", "workos-docs-10" ] }, { "productId": "workos", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "WorkOS is an identity/auth infrastructure platform, not an AI agent or agent framework — it has no functionality where a user 'plugs in' MCP servers so *it* can consume their tools. Its only MCP-related feature is the reverse: exposing its own MCP server so external AI agents can call the WorkOS API (workos-docs-8/9/15/44), which is a different axis (serving, not consuming, MCP tools).", "evidenceIds": [] }, { "productId": "workos", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "WorkOS documents an official remote MCP server that lets MCP-compatible AI agents act on the WorkOS workspace, managing organizations, connections, users, and branding, secured via AuthKit as an OAuth authorization server. This is corroborated by a direct probe confirming the docs page exists. missing for 10: independent/hands-on community confirmation of connecting an agent to the MCP server (only first-party docs and a probe, no third-party usage reports).", "evidenceIds": [ "workos-docs-8", "workos-docs-9", "workos-docs-15", "workos-docs-27", "workos-docs-31", "workos-docs-44", "workos-probe-4" ] }, { "productId": "workos", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "WorkOS ships an official remote MCP server that lets MCP-compatible AI agents manage WorkOS workspace data (organizations, connections, users, branding) via natural-language driven agent tools, effectively allowing natural-language operation through a connected agent rather than a built-in chat UI. Missing for 10: no first-party natural-language chat/command interface within the WorkOS dashboard itself, and no independent/hands-on evidence validating the MCP-driven natural-language workflow in practice.", "evidenceIds": [ "workos-docs-9", "workos-docs-15", "workos-docs-44", "workos-probe-4", "workos-docs-8" ] }, { "productId": "workos", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "WorkOS documents a dedicated 'WorkOS CLI migrations tool' for exporting Auth0 data, showing an official CLI exists, but there is no evidence of a general-purpose CLI for managing WorkOS resources or built for AI-native/agentic workflows (the AI-facing surface is instead an MCP server, not a CLI). Missing for 10: a general-purpose official CLI beyond migration tooling, CLI documentation/reference, and any AI-native use-case tailoring of the CLI.", "evidenceIds": [ "workos-docs-37", "workos-docs-15", "workos-docs-44" ] }, { "productId": "workos", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "WorkOS exposes a documented public REST API (confirmed via live OpenAPI spec probe), official SDKs (workos-node), and even an official MCP server plus AuthKit MCP auth guides enabling AI agents to programmatically manage the WorkOS workspace. missing for 10: independent third-party developer corroboration of API completeness/stability beyond WorkOS's own docs and probes.", "evidenceIds": [ "workos-probe-3", "workos-probe-1", "workos-probe-2", "workos-docs-15", "workos-docs-44", "workos-docs-8", "workos-gh-1", "workos-probe-4" ] }, { "productId": "workos", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "WorkOS supports M2M applications for issuing API access credentials to third parties/agents (workos-docs-32), and AuthKit acts as a spec-compliant OAuth authorization server for MCP access, implying scoped OAuth grants for AI agents (workos-docs-8, workos-docs-27, workos-docs-9). Custom RBAC roles/permissions can further restrict access at the org/tenant level (workos-docs-22, workos-docs-40). However, there is no explicit documentation of fine-grained scope definitions or least-privilege token minting specifically tailored for agent workloads (e.g., scope lists, token TTL controls for agents). Missing for 10: explicit least-privilege scope configuration for agent credentials, documented examples of restricting an agent's API surface, and independent verification of this in practice.", "evidenceIds": [ "workos-docs-32", "workos-docs-8", "workos-docs-27", "workos-docs-9", "workos-docs-22", "workos-docs-40", "workos-docs-44" ] }, { "productId": "workos", "storyId": "agentic-sdks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "WorkOS ships an official, actively maintained SDK (@workos-inc/node) with clear install instructions, safe client-side initialization for secret-less environments, and built-in resilient retry/backoff logic — solid engineering signals for building production apps, including AI-native ones. Missing for 10: evidence of multiple official language SDKs beyond Node, AI-specific SDK ergonomics (e.g., agent/function-calling helpers), and independent third-party reviews specifically praising SDK quality (community quotes are about the SSO/SCIM wizard, not the SDK itself).", "evidenceIds": [ "workos-gh-1", "workos-gh-2", "workos-gh-3", "workos-gh-4" ] }, { "productId": "workos", "storyId": "agentic-webhooks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm WorkOS supports webhook-based event delivery specifically for Directory Sync (\"Directory Sync sends automatic updates to your app for changes to directories, groups, users, or access rules\" and \"Directory updates can be delivered to you via webhooks or retrieved using the Events API\"), giving genuine webhook subscription capability. However, evidence does not show a general-purpose webhooks system covering other resources (e.g., audit logs are described as export-only, not webhook-push), nor any webhook signature/verification docs or SDK helpers for consuming webhooks. Missing for 10: documentation of a unified/general webhooks API across all WorkOS resources, webhook signature verification guidance, and independent/hands-on confirmation of webhook reliability.", "evidenceIds": [ "workos-docs-21", "workos-supp-scim-webhooks", "workos-docs-4", "workos-docs-23" ] }, { "productId": "workos", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence confirms an OpenAPI spec exists and machine-readable docs (llms.txt, docs.md) are served, but there is no evidence of an interactive API reference with runnable/try-it examples (e.g., a Swagger-style explorer) — missing for 10: an interactive docs UI, runnable code/API call examples, and any confirmation of an in-browser 'try it' feature.", "evidenceIds": [ "workos-probe-3", "workos-probe-2", "workos-probe-1" ] }, { "productId": "workos", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe for standard OpenAPI/Swagger spec paths (workos.com/openapi.json, swagger.json, etc.) returned 404 across the board, and no docs or GitHub evidence mention a downloadable OpenAPI spec despite WorkOS being an API-first product where this would be a reasonable ask.", "evidenceIds": [ "workos-probe-3" ] }, { "productId": "workos", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions a sandbox, staging, or test-mode environment for WorkOS that would let an AI-native user test integrations without touching production data; all citations focus on SSO, RBAC, directory sync, audit logs, migrations, and MCP server access rather than environment isolation.", "evidenceIds": [] }, { "productId": "workos", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item documents API versioning scheme or a deprecation policy; only an OpenAPI spec and SDK retry logic are mentioned, neither of which addresses version lifecycle or deprecation commitments.", "evidenceIds": [] }, { "productId": "workos", "storyId": "audit-logs-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "WorkOS documents Audit Log Events for authentication/admin activity, configuration/export flows, and explicit 'Log streaming (per SIEM connection)' pricing showing direct SIEM export capability, plus framing as a compliance 'paper trail'. However, no evidence addresses tamper-evidence (immutability, hashing, cryptographic sealing) of the logs, and there's no independent/hands-on confirmation of SIEM streaming reliability. Missing for 10: explicit tamper-evident/immutable log guarantees, third-party validation of SIEM export in practice, details on retention/integrity controls.", "evidenceIds": [ "workos-docs-4", "workos-docs-23", "workos-docs-35", "workos-docs-42", "workos-docs-14", "workos-supp-radar" ] }, { "productId": "workos", "storyId": "auth-event-webhooks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "WorkOS documents webhooks/Events API for Directory Sync changes (user/group/directory updates) and an Audit Logs system that captures 'notable actions taken by users' for compliance, which can include auth-related events, but there's no explicit documentation of a dedicated webhook/event stream specifically for sign-up/sign-in/session events outside these two systems. missing for 10: explicit auth-specific event types (e.g., 'user.signed_in', 'user.created') webhook payload docs, and independent confirmation of real-time auth event delivery.", "evidenceIds": [ "workos-docs-21", "workos-supp-scim-webhooks", "workos-docs-23", "workos-docs-35", "workos-docs-42", "workos-docs-4" ] }, { "productId": "workos", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The Auth0 migration tool bulk-exports/imports users, organizations, roles, and SSO connections in one package (workos-docs-6, workos-docs-37), and the MCP server lets an AI agent call the WorkOS API to manage organizations, connections, users, and branding (workos-docs-9, workos-docs-44), implying some capacity for programmatic multi-item actions. However there is no direct evidence of dedicated bulk/batch API endpoints (e.g., bulk create/update/delete users or roles) for everyday automation beyond the one-off migration flow. Missing for 10: explicit bulk/batch API documentation, evidence of an agent performing bulk operations across many records in a single call, and independent confirmation of this working in practice.", "evidenceIds": [ "workos-docs-6", "workos-docs-37", "workos-docs-9", "workos-docs-44" ] }, { "productId": "workos", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "WorkOS exposes events (Directory Sync updates, Audit Log events) via webhooks/Events API, but there is no evidence of a user-defined rules engine that lets an AI-native user specify conditional triggers and automated actions — it only ships raw event delivery, not rule authoring or automation logic.", "evidenceIds": [ "workos-docs-21", "workos-docs-23", "workos-supp-scim-webhooks" ] }, { "productId": "workos", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "WorkOS is an identity/auth platform (SSO, RBAC, audit logs, MCP admin access), not a workflow/job-scheduling engine; scheduling recurring jobs or automations is outside its product category and not evidenced anywhere in the pack.", "evidenceIds": [] }, { "productId": "workos", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "WorkOS is an identity/auth platform, not an automation/workflow builder; versioning, reviewing, and rolling back 'automations' is not a concept present anywhere in its product surface (SSO, directory sync, RBAC, audit logs, vault, MCP admin access). This axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "workos", "storyId": "bot-attack-protection", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "WorkOS documents a built-in 'Radar' feature explicitly for bot/fraud/abuse protection (workos-docs-12, workos-supp-radar) alongside MFA (workos-docs-11), but the evidence never specifically describes credential-stuffing or brute-force mitigations (e.g., rate limiting, anomaly detection specifics) or provides independent/hands-on validation of Radar's effectiveness. missing for 10: explicit credential-stuffing/brute-force protection mechanics, independent third-party validation of Radar's efficacy.", "evidenceIds": [ "workos-docs-12", "workos-supp-radar", "workos-docs-11", "workos-docs-18" ] }, { "productId": "workos", "storyId": "bulk-user-import-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "WorkOS documents a concrete bulk-import path from Auth0 (users, organizations, memberships, roles, SSO handoff files) via a CLI migrations tool, which implies handling credentials like SSO/password data, but no docs explicitly confirm password-hash import fidelity or any reverse 'export your users out of WorkOS' capability. A community thread explicitly raises the lock-in/export question without a documented resolution, leaving the 'export again' half of the story unaddressed. missing for 10: explicit documentation of password-hash import support, and a documented user-data export/portability path out of WorkOS to prevent lock-in.", "evidenceIds": [ "workos-docs-6", "workos-docs-16", "workos-docs-18", "workos-docs-20", "workos-docs-26", "workos-docs-37", "workos-comm-8" ] }, { "productId": "workos", "storyId": "deployment-data-control", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "WorkOS is presented throughout as a hosted, multi-tenant SaaS platform (dashboard, hosted UI, API-based integration) with no docs offering self-hosted/on-prem deployment, a private instance, or customer-controlled database. Community evidence (workos-comm-8) explicitly flags that customer login data 'lives in WorkOS's database' and raises vendor lock-in concerns about exporting it — the opposite of the deployment control this story asks for. Vault's BYOK (workos-docs-36) only lets you supply encryption keys, not run or host the auth system yourself.", "evidenceIds": [ "workos-comm-8", "workos-docs-36", "workos-docs-25", "workos-probe-1" ] }, { "productId": "workos", "storyId": "device-flow-for-cli-agents", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "WorkOS AuthKit explicitly documents CLI Auth using the OAuth 2.0 Device Authorization Flow (RFC 8628) for authenticating command-line apps/headless clients via a device code and user code, avoiding pasted long-lived secrets — directly matching the story. This is documented in dedicated reference docs with the flow mechanics (device code, user code, polling for tokens). Missing for 10: no independent/hands-on community confirmation of the device-flow CLI Auth specifically (community evidence covers SSO/SCIM, not CLI Auth).", "evidenceIds": [ "workos-docs-10", "workos-docs-24", "workos-docs-34", "workos-supp-cli-auth" ] }, { "productId": "workos", "storyId": "edge-middleware-route-protection", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers WorkOS's Node SDK, SSO, Directory Sync, RBAC, Audit Logs, Vault, and MCP server, but contains no mention of a Next.js-specific SDK, middleware, or edge-based session verification. Since framework-specific route protection is a plausible and expected capability for an auth platform, the lack of any supporting evidence makes this 'none' rather than 'na'.", "evidenceIds": [ "workos-gh-1", "workos-docs-27", "workos-docs-25" ] }, { "productId": "workos", "storyId": "enterprise-saml-oidc-sso", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "WorkOS's core SSO product explicitly supports any IdP using SAML or OIDC protocols (Okta, Entra/Azure, Google Workspace included), backed by strong first-party docs and independent community testimonials confirming ease of SAML/SCIM setup for enterprise workforce sign-in. Missing for 10: no explicit named Okta/Entra/Google Workspace integration walkthroughs cited, and only community anecdotes (not formal audits) corroborate real-world reliability.", "evidenceIds": [ "workos-docs-1", "workos-docs-25", "workos-docs-38", "workos-comm-3", "workos-comm-4", "workos-comm-5" ] }, { "productId": "workos", "storyId": "fine-grained-authorization", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "WorkOS RBAC docs show custom roles, permission assignment, and organization/tenant-scoped roles enforced via JWTs, which goes beyond simple flat roles, but there is no evidence of relationship-based (ReBAC) authorization or a policy engine (e.g., ABAC/OPA-style) for arbitrary resource-level grants. Missing for 10: relationship-graph based authorization, fine-grained per-resource ACLs, and any policy-language/attribute-based access control beyond role+organization scoping.", "evidenceIds": [ "workos-docs-3", "workos-docs-22", "workos-docs-29", "workos-docs-40", "workos-supp-rbac-jwt" ] }, { "productId": "workos", "storyId": "guided-competitor-migration", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "WorkOS has a dedicated Auth0 migration guide/CLI tool that exports and imports users, organizations, roles, and enterprise SSO connections, producing a migration package with warnings—clearly vendor-maintained tooling for moving off a competing provider. However, the evidence never explicitly states passwords/password hashes are migrated or that end users won't be forced to reset passwords, and guides are limited to Auth0 (no mention of other competitors like Okta, Cognito, Firebase). missing for 10: explicit confirmation that password hashes are migrated without forcing resets, migration guides/tooling for providers beyond Auth0, and independent/hands-on confirmation the migration tool works as described in practice.", "evidenceIds": [ "workos-docs-6", "workos-docs-16", "workos-docs-18", "workos-docs-20", "workos-docs-26", "workos-docs-37", "workos-docs-43" ] }, { "productId": "workos", "storyId": "human-approval-for-agent-actions", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "WorkOS documents agent-facing capabilities like an MCP server, OAuth/device-code CLI auth, and MFA, but nothing in the evidence describes a CIBA-style out-of-band/asynchronous human-approval step gating an agent's completion of a sensitive transaction. This is a fair axis for an auth platform enabling agent authorization, but no such flow is evidenced.", "evidenceIds": [ "workos-docs-8", "workos-docs-9", "workos-docs-15", "workos-docs-10", "workos-docs-11" ] }, { "productId": "workos", "storyId": "m2m-client-credentials", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "WorkOS explicitly documents M2M applications for client-credentials-style machine authentication, describing them as used to provide programmatic API access credentials to customers/partners without human involvement (workos-docs-32), alongside supporting RBAC/session JWT enforcement infrastructure. missing for 10: no explicit mention of the OAuth 'client_credentials' grant name or token endpoint details, and no independent/hands-on corroboration of the M2M flow working in practice.", "evidenceIds": [ "workos-docs-32", "workos-docs-45", "workos-supp-rbac-jwt" ] }, { "productId": "workos", "storyId": "mfa-second-factors", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "WorkOS docs confirm MFA enrollment is supported and can be enabled via the dashboard, but the evidence never specifies TOTP authenticator app support, backup codes, or step-up (adaptive) enforcement mechanics. missing for 10: explicit documentation of TOTP authenticator app enrollment, backup code generation/recovery flow, and step-up MFA enforcement triggers or APIs, plus independent/hands-on confirmation.", "evidenceIds": [ "workos-docs-11", "workos-docs-46" ] }, { "productId": "workos", "storyId": "oauth-oidc-standard-flows", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "WorkOS's AuthKit is documented as a 'spec-compatible OAuth authorization server' (workos-docs-27) and supports device authorization flow (RFC 8628) for CLI apps (workos-docs-10, workos-supp-cli-auth), plus M2M and third-party OAuth app patterns (workos-docs-32, workos-docs-33, workos-docs-45), and SDKs handle retries/backoff (workos-gh-3, workos-gh-4). However, there's no explicit documentation of standard authorization-code-with-PKCE flow mechanics or refresh token handling/rotation for typical web/mobile apps — the evidence focuses on SSO/SAML, device flow, and M2M rather than the classic PKCE+refresh-token developer flow. missing for 10: explicit docs/SDK code samples showing authorization code + PKCE flow implementation, refresh token issuance/rotation details, and independent developer corroboration of hand-rolling-free OAuth/OIDC flow usage outside SSO context.", "evidenceIds": [ "workos-docs-27", "workos-docs-10", "workos-supp-cli-auth", "workos-docs-32", "workos-docs-33", "workos-docs-45", "workos-gh-3", "workos-gh-4" ] }, { "productId": "workos", "storyId": "openness-api-parity", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "WorkOS ships a documented OpenAPI spec (workos-probe-3) and explicit docs stating configuration (roles, permissions, orgs) can be done 'directly in the WorkOS Dashboard or using the API' (workos-docs-40), plus an official MCP server that lets an agent 'read and change the same dashboard data you can (managing organizations, connections, users, branding, and more)' via the API (workos-docs-9, workos-docs-44). This shows the API is a first-class, near-complete mirror of the UI, not just SDK convenience wrappers (workos-gh-1..4). Missing for 10: independent hands-on confirmation that literally every dashboard action (e.g., billing/plan changes, full self-serve IT admin onboarding) has an API equivalent, and any third-party audit of full parity.", "evidenceIds": [ "workos-docs-9", "workos-docs-40", "workos-docs-44", "workos-probe-3", "workos-probe-4", "workos-docs-15" ] }, { "productId": "workos", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "WorkOS documents only importing data (e.g., migrating FROM Auth0 INTO WorkOS) and offers audit-log export, but there is no documented capability for a user/customer to export all their identity, SSO, directory, RBAC, or vault data in open formats to leave the platform; a community thread explicitly raises this exact lock-in/export concern without any WorkOS response in the evidence pack.", "evidenceIds": [ "workos-docs-18", "workos-docs-20", "workos-docs-37", "workos-comm-8" ] }, { "productId": "workos", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "WorkOS is a closed-source SaaS platform; its SDKs (workos-node) are open-source client libraries, but there is no evidence the core WorkOS product/service source is available under an open license. The evidence pack shows only docs, APIs, and SDK wrappers, not source availability of the platform itself.", "evidenceIds": [] }, { "productId": "workos", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "WorkOS is presented entirely as a hosted SaaS platform with an API/SDK model; there is no evidence of a self-hostable core product, open-source server, or on-prem deployment option. Community comments even raise vendor lock-in concerns about data living in WorkOS's own database, reinforcing that it is not self-hostable.", "evidenceIds": [ "workos-comm-8", "workos-probe-1", "workos-probe-2" ] }, { "productId": "workos", "storyId": "org-level-sso-domains", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm WorkOS SSO is IdP-agnostic (SAML/OIDC) and organization-scoped constructs exist (org-scoped RBAC roles, per-org SSO connections referenced in the Auth0 migration guide, self-serve IT admin onboarding), supporting the idea that each customer org can have its own SSO connection. However, the evidence pack never explicitly documents 'verified domains' or 'just-in-time (JIT) provisioning' mechanics for SSO connections specifically (JIT-like behavior is only shown for Directory Sync/SCIM user lifecycle, not SSO login-time provisioning). Missing for 10: explicit verified-domain configuration docs, explicit JIT user/org provisioning on SSO login, and independent confirmation beyond vendor docs.", "evidenceIds": [ "workos-docs-1", "workos-docs-7", "workos-docs-22", "workos-docs-40", "workos-docs-26", "workos-supp-rbac-jwt" ] }, { "productId": "workos", "storyId": "organizations-multitenant", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "WorkOS docs explicitly reference organizations, memberships, and org/tenant-scoped roles (e.g., migration package includes 'users, organizations, memberships, roles' and RBAC roles 'at the organization or tenant level'), and self-serve IT admin onboarding is mentioned, showing multi-tenant B2B modeling out of the box. However, there is no direct evidence of a dedicated invitation flow API or UI for inviting members to organizations. missing for 10: explicit invitation flow documentation/API, hands-on/independent confirmation of the org+membership+invite workflow end-to-end.", "evidenceIds": [ "workos-docs-6", "workos-docs-3", "workos-docs-22", "workos-docs-7", "workos-supp-rbac-jwt" ] }, { "productId": "workos", "storyId": "passkeys-passwordless-login", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "WorkOS/AuthKit explicitly lists passkeys, magic auth (email OTP), and passwordless as first-class sign-in methods alongside SSO, bundled into a single integration per pricing docs. Evidence confirms these are core AuthKit features, not add-ons requiring separate tooling. Missing for 10: independent hands-on developer testimony specifically about implementing passkeys/WebAuthn or magic link flows (community quotes focus on SSO/SCIM, not passwordless), and no detailed WebAuthn API/config documentation excerpt.", "evidenceIds": [ "workos-docs-28", "workos-docs-39", "workos-supp-passkeys-pricing", "workos-docs-38" ] }, { "productId": "workos", "storyId": "prebuilt-auth-ui", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "WorkOS/AuthKit is documented as a 'full-fledged authentication platform' with 'customizable hosted UI for authentication at any size' covering SSO, passkeys, social sign-in, passwords, and magic auth, plus APIs to 'store additional information about users and organizations' for profile management, and community testimonials confirm ease of adopting its SSO/SCIM UI wrappers in production apps. Missing for 10: explicit documentation of prebuilt embeddable UI *components* (vs. only hosted pages) for profile management specifically, and independent hands-on verification of UI customization depth beyond vendor docs.", "evidenceIds": [ "workos-docs-17", "workos-docs-25", "workos-docs-28", "workos-docs-39", "workos-docs-30", "workos-supp-passkeys-pricing", "workos-comm-3", "workos-comm-4" ] }, { "productId": "workos", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, regional storage options, or geographic data location controls for WorkOS-managed data; the closest topics (Vault/BYOK, migration, audit logs) address encryption/key management, not residency choice. Missing for 10: any documentation of region selection, data residency guarantees, or geo-specific hosting options.", "evidenceIds": [] }, { "productId": "workos", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "WorkOS is an enterprise auth/identity infrastructure platform (SSO, directory sync, RBAC, audit logs, MFA), not an AI model or AI product that trains on user data; the story about preventing data use for AI model training is a category error for this kind of product.", "evidenceIds": [] }, { "productId": "workos", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "WorkOS stores customer/user identity data (SSO, directory sync, Vault-encrypted secrets) so data retention/deletion is a fair axis, but the evidence pack contains no documentation of deletion APIs, retention policies, or data export/erasure controls — only a community comment raising unresolved concern about how to export data when leaving WorkOS (workos-comm-8), which is not evidence of a working capability.", "evidenceIds": [ "workos-comm-8", "workos-docs-5" ] }, { "productId": "workos", "storyId": "privacy-telemetry-optout", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "WorkOS is an enterprise auth/identity platform, not an AI agent/tool whose own telemetry collection is a relevant axis; the evidence pack contains no mention of telemetry opt-out at all, and this story is a category mismatch for this type of infrastructure product.", "evidenceIds": [] }, { "productId": "workos", "storyId": "rbac-roles-in-tokens", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "WorkOS RBAC docs explicitly describe defining custom roles and permissions at the org/tenant level, assigning them via API or dashboard, and enforcing access through session JWTs, including organization-scoped roles and IdP role assignment via SSO/Directory Sync — directly matching the story's requirements. missing for 10: independent/hands-on developer corroboration of JWT-based permission checks in production and concrete code samples showing authorization checks against token claims.", "evidenceIds": [ "workos-docs-3", "workos-docs-22", "workos-docs-40", "workos-supp-rbac-jwt", "workos-docs-29" ] }, { "productId": "workos", "storyId": "scim-directory-sync", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "WorkOS Directory Sync explicitly supports SCIM protocol integration with dozens of IdPs, delivers automatic updates for user/group/directory changes via webhooks or Events API, and enables IT admins to deactivate accounts (revoking access) directly reflecting IdP-side deprovisioning; community testimonials corroborate real-world ease of SCIM/SSO setup. missing for 10: no independent hands-on test specifically demonstrating deprovisioning propagation latency or failure edge cases.", "evidenceIds": [ "workos-docs-2", "workos-docs-21", "workos-docs-41", "workos-supp-scim-webhooks", "workos-comm-3", "workos-comm-4" ] }, { "productId": "workos", "storyId": "scoped-agent-identities", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "WorkOS offers building blocks that could support this story — M2M applications for programmatic credential issuance (workos-docs-32), RBAC with custom roles and organization-scoped permissions enforced via session JWTs (workos-docs-3, workos-docs-22, workos-supp-rbac-jwt), and AuthKit as a spec-compliant OAuth authorization server for MCP agent connections (workos-docs-27, workos-docs-8). However, none of this is packaged as an agent-specific 'least-privilege machine identity' feature; the MCP docs describe agents getting broad dashboard-equivalent access (workos-docs-9, workos-docs-44) rather than narrowly scoped per-agent permissions, and there is no explicit documentation of instant/one-click revocation for M2M or agent credentials. missing for 10: dedicated agent-identity primitive distinct from human/M2M app credentials, explicit least-privilege scoping guidance for AI agents specifically, and documented instant revocation mechanism for agent credentials.", "evidenceIds": [ "workos-docs-32", "workos-docs-3", "workos-docs-22", "workos-supp-rbac-jwt", "workos-docs-27", "workos-docs-8", "workos-docs-9", "workos-docs-44" ] }, { "productId": "workos", "storyId": "secure-mcp-server-auth", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "WorkOS explicitly documents using AuthKit as a spec-compatible OAuth authorization server to protect MCP servers, with tokens and scoped access for agent connections (workos-docs-8, workos-docs-27, workos-docs-31), backed by a first-party MCP server product (workos-docs-15, workos-probe-4). This directly matches the story of putting a spec-compliant OAuth flow in front of an MCP server for remote agent connections with scoped, verifiable tokens. Missing for 10: independent/hands-on verification of token scoping in practice and no community corroboration specifically about MCP+OAuth setup.", "evidenceIds": [ "workos-docs-8", "workos-docs-27", "workos-docs-15", "workos-docs-31", "workos-docs-44", "workos-probe-4" ] }, { "productId": "workos", "storyId": "session-lifecycle-revocation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "WorkOS/AuthKit is an authentication platform where session lifecycle management (expiry, refresh, revocation) would be a natural and expected capability, so the axis applies — but the evidence pack contains no documentation of session expiry policies, token refresh mechanics, or server-side session/token revocation for compromised sessions.", "evidenceIds": [] }, { "productId": "workos", "storyId": "social-login-breadth", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "WorkOS/AuthKit is explicitly documented as compatible with any SAML/OIDC IdP and bundles social sign-in (Google, Microsoft, and more) alongside SSO in one configurable integration, with no custom code required — corroborated by developer testimonials praising the 'wizard' setup for SSO/social auth. missing for 10: an exhaustive enumerated list of supported social/OAuth providers, and one community report (workos-comm-9) noting a specific provider (Slack) advertised but not found in the SSO setup UI, indicating some provider coverage gaps in practice.", "evidenceIds": [ "workos-docs-1", "workos-docs-38", "workos-docs-28", "workos-supp-passkeys-pricing", "workos-docs-25", "workos-comm-3", "workos-comm-4", "workos-comm-9" ] }, { "productId": "workos", "storyId": "token-vending-for-agent-tools", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "WorkOS's 'Connect' feature lets an app obtain third-party OAuth credentials (docs-19, docs-33) and M2M API credentials (docs-32), and Vault can encrypt/store tokens (workos-docs-5), which are the building blocks of a token-vault/exchange pattern, but none of the evidence describes short-lived token issuance, refresh/rotation, or an agent-specific consent flow tying tool calls to delegated user authority. The MCP-related docs (workos-docs-8, workos-docs-27) cover securing access to WorkOS's own MCP server via AuthKit OAuth, not obtaining tokens for arbitrary third-party APIs on a user's behalf. Missing for 10: explicit short-lived/expiring token semantics, agent-specific consent UX, documented exchange/refresh API, and independent confirmation of the Connect flow in an agent context.", "evidenceIds": [ "workos-docs-19", "workos-docs-32", "workos-docs-33", "workos-docs-5", "workos-docs-8", "workos-docs-27" ] } ]