[ { "productId": "bigquery", "storyId": "agent-runs-sql-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "BigQuery ships a documented, non-interactive bq CLI and a machine-readable REST API (verified live: keyless discovery doc exposes datasets/tables/routines/jobs resources) alongside IAM-based authentication/authorization, letting an agent authenticate, discover schemas, run SQL, and pull results with zero dashboard interaction. Google also ships an official MCP Toolbox with a prebuilt BigQuery toolset for agent access, confirmed running via npx (credential-gated as expected). Missing for 10: a first-party end-to-end hands-on trace showing an agent chaining auth to schema discovery to query to result-read purely via CLI/API without any human/dashboard step, and independent non-Google confirmation of this exact workflow.", "evidenceIds": [ "bigquery-docs-12", "bigquery-docs-49", "bigquery-docs-31", "bigquery-docs-65", "bigquery-probe-rt-1", "bigquery-probe-rt-2", "bigquery-probe-3", "bigquery-probe-4" ] }, { "productId": "bigquery", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe for llms.txt at cloud.google.com returned 404, and no evidence pack item shows any agent-oriented docs manifest for BigQuery (only human-oriented docs pages in various languages).", "evidenceIds": [ "bigquery-probe-1" ] }, { "productId": "bigquery", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "BigQuery's Gemini integration provides conversational analytics (natural language Q&A over data), data canvas for exploration, SQL/Python code generation and query explanation, plus AI functions for summarization/sentiment/enrichment and BigQuery ML predictive insights — directly matching the story of in-product AI-generated insights and suggestions. missing for 10: independent/hands-on community validation of Gemini-in-BigQuery's actual output quality (only first-party docs cited) and more detail on proactive 'suggestions' beyond query generation.", "evidenceIds": [ "bigquery-docs-3", "bigquery-docs-4", "bigquery-docs-5", "bigquery-docs-13", "bigquery-docs-16", "bigquery-docs-17", "bigquery-docs-39", "bigquery-docs-51", "bigquery-docs-66" ] }, { "productId": "bigquery", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "BigQuery offers background-automation primitives — continuous queries that run indefinitely to process streaming data in real time, scheduled load jobs, and Dataform for scheduling and orchestrating multi-step transformation workflows — which can run autonomously without a user actively invoking them. However, none of this is framed or documented as 'AI-native' agentic automation (e.g., AI-triggered actions, agent orchestration, or autonomous decision loops using BigQuery's AI/ML functions), and there is no independent/community corroboration of these automation features being used this way. Missing for 10: AI-agent-specific automation/orchestration evidence, independent hands-on validation of continuous queries/Dataform running unattended long-term, and any framing tying automation to autonomous AI workflows rather than plain data-pipeline scheduling.", "evidenceIds": [ "bigquery-docs-11", "bigquery-docs-54", "bigquery-docs-41", "bigquery-docs-7", "bigquery-docs-50", "bigquery-docs-40", "bigquery-docs-61" ] }, { "productId": "bigquery", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "BigQuery ships Gemini in BigQuery, a built-in AI assistant offering conversational analytics (natural-language Q&A over data), a data canvas for exploring/transforming/querying/visualizing data, and SQL/Python code generation, explanation, and suggestion — directly matching the story of delegating tasks to an in-product AI assistant. Release notes also show ongoing expansion (e.g., market basket analysis support in conversational analytics).\n\nmissing for 10: independent/hands-on user reports validating Gemini in BigQuery's assistant behavior (only first-party docs cited), and no evidence of broader agentic task delegation (e.g., multi-step autonomous execution) beyond query/code assistance.", "evidenceIds": [ "bigquery-docs-3", "bigquery-docs-4", "bigquery-docs-5", "bigquery-docs-13", "bigquery-docs-39", "bigquery-docs-66" ] }, { "productId": "bigquery", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "BigQuery ships a documented Python-based CLI (`bq`) for scriptable dataset/table/query operations, a fully machine-readable REST API (verified live via discovery document), ODBC/JDBC drivers, and scheduled/batch load jobs — all standard building blocks for headless CI automation. missing for 10: explicit CI/CD pipeline examples (e.g. GitHub Actions/Cloud Build recipes) and independent hands-on reports of using bq/REST in automated pipelines.", "evidenceIds": [ "bigquery-docs-12", "bigquery-docs-49", "bigquery-probe-4", "bigquery-docs-41", "bigquery-docs-55", "bigquery-probe-rt-1" ] }, { "productId": "bigquery", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence only shows Google's MCP Toolbox exposing BigQuery *as* an MCP server/tool provider for external AI agents (bigquery-probe-3, bigquery-probe-rt-2) — the opposite direction from this story, which asks whether a user can plug external MCP servers *into* BigQuery so BigQuery itself can consume their tools (e.g., within Gemini in BigQuery's conversational analytics). No evidence shows BigQuery or its Gemini features acting as an MCP client that ingests external tool servers.", "evidenceIds": [ "bigquery-probe-3", "bigquery-probe-rt-2", "bigquery-docs-3", "bigquery-docs-39" ] }, { "productId": "bigquery", "storyId": "agentic-mcp-server", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Google's official MCP Toolbox for Databases (googleapis/mcp-toolbox) provides a documented, runnable MCP server with a prebuilt BigQuery toolset, confirmed to run via npx and require ADC credentials plus a project ID. However, it is a general 'Toolbox for Databases' rather than a BigQuery-specific first-party product page, and the runtime probe shows the handshake is credential-gated with no independent hands-on confirmation of full agent connectivity in production use. Missing for 10: independent/third-party corroboration of agent use, BigQuery-specific (not generic toolbox) branding, and evidence of production-scale reliability or broader ecosystem adoption.", "evidenceIds": [ "bigquery-probe-3", "bigquery-probe-rt-2" ] }, { "productId": "bigquery", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "BigQuery's Gemini/Conversational Analytics features let users query, join, and visualize data using natural language, and generate/explain SQL or Python code from NL prompts, which directly supports natural-language operation of the product. However, this is all first-party doc-based evidence with no independent/hands-on corroboration, and NL support seems scoped to querying/exploration rather than full operational control (e.g., admin, pipeline management). Missing for 10: independent verification of NL feature reliability, evidence of NL commands controlling broader BigQuery operations beyond analytics/query generation.", "evidenceIds": [ "bigquery-docs-3", "bigquery-docs-4", "bigquery-docs-5", "bigquery-docs-13", "bigquery-docs-39", "bigquery-docs-66" ] }, { "productId": "bigquery", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "BigQuery ships 'bq', an official Python-based CLI for creating datasets, loading data, and querying tables, well documented across multiple doc pages and confirmed as a distinct probe finding. This is a general-purpose CLI rather than one purpose-built for AI-native agentic workflows (e.g., no agent-specific CLI features), so missing for 10: evidence of AI-agent-specific CLI features or tooling, independent hands-on community validation of CLI usage in agentic contexts.", "evidenceIds": [ "bigquery-docs-12", "bigquery-docs-49", "bigquery-docs-58", "bigquery-probe-4" ] }, { "productId": "bigquery", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "BigQuery exposes a documented, machine-readable REST API (v2 discovery doc confirmed live with resources for datasets/jobs/models/tables), plus gRPC Storage Write API, client SDKs, ODBC/JDBC drivers, and a Python-based bq CLI, all publicly documented and independently verified via runtime probe. missing for 10: no llms.txt or standalone OpenAPI spec file was found (404s), and the MCP server integration requires credential-gated setup rather than being fully turnkey.", "evidenceIds": [ "bigquery-probe-rt-1", "bigquery-docs-1", "bigquery-docs-35", "bigquery-docs-55", "bigquery-probe-4", "bigquery-probe-1", "bigquery-probe-2", "bigquery-probe-rt-2" ] }, { "productId": "bigquery", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "BigQuery documents predefined IAM roles/permissions scoped to specific resources (datasets, tables, views, routines) and custom quotas, which are the building blocks for issuing least-privilege access to any caller including an agent, but there is no explicit documentation of a workflow for minting scoped/short-lived API credentials specifically for an AI agent (e.g., service account impersonation, workload identity federation, or OAuth scope restriction guidance tied to agentic use). Missing for 10: explicit agent-credential-issuance guidance, short-lived/ephemeral token support, workload identity federation docs, and IAM Conditions examples for fine-grained scoping.", "evidenceIds": [ "bigquery-docs-31", "bigquery-docs-65", "bigquery-docs-63" ] }, { "productId": "bigquery", "storyId": "agentic-sdks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "BigQuery documents multiple official SDK/API surfaces for building integrations: the Storage Write API (gRPC) client, a Rust SDK now in Preview, ODBC/JDBC drivers, the Python-based bq CLI, and a machine-readable REST v2 discovery document verified reachable keylessly at runtime. This gives an AI-native builder concrete, documented, and runtime-verified official surfaces to build against. Missing for 10: explicit listing/mention of mainstream language client libraries (Python, Java, Node.js, Go, .NET) by name and any independent hands-on developer report confirming SDK ergonomics beyond docs.", "evidenceIds": [ "bigquery-docs-14", "bigquery-docs-22", "bigquery-docs-35", "bigquery-docs-55", "bigquery-probe-4", "bigquery-probe-rt-1" ] }, { "productId": "bigquery", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows BigQuery streaming ingestion via Pub/Sub subscriptions and continuous queries for real-time analysis of incoming data, but nothing documents an outbound webhook mechanism for subscribing external clients to BigQuery events (e.g., job completion, table changes) via HTTP callbacks. Pub/Sub subscriptions described here are for loading data in, not for AI-native agents subscribing to events out.", "evidenceIds": [ "bigquery-docs-18", "bigquery-docs-29", "bigquery-docs-42", "bigquery-docs-54" ] }, { "productId": "bigquery", "storyId": "ai-sql-assistant", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Gemini in BigQuery provides an in-product natural-language assistant that generates/suggests SQL and Python code and explains existing queries, plus conversational analytics for chatting with data (docs-3,4,5,39,66,21,13). This matches the story of a built-in assistant writing, fixing, and explaining SQL from natural language directly inside the product. missing for 10: independent/hands-on user reports specifically validating the assistant's SQL-writing/fixing/explaining quality (only vendor docs cited).", "evidenceIds": [ "bigquery-docs-5", "bigquery-docs-3", "bigquery-docs-4", "bigquery-docs-39", "bigquery-docs-66", "bigquery-docs-13" ] }, { "productId": "bigquery", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows only a machine-readable REST discovery document (bigquery-probe-rt-1) and standard docs pages, with explicit probe failures for llms.txt and openapi.json (bigquery-probe-1, bigquery-probe-2). There is no evidence of an interactive, runnable-example API reference (e.g., a try-it console or embedded runnable code snippets) for AI-native exploration.", "evidenceIds": [ "bigquery-probe-1", "bigquery-probe-2", "bigquery-probe-rt-1" ] }, { "productId": "bigquery", "storyId": "api-machine-spec", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "BigQuery does not host a standard OpenAPI/swagger.json file (probe confirms openapi.json/swagger.json paths 404), but it exposes Google's own machine-readable API Discovery Document (bigquery:v2) at the REST reference which was verified to download keylessly and parse cleanly into resources/methods — functionally equivalent to an OpenAPI spec though in a different format. Missing for 10: a canonical OpenAPI/Swagger file, first-party statement equating the Discovery doc to OpenAPI, and independent community confirmation of AI-native tooling consuming it.", "evidenceIds": [ "bigquery-probe-2", "bigquery-probe-rt-1" ] }, { "productId": "bigquery", "storyId": "api-sandbox", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "BigQuery Sandbox is explicitly documented as a way to explore and test BigQuery capabilities without a billing account or credit card, functioning as an isolated environment separate from a paid production setup. This directly matches the story of testing without touching production data, though evidence is purely first-party docs. Missing for 10: independent/hands-on confirmation of sandbox isolation guarantees, and specifics on how sandbox environments map to keeping AI-agent test workloads fully separate from production datasets.", "evidenceIds": [ "bigquery-docs-6", "bigquery-docs-48", "bigquery-docs-62" ] }, { "productId": "bigquery", "storyId": "api-versioning-policy", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "BigQuery exposes a versioned, machine-readable REST API (v2 discovery document) confirmed via runtime probe, showing a genuine versioned API surface, but there is no evidence in the pack of a documented deprecation policy, versioning cadence, or sunset guarantees for that API. missing for 10: explicit deprecation/sunset policy docs, versioning changelog commitments, and independent confirmation of long-term API stability guarantees.", "evidenceIds": [ "bigquery-probe-rt-1", "bigquery-docs-14" ] }, { "productId": "bigquery", "storyId": "audit-lineage", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack only covers IAM roles/permissions (bigquery-docs-31, bigquery-docs-65) and compliance certifications, with no mention of Cloud Audit Logs, job history, or column-level lineage tracking (e.g., Data Catalog/Dataplex lineage) for BigQuery. Governance/audit-and-lineage is a plausible and expected axis for a data warehouse, but no evidence in this pack substantiates it.", "evidenceIds": [ "bigquery-docs-31", "bigquery-docs-65" ] }, { "productId": "bigquery", "storyId": "automation-bulk-operations", "verdict": "full", "quality": 7, "confidence": "high", "rationale": "BigQuery natively supports bulk operations via DML for batch insert/update/delete, batch load jobs with scheduling, and the high-throughput Storage Write API for large-scale bulk ingestion, plus the bq CLI for scripting these operations across many items/tables. Community evidence corroborates BigQuery's ability to process petabyte-scale data efficiently. Missing for 10: explicit evidence of AI-agent-driven orchestration of bulk operations (e.g., an agent issuing many bulk actions via MCP/API in one flow) and independent benchmarks specifically on DML/bulk-write scale rather than just query scale.", "evidenceIds": [ "bigquery-docs-2", "bigquery-docs-35", "bigquery-docs-59", "bigquery-docs-41", "bigquery-docs-49", "bigquery-comm-4", "bigquery-docs-47" ] }, { "productId": "bigquery", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "BigQuery's continuous queries feature runs SQL statements continuously to analyze incoming data in real time, which is the closest capability to event-triggered automation, but there is no evidence of a general rules/condition-action engine that fires arbitrary actions (notifications, function calls, workflows) on events—continuous queries are limited to SQL-based streaming analysis. Missing for 10: explicit rule definition UI/API, broader action types (not just SQL output), and evidence of integration with external triggers/alerts as an automation mechanism.", "evidenceIds": [ "bigquery-docs-11", "bigquery-docs-54" ] }, { "productId": "bigquery", "storyId": "automation-scheduled-jobs", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "BigQuery documents native support for recurring automation: scheduled load jobs ('schedule one-time or batch data transfers at regular intervals') and Dataform for developing, testing, versioning, and scheduling complex data-transformation workflows, plus continuous queries for always-on real-time processing. These are first-party, concrete capabilities directly matching the story. Missing for 10: explicit mention of the dedicated BigQuery Scheduled Queries feature by name, and independent/hands-on confirmation of scheduling reliability in production.", "evidenceIds": [ "bigquery-docs-41", "bigquery-docs-7", "bigquery-docs-50", "bigquery-docs-11", "bigquery-docs-54", "bigquery-docs-40", "bigquery-docs-61" ] }, { "productId": "bigquery", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Dataform (BigQuery's workflow/transformation service) explicitly supports version control and Git-based collaboration for data pipelines, plus Git folders for scheduled pipelines, which cover versioning and team review to a degree. However, there's no explicit documentation of a rollback mechanism or formal review/approval workflow for automations beyond generic Git usage. Missing for 10: explicit rollback/revert documentation, structured review/approval process (e.g., PR-based approvals) for automations, and independent/hands-on evidence confirming these workflows in practice.", "evidenceIds": [ "bigquery-docs-7", "bigquery-docs-40", "bigquery-docs-61", "bigquery-docs-15", "bigquery-docs-50" ] }, { "productId": "bigquery", "storyId": "bi-driver-connectivity", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "BigQuery documents official Simba JDBC and ODBC drivers explicitly for connecting BI tools and applications to BigQuery, plus native integrations (bq CLI, Sheets, Analytics Hub/sharing) that support dashboard connectivity without custom glue. Missing for 10: no independent hands-on report confirming smooth BI-tool dashboard connection experience beyond vendor docs.", "evidenceIds": [ "bigquery-docs-55", "bigquery-probe-4" ] }, { "productId": "bigquery", "storyId": "bulk-load-object-storage", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "BigQuery's documented `bq load` command and load jobs support single-command bulk loading of CSV, JSON, and Parquet directly from Cloud Storage, with scheduling and batch options documented (bigquery-docs-41, bigquery-docs-12/49/58). This is a well-established, heavily documented core capability. Missing for 10: independent hands-on confirmation specifically of the load command (community evidence covers pricing/perf but not load-command usage) and explicit mention of all three formats in one citation.", "evidenceIds": [ "bigquery-docs-41", "bigquery-docs-12", "bigquery-docs-49", "bigquery-docs-58", "bigquery-probe-4" ] }, { "productId": "bigquery", "storyId": "compliance-attestations", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Google Cloud's compliance resource center explicitly documents BigQuery-covered attestations including SOC 1/2/3, HIPAA support, ISO 27001-family certs, and downloadable audit reports, directly satisfying a platform-engineer's need for security-review documentation. Missing for 10: explicit PCI DSS attestation mention (only general 'sector/regional programs' referenced) and no independent third-party corroboration of the compliance claims.", "evidenceIds": [ "bigquery-supp-compliance" ] }, { "productId": "bigquery", "storyId": "connector-ecosystem", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "BigQuery's docs show database replication ('replicating data from databases to BigQuery in near real time'), scheduled load jobs, Pub/Sub streaming ingestion, Dataform for ELT transformation workflows, and ODBC/JDBC drivers for tool connectivity — a real but partial connector story. However, there's no evidence of BigQuery Data Transfer Service or a documented catalog of first-party/partner SaaS-app connectors (e.g. Google Ads, Analytics, Salesforce, Fivetran/Stitch-style partner ecosystem), and one community report notes CloudSQL-to-BigQuery ingestion historically required a third-party tool rather than a native connector. Missing for 10: explicit SaaS-app connector documentation, a partner-connector directory/marketplace, and independent confirmation that documented setup for diverse source types (beyond DBs/Pub/Sub) is straightforward.", "evidenceIds": [ "bigquery-docs-36", "bigquery-docs-41", "bigquery-docs-18", "bigquery-docs-29", "bigquery-docs-42", "bigquery-docs-7", "bigquery-docs-50", "bigquery-docs-55", "bigquery-comm-14" ] }, { "productId": "bigquery", "storyId": "continuous-transformations", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "BigQuery provides continuous queries (SQL statements running continuously to analyze streaming data in real time) as a first-class feature, plus Dataform for declarative, version-controlled, scheduled transformation pipelines with dependency graphs, and streaming ingestion via Storage Write API/Pub/Sub subscriptions for near-real-time pipelines. Together these cover streams, tasks/scheduled pipelines, declarative pipelines, and continuous queries as described in the story. Missing for 10: independent/hands-on validation of continuous queries at scale and clearer documentation of latency/cost tradeoffs in production use.", "evidenceIds": [ "bigquery-docs-11", "bigquery-docs-54", "bigquery-docs-7", "bigquery-docs-40", "bigquery-docs-61", "bigquery-docs-50", "bigquery-docs-18", "bigquery-docs-29", "bigquery-docs-42", "bigquery-docs-36", "bigquery-docs-41", "bigquery-docs-1", "bigquery-docs-59" ] }, { "productId": "bigquery", "storyId": "cost-model-clarity", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Google documents pricing clearly (on-demand per-TiB, first 1TiB free, slot-based flat-rate reservations, custom quotas to cap spend) and community reports confirm a dry-run/bytes-scanned estimator exists that lets users predict costs before running queries. However, a concrete hands-on report describes a user being billed $14,000 with zero warning, and community critique that BigQuery 'hides query cost behind abstracted TBs scanned or slots' making bill estimation harder than it should be — a direct contradiction to the claim that pricing is 'documented clearly enough to estimate a bill' in practice. missing for 10: independent case studies showing accurate monthly forecasting at scale, clearer treatment of storage/streaming/BI Engine cost interactions, and resolution of the surprise-billing complaint.", "evidenceIds": [ "bigquery-docs-19", "bigquery-docs-28", "bigquery-docs-46", "bigquery-docs-63", "bigquery-comm-1", "bigquery-comm-2", "bigquery-comm-3", "bigquery-comm-4", "bigquery-comm-9", "bigquery-comm-10" ] }, { "productId": "bigquery", "storyId": "dataset-marketplace", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "BigQuery sharing (formerly Analytics Hub) is explicitly documented as a data exchange platform for discovering, sharing and accessing third-party/Google curated datasets and combining them with internal data, directly inside the platform. Multiple docs confirm this cross-org discovery and combination workflow. Missing for 10: independent hands-on evidence/user reports specifically validating the marketplace discovery/enrichment workflow (community evidence only covers other BigQuery aspects like pricing/performance, not the marketplace).", "evidenceIds": [ "bigquery-docs-9", "bigquery-docs-32", "bigquery-docs-52", "bigquery-docs-25" ] }, { "productId": "bigquery", "storyId": "dbt-first-class", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no vendor documentation from Google about a dbt adapter or native dbt project support for BigQuery; the only related item is a community comment casually noting BigQuery's pipe syntax is 'great for dbt macros,' which is a third-party observation, not vendor-documented adapter support. Absence of evidence for this applicable ecosystem-integration axis yields 'none.'", "evidenceIds": [ "bigquery-comm-15" ] }, { "productId": "bigquery", "storyId": "free-evaluation-path", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "BigQuery Sandbox explicitly lets analysts explore BigQuery capabilities and run real queries without providing a credit card or creating a billing account, and on-demand pricing includes 1 TiB of free query processing per month — directly matching the story of a no-card, no-sales-call evaluation path. Missing for 10: independent/hands-on user reports confirming the sandbox experience in practice (only first-party docs are cited, and community evidence focuses on billing surprises for paid usage rather than the sandbox itself).", "evidenceIds": [ "bigquery-docs-6", "bigquery-docs-48", "bigquery-docs-62", "bigquery-docs-19" ] }, { "productId": "bigquery", "storyId": "full-sql-surface", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "BigQuery's GoogleSQL is documented as ANSI-compliant with support for DML, query statements, and a full analytical surface including window functions, CTEs, arrays/structs, JSON handling, and date/time types (implied by GoogleSQL SQL reference and confirmed by community praise of pipe syntax and table-valued functions). Independent hands-on testimony (bigquery-comm-15, bigquery-comm-16) confirms real analyst usage of advanced SQL constructs without extensions. Missing for 10: explicit doc citations enumerating window functions, JSON/array functions, and DATETIME/TIMESTAMP types individually rather than inferred from general SQL intro docs.", "evidenceIds": [ "bigquery-docs-23", "bigquery-docs-30", "bigquery-docs-2", "bigquery-comm-15", "bigquery-comm-16" ] }, { "productId": "bigquery", "storyId": "granular-rbac-masking", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "IAM predefined roles for BigQuery resources (datasets, tables, views, routines) are documented, and the REST API surface exposes a rowAccessPolicies resource confirming row-level security exists, but the evidence pack contains no documentation of column-level access controls or masking policies (e.g., taxonomy-based dynamic data masking) needed to fully satisfy the story. missing for 10: column-level security/masking policy documentation, examples of combining row+column policies with IAM roles for multi-tenant governance, independent/hands-on corroboration of masking behavior.", "evidenceIds": [ "bigquery-docs-31", "bigquery-docs-65", "bigquery-probe-rt-1" ] }, { "productId": "bigquery", "storyId": "live-data-sharing", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "BigQuery sharing (formerly Analytics Hub) is explicitly documented as a data exchange platform enabling secure sharing, discovery, and access to data across organizational boundaries without replicating data—exactly matching the story of sharing live datasets with another account/org without copying or building an export pipeline. Missing for 10: independent/hands-on corroboration beyond first-party docs.", "evidenceIds": [ "bigquery-docs-9", "bigquery-docs-25", "bigquery-docs-32", "bigquery-docs-52" ] }, { "productId": "bigquery", "storyId": "local-dev-loop", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "BigQuery offers a free 'Sandbox' mode that lets users try queries and transformations without a credit card or billing account, and on-demand pricing gives 1 TiB/month free — both lowering the barrier to a free dev loop. However, there is no evidence of a local engine or offline emulator (unlike e.g. Firestore's emulator); the Sandbox is still a hosted, quota-limited slice of the real cloud service, not a local dev-loop tool. Missing for 10: a documented local/offline emulator, evidence of local development without any cloud dependency, and any hands-on account confirming the sandbox is sufficient for full transformation development before touching production.", "evidenceIds": [ "bigquery-docs-6", "bigquery-docs-48", "bigquery-docs-62", "bigquery-docs-19" ] }, { "productId": "bigquery", "storyId": "managed-continuous-ingestion", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "BigQuery provides multiple fully-managed ingestion paths for continuously arriving data: the Storage Write API with exactly-once semantics for streaming writes, a native Pub/Sub-to-BigQuery subscription for high-throughput event streaming, Continuous Queries for real-time analysis of incoming data, and scheduled or near-real-time database replication load jobs, all without the user provisioning or operating pipeline infrastructure. Docs explicitly market BigQuery as having no infrastructure to set up or manage. Missing for 10: independent or hands-on evidence validating continuous streaming ingestion at scale, and more detail on CDC or file-arrival-triggered ingestion beyond Pub/Sub and replication mentions.", "evidenceIds": [ "bigquery-docs-18", "bigquery-docs-29", "bigquery-docs-42", "bigquery-docs-35", "bigquery-docs-59", "bigquery-docs-11", "bigquery-docs-54", "bigquery-docs-36", "bigquery-docs-41", "bigquery-docs-34" ] }, { "productId": "bigquery", "storyId": "native-notebooks", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "BigQuery's notebooks feature explicitly combines SQL queries, Python code, rich text, and inline visualizations, with Colab Enterprise integration enabling end-to-end data science/ML workflows in one interface. This directly matches the analyst story of mixing SQL/Python with inline results and charts. Missing for 10: independent/hands-on corroboration of the notebook experience beyond first-party docs, and detail on how seamlessly SQL and Python cells interoperate in practice.", "evidenceIds": [ "bigquery-docs-8", "bigquery-docs-24", "bigquery-docs-43" ] }, { "productId": "bigquery", "storyId": "nl-governed-answers", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "BigQuery's Gemini-powered conversational analytics and data canvas explicitly let business users ask natural-language questions to find, join, and query tables, with collaboration and visualization built in (docs-3,4,5,39,66,13/21), and IAM-based access control (docs-31,65) provides a governance layer over which data can be queried. However, there's no first-party evidence of a dedicated semantic/business-glossary layer preventing hallucinated joins, and no independent or hands-on corroboration that these NL answers are reliably 'governed' or free of hallucination. Missing for 10: evidence of an explicit semantic modeling/metadata layer enforcing join correctness, and independent validation of answer accuracy/hallucination rates.", "evidenceIds": [ "bigquery-docs-3", "bigquery-docs-4", "bigquery-docs-5", "bigquery-docs-39", "bigquery-docs-66", "bigquery-docs-13", "bigquery-docs-31", "bigquery-docs-65" ] }, { "productId": "bigquery", "storyId": "open-table-formats", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "BigQuery explicitly supports querying open table formats (Iceberg, Delta, Hudi) and files in object storage without ingesting them into proprietary storage — Iceberg managed tables store data in customer-owned buckets with schema evolution and time travel, and BigQuery natively supports open table formats per its intro docs. Missing for 10: independent hands-on validation of querying Delta/Parquet files directly (only Iceberg is detailed with specifics), and no community corroboration of external-table performance/limitations for these formats.", "evidenceIds": [ "bigquery-docs-57", "bigquery-docs-10", "bigquery-docs-26", "bigquery-docs-53", "bigquery-docs-60", "bigquery-docs-44" ] }, { "productId": "bigquery", "storyId": "openness-api-parity", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "BigQuery exposes a comprehensive, self-describing REST API (datasets/jobs/models/tables/routines) plus the bq CLI and client libraries, and SQL/DML/ML training operations that appear in the UI are all reachable via API or SQL statements — core data and query operations have strong API parity. However, several newer UI-centric features (Gemini-powered conversational analytics, BigQuery Data Canvas, notebook-based workflows) are documented as console experiences without clear evidence of an equivalent programmatic/API path for all their functionality. Missing for 10: explicit documentation that conversational analytics/Data Canvas/notebook features are fully accessible via API rather than console-only, and independent hands-on confirmation of full UI/API parity.", "evidenceIds": [ "bigquery-probe-rt-1", "bigquery-probe-4", "bigquery-docs-12", "bigquery-docs-2", "bigquery-docs-31", "bigquery-docs-4", "bigquery-docs-3", "bigquery-docs-8" ] }, { "productId": "bigquery", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "BigQuery documents Iceberg managed tables that store data in customer-owned buckets in the open Apache Iceberg format for interoperability with third-party/open-source engines, and states general support for open table formats like Iceberg, Delta, and Hudi, addressing data portability/lock-in concerns. However, evidence does not show a dedicated bulk-export mechanism (e.g. bq extract to CSV/Avro/Parquet/JSON) for standard native BigQuery tables, nor confirmation that ALL data (not just tables opted into Iceberg format) can be freely exported without proprietary lock-in. Missing for 10: documented export/extract tooling for native tables in open formats, evidence of full-dataset export workflows, and independent confirmation that migrating away is friction-free.", "evidenceIds": [ "bigquery-docs-10", "bigquery-docs-26", "bigquery-docs-53", "bigquery-docs-57" ] }, { "productId": "bigquery", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "BigQuery is a proprietary, closed-source managed cloud data warehouse service; there is no source code released under an open license for users to inspect. This is a category error — BigQuery is a hosted SaaS product, not open-source software, so the axis of 'reading source under an open license' does not apply to the product itself.", "evidenceIds": [] }, { "productId": "bigquery", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "BigQuery is a fully managed, proprietary Google Cloud data warehouse with no self-hostable open-source core — self-hosting is a category error for this SaaS product, not an unmet capability.", "evidenceIds": [] }, { "productId": "bigquery", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation or mention of choosing a dataset/table region, multi-region options, or data residency controls for BigQuery; only general compliance certifications and unrelated feature descriptions are provided. Missing for 10: explicit docs on selecting dataset location/region, data residency guarantees, or region-locking configuration.", "evidenceIds": [] }, { "productId": "bigquery", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers BigQuery's features (SQL, Gemini in BigQuery, pricing, sharing, compliance certifications) but contains no documentation of a specific data-usage policy or customer control for opting out of having data used to train Google's AI models, despite BigQuery now integrating Gemini AI features. Missing for 10: explicit AI-training data-usage policy, opt-out/opt-in controls, and any documentation addressing whether customer data feeds model training.", "evidenceIds": [] }, { "productId": "bigquery", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "BigQuery's DML supports DELETE operations on tables (bigquery-docs-2) and Iceberg managed tables reference 'time travel for historical data access' (bigquery-docs-44), giving users mechanisms to remove or manage historical data, and IAM/access-control docs show governance controls exist (bigquery-docs-31). However, there is no direct documentation of retention policy configuration (e.g., table/partition expiration settings) or a dedicated data-deletion/GDPR compliance workflow in the evidence pack. Missing for 10: explicit table/dataset expiration policy docs, a dedicated data retention & deletion API/console feature, and independent confirmation that deletion requests are honored end-to-end.", "evidenceIds": [ "bigquery-docs-2", "bigquery-docs-44", "bigquery-docs-31" ] }, { "productId": "bigquery", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of BigQuery collecting telemetry/usage data from client tools (bq CLI, client libraries, notebooks) nor any documented opt-out mechanism, despite this being a plausible axis for a cloud data platform with CLI/SDK tooling. missing for 10: any documentation of telemetry collection practices, an opt-out flag/setting (e.g., in bq CLI or client SDKs), or privacy-posture statements addressing usage tracking.", "evidenceIds": [] }, { "productId": "bigquery", "storyId": "query-profiling", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "BigQuery documents dedicated query plan/execution diagnostics via EXPLAIN-like output and timing information, letting engineers inspect stages, slot usage, and bottlenecks; the pricing model also exposes bytes-scanned/dry-run cost estimation used to diagnose expensive queries. Missing for 10: independent hands-on walkthroughs of using the query plan UI to debug a real slow query (only docs mention the feature, no community corroboration of using execution plans specifically).", "evidenceIds": [ "bigquery-docs-56", "bigquery-docs-64", "bigquery-comm-3", "bigquery-comm-4", "bigquery-docs-46" ] }, { "productId": "bigquery", "storyId": "runaway-cost-controls", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "BigQuery docs describe custom quotas that cap daily bytes processed per project/user as a cost-control mechanism, and community discussion confirms dry-run/bytes-scanned previews exist to estimate cost before running a query — but a documented real-world case shows a runaway query still resulted in a surprise $14,000 bill with 'zero warning' and Google refusing to waive it, meaning no automatic budget/resource-monitor/auto-suspend actually intervened. Missing for 10: evidence of real-time Cloud Billing budget alerts wired to auto-suspend BigQuery usage, a per-query 'maximum bytes billed' kill-switch, or any documented fix/response after the community-reported billing incident.", "evidenceIds": [ "bigquery-docs-63", "bigquery-comm-1", "bigquery-comm-3", "bigquery-comm-4" ] }, { "productId": "bigquery", "storyId": "semantic-model-metrics", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows BigQuery's SQL engine, Dataform (ELT transformation), notebooks, and Gemini natural-language querying, but no dedicated semantic-layer feature where metrics, dimensions, and joins are declared once and consistently reused across queries and AI tools (e.g., no LookML/metrics-layer equivalent is documented here). Dataform manages transformation pipelines, not a governed semantic/metrics model, so the specific capability described in the story is unevidenced.", "evidenceIds": [ "bigquery-docs-7", "bigquery-docs-50", "bigquery-docs-23" ] }, { "productId": "bigquery", "storyId": "streaming-ingest-latency", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "BigQuery's Storage Write API (gRPC) is documented as a real-time streaming ingestion API with exactly-once delivery semantics, and continuous queries/Pub/Sub subscriptions confirm data becomes queryable in near real time. missing for 10: no independent/hands-on benchmark confirming actual seconds-level latency from write to queryability, and no explicit SLA/numeric latency figure in docs.", "evidenceIds": [ "bigquery-docs-1", "bigquery-docs-35", "bigquery-docs-59", "bigquery-docs-54", "bigquery-docs-11", "bigquery-docs-42", "bigquery-docs-18" ] }, { "productId": "bigquery", "storyId": "time-travel-restore", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Only one brief doc mention confirms BigQuery supports time travel for historical data access, but the evidence pack lacks detail on the retention window, querying historical snapshots via SQL syntax, or the fail-safe/restore-dropped-table mechanism, and has no independent/hands-on corroboration. missing for 10: documentation on the time-travel query syntax (FOR SYSTEM_TIME AS OF), the default/configurable retention window, explicit restore-dropped-table (fail-safe) workflow, and any community validation of these mechanisms.", "evidenceIds": [ "bigquery-docs-44" ] }, { "productId": "databricks", "storyId": "agent-runs-sql-headless", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Databricks documents a full non-UI path: CLI/API authentication (databricks-docs-2/48/64), schema and object discovery governed by Unity Catalog accessible via CLI/SQL/REST (databricks-docs-56, 19, 38, 70, 75), SQL execution via the Databricks SQL API/CLI (databricks-supp-api-versioning, databricks-docs-32), and reading results back programmatically (workspace REST API reference, databricks-docs-65), with runtime evidence that the CLI actually installs and exposes a raw `databricks api` wrapper keylessly (databricks-probe-rt-1). Missing for 10: an end-to-end documented/hands-on example chaining auth→discover→query→result specifically for an agent (no single walkthrough), and no independent corroboration of the full flow outside vendor docs.", "evidenceIds": [ "databricks-docs-2", "databricks-docs-48", "databricks-docs-56", "databricks-docs-19", "databricks-docs-38", "databricks-docs-70", "databricks-docs-75", "databricks-docs-32", "databricks-supp-api-versioning", "databricks-docs-65", "databricks-probe-rt-1" ] }, { "productId": "databricks", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Databricks hosts an llms.txt file (probe confirms HTTP 200 with a structured documentation index) and also ships agent-oriented skill docs (SKILL.md files installable via `databricks aitools install`) that agents can be pointed at, going beyond a bare llms.txt. missing for 10: no independent third-party report of an agent successfully consuming llms.txt or the skills in practice, and no evidence of an /llms-full.txt or deeper machine-readable agent doc index.", "evidenceIds": [ "databricks-probe-1", "databricks-probe-rt-2", "databricks-docs-3" ] }, { "productId": "databricks", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Databricks ships multiple first-party AI-insight surfaces: Genie for natural-language Q&A grounded in org data, AI/BI dashboards with AI-assisted authoring, automatic query performance insights/recommendations, and a semantic layer (metric views) that standardizes metrics for consistent AI-driven analysis. This directly satisfies the story of getting AI-generated insights/suggestions inside the product. Missing for 10: independent/hands-on user corroboration specifically validating insight quality (community evidence is generic platform sentiment, not about Genie/AI-BI insight accuracy), and no benchmark of suggestion usefulness.", "evidenceIds": [ "databricks-docs-8", "databricks-docs-9", "databricks-docs-6", "databricks-docs-22", "databricks-docs-42", "databricks-docs-72" ] }, { "productId": "databricks", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Databricks supports autonomous background automation via Lakeflow pipelines/jobs scheduling, Genie Code running as an autonomous agent that plans, runs code, fixes errors and asks approval before tool use, and CLI/SDK scripting for automation, plus MCP-based agent tooling for background agent workflows. missing for 10: independent/hands-on verification of fully unattended (no-human-in-loop) agent runs, and clearer documentation of scheduling/triggers specifically for autonomous agent tasks rather than just pipelines.", "evidenceIds": [ "databricks-docs-29", "databricks-docs-1", "databricks-docs-2", "databricks-docs-49", "databricks-docs-43", "databricks-docs-50" ] }, { "productId": "databricks", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Databricks ships Genie Code, a built-in AI assistant that generates/runs code, builds pipelines and dashboards, debugs errors, and can run as an autonomous agent that plans, executes code, fixes errors and asks approval before tool use — directly matching 'delegate tasks to a built-in AI assistant.' It also supports parallel chats and can be given governed tool access (Slack, GitHub, etc.) via MCP for agentic task execution. missing for 10: independent/hands-on user validation of agentic delegation quality is absent from community evidence.", "evidenceIds": [ "databricks-docs-15", "databricks-docs-28", "databricks-docs-29", "databricks-docs-30", "databricks-docs-63", "databricks-docs-31" ] }, { "productId": "databricks", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Databricks provides a documented, keylessly-installable CLI supporting scripting/automation of workspace, jobs, and Lakeflow pipelines (databricks-docs-2, databricks-docs-48, databricks-probe-rt-1), plus REST APIs and Python SDK explicitly for automating operations (databricks-docs-49, databricks-docs-65), enabling headless/CI use for orchestrating pipelines, jobs, and ML training. Jobs/pipelines can be triggered/orchestrated without UI interaction, matching CI automation needs. Missing for 10: no explicit CI/CD pipeline integration guide (e.g., GitHub Actions) or independent case study confirming CI usage beyond docs and CLI probes.", "evidenceIds": [ "databricks-docs-2", "databricks-docs-48", "databricks-docs-49", "databricks-docs-65", "databricks-probe-rt-1", "databricks-docs-40" ] }, { "productId": "databricks", "storyId": "agentic-mcp-client", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Databricks documents connecting agents to external, managed, and custom MCP servers (docs-45, docs-50, docs-71) and using their tools/third-party SaaS integrations (docs-31, docs-43), plus a Marketplace listing MCP servers for discovery (docs-34/69). This directly matches the story of plugging in MCP servers to gain tool access. Missing for 10: independent/hands-on community confirmation of the MCP-client integration working end-to-end (only vendor docs, no third-party validation).", "evidenceIds": [ "databricks-docs-45", "databricks-docs-50", "databricks-docs-71", "databricks-docs-43", "databricks-docs-34", "databricks-probe-3" ] }, { "productId": "databricks", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Databricks documents official MCP server support extensively: it hosts managed/custom MCP servers as Databricks Apps, exposes Unity Catalog functions as MCP tools, provides built-in system.ai MCP Services for third-party SaaS tools, and explicitly supports wiring external clients (Claude, Cursor, MCP Inspector) to MCP servers hosted on Databricks; Marketplace also lists MCP servers as discoverable assets. Missing for 10: independent/hands-on third-party verification of the MCP connection flow (only first-party docs available).", "evidenceIds": [ "databricks-docs-4", "databricks-docs-31", "databricks-docs-43", "databricks-docs-45", "databricks-docs-50", "databricks-docs-69", "databricks-docs-71", "databricks-probe-3" ] }, { "productId": "databricks", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Databricks documents multiple natural-language interaction surfaces: Genie for asking data questions in plain English (docs-8), Genie Code as an autonomous coding/data agent that plans, runs code, fixes errors, and builds pipelines/dashboards from NL prompts (docs-15, docs-28, docs-29, docs-30), and NL-driven dashboard authoring (docs-22). This covers the core of 'operate via natural language' across both business and technical personas. Missing for 10: independent/hands-on validation of NL command reliability and no evidence of NL control over broader ops (e.g., cluster/job management) beyond data/coding tasks.", "evidenceIds": [ "databricks-docs-8", "databricks-docs-15", "databricks-docs-22", "databricks-docs-28", "databricks-docs-29", "databricks-docs-30" ] }, { "productId": "databricks", "storyId": "agentic-official-cli", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Databricks ships an official CLI (docs, migration guides, REST wrapper) confirmed by hands-on runtime probe (v1.15.0), and it is explicitly AI-native: the CLI ships an `aitools install` subcommand that installs vendor agent skills (SKILL.md files) for Claude Code, Codex, Cursor, Copilot, etc. missing for 10: independent/community corroboration beyond the vendor docs and runtime probe.", "evidenceIds": [ "databricks-docs-2", "databricks-docs-48", "databricks-docs-11", "databricks-probe-4", "databricks-probe-rt-1", "databricks-probe-rt-2" ] }, { "productId": "databricks", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Databricks publishes a documented REST API reference (workspace-level APIs, versioned Jobs/SQL APIs with request/response examples), plus CLI, SDKs (Python), and Databricks Connect that all wrap this public API for automation and agentic use, and this is corroborated by a runtime probe confirming a working CLI with a raw `databricks api` REST wrapper. Missing for 10: a live interactive API try-it console or independent third-party API-quality corroboration beyond vendor docs.", "evidenceIds": [ "databricks-docs-65", "databricks-supp-versioned-apis", "databricks-supp-api-reference-examples", "databricks-docs-48", "databricks-docs-49", "databricks-docs-64", "databricks-probe-rt-1", "databricks-probe-4" ] }, { "productId": "databricks", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Databricks documents Unity Catalog governance that enforces access control on data/AI assets and lets agents get 'governed access' to third-party tools via Unity Catalog functions and MCP services (docs-19, 38, 70, 71, 43), implying a permissions model that could scope agent access. However, there is no explicit documentation of an API/credential-issuance mechanism (e.g., scoped service-principal tokens or OAuth scopes specifically for agents) described as 'least-privilege API credentials for an agent.' Missing for 10: explicit credential/token issuance workflow scoped to an agent identity, documentation of OAuth/service-principal scoping granularity, and any hands-on or independent confirmation that credentials can be narrowly scoped per-agent.", "evidenceIds": [ "databricks-docs-19", "databricks-docs-38", "databricks-docs-70", "databricks-docs-71", "databricks-docs-43", "databricks-docs-75" ] }, { "productId": "databricks", "storyId": "agentic-sdks", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Databricks publishes official SDKs (e.g., Databricks SDK for Python), a Databricks Connect library embeddable in any IDE, official CLI, and REST API reference with SDK-based examples, all first-party documented and independently verified via CLI runtime probes. This directly supports AI-native developers building against official SDKs/tools, further reinforced by agent-skill installs and MCP tool integration docs. missing for 10: no independent third-party benchmark of SDK reliability/coverage across languages beyond Python.", "evidenceIds": [ "databricks-docs-49", "databricks-docs-47", "databricks-docs-64", "databricks-docs-65", "databricks-probe-rt-1", "databricks-probe-rt-2", "databricks-supp-versioned-apis" ] }, { "productId": "databricks", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack describes any webhook subscription mechanism (e.g., event notifications pushed to external endpoints); Databricks docs reference REST APIs, SQL alerts, and MCP tools but nothing about webhooks for event subscription.", "evidenceIds": [] }, { "productId": "databricks", "storyId": "ai-sql-assistant", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Databricks documents a built-in AI assistant (Genie/Genie Code) that writes, debugs, and explains code and queries directly inside notebooks/SQL editor against Unity Catalog tables, columns, and lineage, plus a separate Genie conversational analytics surface for asking natural-language data questions grounded in org data. This is native, in-product functionality (not a bolt-on), covering SQL generation, fixing errors, and natural-language explanation/answers over schemas. Missing for 10: independent/hands-on third-party verification of SQL-writing accuracy and no explicit example transcript showing it explaining SQL syntax step-by-step.", "evidenceIds": [ "databricks-docs-3", "databricks-docs-5", "databricks-docs-8", "databricks-docs-15", "databricks-docs-28", "databricks-docs-29", "databricks-docs-30", "databricks-docs-22" ] }, { "productId": "databricks", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Databricks publishes a detailed REST API reference with request/response payload examples and code samples via CLI/Terraform/SDKs (databricks-docs-65, databricks-supp-api-reference-examples), but the same doc explicitly states 'No live try-it runner is documented,' and probes found no OpenAPI/Swagger endpoint to power an interactive explorer (databricks-probe-2). This gives static, copyable examples rather than a true in-browser runnable API reference. Missing for 10: an in-page 'try it now' execution console, OpenAPI-based interactive explorer, and confirmation of live request execution against a user's workspace from the docs site.", "evidenceIds": [ "databricks-docs-65", "databricks-supp-api-reference-examples", "databricks-supp-versioned-apis", "databricks-probe-2" ] }, { "productId": "databricks", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Databricks publishes an extensive human-readable REST API reference (databricks-docs-65, databricks-supp-versioned-apis, databricks-supp-api-reference-examples) but there is no evidence of a downloadable machine-readable spec — the probe explicitly found openapi.json/swagger.json/api/openapi.json/.well-known/openapi.json all return 404 (databricks-probe-2), and the API reference page confirms 'No live try-it runner is documented.' This is an applicable axis for a platform with a large REST API surface, so absent evidence of an OpenAPI/Swagger artifact this is 'none' rather than 'na'.", "evidenceIds": [ "databricks-probe-2", "databricks-docs-65", "databricks-supp-api-reference-examples" ] }, { "productId": "databricks", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence shows Unity Catalog governance, Free Edition for learning, and an AI Playground for prototyping agents, but nothing documents a dedicated sandbox/test environment that isolates AI-native testing from production data (e.g., dev/test catalog cloning, data masking for test runs, or a documented sandbox mode). Missing for 10: explicit sandbox/test-environment feature, data isolation guarantees for testing, and any customer/community confirmation of safe non-production testing workflows.", "evidenceIds": [] }, { "productId": "databricks", "storyId": "api-versioning-policy", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Databricks documents versioned REST APIs (e.g., Jobs v2.0) with explicit recommendations to use the latest version, and shows a concrete deprecation/migration case (legacy SQL API path deprecated in favor of api/2.0/sql/queries with a migration guide). This shows real-world versioning and deprecation practice, but there's no single consolidated deprecation policy document, no stated support timelines/sunset dates, and no explicit versioning scheme description (e.g., semver, LTS windows) across the whole API surface. missing for 10: a unified deprecation-policy page with timelines/sunset commitments, explicit API versioning scheme documentation, and independent confirmation that deprecations are reliably telegraphed in advance across all APIs.", "evidenceIds": [ "databricks-supp-api-versioning", "databricks-supp-versioned-apis", "databricks-supp-api-reference-examples", "databricks-docs-65" ] }, { "productId": "databricks", "storyId": "audit-lineage", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Unity Catalog docs explicitly state it enforces access control, tracks lineage of data and AI assets, and logs activity for auditing automatically across the workspace, and Delta history supports auditing operations. missing for 10: no independent/hands-on corroboration of column-level lineage granularity or audit log query examples, and no explicit mention of 'who ran what' query-level attribution beyond general activity logging.", "evidenceIds": [ "databricks-docs-19", "databricks-docs-38", "databricks-docs-70", "databricks-docs-54", "databricks-docs-56" ] }, { "productId": "databricks", "storyId": "automation-bulk-operations", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Databricks exposes bulk/automation capabilities via CLI, SDKs, REST APIs, and Lakeflow pipelines that let AI-native users script operations across many jobs, tables, or workspace objects at once (databricks-docs-2, databricks-docs-48, databricks-docs-49, databricks-docs-56, databricks-docs-65), and Unity Catalog governs bulk operations across catalogs/schemas with consistent access control. Runtime probes confirm the CLI actually works keylessly and covers workspace/compute/jobs/pipelines plus a raw REST wrapper (databricks-probe-rt-1). missing for 10: explicit documentation or example of a single bulk-operation command (e.g., batch update/delete across many items in one call) rather than iterating via scripts, and independent/hands-on evidence of bulk-operation reliability at scale.", "evidenceIds": [ "databricks-docs-2", "databricks-docs-48", "databricks-docs-49", "databricks-docs-56", "databricks-docs-65", "databricks-probe-rt-1", "databricks-docs-38" ] }, { "productId": "databricks", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Databricks supports rule-like automation via SQL Alerts (monitor query results, evaluate conditions, deliver notifications automatically) and event-driven Lakeflow/Auto Loader pipelines that trigger on new file arrival, plus job triggers and the CLI/SDK for scripting automated actions. However, there is no dedicated general-purpose 'if event then action' rules engine documented beyond these specific mechanisms (alerts, streaming triggers, job schedules). Missing for 10: a unified event-rule/trigger API spanning arbitrary events, independent hands-on validation of alert/trigger reliability, and more detail on custom action types beyond notifications.", "evidenceIds": [ "databricks-docs-26", "databricks-docs-13", "databricks-docs-36", "databricks-docs-1", "databricks-docs-48", "databricks-docs-49" ] }, { "productId": "databricks", "storyId": "automation-scheduled-jobs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Databricks Lakeflow Jobs (referenced via docs-1, docs-40, docs-4/45/50 MCP/agent orchestration context, and CLI docs-2/48/rt-1) natively supports scheduling recurring jobs/workflows, and the CLI/SDK (docs-49) allow programmatic/automated job management for AI-native workflows. Missing for 10: no explicit docs excerpt detailing cron/schedule syntax or trigger configuration, and no independent/hands-on confirmation of scheduling reliability beyond vendor docs.", "evidenceIds": [ "databricks-docs-1", "databricks-docs-40", "databricks-docs-2", "databricks-docs-48", "databricks-docs-49", "databricks-probe-rt-1" ] }, { "productId": "databricks", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Databricks documents automatic versioning for notebooks, Delta table time-travel/rollback, and Unity Catalog lineage/audit logging, which together cover generic version/review/rollback capabilities, but none of this evidence explicitly ties into versioning or rolling back Lakeflow pipelines/Jobs (the actual 'automations') rather than notebooks/tables. Missing for 10: explicit job/pipeline version history and rollback workflow, CI/CD or git-based pipeline versioning evidence, and any hands-on confirmation of rolling back an automation run.", "evidenceIds": [ "databricks-docs-12", "databricks-docs-37", "databricks-docs-54", "databricks-docs-19", "databricks-docs-38" ] }, { "productId": "databricks", "storyId": "bi-driver-connectivity", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs show that external BI tools (Power BI, Tableau, Sigma) can query Databricks metric views directly, and a JDBC-style connection library (Databricks Connect) is documented, indicating standard driver-based BI connectivity without custom glue. However, no dedicated JDBC/ODBC driver certification page or explicit ODBC integration guide is cited in the pack. missing for 10: a direct JDBC/ODBC driver download/certification doc, independent BI-tool hands-on validation.", "evidenceIds": [ "databricks-docs-33", "databricks-docs-64", "databricks-docs-32", "databricks-docs-42" ] }, { "productId": "databricks", "storyId": "bulk-load-object-storage", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Databricks documents ingestion tooling (a connector-selection page for cloud object storage sources) and Auto Loader for incrementally/bulk processing new files as they arrive in cloud storage, which covers CSV/JSON/Parquet ingestion, but no evidence names a single specific command (e.g., COPY INTO) or shows a concrete one-line bulk-load example spanning all three formats. Missing for 10: explicit single documented command syntax for bulk-loading CSV/JSON/Parquet, and independent/hands-on confirmation of ease-of-use.", "evidenceIds": [ "databricks-docs-20", "databricks-docs-13", "databricks-docs-1" ] }, { "productId": "databricks", "storyId": "compliance-attestations", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Databricks' Trust Center compliance page explicitly documents SOC 2 Type II reports and a broader certification portfolio, directly addressing platform-engineer compliance-review needs, and this is reinforced by Unity Catalog's built-in access control, lineage, and audit logging documentation which supports such attestations operationally. Missing for 10: explicit mention of HIPAA and PCI attestations/BAA details in the evidence pack, and no independent/third-party audit confirmation beyond the vendor's own trust page.", "evidenceIds": [ "databricks-supp-compliance", "databricks-docs-19", "databricks-docs-38" ] }, { "productId": "databricks", "storyId": "connector-ecosystem", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Databricks documents an ingestion hub for choosing 'standard connectors by data source' (Lakeflow Connect/Auto Loader) and a documented partner integration for dbt, plus a Marketplace listing partner data/tool assets — showing first-party and partner connector coverage with setup docs. However, the evidence pack lacks an explicit named catalog of SaaS-app/database connectors (e.g., Salesforce, Workday, specific DB connectors) with individual setup guides beyond dbt and generic Auto Loader ingestion. Missing for 10: an exhaustive/documented list of named SaaS and database connectors with per-connector setup instructions, and independent confirmation that connector coverage meets diverse source needs.", "evidenceIds": [ "databricks-docs-20", "databricks-docs-10", "databricks-docs-14", "databricks-docs-35", "databricks-docs-13", "databricks-docs-55", "databricks-docs-69" ] }, { "productId": "databricks", "storyId": "continuous-transformations", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Databricks documents Structured Streaming for continuous/incremental processing with exactly-once guarantees and low-latency options, Lakeflow declarative pipelines with Auto Loader for incremental ingestion/ETL, and Databricks Workflows/Jobs orchestration referenced via CLI/tasks. This directly covers streams, declarative pipelines, and continuous/incremental transformations inside the platform. Missing for 10: no independent/hands-on benchmark validating claimed low-latency figures, and no explicit mention of scheduled/triggered task orchestration UI beyond CLI/Lakeflow docs.", "evidenceIds": [ "databricks-docs-1", "databricks-docs-13", "databricks-docs-36", "databricks-docs-51", "databricks-docs-68", "databricks-docs-40" ] }, { "productId": "databricks", "storyId": "cost-model-clarity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Databricks documents a pay-as-you-go, per-second-billing model (docs-21) and provides account/workspace-level budget tracking tools (docs-58, docs-61) that help monitor spend, but there is no evidence of a pricing calculator, DBU rate tables, or concrete per-workload cost examples that would let a platform engineer pre-estimate a monthly bill before committing. Community comments (databricks-comm-2, databricks-comm-4) describing the platform as 'crazy expensive' with 'surprise gotchas' around serverless pricing further suggest cost estimation in practice is harder than the high-level docs imply, though this falls short of a concrete documented failure of the pricing docs themselves. Missing for 10: a pricing/cost calculator, itemized DBU/unit rate tables, example workload cost breakdowns, and independent confirmation that pre-commitment estimates match actual bills.", "evidenceIds": [ "databricks-docs-21", "databricks-docs-58", "databricks-docs-61", "databricks-comm-2", "databricks-comm-4" ] }, { "productId": "databricks", "storyId": "dataset-marketplace", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Databricks Marketplace is documented as an in-platform catalog where customers can discover, evaluate, and connect directly to third-party datasets (plus AI models, notebooks, apps, MCP servers) from within their workspace, and an Open Marketplace variant even works without a workspace. This directly matches the analyst story of enriching own data with third-party datasets inside the platform. Missing for 10: no independent/hands-on evidence of the enrichment workflow in practice, and no detail on how a discovered dataset is joined/merged with an analyst's own tables.", "evidenceIds": [ "databricks-docs-18", "databricks-docs-34", "databricks-docs-55", "databricks-docs-69" ] }, { "productId": "databricks", "storyId": "dbt-first-class", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Databricks has a dedicated vendor docs page for dbt covering installation, connection, and dbt Core usage (databricks-docs-10, -14, -35), making dbt a documented first-class integration path for data engineers. missing for 10: no evidence of a native dbt-databricks adapter maintenance page, independent community corroboration of dbt workflow quality, or dbt Cloud-specific integration details.", "evidenceIds": [ "databricks-docs-10", "databricks-docs-14", "databricks-docs-35" ] }, { "productId": "databricks", "storyId": "free-evaluation-path", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Databricks offers a genuinely free, self-serve 'Free Edition' (replacing Community Edition) plus a free trial, explicitly for exploring datasets and running real queries/ML without needing a workspace purchase or sales call (docs-66, docs-73, docs-74, docs-39). This directly matches the analyst story of self-serve evaluation on real data. Missing for 10: explicit confirmation that no credit card is required to sign up, and independent/hands-on user corroboration of the free-tier experience (community evidence only discusses paid usage/cost complaints, not the free tier).", "evidenceIds": [ "databricks-docs-66", "databricks-docs-73", "databricks-docs-74", "databricks-docs-39" ] }, { "productId": "databricks", "storyId": "full-sql-surface", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Databricks SQL is documented as ANSI-SQL compliant with Delta Lake extensions (databricks-docs-32) and Unity Catalog governs SQL objects (databricks-docs-56), implying a broad native SQL surface without bolt-on tools, but the evidence pack never explicitly calls out window functions, CTEs, JSON/semi-structured handling, array types, or date/time type richness. missing for 10: explicit doc citations for window functions, CTE support, JSON/semi-structured query functions, array manipulation functions, and native date/time types.", "evidenceIds": [ "databricks-docs-32", "databricks-docs-56", "databricks-docs-19" ] }, { "productId": "databricks", "storyId": "granular-rbac-masking", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Unity Catalog docs describe enforcing access control on tables and objects with grants (databricks-docs-19/38/46/56/70), which supports role-based table-level governance for many teams, but the evidence pack contains no first-party documentation of column-level masking policies or row-level filters. A community report explicitly contradicts the row/column-masking claim, stating Databricks security 'seems lacking - just table level, only in SQL and Spark, none in R' compared to competitors offering table/column/row-level security and dynamic masking (databricks-comm-9). Missing for 10: first-party docs on column masking policies, row-level security/filters, and independent confirmation these work as claimed to resolve the community-reported gap.", "evidenceIds": [ "databricks-docs-19", "databricks-docs-38", "databricks-docs-46", "databricks-docs-70", "databricks-comm-9" ] }, { "productId": "databricks", "storyId": "live-data-sharing", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Delta Sharing (OpenSharing) is documented as a secure data sharing platform for sharing data and AI assets outside the organization without requiring the recipient to be on Databricks, explicitly avoiding data copying/export pipelines, and the Marketplace/Open Marketplace extends this to discovery and listing of shared datasets across organizations. missing for 10: independent hands-on validation of cross-account sharing beyond vendor docs.", "evidenceIds": [ "databricks-docs-17", "databricks-docs-18", "databricks-docs-34", "databricks-docs-55", "databricks-docs-69" ] }, { "productId": "databricks", "storyId": "local-dev-loop", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Databricks offers a free-tier workspace (Free Edition) and local-IDE tooling (Databricks Connect, dbt Core, CLI/SDK) that let a data-engineer author and test transformations from their laptop, but Databricks Connect still executes against real remote (Databricks) compute rather than a true local engine/emulator, and Free Edition is a hosted cloud workspace, not an offline sandbox. Community commentary even notes some engineers prefer spinning up their own local notebook/storage instead, underscoring the lack of a genuine local execution engine. missing for 10: a true local/offline execution engine or emulator that fully mimics production compute without any cloud dependency, and evidence of hands-on validation of that local dev loop.", "evidenceIds": [ "databricks-docs-47", "databricks-docs-64", "databricks-docs-66", "databricks-docs-73", "databricks-docs-14", "databricks-docs-35", "databricks-comm-10" ] }, { "productId": "databricks", "storyId": "managed-continuous-ingestion", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Databricks provides Auto Loader/Lakeflow pipelines to incrementally and efficiently ingest new files as they arrive in cloud storage, plus Structured Streaming for continuous event processing with low latency and exactly-once guarantees on Delta Lake, all as managed serverless/DLT-style pipelines rather than self-managed infra, and a broader ingestion connector catalog for various sources. Missing for 10: independent/hands-on evidence of production reliability at scale for continuous ingestion, and more detail on serverless auto-scaling/operational overhead reduction claims beyond docs.", "evidenceIds": [ "databricks-docs-13", "databricks-docs-1", "databricks-docs-36", "databricks-docs-51", "databricks-docs-68", "databricks-docs-20" ] }, { "productId": "databricks", "storyId": "native-notebooks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Databricks notebooks docs explicitly describe multi-language coauthoring, automatic versioning, and built-in visualizations, and the getting-started guide shows querying Unity Catalog data and visualizing results inline in a notebook — directly matching the analyst story of mixing SQL/Python with inline charts. Missing for 10: explicit documentation of magic-command language switching within a single notebook cell (%sql/%python) and independent/hands-on corroboration beyond first-party docs.", "evidenceIds": [ "databricks-docs-12", "databricks-docs-37", "databricks-docs-23" ] }, { "productId": "databricks", "storyId": "nl-governed-answers", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Genie provides natural language Q&A grounded in organizational data (docs-8, docs-22, docs-57), backed by Unity Catalog governance enforcing access control and lineage (docs-19, docs-38) and metric views providing a semantic layer so metrics are defined once and computed consistently rather than via ad-hoc joins (docs-7, docs-16, docs-42, docs-72). Business-user-focused Genie One interface is explicitly documented for non-technical users (docs-57). missing for 10: independent/hands-on evidence of Genie's accuracy avoiding hallucinated joins in practice, and no community corroboration of semantic grounding quality specifically (community evidence pack is generic platform commentary, not about Genie/semantic layer).", "evidenceIds": [ "databricks-docs-8", "databricks-docs-22", "databricks-docs-57", "databricks-docs-7", "databricks-docs-16", "databricks-docs-42", "databricks-docs-72", "databricks-docs-19", "databricks-docs-38" ] }, { "productId": "databricks", "storyId": "open-table-formats", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm Databricks SQL runs directly on the data lake with ANSI SQL and Delta Lake extensions 'without moving your data' (databricks-docs-32), and Unity Catalog governs tables/external locations directly in object storage (databricks-docs-19, databricks-docs-38, databricks-docs-46). However, the evidence pack never explicitly mentions querying Iceberg tables or plain Parquet files in-place (e.g., via UniForm or external tables) — missing for 10: explicit Iceberg format support, Parquet file querying without ingestion, and independent/hands-on confirmation of in-place multi-format querying.", "evidenceIds": [ "databricks-docs-32", "databricks-docs-19", "databricks-docs-38", "databricks-docs-46", "databricks-docs-56" ] }, { "productId": "databricks", "storyId": "openness-api-parity", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Databricks documents an extensive, versioned REST API surface (workspace-level API reference, SDKs, Terraform provider, CLI that itself is a thin wrapper around the REST API — `databricks api`) and states that Unity Catalog objects can be managed via 'Catalog Explorer, SQL, the Databricks CLI, and REST APIs,' suggesting broad UI/API parity. However, a hands-on community report specifically contradicts full parity, stating 'no good way to get usage info programmatically' and calling out other CLI/SQL config gaps (e.g., can't set Spark config easily), showing some UI-visible functionality isn't cleanly exposed via API. Missing for 10: a documented comprehensive parity guarantee, independent confirmation that admin/UI-only features (budgets, workspace settings) are fully scriptable, and resolution of the community-reported programmatic usage-info gap.", "evidenceIds": [ "databricks-docs-65", "databricks-docs-56", "databricks-docs-49", "databricks-probe-rt-1", "databricks-comm-5" ] }, { "productId": "databricks", "storyId": "openness-full-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Databricks stores data in the open Delta Lake/Parquet format and offers Delta Sharing (OpenSharing) to share or export data outside the organization regardless of platform, plus CLI/SDK/REST API access for programmatic extraction of data and metadata. However, there's no single documented 'export everything and leave' workflow or bulk account-export tool, and community commentary (e.g., migrating workloads to Postgres) suggests migration is done piecemeal rather than via a turnkey export feature. missing for 10: a dedicated full-account/bulk data export or migration tool, and independent hands-on confirmation of frictionless full data egress.", "evidenceIds": [ "databricks-docs-17", "databricks-docs-18", "databricks-docs-56", "databricks-docs-65", "databricks-comm-12" ] }, { "productId": "databricks", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Databricks is a proprietary commercial data/AI platform; there is no evidence of its core source code being available under an open license, and 'read the product's source' is not a fair expectation for this category of SaaS platform (unlike an open-source library or framework). This axis does not apply.", "evidenceIds": [] }, { "productId": "databricks", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Databricks is offered exclusively as a managed cloud service (AWS/Azure/GCP workspaces, pay-as-you-go billing) with no documented option to self-host the core platform on your own infrastructure; the closest analog (Free Edition) is still a hosted SaaS trial, not a self-hostable deployment. Community evidence even references how self-hosting Spark used to be painful specifically because Databricks replaced that with a hosted service, reinforcing that the core product is not self-hostable.", "evidenceIds": [ "databricks-docs-21", "databricks-docs-73", "databricks-comm-6" ] }, { "productId": "databricks", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses regional deployment, data residency controls, or workspace region selection for Databricks; the compliance mention only covers SOC 2 attestations, not data-location choice.", "evidenceIds": [] }, { "productId": "databricks", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers Databricks' data platform, governance, agent, and dev-tool features but contains no mention of any control to opt out of, or prevent, customer data being used to train Databricks' (or third-party) AI models — no data-use policy, model-training opt-out setting, or contractual guarantee is cited. missing for 10: an explicit AI-training opt-out/data-use policy, documentation of contractual or technical controls preventing model training on customer data, and any independent confirmation of such a guarantee.", "evidenceIds": [] }, { "productId": "databricks", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence covers Unity Catalog governance (access control, lineage, audit logging) and Delta Lake time-travel/versioning, but nothing documents user-controllable data retention periods, deletion/erasure APIs, or lifecycle policies for AI-native privacy control. Missing for 10: explicit retention configuration, right-to-delete/erasure mechanisms, data lifecycle/expiry policy documentation, and any independent verification of deletion behavior.", "evidenceIds": [ "databricks-docs-19", "databricks-docs-38", "databricks-docs-54", "databricks-docs-70" ] }, { "productId": "databricks", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item addresses telemetry/usage-tracking opt-out settings for Databricks; documentation covers compliance/SOC2 but not a specific telemetry toggle. missing for 10: any mention of telemetry collection, opt-out settings, or usage-tracking controls.", "evidenceIds": [] }, { "productId": "databricks", "storyId": "query-profiling", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly describe inspecting execution plans to find bottlenecks/optimization opportunities and automatic insights/recommendations for inefficient queries, directly matching the story. This is first-party documentation without independent hands-on corroboration or deeper detail on cost/spill/skew diagnostics. Missing for 10: independent/community validation of query profile usability, and more detail on cost breakdown metrics beyond the brief doc snippets.", "evidenceIds": [ "databricks-docs-6", "databricks-docs-60", "databricks-docs-32" ] }, { "productId": "databricks", "storyId": "runaway-cost-controls", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Databricks docs show account-level budgets for tracking spend by team/project/workspace and query-level alerting when conditions are met (databricks-docs-58, -61, -26), which partially covers cost monitoring, but the evidence never documents an automatic enforcement mechanism (e.g., warehouse auto-stop for idle compute or query kill/timeout) that actually halts a runaway query or idle cluster overnight — budgets are described as tracking/notifying, not stopping. missing for 10: explicit auto-suspend/auto-stop behavior for idle compute, automatic termination of runaway queries, and independent confirmation that budgets can enforce hard spend caps rather than just alert.", "evidenceIds": [ "databricks-docs-58", "databricks-docs-61", "databricks-docs-26" ] }, { "productId": "databricks", "storyId": "semantic-model-metrics", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Databricks Unity Catalog Metric Views let analysts define metrics, dimensions, and joins once ('you define the metric once... users can group by any available field. The query engine generates the correct computation') and this governed semantic layer is queryable consistently from SQL, AI/BI dashboards, Genie natural-language Q&A, and external BI tools like Power BI, Tableau, and Sigma. missing for 10: independent/hands-on corroboration of consistency across tools, and detail on join declaration beyond metric definition.", "evidenceIds": [ "databricks-docs-7", "databricks-docs-16", "databricks-docs-72", "databricks-docs-33", "databricks-docs-42", "databricks-docs-8" ] }, { "productId": "databricks", "storyId": "streaming-ingest-latency", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Databricks documents Structured Streaming as a first-class streaming API that treats streaming like batch, supports Delta Lake tables as streaming sources/sinks with exactly-once guarantees, and Auto Loader for incremental ingestion from cloud storage, with end-to-end latency claimed as low as 5ms — directly supporting near-real-time queryable ingestion. missing for 10: independent/hands-on benchmark confirming 'queryable within seconds' end-to-end, and no community corroboration specific to streaming latency claims.", "evidenceIds": [ "databricks-docs-51", "databricks-docs-13", "databricks-docs-68", "databricks-docs-36", "databricks-docs-1" ] }, { "productId": "databricks", "storyId": "time-travel-restore", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Databricks documents Delta Lake history/time travel explicitly: querying tables as of a past version/timestamp and restoring/rolling back dropped or corrupted tables using history information, directly matching the story. This is a core, well-documented Delta Lake feature integrated into Databricks SQL/Unity Catalog. Missing for 10: no independent/hands-on corroboration of restore-after-drop specifically, beyond first-party docs.", "evidenceIds": [ "databricks-docs-54" ] }, { "productId": "motherduck", "storyId": "agent-runs-sql-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "MotherDuck ships official CLI/SDK/API access (docs-1, docs-2, docs-25), a documented auth flow with local credential caching (docs-8, docs-20), and both local and remote MCP servers that execute read-only/read-write SQL and support schema/context discovery via 'Guides' (docs-30, docs-49, docs-38), letting an agent authenticate, discover, query and read results with no dashboard involved. This is corroborated by runtime probes showing a real authenticated MCP handshake plus executed aggregate SQL query (motherduck-probe-rt-2) and a live OAuth-gated remote MCP endpoint (motherduck-probe-rt-3). Missing for 10: an end-to-end demonstration against an authenticated (non-keyless) MotherDuck cloud database showing schema discovery + query + result retrieval in one continuous agent session, and clearer documentation of a dedicated schema-listing API beyond Guides.", "evidenceIds": [ "motherduck-docs-1", "motherduck-docs-8", "motherduck-docs-20", "motherduck-docs-30", "motherduck-docs-31", "motherduck-docs-38", "motherduck-docs-49", "motherduck-probe-rt-2", "motherduck-probe-rt-3", "motherduck-probe-5" ] }, { "productId": "motherduck", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "MotherDuck publishes a working llms.txt (HTTP 200, verified content) and goes further with a first-class agent-discovery surface: an agentskills.io-conformant manifest at /.well-known/agent-skills/index.json, /.well-known/mcp.json, and an every-page-as-markdown convention, explicitly framed as agent-oriented docs. This directly matches the story of pointing an agent at llms.txt or agent docs. Missing for 10: independent/community confirmation that an agent successfully consumed these files in practice (only first-party probes cited).", "evidenceIds": [ "motherduck-probe-1", "motherduck-probe-rt-4", "motherduck-probe-4" ] }, { "productId": "motherduck", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "MotherDuck's 'Dives' feature and MCP integration let users generate charts, dashboards, and insights via natural language prompts ('build a Dive with a chart of top 10 products', 'add a filter for US region'), with conversational iteration and no SQL required, confirmed by both docs and a runtime probe showing a working MCP tool call returning real query results. Missing for 10: independent/community hands-on validation specifically of the Dives/insights UX (community evidence covers MCP and SQL editor features but not Dives directly), and no evidence of proactive/unprompted AI-generated suggestions (all examples are user-initiated prompts).", "evidenceIds": [ "motherduck-docs-4", "motherduck-docs-14", "motherduck-docs-15", "motherduck-docs-23", "motherduck-docs-26", "motherduck-docs-27", "motherduck-docs-35", "motherduck-docs-40", "motherduck-probe-rt-2", "motherduck-probe-rt-3" ] }, { "productId": "motherduck", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "MotherDuck documents scheduling Python jobs for ingest/transformation/operational tasks (docs-3), which supports background automations, but there is no evidence of a broader autonomous agent-scheduling framework, triggers, or orchestration for AI-driven background workflows beyond scheduled jobs. Missing for 10: evidence of autonomous agent-initiated scheduling/triggers, monitoring/retry of automated jobs, and independent confirmation of reliability of scheduled runs.", "evidenceIds": [ "motherduck-docs-3", "motherduck-docs-30", "motherduck-docs-49" ] }, { "productId": "motherduck", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "MotherDuck ships 'Dives,' a native feature that lets users build and iteratively refine dashboards via natural-language prompts ('add a filter for US region', 'switch to a bar chart') directly inside the product, and it also exposes a remote MCP server for natural-language SQL and visualization generation. This is real built-in agentic behavior, but the natural-language SQL/agent capability is delivered mainly via MCP, which requires an external AI client (e.g., Claude) rather than a self-contained chat assistant, and there's no independent/hands-on corroboration of the Dives assistant experience. Missing for 10: independent hands-on validation of Dives as a true in-product chat assistant, and clarity on whether any AI runs natively without an external MCP-connected LLM client.", "evidenceIds": [ "motherduck-docs-4", "motherduck-docs-14", "motherduck-docs-15", "motherduck-docs-23", "motherduck-docs-27", "motherduck-docs-35", "motherduck-docs-26", "motherduck-docs-5" ] }, { "productId": "motherduck", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "MotherDuck supports headless/CI automation via CLI, Python SDKs, saved credentials for non-interactive sessions, and scheduled Python jobs for ingest/transformation; dbt integration and SaaS Mode further support scripted, non-interactive pipelines. Runtime probes confirm a keyless CLI roundtrip and a fully automatable MCP server flow, both consistent with CI usage. Missing for 10: an explicit CI/CD example (e.g., GitHub Actions workflow) or documented non-interactive auth token flow for pipelines.", "evidenceIds": [ "motherduck-docs-3", "motherduck-docs-8", "motherduck-docs-20", "motherduck-docs-9", "motherduck-docs-19", "motherduck-probe-rt-1", "motherduck-probe-rt-2" ] }, { "productId": "motherduck", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "motherduck", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "MotherDuck ships both a remote hosted MCP server (api.motherduck.com/mcp, OAuth-secured) and an official local MCP server (mcp-server-motherduck), both well documented with read/write SQL tool access, and runtime probes confirm the local server completes a full stdio handshake and executes a real query, while the remote endpoint is live and speaks proper MCP OAuth. Missing for 10: no independent third-party hands-on report of the remote MCP server itself (only official docs and probes) and no community corroboration of production-grade reliability at scale.", "evidenceIds": [ "motherduck-docs-5", "motherduck-docs-13", "motherduck-docs-22", "motherduck-docs-28", "motherduck-docs-38", "motherduck-docs-49", "motherduck-probe-4", "motherduck-probe-rt-2", "motherduck-probe-rt-3" ] }, { "productId": "motherduck", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "MotherDuck ships natural-language interaction via 'Dives' (NL-to-dashboard generation with conversational refinement like 'add a filter for US region') and both local and remote MCP servers that let AI agents run SQL from natural-language prompts without writing SQL themselves; runtime probes confirm the MCP server actually executes real analytical queries end-to-end. missing for 10: independent/community validation specifically of the natural-language accuracy (only vendor docs and technical probes, no hands-on user review of NL command quality).", "evidenceIds": [ "motherduck-docs-4", "motherduck-docs-5", "motherduck-docs-14", "motherduck-docs-23", "motherduck-docs-26", "motherduck-docs-27", "motherduck-docs-35", "motherduck-docs-40", "motherduck-probe-rt-2", "motherduck-probe-rt-3", "motherduck-probe-rt-4" ] }, { "productId": "motherduck", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "MotherDuck relies on the DuckDB CLI (an official, third-party-maintained but DuckDB-branded CLI) to connect directly to MotherDuck, and docs describe CLI-based authentication/credential caching, but there is no MotherDuck-specific CLI (e.g., a 'motherduck' binary) with agentic/AI-native command features documented. missing for 10: a dedicated MotherDuck-branded CLI tool, evidence of AI-native/agentic command support within the CLI itself, and independent hands-on confirmation of CLI workflows beyond basic connection.", "evidenceIds": [ "motherduck-docs-2", "motherduck-docs-8", "motherduck-docs-20", "motherduck-probe-5" ] }, { "productId": "motherduck", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "MotherDuck publishes a documented public API surface: official SDKs/drivers in multiple languages, a live OpenAPI spec (probe confirms HTTP 200 with 'openapi' key), and a documented SQL/MCP interface with both local and remote MCP servers verified working end-to-end at runtime. Machine-readable agent-skills and llms.txt manifests further support programmatic/AI-native access.\n\nmissing for 10: no independent third-party API client review beyond MotherDuck's own docs/probes.", "evidenceIds": [ "motherduck-probe-3", "motherduck-docs-1", "motherduck-docs-25", "motherduck-probe-rt-2", "motherduck-probe-rt-3", "motherduck-probe-rt-4", "motherduck-docs-49" ] }, { "productId": "motherduck", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "MotherDuck documents concrete least-privilege options for agent access: restricting an AI assistant/MCP client to read-only SQL access, SaaS Mode to limit local environment interaction, and role/account-scoped Shares for multi-tenant isolation, plus a probed OAuth-protected resource flow on the hosted remote MCP endpoint (401 + WWW-Authenticate Bearer) confirming a real scoped-auth mechanism. However, there is no documented ability to mint a distinct, narrowly-scoped API key/token per agent (e.g. table- or query-level permission grants beyond read/write toggle), so it's read-only-vs-read-write plus share-based isolation rather than fine-grained credential issuance. Missing for 10: granular per-resource API key scoping beyond binary read-only/read-write, first-class 'create scoped credential for agent X' workflow, independent verification of the read-only restriction in practice.", "evidenceIds": [ "motherduck-docs-31", "motherduck-docs-38", "motherduck-docs-49", "motherduck-docs-9", "motherduck-docs-47", "motherduck-docs-43", "motherduck-probe-rt-3" ] }, { "productId": "motherduck", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "MotherDuck documents official SDKs across multiple languages (Python, Node.js, Go, Rust, R, Java) built on DuckDB, plus an official CLI, MCP servers (local and remote), and an OpenAPI spec confirmed by probes. This directly supports AI-native/agentic developers building against official interfaces, with runtime probes confirming keyless SDK, CLI, and MCP functionality. Missing for 10: independent third-party benchmarking or reviews of SDK completeness/quality beyond vendor docs.", "evidenceIds": [ "motherduck-docs-25", "motherduck-docs-1", "motherduck-probe-3", "motherduck-probe-4", "motherduck-probe-5", "motherduck-probe-rt-1", "motherduck-probe-rt-2", "motherduck-probe-rt-3" ] }, { "productId": "motherduck", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "MotherDuck offers MCP servers, scheduled jobs, and data sharing, but nothing in the evidence describes a webhook subscription mechanism for event notifications. Missing for 10: any webhook endpoint registration, event-driven push notification system, or documentation of subscribable events.", "evidenceIds": [] }, { "productId": "motherduck", "storyId": "ai-sql-assistant", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "MotherDuck ships both a remote MCP server and a Dives natural-language interface that write, execute, and iteratively refine SQL/visualizations against user schemas via conversational prompts (e.g., 'add a filter for US region', 'build a chart of top 10 products'), and runtime probes confirm the MCP handshake and real query execution work end-to-end. Guides let the assistant learn schema/metric context to produce accurate SQL without repetitive prompting, directly matching the story's 'against my schemas' requirement. missing for 10: independent hands-on evidence specifically of the assistant 'explaining' or fixing broken SQL (evidence focuses on generation/iteration, not debugging existing queries), and broader third-party corroboration beyond vendor docs.", "evidenceIds": [ "motherduck-docs-5", "motherduck-docs-14", "motherduck-docs-23", "motherduck-docs-26", "motherduck-docs-30", "motherduck-docs-38", "motherduck-probe-rt-2", "motherduck-probe-rt-3" ] }, { "productId": "motherduck", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows an OpenAPI spec exists at /openapi.json (a machine-readable schema) and various driver/API docs, but there is no evidence of an actual interactive reference UI (e.g., Swagger-style 'try it' console) or runnable in-browser examples tied to that spec. missing for 10: an interactive API explorer page, runnable/live code examples, and any confirmation the OpenAPI spec is surfaced as a browsable interactive reference.", "evidenceIds": [ "motherduck-probe-3", "motherduck-docs-1" ] }, { "productId": "motherduck", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A live OpenAPI JSON spec is confirmed at https://motherduck.com/openapi.json (HTTP 200, contains 'openapi' key), directly satisfying the machine-readable API spec requirement; MotherDuck also publishes additional machine-readable discovery surfaces (llms.txt, agent-skills manifest, mcp.json). Missing for 10: independent third-party confirmation/discussion of the OpenAPI spec's completeness or usage in the community evidence.", "evidenceIds": [ "motherduck-probe-3", "motherduck-probe-rt-4", "motherduck-probe-1" ] }, { "productId": "motherduck", "storyId": "api-sandbox", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "MotherDuck supports local/in-memory DuckDB development (via CLI or local MCP server) that lets an agent run real analytical queries with zero MotherDuck account or production data touched, and offers read-only restriction and SaaS Mode to limit AI/agent access to local environment before syncing to cloud (docs-18, docs-31, docs-9, probe-rt-2). However, there is no dedicated 'sandbox' or database-branching feature that clones or isolates production data for testing—developers must manually use local files or in-memory DBs instead. Missing for 10: an explicit sandbox/branch/clone-of-production feature, first-party guidance framing this as a test-vs-prod workflow, and independent confirmation of safe isolation from prod data.", "evidenceIds": [ "motherduck-docs-18", "motherduck-docs-9", "motherduck-docs-31", "motherduck-probe-rt-2", "motherduck-docs-21" ] }, { "productId": "motherduck", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While MotherDuck exposes an OpenAPI spec and various SDKs/drivers, the evidence pack contains no mention of API versioning scheme or a documented deprecation policy for its APIs, drivers, or MCP servers. Missing for 10: any docs describing API version numbers, backward-compatibility guarantees, or a deprecation/sunset policy.", "evidenceIds": [ "motherduck-probe-3" ] }, { "productId": "motherduck", "storyId": "audit-lineage", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows billing/usage dashboards (docs-44) and secret scoping/sharing permissions (docs-43, docs-47) but nothing describing query-level audit logs tied to user identity or column-level data lineage tracking. missing for 10: audit log documentation (who ran what query, when), column-level lineage tracking or metadata catalog, any independent verification of these governance features.", "evidenceIds": [ "motherduck-docs-44", "motherduck-docs-43", "motherduck-docs-47" ] }, { "productId": "motherduck", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "SQL is inherently set-based, and MotherDuck documents bulk data loading (from Postgres/MySQL, S3, DuckDB files) and read-write SQL execution via both local and remote MCP servers, which lets an AI agent issue bulk INSERT/UPDATE/COPY operations across many rows or tables in one call. However, there's no explicit documentation of an AI-native 'bulk operations across many items' primitive (e.g., batch MCP tool calls, bulk-apply-to-many-objects UX) — the evidence shows general bulk data movement/SQL rather than an agent-oriented batch-processing feature. Missing for 10: explicit agent-facing batch/bulk-operation tooling or examples, and independent confirmation of large-scale bulk SQL runs via MCP.", "evidenceIds": [ "motherduck-docs-6", "motherduck-docs-12", "motherduck-docs-29", "motherduck-docs-42", "motherduck-docs-48", "motherduck-docs-49", "motherduck-probe-rt-2" ] }, { "productId": "motherduck", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows scheduled Python jobs (time-based cron-style automation) but no capability for defining rules that trigger actions automatically on data or system events (e.g., event-driven triggers, alerts, webhooks on conditions). This is a fair axis for a data platform, but no evidence supports event-driven rule automation.", "evidenceIds": [ "motherduck-docs-3" ] }, { "productId": "motherduck", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs mention running Python jobs 'on a schedule for ingest, transformation, and operational tasks' (motherduck-docs-3), indicating some support for recurring automation, but there is no detail on how scheduling is configured (cron syntax, UI, triggers) nor independent verification of this capability. Missing for 10: documented scheduling mechanism/UI, examples of workflow definitions, and independent or hands-on confirmation that recurring jobs actually run reliably.", "evidenceIds": [ "motherduck-docs-3" ] }, { "productId": "motherduck", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "MotherDuck documents scheduled Python jobs and AI-generated Dives, but nothing in the evidence describes versioning, reviewing, or rolling back these automations/schedules — no changelog, diff, approval, or rollback mechanism is mentioned anywhere in the docs or community evidence.", "evidenceIds": [] }, { "productId": "motherduck", "storyId": "bi-driver-connectivity", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs claim Postgres wire-protocol compatibility ('Use any Postgres-compatible tool, no DuckDB install needed') which implies some BI tools could connect without custom glue, and DuckDB SDKs exist for several languages, but there is no explicit mention of JDBC/ODBC drivers or named, documented BI-tool integrations (e.g., Tableau, Power BI, Looker) in the evidence pack. Missing for 10: explicit JDBC/ODBC driver documentation, named/certified BI-tool integration guides, and any hands-on confirmation that dashboards connect without custom glue.", "evidenceIds": [ "motherduck-docs-17", "motherduck-docs-25" ] }, { "productId": "motherduck", "storyId": "bulk-load-object-storage", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "MotherDuck's docs reference loading data from object storage and databases, and support S3 credential/secret management for cloud storage access, implying use of DuckDB's native read_csv/read_parquet/read_json functions against S3 paths. However, the evidence pack never shows an explicit single documented command example bulk-loading CSV, JSON, and Parquet uniformly from cloud storage. Missing for 10: explicit command syntax/example for each format (CSV, JSON, Parquet) from S3/GCS, and any independent hands-on confirmation of a single-command bulk load.", "evidenceIds": [ "motherduck-docs-6", "motherduck-docs-43", "motherduck-docs-48", "motherduck-docs-29", "motherduck-docs-42" ] }, { "productId": "motherduck", "storyId": "compliance-attestations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack entry references SOC 2, HIPAA, PCI, or any formal compliance/attestation program, trust center, or security review documentation; the pack covers connectivity, MCP, sharing, and billing but not compliance certifications. missing for 10: SOC 2 report/attestation, HIPAA compliance statement, PCI attestation, trust center or security whitepaper, audit/certification documentation.", "evidenceIds": [] }, { "productId": "motherduck", "storyId": "connector-ecosystem", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "MotherDuck documents loading from other databases (Postgres/MySQL) and object storage (S3), plus dbt as an ETL/transformation integration point, all with setup docs — covering the 'databases' and 'ETL/ELT tools' parts of the story. However there is no evidence of a broad partner/connector catalog for SaaS apps (e.g., Salesforce, Stripe, HubSpot) or a marketplace of pre-built connectors comparable to dedicated ETL platforms. Missing for 10: documented SaaS-app connectors, a partner connector directory/marketplace, and coverage beyond Postgres/MySQL/S3/dbt.", "evidenceIds": [ "motherduck-docs-6", "motherduck-docs-42", "motherduck-docs-48", "motherduck-docs-19", "motherduck-docs-7", "motherduck-docs-29" ] }, { "productId": "motherduck", "storyId": "continuous-transformations", "verdict": "partial", "quality": 3, "confidence": "medium", "rationale": "MotherDuck's docs mention running scheduled Python jobs for ingest/transformation (docs-3) and dbt integration supporting incremental models (docs-7/19/32), which gives some incremental transformation capability, but there is no evidence of native streaming primitives, continuous queries, declarative pipeline objects, or task/stream constructs (à la Snowflake Streams & Tasks or Materialize) — MotherDuck is positioned as a serverless analytical warehouse, not a streaming engine. Missing for 10: native stream/CDC ingestion, continuous query execution, declarative pipeline DSL, task scheduling beyond generic Python cron jobs, and any independent evidence of these being used for continuous transformations.", "evidenceIds": [ "motherduck-docs-3", "motherduck-docs-7", "motherduck-docs-19", "motherduck-docs-32" ] }, { "productId": "motherduck", "storyId": "cost-model-clarity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "MotherDuck documents billing mechanics well—per-second billing, sized instances (Pulse–Giga), a free Lite tier, and a billing dashboard with SQL views to monitor consumption and identify savings—giving a platform engineer real levers to reason about cost drivers. However, the evidence pack never surfaces an actual price list or $/instance-hour rate card needed to project a concrete monthly bill before signing up. Missing for 10: published per-instance-size dollar rates, a documented cost-estimation worksheet/example calculation, and independent confirmation that the dashboard estimates match real invoices.", "evidenceIds": [ "motherduck-docs-39", "motherduck-docs-50", "motherduck-docs-41", "motherduck-docs-44", "motherduck-supp-free-plan", "motherduck-supp-cost-controls" ] }, { "productId": "motherduck", "storyId": "dataset-marketplace", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "MotherDuck's docs describe secure data *sharing* (grants, roles, presets, share links like md:_share/hacker_news/...) for collaboration within or across an organization, but there is no evidence of a curated marketplace of third-party datasets from external providers that an analyst could browse and attach for enrichment.", "evidenceIds": [ "motherduck-docs-10", "motherduck-docs-11", "motherduck-docs-37", "motherduck-docs-47", "motherduck-docs-24" ] }, { "productId": "motherduck", "storyId": "dbt-first-class", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "MotherDuck's docs explicitly document dbt integration via the dbt-duckdb adapter, noting the only change needed is the `path:` setting for MotherDuck, and evidence shows dbt workloads sized to specific instances (e.g. 'run dbt workloads on a Standard'), indicating dbt is a recognized first-class workflow. missing for 10: no independent/hands-on report of a real dbt run against MotherDuck, and no dedicated MotherDuck-branded dbt adapter (relies on community dbt-duckdb rather than a vendor-authored adapter).", "evidenceIds": [ "motherduck-docs-7", "motherduck-docs-19", "motherduck-docs-32", "motherduck-docs-50" ] }, { "productId": "motherduck", "storyId": "free-evaluation-path", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly confirm a 7-day free trial with no credit card, followed by a free Lite plan (no credit card, 10GB storage, free monthly compute), and billing/monitoring tools let analysts run real queries on real data without a sales call. Runtime probes further show real analytical queries executing without any account at all via local DuckDB/MCP. Missing for 10: independent third-party confirmation of the no-CC signup flow beyond vendor docs.", "evidenceIds": [ "motherduck-supp-free-plan", "motherduck-supp-cost-controls", "motherduck-docs-44", "motherduck-probe-rt-1" ] }, { "productId": "motherduck", "storyId": "full-sql-surface", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "MotherDuck runs on DuckDB and evidence confirms native CTE support (praised CTE inspection feature) and a real keyless aggregate GROUP BY analytical query executed without extensions, suggesting a native, non-bolt-on SQL engine. However, the pack contains no explicit documentation or examples of window functions, semi-structured JSON handling, array types, or rich date/time types. Missing for 10: explicit docs/examples of window functions, JSON/semi-structured querying, array types, and date/time type support.", "evidenceIds": [ "motherduck-comm-7", "motherduck-probe-rt-1", "motherduck-probe-1", "motherduck-docs-36" ] }, { "productId": "motherduck", "storyId": "granular-rbac-masking", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "MotherDuck documents role-based Shares that grant read access to users/roles and can restrict exposure to specific tables/views via include patterns, plus preset roles for tenant isolation — covering table-level RBAC for multi-tenant use. However, there is no evidence of column-level masking policies or row-level security filters, which the story explicitly requires. Missing for 10: column masking policies, row-level security/filtering, unified role hierarchy spanning column+row+table grants.", "evidenceIds": [ "motherduck-docs-10", "motherduck-docs-11", "motherduck-docs-47", "motherduck-docs-37" ] }, { "productId": "motherduck", "storyId": "live-data-sharing", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "MotherDuck's Sharing Data docs describe granting read access to specific users, roles, or accounts (including cross-org) via Shares without copying data, with the shared database attachable directly (e.g. `md:_share/...`), and include ways to restrict exposed tables/views — exactly a live-share model without export pipelines. This is corroborated by concrete attach syntax in the MCP getting-started doc showing a live shared dataset being queried directly. missing for 10: independent/hands-on confirmation of cross-organization (not just cross-user) sharing in production and any limits/latency of live share updates.", "evidenceIds": [ "motherduck-docs-37", "motherduck-docs-10", "motherduck-docs-11", "motherduck-docs-47", "motherduck-docs-24" ] }, { "productId": "motherduck", "storyId": "local-dev-loop", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "MotherDuck is built on DuckDB, which runs fully local/embedded (confirmed by runtime probe showing a keyless local DuckDB engine executing real analytical queries with zero accounts or config), and docs explicitly describe a 'develop and iterate locally, then share and scale in the cloud when you need it' workflow plus a free Lite plan/trial with no credit card for early dev use. This gives data engineers a genuine free/local dev loop before touching production compute. Missing for 10: no first-party 'emulator' or sandbox mode specific to MotherDuck's cloud extensions (e.g., DuckLake, MotherDuck-specific SQL features) that fully mirrors production compute locally, and no independent case study explicitly describing a local-dev-then-promote-to-prod workflow.", "evidenceIds": [ "motherduck-docs-18", "motherduck-probe-rt-1", "motherduck-supp-free-plan", "motherduck-docs-16" ] }, { "productId": "motherduck", "storyId": "managed-continuous-ingestion", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "MotherDuck documents scheduled Python jobs for ingest/transformation and connectors for loading data from other databases and object storage (S3, Postgres, MySQL) without managing your own infra, which reduces pipeline-ops burden. However, this is scheduled/batch-oriented rather than a managed continuous/event-driven ingestion service that automatically picks up new files or streaming events as they arrive — there's no CDC, file-watcher, or event-trigger ingestion documented. Missing for 10: evidence of true event-driven/streaming ingestion (e.g., auto-detecting new files in a bucket, Kafka/CDC connectors, or push-based triggers) rather than cron-style scheduled jobs.", "evidenceIds": [ "motherduck-docs-3", "motherduck-docs-6", "motherduck-docs-42", "motherduck-docs-48", "motherduck-docs-45", "motherduck-docs-36" ] }, { "productId": "motherduck", "storyId": "native-notebooks", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "MotherDuck's UI supports SQL notebooks with inline results and AI-generated charts (Instant SQL, Dives with GROUP BY visualizations, conversational chart edits), and Python is supported as a first-class SDK/scheduled-job language, but there is no evidence of a single first-party notebook interface that mixes SQL and Python cells together with inline charts as the story describes. missing for 10: evidence of Python cells inline within the same notebook/workspace as SQL cells, evidence of mixed-language notebook execution rather than separate SQL editor vs Python SDK/job usage.", "evidenceIds": [ "motherduck-docs-3", "motherduck-docs-4", "motherduck-docs-14", "motherduck-docs-23", "motherduck-comm-7" ] }, { "productId": "motherduck", "storyId": "nl-governed-answers", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "MotherDuck ships concrete building blocks for this story: MCP-based natural-language querying (Dives, remote/local MCP servers) and 'Guides' — markdown docs capturing org metric definitions, join conventions, and domain context so agents produce accurate SQL without repetitive prompting, i.e. a governance/semantic-grounding mechanism directly aimed at preventing hallucinated joins. Governance controls (read-only restriction, SaaS Mode, role-based Shares) reinforce this for business users. Missing for 10: independent/hands-on evidence that Guides actually reduce hallucinated joins in practice, no case study or benchmark showing accuracy improvement, and the semantic layer relies on manually authored Guides rather than an automated governed semantic model.", "evidenceIds": [ "motherduck-docs-30", "motherduck-docs-31", "motherduck-docs-38", "motherduck-docs-49", "motherduck-docs-26", "motherduck-docs-14", "motherduck-docs-23", "motherduck-probe-rt-2", "motherduck-probe-rt-3" ] }, { "productId": "motherduck", "storyId": "open-table-formats", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "MotherDuck's docs show S3 object-storage integration (secrets for S3 credentials) and DuckLake for building a lakehouse on top of 'your own files' or bring-your-own object storage, implying in-place querying of external storage rather than mandatory ingestion. However, the pack never explicitly names Iceberg or Delta table format support, or demonstrates querying Parquet/Iceberg/Delta files directly without first loading them into MotherDuck-managed storage — most 'loading data' docs (docs-6, docs-29, docs-42) describe explicit load/upload workflows instead. Missing for 10: explicit documentation or example of querying Iceberg/Delta tables in place, a worked example reading Parquet directly from S3 without a load step, and independent/hands-on confirmation of zero-copy querying against object storage.", "evidenceIds": [ "motherduck-docs-45", "motherduck-docs-48", "motherduck-docs-36", "motherduck-docs-6", "motherduck-docs-43" ] }, { "productId": "motherduck", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "MotherDuck exposes a broad API surface (SDKs in Python/Node/Go/Rust/R/Java, a documented OpenAPI spec, SQL access, and both local/remote MCP servers that can execute read/write SQL and even generate visualizations conversationally), giving AI-native users substantial parity with UI capabilities like Dives and querying. However, several UI-only surfaces (billing dashboard, org/user management console, Instant SQL editor features) have no documented API/MCP equivalent, and no evidence explicitly confirms full feature parity across the board. Missing for 10: explicit documentation or confirmation that admin/billing/org-management UI actions are also API-accessible, and independent verification of full parity beyond query/visualization workflows.", "evidenceIds": [ "motherduck-docs-1", "motherduck-docs-25", "motherduck-docs-26", "motherduck-docs-49", "motherduck-probe-3", "motherduck-probe-rt-2", "motherduck-probe-rt-3", "motherduck-docs-44" ] }, { "productId": "motherduck", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "MotherDuck is built on DuckDB and documents local-first workflows (upload/download DuckDB files, local CLI/SDKs in Python/Node/Go/Rust/R/Java), implying data can move between local open DuckDB files and the cloud service, but there is no explicit documentation of a bulk 'export all your data to open formats (Parquet/CSV) and migrate away' workflow or data-portability guarantee. missing for 10: an explicit export-all-data/backup feature or migration guide, independent confirmation that a full account's data can be extracted losslessly, and any statement addressing vendor lock-in/exit.", "evidenceIds": [ "motherduck-docs-16", "motherduck-docs-18", "motherduck-docs-25", "motherduck-docs-6", "motherduck-probe-rt-1" ] }, { "productId": "motherduck", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "MotherDuck is a proprietary managed SaaS data warehouse; the evidence pack shows open-source ancillary tools (mcp-server-motherduck) but no indication that MotherDuck's core service/source itself is released under an open license. Absence of evidence for an applicable openness axis means the verdict is none.", "evidenceIds": [] }, { "productId": "motherduck", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "MotherDuck is explicitly a managed serverless cloud data warehouse; while DuckDB itself is open-source and can run locally, the core MotherDuck service (multi-tenant cloud engine, billing, sharing, dives, remote MCP) is not offered as a self-hosted deployment. Community discussion explicitly surfaces confusion about whether MotherDuck's editor/UI features are available on-prem, with no confirming answer that self-hosting is possible.", "evidenceIds": [ "motherduck-comm-8", "motherduck-docs-18", "motherduck-probe-1" ] }, { "productId": "motherduck", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack material addresses data region/residency selection, geographic data center choice, or compliance certifications tied to storage location; the pack covers connectivity, sharing, MCP, and billing but nothing about where data physically resides. missing for 10: region selection options, data residency documentation, compliance/geo controls, any mention of storage location choice.", "evidenceIds": [] }, { "productId": "motherduck", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no statement about whether MotherDuck uses customer data to train AI models, nor any opt-out/data-training policy control; this is a fair question given MotherDuck's AI features (Dives, natural language MCP querying) but no documentation addresses it.", "evidenceIds": [] }, { "productId": "motherduck", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers authentication, sharing, secrets scoping, and MCP access controls but contains no documentation of data retention periods, deletion APIs, or purge/right-to-be-forgotten mechanisms for a MotherDuck-hosted database. As a cloud data warehouse, retention/deletion controls are a fair and expected axis, but no citation here shows explicit user-facing retention or deletion controls beyond generic DROP TABLE-type SQL implied by being a database.", "evidenceIds": [] }, { "productId": "motherduck", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers SaaS Mode (limiting local environment interaction), secret scoping, and billing/usage dashboards, but none of it addresses telemetry or product usage tracking opt-out specifically — these are different concerns (environment access control and cost monitoring, not telemetry collection).", "evidenceIds": [] }, { "productId": "motherduck", "storyId": "query-profiling", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack mentions a query-performance doc and billing/usage dashboards (docs-44, docs-46) but none of it describes actual query profiling tools, EXPLAIN/ANALYZE output, or execution-plan visualization for diagnosing slow or expensive queries. Missing for 10: any documentation of an EXPLAIN/PROFILE command, a query plan visualizer, or per-query cost/latency breakdown tool for MotherDuck specifically.", "evidenceIds": [ "motherduck-docs-44", "motherduck-docs-46" ] }, { "productId": "motherduck", "storyId": "runaway-cost-controls", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "MotherDuck docs confirm auto-suspend behavior ('idle instances shut down automatically, zero idle cost') and a billing dashboard/SQL views for monitoring compute and storage consumption, addressing the 'idle compute burning money' half of the story. However there is no evidence of configurable budgets, spend alerts/caps, or resource monitors that can kill a runaway query in progress — missing for 10: budget/spend-limit enforcement, automatic query cancellation on runaway cost, alerting thresholds tied to actions.", "evidenceIds": [ "motherduck-supp-cost-controls", "motherduck-docs-44" ] }, { "productId": "motherduck", "storyId": "semantic-model-metrics", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "MotherDuck documents \"Guides\" — markdown documents capturing metric definitions, join conventions, and domain context so AI agents produce consistent SQL without repetitive prompting — which is a lightweight step toward a governed semantic layer for AI consumers via MCP. However, this is informal markdown context rather than a structured semantic model with declared metrics/dimensions objects enforced across all query paths (SQL, BI tools, dashboards), and there's no evidence of validation, versioning, or joins-as-schema enforcement outside the AI/MCP context. Missing for 10: a structured metric/dimension definition schema (not just markdown guides), enforcement of consistency across non-AI query paths (BI/dashboard tools), and independent verification that guides actually produce consistent results across different AI tools.", "evidenceIds": [ "motherduck-docs-30" ] }, { "productId": "motherduck", "storyId": "streaming-ingest-latency", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows batch-style ingestion (file uploads, scheduled Python jobs, database/object-storage loads) but no documented streaming ingestion API with seconds-level queryability guarantees. No mention of Kafka/CDC connectors, streaming inserts, or latency SLAs for streamed data.", "evidenceIds": [] }, { "productId": "motherduck", "storyId": "time-travel-restore", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers connectivity, sharing, MCP, dbt integration, pricing, and dashboards, but contains no mention of time-travel querying, point-in-time recovery, or restoring dropped/corrupted tables from history — a capability common in some cloud warehouses but not documented here for MotherDuck.", "evidenceIds": [] }, { "productId": "snowflake", "storyId": "agent-runs-sql-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Snowflake provides multiple governed, non-dashboard paths for an agent to run SQL end-to-end: the SQL API (REST) for authenticating and executing queries/managing objects, the Snowflake CLI for programmatic access, RBAC/access history for governance, and a Snowflake-managed MCP server that lets agents securely retrieve data and execute SQL as tools without deploying infrastructure. missing for 10: independent/hands-on third-party validation of the full agent workflow (auth→discover→query→read-results) via CLI/API/MCP, and explicit documentation of schema-discovery APIs (e.g., information_schema access) tied directly to the MCP/SQL API flow.", "evidenceIds": [ "snowflake-docs-65", "snowflake-docs-66", "snowflake-docs-73", "snowflake-docs-72", "snowflake-docs-14", "snowflake-docs-57", "snowflake-docs-8", "snowflake-docs-52", "snowflake-probe-3", "snowflake-probe-4" ] }, { "productId": "snowflake", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Snowflake publishes a root llms.txt (confirmed via live probe, HTTP 200) summarizing the platform, plus dedicated llms.txt files for individual products like Snowpark, Streamlit, and Snowflake Postgres, giving agents a structured, agent-oriented entry point into the docs. missing for 10: independent/third-party confirmation that agents successfully consume these llms.txt files in practice.", "evidenceIds": [ "snowflake-probe-1", "snowflake-docs-75", "snowflake-docs-77", "snowflake-docs-80" ] }, { "productId": "snowflake", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Snowflake ships Cortex AI Functions for LLM-powered unstructured analytics on data in-place, and Cortex Analyst lets business users ask natural-language questions and get direct answers without SQL, plus semantic views to ground these insights in business context — all inside the product. missing for 10: independent/hands-on validation of insight quality and no evidence of proactive/automated suggestion surfacing (e.g., anomaly-based recommendations) beyond query-driven Q&A.", "evidenceIds": [ "snowflake-docs-12", "snowflake-docs-13", "snowflake-docs-25", "snowflake-docs-56", "snowflake-docs-11", "snowflake-docs-47" ] }, { "productId": "snowflake", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Snowflake documents several building blocks for background automation — Snowpipe auto-ingests files without manual scheduling, Dynamic Tables (GA) auto-refresh on a target lag, Streams capture DML changes to trigger downstream actions, and dbt Projects on Snowflake supports full orchestration of pipelines inside the platform. Cortex Agents plus the managed MCP server let AI agents query Snowflake data, but the evidence shows MCP as a request/response tool-serving interface rather than a documented mechanism for an AI-native user to schedule agents that run continuously and autonomously in the background. Missing for 10: explicit documentation of a Tasks/scheduler feature for autonomous job execution, evidence of agent-triggered/looping automations (not just query-based interactions), and independent/hands-on confirmation that these pieces compose into unattended AI-driven automation.", "evidenceIds": [ "snowflake-docs-6", "snowflake-docs-15", "snowflake-docs-7", "snowflake-docs-55", "snowflake-docs-61", "snowflake-docs-14", "snowflake-docs-57" ] }, { "productId": "snowflake", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Snowflake's homepage advertises 'Snowflake CoWork' ('Ask anything') and 'Snowflake CoCo' ('Build anything') as built-in enterprise AI assistants, and Cortex Analyst lets business users get natural-language answers without SQL — both suggest task delegation to an in-product assistant. However, the evidence pack has no dedicated docs page, feature list, or independent/hands-on account of CoWork/CoCo's actual task-delegation capabilities, so the claim rests on thin marketing taglines rather than substantiated documentation. Missing for 10: dedicated CoWork/CoCo documentation detailing task-delegation scope, hands-on or independent corroboration of the assistant performing multi-step tasks, and clarity on how it differs from Cortex Analyst's NL query feature.", "evidenceIds": [ "snowflake-docs-18", "snowflake-docs-19", "snowflake-docs-13", "snowflake-docs-25" ] }, { "productId": "snowflake", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Snowflake ships an official CLI for managing apps/workloads and REST APIs (SQL API and Resource Management API) explicitly designed for programmatic, non-interactive integration and automation, which are the standard mechanisms for running Snowflake operations headlessly in CI pipelines. missing for 10: explicit CI/CD pipeline examples (e.g., GitHub Actions integration) and documentation of non-interactive auth flows (key-pair/OAuth) tailored for headless automation.", "evidenceIds": [ "snowflake-docs-1", "snowflake-docs-65", "snowflake-docs-66", "snowflake-docs-72", "snowflake-docs-73", "snowflake-probe-4" ] }, { "productId": "snowflake", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence documents Snowflake's own MCP server that exposes Snowflake's Cortex tools to external AI agents (server-side), but there is no evidence Snowflake can act as an MCP client consuming external MCP servers' tools to extend Cortex Agents or CoWork/CoCo's capabilities.", "evidenceIds": [ "snowflake-docs-14", "snowflake-docs-57", "snowflake-docs-74", "snowflake-probe-3" ] }, { "productId": "snowflake", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Snowflake documents an official, Snowflake-managed MCP server (Cortex Agents MCP) that lets AI agents securely retrieve data without deploying separate infrastructure, and it can be configured to serve Cortex Analyst, Cortex Search, Cortex Agents, custom tools, and SQL execution as MCP tools. This is a first-party, GA-documented feature confirmed by a direct probe of the docs URL. Missing for 10: independent/hands-on third-party corroboration of real-world agent connections beyond vendor docs.", "evidenceIds": [ "snowflake-docs-14", "snowflake-docs-57", "snowflake-docs-74", "snowflake-probe-3" ] }, { "productId": "snowflake", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Snowflake documents genuine natural-language operation via Cortex Analyst (ask business questions in NL, get answers without writing SQL) and Cortex AI Functions for NL-driven analytics on text/images, plus a marketed 'CoWork'/'CoCo' NL agent interface and an MCP server exposing these NL/agent capabilities to external AI agents. This is solid support for AI-native NL interaction, but most of it is scoped to querying/analytics rather than full product administration (warehouses, security, pipelines) via natural language, and CoWork/CoCo are only marketing taglines with no detailed docs in the pack. Missing for 10: independent/hands-on validation of NL breadth, detailed CoWork/CoCo documentation, and evidence that operational/admin tasks (not just analytics) can be driven by natural language.", "evidenceIds": [ "snowflake-docs-13", "snowflake-docs-25", "snowflake-docs-12", "snowflake-docs-56", "snowflake-docs-18", "snowflake-docs-19", "snowflake-docs-14", "snowflake-docs-57" ] }, { "productId": "snowflake", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Snowflake ships an official Snowflake CLI documented for creating, managing, updating, and viewing apps across Streamlit, Native Apps, Snowpark Container Services, and Snowpark, confirmed by a dedicated docs probe. It's a general developer CLI rather than one purpose-built for AI-native/agentic workflows specifically. Missing for 10: independent/hands-on corroboration of CLI usage, and explicit agentic-workflow-specific CLI features (vs. general app management).", "evidenceIds": [ "snowflake-docs-1", "snowflake-docs-26", "snowflake-docs-28", "snowflake-probe-4" ] }, { "productId": "snowflake", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Snowflake documents a public REST API for resource management and a SQL API (both OpenAPI-compliant with an official spec repo and Postman collections) allowing programmatic querying, deployment management, and integration building, plus Snowpark APIs for Python/Java/Scala. This is well-documented first-party evidence of a documented public API surface usable by AI-native/automated clients. Missing for 10: independent hands-on developer corroboration of API robustness/limits, and a discoverable machine-readable openapi.json at the probed standard paths (probe returned 404s despite claims of OpenAPI compliance).", "evidenceIds": [ "snowflake-docs-65", "snowflake-docs-66", "snowflake-docs-72", "snowflake-docs-73", "snowflake-supp-openapi-specs", "snowflake-probe-2" ] }, { "productId": "snowflake", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Snowflake's RBAC system lets admins scope privileges to specific roles that can then be attached to service/API credentials, and its REST APIs plus the Cortex Agents MCP server explicitly support secure, credential-based programmatic access for AI agents. However, no evidence directly documents an agent-specific credential-issuance workflow (e.g., short-lived scoped tokens purpose-built for agent identities) — the story is inferred from general RBAC/API docs rather than an agent-specific credentialing feature. Missing for 10: explicit documentation of issuing least-privilege, agent-scoped API keys/tokens (vs. general RBAC/roles), and any hands-on/independent confirmation of this workflow.", "evidenceIds": [ "snowflake-docs-8", "snowflake-docs-32", "snowflake-docs-45", "snowflake-docs-65", "snowflake-docs-72", "snowflake-docs-14", "snowflake-docs-57", "snowflake-docs-74" ] }, { "productId": "snowflake", "storyId": "agentic-sdks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Snowflake documents multiple official SDKs/APIs for developers—Snowpark API for Python, Java, and Scala, the Snowflake SQL API, and REST APIs for resource management with OpenAPI specifications hosted in an official GitHub repo—plus a managed MCP server letting AI agents interact with Snowflake without custom infrastructure, directly supporting AI-native build patterns. Missing for 10: independent/hands-on developer corroboration of SDK quality/ergonomics and more explicit agent-framework integration examples beyond MCP.", "evidenceIds": [ "snowflake-docs-75", "snowflake-docs-65", "snowflake-docs-66", "snowflake-docs-72", "snowflake-docs-73", "snowflake-supp-openapi-specs", "snowflake-docs-14", "snowflake-docs-57", "snowflake-probe-3" ] }, { "productId": "snowflake", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Snowflake supports streams for CDC (polled) and notification integrations for cost alerts, but there is no documentation of a general-purpose webhook subscription mechanism that lets an AI-native user subscribe to events pushed via webhooks. Absence of evidence for this applicable capability yields 'none'.", "evidenceIds": [ "snowflake-docs-7", "snowflake-docs-33", "snowflake-docs-54" ] }, { "productId": "snowflake", "storyId": "ai-sql-assistant", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Cortex Analyst is a built-in, documented Snowflake feature that lets users ask questions in natural language and get answers 'without writing SQL,' implying it generates and runs SQL against the user's schema inside Snowsight; Cortex AI Functions and Notebooks further support SQL/Python assistance in-product. However, the evidence never explicitly shows the assistant 'fixing' or 'explaining' existing SQL (vs. generating it from scratch), and there's no independent/hands-on corroboration of accuracy or workflow beyond first-party docs. Missing for 10: explicit fix/explain-SQL capability documentation, and independent/community validation of the NL-to-SQL experience.", "evidenceIds": [ "snowflake-docs-13", "snowflake-docs-25", "snowflake-docs-12", "snowflake-docs-56", "snowflake-docs-10", "snowflake-docs-39" ] }, { "productId": "snowflake", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Snowflake publishes OpenAPI-compliant spec files and importable Postman collections for its REST/SQL APIs, which gives developers a way to run example requests, but there is no evidence of an embedded, interactive 'try-it' API reference on the docs site itself — a direct probe for openapi.json/swagger endpoints returned 404s. Missing for 10: an in-browser interactive API explorer (e.g., Swagger/Redoc UI) hosted by Snowflake, and any independent confirmation that the Postman-based workflow is commonly used as a live API reference.", "evidenceIds": [ "snowflake-supp-openapi-specs", "snowflake-docs-65", "snowflake-docs-66", "snowflake-docs-72", "snowflake-docs-73", "snowflake-probe-2" ] }, { "productId": "snowflake", "storyId": "api-machine-spec", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Snowflake publishes an official, first-party GitHub repository (snowflakedb/snowflake-rest-api-specs) containing downloadable OpenAPI-compliant spec files plus Postman collections for its REST APIs, and its own docs explicitly state the REST APIs are 'compliant with the OpenAPI specification' and enable building integrations 'using the openAPI specifications.' Missing for 10: a directly hosted /openapi.json or /swagger.json endpoint on the docs site itself (probe found 404s there), so the machine-readable spec must be fetched from the separate GitHub repo rather than a single canonical URL.", "evidenceIds": [ "snowflake-supp-openapi-specs", "snowflake-docs-72", "snowflake-docs-65", "snowflake-probe-2" ] }, { "productId": "snowflake", "storyId": "api-sandbox", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Snowflake's zero-copy cloning (object-clone) lets users create full database/schema clones inheriting privileges without duplicating or altering production data, which supports building an isolated sandbox for testing, and RBAC plus resource monitors help isolate/limit that environment. However, the evidence never frames this explicitly as an 'AI-native sandbox' workflow or ties cloning to agentic/AI testing use cases. missing for 10: explicit sandbox/dev-test documentation framed for AI-native or agent workflows, guidance on isolating agent test runs from prod data, and independent corroboration of clone-based sandboxing in practice.", "evidenceIds": [ "snowflake-docs-67", "snowflake-docs-71", "snowflake-docs-8", "snowflake-docs-32", "snowflake-docs-45" ] }, { "productId": "snowflake", "storyId": "api-versioning-policy", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Snowflake documents official REST APIs (SQL API, resource-management API) built on OpenAPI specs, and maintains a dedicated 'Behavior change announcements' page describing dated bundles of changes meant to minimize disruption — functioning as a de facto deprecation/change policy. However, there's no explicit documentation of API version numbers (v1/v2) or formal deprecation timelines/sunset dates for the APIs themselves. Missing for 10: explicit API versioning scheme documentation, formal deprecation timeline/sunset policy tied to specific API versions, independent corroboration of API stability guarantees.", "evidenceIds": [ "snowflake-docs-65", "snowflake-docs-66", "snowflake-supp-openapi-specs", "snowflake-supp-behavior-changes", "snowflake-docs-72", "snowflake-docs-73" ] }, { "productId": "snowflake", "storyId": "audit-lineage", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Snowflake provides strong audit-log coverage via Query History (who ran what queries) and Access History (ACCESS_HISTORY view tracking reads/writes for compliance auditing, including per-column access insights), which addresses the 'who ran what' half of the story well. However, the evidence pack does not document an explicit column-level lineage graph/feature (e.g., Snowflake's Horizon/lineage explorer) showing upstream data provenance — only that Access History surfaces 'popular and frequently accessed tables and columns,' which is usage insight, not full lineage tracing. Missing for 10: explicit column-level lineage/graph documentation, independent validation of lineage accuracy across transformations.", "evidenceIds": [ "snowflake-docs-51", "snowflake-docs-59", "snowflake-docs-52", "snowflake-docs-63", "snowflake-docs-78", "snowflake-docs-70" ] }, { "productId": "snowflake", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Snowflake supports bulk/batch operations via SQL (COPY INTO, Snowpipe micro-batches), Snowpark for programmatic bulk data processing, REST/SQL APIs for scripted batch operations, and CLI/dbt for orchestrating multi-object workflows, but these are general data-platform bulk operations rather than an AI-native agent-oriented bulk-action capability explicitly designed for AI agents to invoke across many items at once. The MCP server (Cortex Agents MCP) exposes tools for agents but evidence doesn't show explicit bulk/batch operation support through that interface. missing for 10: explicit documentation of AI-agent-initiated bulk operations (e.g., MCP server batch tool calls across many items), hands-on/independent evidence of AI-native bulk workflows succeeding at scale, and any agent-specific bulk-action API beyond generic SQL/REST batch capabilities.", "evidenceIds": [ "snowflake-docs-6", "snowflake-docs-7", "snowflake-docs-65", "snowflake-docs-66", "snowflake-docs-75", "snowflake-docs-14", "snowflake-docs-57" ] }, { "productId": "snowflake", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Snowflake offers building blocks that support event-driven automation — Streams capture DML changes for downstream actions, Snowpipe auto-ingests new files, resource monitors can auto-suspend warehouses on cost thresholds, and anomaly monitors trigger notification alerts — but there is no single unified 'rules engine' or declarative trigger-action framework exposed as one product for AI-native users to define arbitrary event-condition-action automations. missing for 10: a first-class, general-purpose rules/trigger-action framework (e.g., 'if X event then do Y') rather than disparate feature-specific mechanisms (streams+tasks, Snowpipe, resource monitors), and independent/hands-on evidence of composing these into end-to-end automated workflows.", "evidenceIds": [ "snowflake-docs-7", "snowflake-docs-54", "snowflake-docs-6", "snowflake-docs-20", "snowflake-docs-53", "snowflake-docs-60", "snowflake-docs-17", "snowflake-docs-33" ] }, { "productId": "snowflake", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence shows dbt Projects on Snowflake can 'orchestrate' transformations with a managed runtime, implying some recurring-job scheduling capability, but the pack contains no direct documentation of Snowflake's native TASK/scheduler object or agent-triggered recurring workflows for AI-native users. Missing for 10: explicit docs on Snowflake Tasks or a scheduler API, evidence of AI agents creating/managing recurring jobs via MCP or CLI, and independent/hands-on confirmation of scheduling reliability.", "evidenceIds": [ "snowflake-docs-2", "snowflake-docs-55", "snowflake-docs-61" ] }, { "productId": "snowflake", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Snowflake docs show building blocks that touch pieces of this story — dbt Projects on Snowflake covers the 'develop, deploy, orchestrate, observe' lifecycle (implying dbt's git-based versioning) [snowflake-docs-2][snowflake-docs-55], Time Travel lets you restore/rollback objects and query past states [snowflake-docs-5][snowflake-docs-24][snowflake-docs-35], and object cloning/behavior-change bundles give some notion of state snapshots and change tracking [snowflake-docs-67][snowflake-supp-behavior-changes]. However there is no dedicated, documented capability for versioning, reviewing, and rolling back 'automations' (e.g., Tasks, Dynamic Tables pipelines, agent workflows) as a first-class AI-native feature — the evidence is inferential rather than a direct product answer to this story. Missing for 10: explicit versioning/review/rollback tooling for Tasks/Dynamic Tables/agent pipelines, a documented approval/review workflow for automation changes, and independent/hands-on confirmation of rollback working for automations specifically.", "evidenceIds": [ "snowflake-docs-2", "snowflake-docs-55", "snowflake-docs-5", "snowflake-docs-24", "snowflake-docs-35", "snowflake-docs-67", "snowflake-supp-behavior-changes" ] }, { "productId": "snowflake", "storyId": "bi-driver-connectivity", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no specific mention of JDBC/ODBC drivers or documented BI-tool integrations (e.g., Tableau, Power BI, Looker) for Snowflake; only a generic reference to 'applications and tools you can use to access Snowflake' (snowflake-docs-27) without detail on standard drivers or BI connectors. This is an axis that clearly applies to a data warehouse product, but no concrete evidence demonstrates the capability.", "evidenceIds": [] }, { "productId": "snowflake", "storyId": "bulk-load-object-storage", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Evidence confirms Snowflake's COPY INTO/Snowpipe ingestion path from cloud stages (referenced via stream and Snowpipe docs, plus a community mention of COPY skipping already-loaded files and JSON Lines usage) and separate support for semi-structured JSON data types, but the pack never explicitly documents a single COPY INTO command example covering CSV, JSON, and Parquet together. missing for 10: explicit doc snippet naming CSV/Parquet file format support in COPY INTO, and a concrete single-command example spanning all three formats.", "evidenceIds": [ "snowflake-docs-6", "snowflake-docs-7", "snowflake-supp-sql-semistructured", "snowflake-comm-4" ] }, { "productId": "snowflake", "storyId": "compliance-attestations", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Snowflake's dedicated Regulatory Compliance documentation explicitly lists per-certification pages for SOC 1 Type II, SOC 2 Type II, PCI DSS, and FedRAMP (a HIPAA-relevant standard), directly addressing the security-review documentation need. This is supported by first-party docs with named certifications, giving platform-engineers concrete attestation references. Missing for 10: an explicit named HIPAA compliance page/BAA reference and independent third-party corroboration of audit currency.", "evidenceIds": [ "snowflake-supp-compliance" ] }, { "productId": "snowflake", "storyId": "connector-ecosystem", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "The pack shows Snowflake integrates with ETL/ELT tooling (dbt Projects on Snowflake, managed dbt runtimes) and general data-loading mechanisms (Snowpipe, COPY INTO, Iceberg tables for external lakes) with documented setup, but it never documents a first-party/partner connector catalog for specific SaaS apps (e.g., Salesforce, HubSpot) or databases (e.g., Postgres, MySQL) — no Partner Connect, native connectors gallery, or marketplace connector docs are cited. Missing for 10: documented SaaS-app connector catalog, database-source connector docs, partner ETL/ELT integration listings beyond dbt.", "evidenceIds": [ "snowflake-docs-2", "snowflake-docs-6", "snowflake-docs-50", "snowflake-docs-61" ] }, { "productId": "snowflake", "storyId": "continuous-transformations", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Snowflake documents Streams for CDC-based incremental change capture (doc-7/21/38/54), Dynamic Tables (GA, doc-15) for declarative continuous transformation pipelines, and Snowpipe for continuous/micro-batch ingestion (doc-6/20/29/34/44/76) — directly matching the story's streams/declarative-pipeline/continuous-processing capabilities inside the platform. Missing for 10: explicit documentation of the Tasks object (scheduling/orchestration companion to Streams), Snowpipe Streaming (true low-latency continuous ingestion) specifics, and independent/hands-on validation of these pipelines at production scale.", "evidenceIds": [ "snowflake-docs-7", "snowflake-docs-15", "snowflake-docs-6", "snowflake-docs-21", "snowflake-docs-38", "snowflake-docs-54", "snowflake-docs-76" ] }, { "productId": "snowflake", "storyId": "cost-model-clarity", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Snowflake docs explain the underlying cost model (credit-consuming virtual warehouses, resizing, resource monitors, cost-anomaly alerts, and a general pricing-tier page) which gives a platform engineer the mechanics needed to model cost, but there is no concrete rate card or calculator evidenced. Community reports concretely contradict the 'estimate before committing' claim: users note costs 'balloon pretty fast', warehouse credit consumption 'doubles each size up', there is 'no query level attribution of costs and no in-built features for monitoring', and it's 'easy... to get to $100k/year bills without massive data' — indicating the documented model is hard to translate into an accurate upfront estimate in practice. missing for 10: a transparent unit-price/rate table, a first-party cost calculator, and independent confirmation that engineers can reliably forecast bills from the docs alone.", "evidenceIds": [ "snowflake-docs-36", "snowflake-docs-40", "snowflake-docs-53", "snowflake-docs-60", "snowflake-docs-42", "snowflake-docs-49", "snowflake-comm-1", "snowflake-comm-7", "snowflake-comm-13" ] }, { "productId": "snowflake", "storyId": "dataset-marketplace", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack only documents Secure Data Sharing (share selected objects between Snowflake accounts) but contains no mention of a marketplace, data exchange, or third-party dataset listings that an analyst could browse and directly enrich their own data with — the specific marketplace capability described in the story is absent from the pack.", "evidenceIds": [ "snowflake-docs-9", "snowflake-docs-46" ] }, { "productId": "snowflake", "storyId": "dbt-first-class", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Snowflake has a dedicated first-party feature, 'dbt Projects on Snowflake,' with vendor docs describing native support for the full dbt lifecycle (develop, deploy, orchestrate, observe), managed dbt Core/Fusion runtimes, and a documented migration path from dbt Core. This goes beyond a community adapter — it's native, vendor-documented dbt support. missing for 10: independent/hands-on validation of dbt Projects on Snowflake in production, and more detail on parity/limitations vs dbt Core for advanced use cases.", "evidenceIds": [ "snowflake-docs-2", "snowflake-docs-3", "snowflake-docs-55", "snowflake-docs-61", "snowflake-comm-9" ] }, { "productId": "snowflake", "storyId": "free-evaluation-path", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Community evidence confirms a real, usable free trial exists (a user ran actual queries burning real credits and storage over the trial period), showing the trial supports real queries on real data — but there is no first-party documentation in the pack describing signup requirements (credit card, sales contact) for that trial, and pricing docs only describe a paid 'Standard' entry tier rather than a no-card free trial. missing for 10: official docs confirming no-credit-card signup, no sales-call requirement, and trial credit/duration terms.", "evidenceIds": [ "snowflake-comm-1", "snowflake-docs-42", "snowflake-docs-49" ] }, { "productId": "snowflake", "storyId": "full-sql-surface", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs confirm window functions, semi-structured VARIANT/OBJECT/ARRAY types, and Time Travel/date handling are native SQL features, and community evidence corroborates JSON handling via COPY/JSON Lines. However, one community report notes a gap versus generate_series-style sequence generation, and CTE support and rich date/time type breadth aren't directly evidenced in the pack. missing for 10: explicit CTE documentation, comprehensive date/time type reference, independent corroboration beyond the noted generate_series gap.", "evidenceIds": [ "snowflake-supp-sql-window", "snowflake-supp-sql-semistructured", "snowflake-comm-2", "snowflake-comm-4", "snowflake-docs-5" ] }, { "productId": "snowflake", "storyId": "granular-rbac-masking", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm Snowflake's core RBAC model (privileges assigned to roles, roles to users) and Access History views tracking table/column-level query and write activity, which supports auditing at the object level. However, the evidence pack contains no mention of dynamic data masking policies or row access policies (row-level security), which are the specific mechanisms the story calls out for column- and row-level control — missing for 10: masking policy documentation, row access policy documentation, and any independent/hands-on confirmation these work as described.", "evidenceIds": [ "snowflake-docs-8", "snowflake-docs-32", "snowflake-docs-45", "snowflake-docs-52", "snowflake-docs-63", "snowflake-docs-70", "snowflake-docs-78" ] }, { "productId": "snowflake", "storyId": "live-data-sharing", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Snowflake's Secure Data Sharing directly delivers this story: it lets accounts share selected database objects with other Snowflake accounts, with docs explicitly stating no actual data is copied or transferred between accounts, eliminating the need for export pipelines. This is a first-party, well-documented core Snowflake feature (also referenced as part of Standard plan capabilities). Missing for 10: independent/hands-on third-party confirmation of cross-organization sharing experience beyond vendor docs.", "evidenceIds": [ "snowflake-docs-9", "snowflake-docs-46", "snowflake-docs-49" ] }, { "productId": "snowflake", "storyId": "local-dev-loop", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Snowflake's evidence pack covers cloud compute (warehouses, Snowpipe, notebooks, dbt-in-Snowflake, Snowpark) but nothing about a local engine, emulator, or free sandbox for developing transformations before touching production compute — everything described runs against live Snowflake compute/credits. Community evidence even notes trial credits get consumed quickly (~$19.60 in credits within days), reinforcing that there's no cost-free local dev loop. Missing for 10: any local engine/emulator, offline dev mode, or a genuinely free (non-credit-consuming) sandbox tier.", "evidenceIds": [ "snowflake-comm-1", "snowflake-docs-42", "snowflake-docs-49", "snowflake-docs-36" ] }, { "productId": "snowflake", "storyId": "managed-continuous-ingestion", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Snowpipe is a first-party managed service that automatically loads files from a stage as soon as they arrive, in micro-batches within minutes, without manual COPY scheduling or customer-run pipeline infrastructure (snowflake-docs-6/20/29/34/44/76). Community evidence corroborates continuous file-based loading behavior (COPY skipping already-loaded files) consistent with this pattern (snowflake-comm-4). Missing for 10: explicit documentation of event-driven/streaming ingestion (e.g., Snowpipe Streaming API or Kafka connector) and independent hands-on validation of the 'no pipeline infra' claim beyond vendor docs.", "evidenceIds": [ "snowflake-docs-6", "snowflake-docs-20", "snowflake-docs-29", "snowflake-docs-34", "snowflake-docs-44", "snowflake-docs-76", "snowflake-comm-4" ] }, { "productId": "snowflake", "storyId": "native-notebooks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Snowflake Notebooks is a first-party, cell-based Snowsight interface supporting Python, SQL, and Markdown in the same notebook with cell-by-cell execution and result comparison, directly matching the story. Missing for 10: explicit documentation/screenshots of inline chart rendering within notebooks and independent/hands-on user corroboration beyond vendor docs.", "evidenceIds": [ "snowflake-docs-10", "snowflake-docs-39", "snowflake-docs-31", "snowflake-comm-3" ] }, { "productId": "snowflake", "storyId": "nl-governed-answers", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Snowflake's Cortex Analyst is documented to let business users ask natural-language questions and get direct answers without writing SQL, and Snowflake Semantic Views let teams define business metrics/entities as governed semantic layer objects that Cortex Analyst can ground against, with RBAC controlling access to underlying data. This directly matches the story's ask for governed, semantically-grounded NL answers rather than raw hallucinated joins. missing for 10: independent/hands-on evidence validating accuracy or hallucination-reduction claims, and no detail on how Cortex Analyst explicitly enforces the semantic model to prevent bad joins (only first-party docs, no third-party corroboration).", "evidenceIds": [ "snowflake-docs-13", "snowflake-docs-25", "snowflake-docs-11", "snowflake-docs-23", "snowflake-docs-47", "snowflake-docs-79", "snowflake-docs-8", "snowflake-docs-32", "snowflake-docs-45" ] }, { "productId": "snowflake", "storyId": "open-table-formats", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Snowflake supports Apache Iceberg tables directly on customer-managed external cloud storage, explicitly positioned for 'existing data lakes that you cannot, or choose not to, store in Snowflake,' and Snowpipe/COPY plus external stages allow querying files in object storage without loading them into proprietary storage. Missing for 10: explicit docs on Delta Lake table querying and independent hands-on benchmarks confirming performance parity for external Iceberg/Delta queries.", "evidenceIds": [ "snowflake-docs-50", "snowflake-docs-58" ] }, { "productId": "snowflake", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Snowflake exposes broad programmatic surfaces — the SQL API and REST APIs for resource management (queries, provisioning users/roles, tables), Snowpark APIs, Snowflake CLI for managing apps/workloads, and OpenAPI-compliant specs published on GitHub — indicating strong API/UI parity ambitions for developers and AI agents (docs-65, docs-66, docs-72, docs-73, snowflake-supp-openapi-specs, docs-1). However there is no explicit vendor claim or independent verification of full 1:1 parity between every Snowsight UI feature (e.g., notebooks cell UI, query profile visualizations, Cortex Analyst chat UI) and API equivalents, and a live probe found no discoverable openapi.json at conventional doc paths. Missing for 10: an explicit parity statement or audit confirming every UI action has an API/CLI equivalent, and independent/hands-on confirmation that AI-native workflows can fully replace UI usage.", "evidenceIds": [ "snowflake-docs-65", "snowflake-docs-66", "snowflake-docs-72", "snowflake-docs-73", "snowflake-supp-openapi-specs", "snowflake-docs-1", "snowflake-probe-2" ] }, { "productId": "snowflake", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Snowflake supports Apache Iceberg tables backed by open, customer-managed cloud storage, which is a genuine open-format, portable-data path, and it exposes SQL/REST APIs for programmatic data extraction. However, there is no direct documentation in the evidence pack of a comprehensive 'export all your data and leave' workflow (e.g., bulk UNLOAD/COPY INTO to open formats for all tables, migration tooling, or explicit anti-lock-in guidance), and community commentary even frames Snowflake's stickiness/lock-in as a concern rather than confirming easy full data egress. Missing for 10: documented bulk export/unload feature for all data, first-party migration/exit tooling, and independent confirmation that a full data exit is straightforward.", "evidenceIds": [ "snowflake-docs-50", "snowflake-docs-58", "snowflake-docs-65", "snowflake-docs-66", "snowflake-comm-11" ] }, { "productId": "snowflake", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Snowflake is a closed-source, proprietary commercial data platform; the story asks about reading the product's own source code under an open license, which is a category error — Snowflake does not distribute or claim to open-source its core platform (evidence only covers clients like the CLI or connectors, not the platform itself).", "evidenceIds": [] }, { "productId": "snowflake", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Snowflake is a fully managed cloud SaaS data platform with no on-premises or self-hostable distribution; self-hosting the core product is a category error for this kind of offering, not a missing feature.", "evidenceIds": [] }, { "productId": "snowflake", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack includes compliance certifications (SOC 2, ISO, FedRAMP, PCI DSS) but contains no mention of region selection, data residency controls, or ability to choose a storage location/cloud region for an account or database. No documentation cited addresses this specific capability.", "evidenceIds": [] }, { "productId": "snowflake", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The axis applies since Snowflake offers AI/Cortex features built on third-party LLMs, making data-use-for-training a fair privacy concern, but no evidence pack item documents an opt-out control, data-processing agreement, or explicit statement that customer data is excluded from model training. Compliance certifications and RBAC/access-history items are about general security/governance, not AI-training data use.", "evidenceIds": [] }, { "productId": "snowflake", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Snowflake's Time Travel docs show a defined retention window for querying/restoring changed or deleted data, and RBAC plus compliance certifications provide governance context, giving an AI-native user some levers over data lifecycle and access. Missing for 10: explicit documentation of configuring the retention period (e.g., DATA_RETENTION_TIME_IN_DAYS parameter), explicit purge/permanent-delete commands, and any GDPR/right-to-erasure specific deletion guarantees.", "evidenceIds": [ "snowflake-docs-5", "snowflake-docs-24", "snowflake-docs-35", "snowflake-docs-48", "snowflake-docs-8", "snowflake-supp-compliance" ] }, { "productId": "snowflake", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions telemetry, usage tracking, or an opt-out mechanism for Snowflake's own product telemetry; only compliance certifications and access-history/audit features are documented, which don't address user-facing telemetry opt-out. This axis is applicable to any AI-native SaaS product, so absence of evidence yields 'none'. Missing for 10: any documentation of a telemetry/usage-data collection policy or an opt-out/opt-in control for such tracking.", "evidenceIds": [] }, { "productId": "snowflake", "storyId": "query-profiling", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Snowsight's Query History page lets data engineers monitor executed queries and drill into a detailed query profile that explores each step of execution, directly supporting root-cause analysis of slow/expensive queries; this is corroborated by community mention of Snowsight's query/visualize UI. missing for 10: no independent hands-on account of using query profile specifically to diagnose a slow query, and no detail on cost-per-query attribution tooling (one community comment even notes lack of query-level cost attribution).", "evidenceIds": [ "snowflake-docs-51", "snowflake-docs-59", "snowflake-docs-62", "snowflake-docs-69", "snowflake-comm-3", "snowflake-comm-7" ] }, { "productId": "snowflake", "storyId": "runaway-cost-controls", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Snowflake ships resource monitors that track credit usage and can auto-suspend a virtual warehouse (or disable an Adaptive Warehouse) when a threshold is hit, plus GA cost-anomaly monitors, per-user quotas, and notification integrations for cost alerts, directly matching the platform-engineer need to stop runaway spend. Community commentary independently corroborates this as standard practice (setting up resource monitors and using small warehouses to control cost), and warehouses can also be manually started, stopped, and resized to control burn. Missing for 10: independent hands-on confirmation that resource monitors reliably kill an in-flight runaway query versus only suspending future warehouse starts, and detail on statement-level timeout controls.", "evidenceIds": [ "snowflake-docs-53", "snowflake-docs-60", "snowflake-docs-64", "snowflake-docs-16", "snowflake-docs-17", "snowflake-docs-33", "snowflake-docs-36", "snowflake-docs-43", "snowflake-comm-1" ] }, { "productId": "snowflake", "storyId": "semantic-model-metrics", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Snowflake's Semantic Views let analysts declare business metrics, entities, and relationships once as schema-level objects, and Cortex Analyst answers natural-language questions against this same semantic layer without writing SQL, while Cortex Agents/MCP expose it to AI tools — directly matching the story of a governed semantic model queried consistently by both SQL and AI. missing for 10: no independent/community corroboration of Semantic Views in practice, and no detail on how BI/SQL query tools (beyond Cortex Analyst) consistently reuse the same semantic definitions.", "evidenceIds": [ "snowflake-docs-11", "snowflake-docs-23", "snowflake-docs-47", "snowflake-docs-79", "snowflake-docs-13", "snowflake-docs-25", "snowflake-docs-14", "snowflake-docs-57" ] }, { "productId": "snowflake", "storyId": "streaming-ingest-latency", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Snowflake documents Snowpipe as a streaming/micro-batch ingestion API, but the documentation explicitly states data becomes available 'within minutes,' not seconds, directly at odds with the story's 'queryable within seconds' requirement. There is no evidence pack mention of the lower-latency Snowpipe Streaming API or REST-based streaming ingestion that could close this gap. Missing for 10: documented sub-second/seconds-level ingestion latency, evidence of a dedicated low-latency streaming ingestion API (e.g., Snowpipe Streaming), and independent/hands-on confirmation of near-real-time query availability.", "evidenceIds": [ "snowflake-docs-6", "snowflake-docs-20", "snowflake-docs-29", "snowflake-docs-34", "snowflake-docs-44", "snowflake-docs-76" ] }, { "productId": "snowflake", "storyId": "time-travel-restore", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Snowflake's documented Time Travel feature explicitly supports querying data as of a past point and restoring accidentally/intentionally deleted objects (tables, schemas, databases) within a retention window, directly matching the story; Clone/UNDROP mechanics and object cloning (inheriting privileges) further support restoration workflows. Missing for 10: no independent/hands-on corroboration of restore success or retention-period limits from community evidence.", "evidenceIds": [ "snowflake-docs-5", "snowflake-docs-24", "snowflake-docs-30", "snowflake-docs-35", "snowflake-docs-48", "snowflake-docs-67", "snowflake-docs-71" ] } ]