[ { "productId": "datalab", "storyId": "accuracy-benchmarks-evals", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only that users can try the product via free allowance/Forge to run their own proof-of-concept (datalab-docs-9, datalab-docs-10, datalab-docs-23, datalab-docs-42), but there is no evidence of vendor-published reproducible accuracy benchmarks (e.g., against ground-truth datasets, competitor comparisons, or accuracy metrics). A community comment even flags difficulty in groundtruthing Datalab's OCR output (datalab-comm-1), reinforcing the absence of published benchmark data.", "evidenceIds": [ "datalab-docs-9", "datalab-docs-10", "datalab-docs-23", "datalab-docs-42", "datalab-comm-1" ] }, { "productId": "datalab", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Datalab hosts a working llms.txt confirmed via direct probe (HTTP 200) at documentation.datalab.to/llms.txt, and also exposes .md variants of docs pages, both signals of agent-oriented documentation. Missing for 10: no independent third-party confirmation of agent usage/adoption, and no dedicated agent-facing onboarding guide beyond the llms.txt file and .md endpoints.", "evidenceIds": [ "datalab-probe-1", "datalab-docs-35", "datalab-docs-36", "datalab-docs-37", "datalab-docs-38", "datalab-docs-39", "datalab-docs-40", "datalab-docs-41", "datalab-docs-43" ] }, { "productId": "datalab", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Datalab offers some AI-assisted automation—auto-generating extraction schemas for a document (datalab-docs-41) and running 'agent processors' on documents (datalab-docs-40)—which could be seen as AI suggesting how to interpret/extract data. However, there's no evidence of broader analytical insights, summaries, trend detection, or proactive suggestions about data content beyond structured extraction. missing for 10: evidence of AI-generated summaries/insights beyond schema suggestion, evidence of in-product analytics or pattern detection, and any independent corroboration of these agentic features actually surfacing insights to users.", "evidenceIds": [ "datalab-docs-40", "datalab-docs-41" ] }, { "productId": "datalab", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Datalab supports background, non-blocking automation via webhooks (eliminating polling) and 'Pipelines' that chain processors into versioned, reusable configurations deployable to production, plus a 'run document agent' endpoint for published agent processors — these together allow document workflows to run unattended once triggered. However there is no evidence of scheduling, event-based triggers, or a true autonomous loop that decides when/what to run without an external caller invoking the API. Missing for 10: scheduling/cron or event-trigger mechanisms, evidence of multi-step autonomous decision-making beyond a single processor call, and independent confirmation of pipelines running unattended in production.", "evidenceIds": [ "datalab-docs-8", "datalab-docs-24", "datalab-docs-4", "datalab-docs-38", "datalab-docs-40" ] }, { "productId": "datalab", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Datalab is a document conversion/extraction API and SDK; the closest evidence is a 'document agent' processor endpoint for running pre-built document pipelines (datalab-docs-40), which is task automation on documents, not an interactive built-in assistant that a user can delegate open-ended tasks to. No docs describe a chat-based or conversational AI assistant embedded in the product for general task delegation.", "evidenceIds": [ "datalab-docs-40" ] }, { "productId": "datalab", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Datalab exposes a REST API, Python SDK, and dedicated CLI for document conversion/extraction that can be scripted headlessly, plus webhooks to avoid polling and health-check endpoints for automation/CI monitoring. Missing for 10: independent hands-on evidence of CI pipeline usage (e.g., GitHub Actions example) and no discoverable OpenAPI spec for programmatic tooling.", "evidenceIds": [ "datalab-docs-14", "datalab-probe-3", "datalab-docs-2", "datalab-docs-8", "datalab-docs-43", "datalab-docs-31" ] }, { "productId": "datalab", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack of an official MCP server or MCP integration for Datalab; documentation covers SDK, CLI, webhooks, API endpoints, and on-prem deployment but never mentions MCP. As a document-processing API/platform, this axis is applicable (an official MCP server is plausible for such a product) but no evidence shows it exists, so absence of evidence means none rather than na.", "evidenceIds": [ "datalab-docs-2", "datalab-docs-14", "datalab-probe-2", "datalab-probe-3" ] }, { "productId": "datalab", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Datalab is a document conversion/extraction API with SDK, CLI, webhooks, and pipelines, but no evidence anywhere in the pack of an official MCP server or MCP integration for connecting AI agents. This axis applies since Datalab is a SaaS/API platform with an ecosystem where an MCP server would be plausible, but no such capability is documented.", "evidenceIds": [] }, { "productId": "datalab", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Datalab's evidence only shows a structured REST API, Python SDK, and CLI for document conversion/extraction — all requiring code or CLI syntax, not natural-language commands. No chat-based interface, NL command layer, or agent-facing conversational control surface is documented anywhere in the evidence pack.", "evidenceIds": [ "datalab-docs-2", "datalab-docs-14", "datalab-probe-3", "datalab-docs-40" ] }, { "productId": "datalab", "storyId": "agentic-official-cli", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Datalab documents an official CLI for converting documents from the command line, confirmed both in docs and a live probe of the CLI page. Missing for 10: no independent/hands-on corroboration of CLI usage, and no detail on CLI feature parity with the SDK (pipelines, extraction, etc.) beyond basic conversion.", "evidenceIds": [ "datalab-docs-14", "datalab-probe-3" ] }, { "productId": "datalab", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Datalab exposes a documented public API/SDK with explicit endpoints (convert, extract, segment, pipelines, webhooks, health check), plus CLI and Python SDK, all backed by extensive first-party API reference docs; a llms.txt probe confirms machine-readable docs discoverability. Minor gap: no formal OpenAPI/swagger spec was found (404s), so machine-consumable schema is less standardized. Missing for 10: a discoverable OpenAPI/swagger spec and independent third-party confirmation of API robustness.", "evidenceIds": [ "datalab-docs-2", "datalab-docs-31", "datalab-docs-32", "datalab-docs-33", "datalab-docs-34", "datalab-docs-14", "datalab-probe-1", "datalab-probe-2", "datalab-probe-3" ] }, { "productId": "datalab", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Datalab's docs cover API keys, 2FA, and BAA/DPA but there is no evidence of scoped or least-privilege API credential issuance (e.g., role-based keys, permission scopes, or agent-specific tokens) for delegating limited access to an agent.", "evidenceIds": [] }, { "productId": "datalab", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Datalab documents an official Python SDK covering conversion, extraction, pipelines, form filling, and file management, plus an official CLI, giving AI-native developers first-party programmatic access to build against. Missing for 10: independent/hands-on developer corroboration beyond vendor docs, and no public OpenAPI/SDKs for other languages confirmed (openapi probe 404s).", "evidenceIds": [ "datalab-docs-2", "datalab-docs-14", "datalab-docs-38", "datalab-docs-37", "datalab-probe-3", "datalab-probe-2" ] }, { "productId": "datalab", "storyId": "agentic-webhooks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Datalab has clear first-party docs for webhooks: real-time job-completion notifications avoiding polling, plus per-request webhook_url override support, directly matching the story's intent for event-driven/agentic integration. Missing for 10: no documented signature verification/security scheme details or independent/hands-on confirmation of webhook delivery reliability.", "evidenceIds": [ "datalab-docs-8", "datalab-docs-24", "datalab-docs-34", "datalab-docs-35" ] }, { "productId": "datalab", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Datalab publishes detailed API reference pages (convert-document, extract-structured-data, segment-document, create-document, run-document-agent, etc.) and offers a separate 'Forge' playground where users can upload documents and see results instantly without an API key, giving some interactive/try-it-now experience. However, a probe for a standard OpenAPI/Swagger spec (which typically powers runnable 'try it' examples embedded directly in API reference pages) returned 404s across all candidate paths, and no evidence shows executable code samples or a live 'run this request' widget within the API reference itself. Missing for 10: an OpenAPI/Swagger spec or embedded runnable request builder directly in the API reference docs, and confirmation that Forge is linked/integrated with the reference pages rather than a separate playground.", "evidenceIds": [ "datalab-docs-31", "datalab-docs-32", "datalab-docs-33", "datalab-docs-9", "datalab-docs-23", "datalab-probe-2" ] }, { "productId": "datalab", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Datalab has a full REST API reference (convert, extract, segment, webhooks, etc.) but a direct probe for standard OpenAPI/Swagger spec locations (openapi.json, swagger.json, etc.) returned 404 across all checked paths, and no evidence of a downloadable machine-readable spec file was found anywhere in the docs.", "evidenceIds": [ "datalab-probe-2", "datalab-probe-1" ] }, { "productId": "datalab", "storyId": "api-sandbox", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Datalab offers a free-tier Forge playground and a monthly usage allowance explicitly framed for running a 'proof of concept' without a credit card, letting users try conversions before committing to production workflows (datalab-docs-9, datalab-docs-10, datalab-docs-23, datalab-docs-42). However, there is no dedicated 'sandbox environment' concept, no separate test API keys/staging tier, and no mention of isolating test runs from production billing/data pipelines. Missing for 10: an explicit sandbox/staging mode, test-vs-production API key separation, and confirmation that sandbox usage doesn't touch production data stores or billing.", "evidenceIds": [ "datalab-docs-9", "datalab-docs-10", "datalab-docs-23", "datalab-docs-42", "datalab-docs-45" ] }, { "productId": "datalab", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While Datalab documents processor/pipeline versioning (e.g., pinning to id@version, versioned pipeline configs) and maintains a changelog, there is no evidence of a documented API versioning scheme (e.g., v1/v2 endpoints) or an explicit deprecation policy for its APIs.", "evidenceIds": [ "datalab-docs-40", "datalab-docs-38", "datalab-docs-15", "datalab-docs-45", "datalab-probe-2" ] }, { "productId": "datalab", "storyId": "async-jobs-webhooks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm webhooks for job completion notifications with overridable webhook_url, and async checkpoint-based processing (save_checkpoint, checkpoint_id) that avoids re-parsing, supporting a non-blocking pipeline pattern. However, evidence doesn't explicitly describe a status-polling endpoint for job state or detail async queue/job semantics beyond webhooks. missing for 10: explicit job-status/polling API documentation, independent/hands-on confirmation that webhooks reliably fire for long-running jobs at scale, and details on job queuing/retry behavior.", "evidenceIds": [ "datalab-docs-8", "datalab-docs-24", "datalab-docs-34", "datalab-docs-35", "datalab-docs-29", "datalab-docs-16" ] }, { "productId": "datalab", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Datalab's SDK, CLI, and pipelines allow scripting repeated calls across many documents, and webhooks let jobs run asynchronously without polling, which supports processing many items programmatically. However, there is no explicit documented 'bulk' or 'batch' API endpoint that accepts multiple files/items in a single call — evidence only shows single-document convert/extract/segment endpoints and file storage for reuse. Missing for 10: a dedicated batch/bulk-submission endpoint, documented concurrency/throughput limits for large-scale runs, and any hands-on/community confirmation of successful large-batch processing.", "evidenceIds": [ "datalab-docs-2", "datalab-docs-14", "datalab-docs-8", "datalab-docs-37", "datalab-docs-38", "datalab-docs-31" ] }, { "productId": "datalab", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Datalab supports webhooks that fire automatically when processing jobs complete, and pipelines let you chain processors into reusable workflows, giving some event-driven automation. However, there's no evidence of a genuine rules engine where users define arbitrary conditions/triggers (e.g., 'if field X extracted equals Y, then run Z') beyond job-completion notifications and static pipeline chains. Missing for 10: conditional rule definition (if/then logic), custom event types beyond job completion, and any UI/API for user-defined trigger conditions.", "evidenceIds": [ "datalab-docs-8", "datalab-docs-24", "datalab-docs-34", "datalab-docs-38", "datalab-docs-21" ] }, { "productId": "datalab", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Datalab is a document conversion/extraction API with webhooks, pipelines, and SDK/CLI, but no evidence exists of a scheduling or recurring-job/workflow trigger mechanism (e.g., cron-like scheduling, recurring pipeline execution). Users must invoke conversions/pipelines via API calls triggered externally, not natively scheduled.", "evidenceIds": [] }, { "productId": "datalab", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Datalab's Pipelines feature is explicitly described as creating 'versioned, reusable configurations' for chaining processors, and agent processors can be pinned to a specific version (id@version), showing some versioning capability. However, there is no evidence of a review workflow (diffing/approving changes) or an explicit rollback mechanism to revert a pipeline to a prior version. Missing for 10: review/audit UI for pipeline changes, explicit rollback/revert functionality, version history browsing.", "evidenceIds": [ "datalab-docs-4", "datalab-docs-21", "datalab-docs-38", "datalab-docs-40" ] }, { "productId": "datalab", "storyId": "batch-throughput", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Datalab documents page-level limits with a workaround (page_range) for large documents, per-processor billing, and Team-tier rate limits (400 req/min per datalab-supp-2), plus an Enterprise option explicitly targeting 'extremely high volume' customers (datalab-supp-1). However there's no dedicated bulk/batch ingestion API, no documented throughput benchmarks for millions of pages, and no independent evidence of sustained high-volume performance. Missing for 10: explicit batch-processing endpoint for large-scale jobs, published throughput/SLA numbers at millions-of-pages scale, and independent corroboration of predictable performance under heavy load.", "evidenceIds": [ "datalab-docs-30", "datalab-supp-1", "datalab-supp-2", "datalab-docs-44", "datalab-docs-8" ] }, { "productId": "datalab", "storyId": "citations-provenance", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Datalab's docs explicitly claim structured extraction returns 'citations back to source bounding boxes for auditability' (datalab-docs-18) and the extract-structured-data API lets you supply a schema and get filled values with provenance-related metadata (datalab-docs-3, datalab-docs-32, datalab-docs-16). Segmentation also returns page ranges (datalab-docs-33), reinforcing page-level references. However, no example response schema or independent/hands-on confirmation is given showing per-field page number + bounding box + source snippet together. Missing for 10: a concrete sample JSON output demonstrating page number, bounding box, and snippet fields together, plus third-party validation that citations are reliable/accurate.", "evidenceIds": [ "datalab-docs-18", "datalab-docs-3", "datalab-docs-32", "datalab-docs-33", "datalab-docs-16" ] }, { "productId": "datalab", "storyId": "classification-splitting", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Datalab's Document Segmentation feature and /segment API explicitly split multi-document PDFs into logical sections and return page ranges per identified segment from a single uploaded file, directly matching the classify-and-split-per-document story (datalab-docs-5, datalab-docs-20, datalab-docs-33). Pipelines can chain convert→segment→extract into a single automated workflow producing per-document structured results (datalab-docs-38). Missing for 10: independent/hands-on validation of segmentation accuracy and a concrete example showing distinct classification labels/output structure per split document.", "evidenceIds": [ "datalab-docs-5", "datalab-docs-20", "datalab-docs-33", "datalab-docs-38" ] }, { "productId": "datalab", "storyId": "complex-layout-parsing", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs confirm Datalab converts PDFs (and other documents) into Markdown/HTML/JSON, which is the baseline capability needed for this story, but the evidence pack contains no specific claims or examples about handling multi-column layouts, headers/footers, or footnote ordering/accuracy. missing for 10: explicit documentation or benchmarks on multi-column layout handling, header/footer suppression, footnote placement, and any independent accuracy evidence for complex real-world PDFs.", "evidenceIds": [ "datalab-docs-17", "datalab-docs-1", "datalab-docs-22", "datalab-docs-31" ] }, { "productId": "datalab", "storyId": "confidence-scores-hitl", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers structured extraction, schema-based field extraction, and citation to bounding boxes for auditability, but nowhere mentions confidence scores on extracted fields or any human-in-the-loop review workflow for low-confidence outputs. Missing for 10: calibrated per-field confidence scores, a review/approval UI or workflow for flagged low-confidence fields, any documentation of human-in-the-loop correction loop.", "evidenceIds": [ "datalab-docs-3", "datalab-docs-18", "datalab-docs-32" ] }, { "productId": "datalab", "storyId": "document-data-compliance", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Datalab documents privacy-oriented options — Team plan includes 'BAA/DPA', and Enterprise offers on-prem/air-gapped deployment for 'highly-regulated environments' — which implicitly supports HIPAA-related contract needs, but there is no explicit mention of a zero-retention data policy or SOC 2 certification anywhere in the evidence pack. Missing for 10: explicit zero-retention/data-deletion policy statement, explicit SOC 2 attestation/report, explicit HIPAA compliance language (only BAA is named).", "evidenceIds": [ "datalab-supp-1", "datalab-supp-2", "datalab-docs-13", "datalab-docs-27" ] }, { "productId": "datalab", "storyId": "figure-chart-handling", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence covers document-to-markdown conversion and field-level bounding-box citations, but nothing in the pack specifically addresses figure/chart extraction, VLM-generated image summaries, or image crop outputs with page-position traceability. Missing for 10: any documentation of figure/image detection, VLM-based figure description, image crop output, or bounding boxes tied specifically to charts/figures.", "evidenceIds": [ "datalab-docs-17", "datalab-docs-18" ] }, { "productId": "datalab", "storyId": "file-format-breadth", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Datalab's single convert/extract API explicitly covers PDFs, images, Word docs (DOCX) and spreadsheets (XLSX) through one endpoint (datalab-docs-17, datalab-docs-31), fitting the 'one API for document mix' story. However, the evidence never explicitly confirms PPTX, HTML-as-input, or email formats being handled by the same pipeline, and community evidence flags OCR round-trip limitations for at least one output type. Missing for 10: explicit PPTX support, HTML/email as input formats, and independent confirmation of true format-agnostic handling without per-format quirks.", "evidenceIds": [ "datalab-docs-17", "datalab-docs-31", "datalab-docs-22", "datalab-docs-2", "datalab-comm-1" ] }, { "productId": "datalab", "storyId": "handwriting-recognition", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers structured extraction, form filling, track changes, and bounding-box citations, but nowhere mentions handwriting recognition specifically or confidence scoring/flagging for uncertain extractions. Missing for 10: any documentation of handwritten text/field recognition, any confidence score or uncertainty flag in extraction output.", "evidenceIds": [] }, { "productId": "datalab", "storyId": "large-document-handling", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Datalab documents explicit size/page limits and instructs users to manually segment large documents using page_range, plus checkpoint/save_checkpoint features to avoid re-parsing — indicating large-file handling requires workarounds rather than being fully automatic and reliable out-of-the-box. Missing for 10: explicit evidence of automatic handling of thousand-page or multi-gigabyte files without truncation, documented timeout limits/behavior, and independent confirmation of reliability at scale.", "evidenceIds": [ "datalab-docs-30", "datalab-docs-29", "datalab-docs-16" ] }, { "productId": "datalab", "storyId": "llm-ready-markdown", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Datalab's Marker-based pipeline is documented to convert PDFs/images/docs into Markdown, HTML, JSON, or LLM-ready chunks (datalab-docs-22, datalab-docs-31), which is the core RAG-chunking use case. However, no evidence explicitly describes removal of repeated headers/footers or other page furniture noise, and there's no independent hands-on validation of output cleanliness (the one community mention, datalab-comm-1, concerns PDF-to-PDF fidelity for OCR groundtruthing, not markdown noise-stripping). Missing for 10: explicit documentation of header/footer/page-furniture stripping logic, and independent benchmarks or user reports confirming clean, low-noise markdown output for LLM ingestion.", "evidenceIds": [ "datalab-docs-22", "datalab-docs-31", "datalab-docs-17", "datalab-comm-1" ] }, { "productId": "datalab", "storyId": "low-latency-sync-mode", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence documents a synchronous/fast processing mode with per-mode latency benchmarks; the API appears async (webhooks, checkpoints, job completion notifications) with no mention of a sync mode or documented latency numbers. Missing for 10: any mention of a synchronous/interactive mode, latency figures per mode, or benchmarks for seconds-level response.", "evidenceIds": [ "datalab-docs-8", "datalab-docs-16", "datalab-docs-29" ] }, { "productId": "datalab", "storyId": "multilingual-documents", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Datalab's conversion, extraction, segmentation, and pipeline features but contains no mention of CJK, right-to-left scripts, or multilingual OCR fidelity at all. Since document parsing/OCR fairly raises the question of non-English script support, this is an applicable axis with no supporting evidence, not a category mismatch.", "evidenceIds": [] }, { "productId": "datalab", "storyId": "openness-api-parity", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Datalab UI (Forge/Playground) is explicitly a preview layer for the same processors (convert, extract, segment, pipelines, form filling, track changes) that are fully documented as API and SDK endpoints, with no UI-only feature lacking an API path. Missing for 10: a discoverable OpenAPI/swagger spec (probe found 404s) and an explicit vendor statement guaranteeing full UI/API parity.", "evidenceIds": [ "datalab-docs-9", "datalab-docs-15", "datalab-docs-2", "datalab-docs-31", "datalab-docs-32", "datalab-docs-33", "datalab-docs-38", "datalab-docs-39", "datalab-docs-40", "datalab-probe-2" ] }, { "productId": "datalab", "storyId": "openness-full-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Datalab's core function is converting documents into open, non-proprietary formats (Markdown, HTML, JSON) and its underlying models (Marker, Surya, Chandra) remain open source, and file/pipeline management APIs let users manage and retrieve their processed data. However, there is no explicit documentation of a bulk 'export all account data' or account-closure data-portability feature—only per-document conversion outputs and file storage APIs. Missing for 10: explicit account-level data export/deletion tooling, evidence of exporting stored pipeline configs/history, and independent confirmation of full data portability.", "evidenceIds": [ "datalab-docs-1", "datalab-docs-17", "datalab-docs-22", "datalab-docs-31", "datalab-docs-37", "datalab-supp-1" ] }, { "productId": "datalab", "storyId": "openness-open-license", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs state that Chandra, Marker, and Surya (Datalab's core document/OCR models) \"remain free open source\" even as paid on-prem/enterprise tiers exist, indicating some source is open — but no explicit license name, no repository link, and no evidence about the core SaaS platform/API code itself being open. missing for 10: explicit license identification (e.g., MIT/Apache), a linked source repository, and confirmation that the broader product (not just underlying models) is open-source.", "evidenceIds": [ "datalab-supp-1" ] }, { "productId": "datalab", "storyId": "openness-self-host", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Datalab documents genuine on-prem deployment (Helm chart for Kubernetes, on-prem containers, air-gapped Enterprise option) confirming self-hosting is technically possible, and the underlying models (Chandra, Marker, Surya) are free open source. However, running the full core product on your own infrastructure is explicitly gated behind an Enterprise sales contract rather than being freely self-hostable by any AI-native user. Missing for 10: evidence of an accessible/self-serve self-hosted deployment path (e.g., public Docker image with docs, license terms) without requiring a sales contract, and independent confirmation of successful self-hosted setups.", "evidenceIds": [ "datalab-docs-13", "datalab-supp-1", "datalab-supp-2", "datalab-docs-27", "datalab-docs-28" ] }, { "productId": "datalab", "storyId": "playground-before-code", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Datalab docs explicitly describe 'Forge', a web playground where you upload a document and see parse/extract results instantly with no API key or code, and the changelog confirms structured extraction/track-changes results are previewable directly in the UI. This directly matches the story's drag-and-drop, no-code preview experience. Missing for 10: independent/hands-on confirmation of the playground UX (e.g. third-party walkthrough or screenshots) beyond vendor docs.", "evidenceIds": [ "datalab-docs-9", "datalab-docs-23", "datalab-docs-15", "datalab-docs-45" ] }, { "productId": "datalab", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Datalab offers Enterprise on-prem/air-gapped deployment so customers can run models on infrastructure they control (implying residency control), and Team plan includes BAA/DPA for compliance, but there is no documented cloud-tier region-selection feature (e.g., choosing US/EU data center) for standard SaaS usage. missing for 10: explicit region/data-residency selector for standard cloud accounts, independent confirmation of on-prem residency guarantees.", "evidenceIds": [ "datalab-supp-1", "datalab-supp-2", "datalab-docs-13", "datalab-docs-27" ] }, { "productId": "datalab", "storyId": "privacy-no-training", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Datalab offers an Enterprise on-prem option explicitly aimed at customers needing data privacy/regulated environments, and the Team plan includes a DPA/BAA which typically constrains data use, implying customer data isn't used for training. However, there is no explicit first-party statement that customer documents are excluded from model training or fine-tuning, and no independent confirmation of this policy. Missing for 10: an explicit no-training-on-customer-data clause/policy statement, independent verification, and any UI/API opt-out toggle for AI training.", "evidenceIds": [ "datalab-supp-1", "datalab-supp-2", "datalab-docs-13" ] }, { "productId": "datalab", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Datalab offers BAA/DPA on Team plan and on-prem/air-gapped Enterprise options implying data control, plus file storage/checkpoint features, but there is no explicit documentation of user-facing data retention windows, deletion controls/APIs, or a way for a self-serve user to purge stored documents/checkpoints. missing for 10: explicit retention period disclosure, self-service deletion/purge API or dashboard control, documentation of default data lifecycle for uploaded files and checkpoints.", "evidenceIds": [ "datalab-supp-2", "datalab-supp-1", "datalab-docs-37", "datalab-docs-29" ] }, { "productId": "datalab", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions telemetry, usage tracking, analytics collection, or an opt-out mechanism; the docs cover product features, security (2FA), billing, and on-prem options but nothing about telemetry opt-out. Missing for 10: any mention of telemetry/analytics collection practices, an opt-out setting or flag, or privacy policy language addressing usage tracking.", "evidenceIds": [] }, { "productId": "datalab", "storyId": "rag-ready-chunking", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Datalab's API/SDK explicitly supports converting documents to 'chunks' as an output format alongside Markdown/HTML/JSON, and has a dedicated Segment endpoint that splits documents into logical sections via a schema with page ranges, plus structured extraction with citations to bounding boxes for metadata/auditability - directly matching the semantic-boundary, metadata, embedding-ready segment requirements of RAG pipelines. Missing for 10: explicit documentation of chunk metadata schema (e.g., embeddings, token counts, overlap settings) and independent/hands-on validation that chunk output is embedding-ready rather than just logically segmented.", "evidenceIds": [ "datalab-docs-22", "datalab-docs-31", "datalab-docs-33", "datalab-docs-5", "datalab-docs-20", "datalab-docs-18" ] }, { "productId": "datalab", "storyId": "reading-order-hierarchy", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Datalab's core conversion outputs (Markdown, HTML, JSON) inherently carry some structural information (headings, sections) and its segmentation feature splits documents into logical sections, implying hierarchy-awareness, but the evidence never explicitly describes preservation of heading levels or reading order fidelity. A community comment on Datalab's Chandra-OCR notes a structural fidelity gap (PDF-in should allow PDF-out for groundtruthing), suggesting output structure isn't fully faithful to source layout in all cases. missing for 10: explicit documentation of heading/section hierarchy retention, reading-order guarantees, and independent benchmarks confirming structural accuracy.", "evidenceIds": [ "datalab-docs-17", "datalab-docs-22", "datalab-docs-31", "datalab-docs-5", "datalab-docs-33", "datalab-comm-1" ] }, { "productId": "datalab", "storyId": "scanned-doc-ocr", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Datalab's docs describe general document/image-to-markdown conversion (Marker, Surya, Chandra) but nowhere claim or demonstrate robustness on skewed pages, stamps, or low-quality/photographed scans specifically. The only community evidence found (HN comment on Chandra-OCR) criticizes a different limitation (lack of PDF-to-PDF round-tripping), not scan-quality accuracy, so it neither confirms nor disputes this specific axis. Missing for 10: any benchmark, docs section, or hands-on report addressing accuracy on skewed/rotated pages, stamped documents, or noisy photographed scans.", "evidenceIds": [ "datalab-docs-1", "datalab-docs-17", "datalab-comm-1" ] }, { "productId": "datalab", "storyId": "schema-driven-extraction", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Datalab has a dedicated structured-extraction API/recipe where developers supply a JSON schema and Marker fills it with extracted values, with an explicit API reference endpoint (extract-structured-data) supporting checkpointed input and even schema auto-generation. Missing for 10: independent/hands-on validation of extraction accuracy and no example of actual JSON output demonstrating field-level correctness.", "evidenceIds": [ "datalab-docs-3", "datalab-docs-16", "datalab-docs-32", "datalab-docs-41", "datalab-docs-18" ] }, { "productId": "datalab", "storyId": "table-structure-extraction", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Datalab's docs describe general PDF/Word/spreadsheet conversion to Markdown/HTML/JSON and structured extraction, but nothing in the evidence pack specifically addresses fidelity for merged cells, nested headers, or tables spanning multiple pages. Missing for 10: any documentation or example demonstrating complex table structure preservation, nested header handling, or multi-page table stitching.", "evidenceIds": [ "datalab-docs-17", "datalab-docs-22", "datalab-docs-31" ] }, { "productId": "datalab", "storyId": "table-to-structured-rows", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Datalab's structured extraction lets users define a JSON schema and get back typed field values (with bounding-box citations for auditability), plus schema auto-generation and checkpointing to reuse parsed state across calls, which maps directly to producing clean typed JSON for downstream DB loading. Missing for 10: independent/hands-on validation of extraction accuracy on real-world tables and any explicit DB-loading connector or schema-validation guarantees beyond docs.", "evidenceIds": [ "datalab-docs-3", "datalab-docs-16", "datalab-docs-18", "datalab-docs-32", "datalab-docs-41", "datalab-docs-29", "datalab-docs-22" ] }, { "productId": "datalab", "storyId": "typed-official-sdks", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Datalab documents a robust Python SDK covering convert/parse, structured extraction, segmentation, pipelines, form filling, and file management with API reference endpoints, plus a CLI — but there is no evidence anywhere in the pack of an official TypeScript/JS SDK, only Python. missing for 10: TypeScript SDK existence/docs, evidence of typed interfaces/type definitions, and independent confirmation of SDK quality or default configs.", "evidenceIds": [ "datalab-docs-2", "datalab-docs-22", "datalab-docs-31", "datalab-docs-32", "datalab-docs-33", "datalab-docs-38", "datalab-docs-36", "datalab-docs-37", "datalab-docs-14" ] }, { "productId": "datalab", "storyId": "vpc-self-host-deployment", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Datalab explicitly offers on-prem/self-hosted deployment via Enterprise contract, including a Helm chart for Kubernetes and air-gapped operation on customer infrastructure, plus BAA/DPA for compliance-sensitive plans. Missing for 10: independent/hands-on verification that self-hosted deployment works as documented, and more detail on VPC-specific deployment beyond 'air-gapped' Enterprise mention.", "evidenceIds": [ "datalab-docs-13", "datalab-supp-1", "datalab-supp-2", "datalab-docs-27", "datalab-docs-28" ] }, { "productId": "extend", "storyId": "accuracy-benchmarks-evals", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Extend documents a full self-serve evaluation framework (evaluation sets, processors, versioning, aggregate metrics from evaluation-set runs) that lets an ML engineer run their own accuracy evals before committing, and community testimony corroborates strong extraction accuracy in practice. However there is no evidence of the vendor publishing its own reproducible benchmark numbers/datasets or accuracy reports for independent verification. Missing for 10: vendor-published benchmark results/methodology, third-party benchmark validation, public accuracy leaderboards or comparable datasets.", "evidenceIds": [ "extend-docs-41", "extend-docs-42", "extend-docs-39", "extend-docs-40", "extend-comm-1" ] }, { "productId": "extend", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Extend has a confirmed live llms.txt at docs.extend.ai/llms.txt (HTTP 200) providing agent-oriented documentation, alongside agent-oriented docs like the Agent Quickstart and MCP integration docs that an AI agent could consume directly. Missing for 10: no independent/community confirmation of an agent actually consuming llms.txt successfully in practice.", "evidenceIds": [ "extend-probe-1", "extend-docs-30", "extend-docs-6" ] }, { "productId": "extend", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Extend's core AI extraction/classification pipeline generates structured data, per-field confidence scores that suggest which values to trust vs. route to human review, and citations pointing back to source locations — a form of AI-generated insight/suggestion from document data. However, there's no evidence of broader analytical insights, trend/summary generation, or proactive suggestions beyond the extraction/review-routing use case. Missing for 10: dedicated 'insights' or summary-generation feature, evidence of proactive suggestions beyond confidence-based review routing, and independent corroboration of this specific capability.", "evidenceIds": [ "extend-docs-34", "extend-docs-35", "extend-docs-23", "extend-docs-1", "extend-docs-11" ] }, { "productId": "extend", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Extend's Workflows feature (versioned, deployable pipelines) combined with async processing and webhook callbacks lets document-processing tasks run without a user waiting live, which is the core of 'background automation' — evidenced by extend-docs-19, extend-docs-12, extend-docs-13, extend-docs-37, extend-docs-43. However there's no explicit mention of scheduling/triggers (e.g., cron, event subscriptions beyond webhooks) or autonomous multi-step agent loops running unattended over time, so the 'autonomous' framing is only partially substantiated. Missing for 10: explicit scheduling/trigger mechanisms, evidence of long-running unattended agent loops, and independent confirmation that workflows execute fully autonomously without manual intervention.", "evidenceIds": [ "extend-docs-19", "extend-docs-12", "extend-docs-13", "extend-docs-37", "extend-docs-43" ] }, { "productId": "extend", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Extend markets built-in agentic features like \"Composer & Review Agent,\" \"Agentic OCR,\" and \"Agent form filling,\" plus an \"Agent Quickstart\" guide, suggesting users can delegate document-processing tasks to built-in AI agents. However, there's no clear evidence of a general-purpose conversational AI assistant inside the product UI for delegating arbitrary tasks — the agentic features are narrowly scoped to document extraction/review workflows. Missing for 10: a documented chat/assistant interface, examples of delegating broader tasks beyond document processing, and independent hands-on confirmation of these agent features in use.", "evidenceIds": [ "extend-docs-20", "extend-docs-23", "extend-docs-28", "extend-docs-30", "extend-docs-31" ] }, { "productId": "extend", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Extend offers a documented CLI, Python SDK (pip install), and REST API with sync/async modes and a test API key mode, all of which support headless/CI-driven automation of document processing. However, there is no explicit CI/CD integration guide, GitHub Actions example, or independent hands-on report of running it in a CI pipeline. missing for 10: explicit CI pipeline examples/integration guide, independent hands-on verification of headless CI usage, and details on exit codes/CLI scripting ergonomics for automation.", "evidenceIds": [ "extend-docs-2", "extend-docs-5", "extend-docs-7", "extend-docs-29", "extend-docs-37", "extend-docs-38", "extend-probe-3" ] }, { "productId": "extend", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Extend documents 'Using Extend via MCP' (docs.extend.ai/mcp), which describes exposing Extend's own tools via MCP to other agents — this is the opposite direction of the story (product consuming external MCP servers as a client). No evidence shows Extend can ingest or plug in third-party MCP servers to use their tools.", "evidenceIds": [ "extend-docs-6", "extend-probe-2" ] }, { "productId": "extend", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Extend is a document-processing platform (not an agent itself), and it documents an official MCP server ('Using Extend via MCP' at docs.extend.ai/mcp), confirmed by probe as live documentation, allowing AI agents to connect via MCP. missing for 10: no independent/hands-on confirmation of the MCP server working in practice, and limited detail on setup specifics beyond the doc title.", "evidenceIds": [ "extend-docs-6", "extend-probe-2" ] }, { "productId": "extend", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Extend documents an official MCP server and Agent Quickstart, which allow AI agents/assistants to drive the platform via natural-language tool calls rather than raw API calls, and it also ships a CLI for programmatic control. However, there is no first-party example or independent corroboration showing an end-user issuing free-form natural-language commands and Extend interpreting/executing them directly. Missing for 10: concrete examples of natural-language command execution, independent/hands-on validation of the MCP/agent workflow.", "evidenceIds": [ "extend-docs-6", "extend-docs-30", "extend-probe-2", "extend-probe-3", "extend-docs-5" ] }, { "productId": "extend", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Extend documents an official CLI at docs.extend.ai/cli, alongside SDKs and MCP support, indicating a first-party CLI exists for AI-native workflows. Missing for 10: no independent/hands-on confirmation of CLI usage or specific command examples beyond the docs listing page.", "evidenceIds": [ "extend-docs-5", "extend-probe-3" ] }, { "productId": "extend", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Extend provides a documented public API with authentication, endpoints (upload, extract, classify, split, parse), sync/async modes, SDKs, CLI, and a dedicated API Quickstart, giving AI-native users a clear path to drive the product programmatically. Community feedback confirms real-world usage of the product's data extraction capabilities. missing for 10: independent third-party validation specifically of API robustness/rate limits beyond docs.", "evidenceIds": [ "extend-docs-4", "extend-docs-7", "extend-docs-29", "extend-docs-32", "extend-docs-37", "extend-docs-5", "extend-comm-1" ] }, { "productId": "extend", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows basic API-key authentication (extend-docs-4) and a separate test-vs-production API key for sandboxing (extend-docs-38), but nothing about issuing scoped, role/permission-limited, or least-privilege credentials for individual agents (e.g., granular scopes, RBAC, per-agent key restrictions).", "evidenceIds": [] }, { "productId": "extend", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Extend documents an official Python SDK (pip install extend-ai), a dedicated SDKs docs page, API quickstart, authentication docs, and API reference endpoints, giving AI-native developers a clear official SDK path alongside CLI and MCP integrations. missing for 10: independent/hands-on developer corroboration of SDK usage beyond first-party docs, and multi-language SDK coverage details.", "evidenceIds": [ "extend-docs-2", "extend-docs-7", "extend-docs-29", "extend-docs-4", "extend-docs-32" ] }, { "productId": "extend", "storyId": "agentic-webhooks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Extend has a dedicated 'Webhook Configuration' documentation page and separately confirms webhooks fire on production runs (and are suppressed in test mode), showing an established webhook subscription mechanism for async events. Missing for 10: specifics on supported event types, payload schema, and independent/hands-on verification of webhook reliability.", "evidenceIds": [ "extend-docs-12", "extend-docs-38", "extend-docs-13" ] }, { "productId": "extend", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Extend has a documented API reference (authentication, endpoints like upload-file, quickstarts) and SDKs/CLI, but there's no evidence of an interactive 'try it' console or runnable code examples embedded in the docs. missing for 10: explicit interactive API playground, in-browser runnable examples, live request/response testing.", "evidenceIds": [ "extend-docs-4", "extend-docs-29", "extend-docs-32", "extend-docs-7", "extend-docs-5" ] }, { "productId": "extend", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Extend documents a full API reference, authentication, and multiple endpoints, but no evidence pack item mentions a downloadable OpenAPI/Swagger spec or any machine-readable API definition file. missing for 10: OpenAPI/Swagger spec file, documented spec download link, machine-readable API schema.", "evidenceIds": [] }, { "productId": "extend", "storyId": "api-sandbox", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Extend explicitly documents a test environment where runs made with a test API key never touch production data and never fire production webhooks, directly matching the sandbox story. missing for 10: no independent/hands-on corroboration of the test-key isolation beyond first-party docs, and no detail on how thoroughly sandbox data mirrors production for realistic testing.", "evidenceIds": [ "extend-docs-38" ] }, { "productId": "extend", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows versioning for internal artifacts (processors, workflows, base models) but no documentation of API endpoint versioning (e.g., v1/v2 paths) or any deprecation/sunset policy for the REST API/SDK itself.", "evidenceIds": [ "extend-docs-40", "extend-docs-43", "extend-docs-45", "extend-docs-4", "extend-docs-29" ] }, { "productId": "extend", "storyId": "async-jobs-webhooks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Extend's docs explicitly state every processing endpoint including parse supports both sync and async modes, and separate docs cover webhook configuration for completion notifications, matching the async-job + webhook pipeline pattern. Missing for 10: explicit mention of a status-polling endpoint/job-id retrieval flow and independent/hands-on confirmation that async+webhook flow works reliably in production.", "evidenceIds": [ "extend-docs-37", "extend-docs-13", "extend-docs-12" ] }, { "productId": "extend", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Extend supports batch-like operations via multifile extraction (run over a collection of files with shared context), async processing endpoints for extract/classify/split/parse/edit, and evaluation set runs that score a processor against every item in a set — all pointing to bulk/batch capability across many documents. However, there's no explicit documentation of a bulk API endpoint for arbitrary large-scale item batches, rate limits/throughput guarantees, or dedicated bulk-operations tooling beyond these features. missing for 10: dedicated bulk-operation API/endpoint documentation, throughput/scale guarantees, independent hands-on evidence of large-scale bulk runs.", "evidenceIds": [ "extend-docs-36", "extend-docs-37", "extend-docs-42", "extend-docs-13" ] }, { "productId": "extend", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Extend supports webhooks to fire on processing events and has a Workflows system with versioning, which lets users react to events (e.g., file processed, extraction complete) with automated actions, but there's no documented general-purpose rule engine for defining custom trigger-condition-action logic across arbitrary events. Missing for 10: an explicit rules/conditions builder beyond simple webhook firing, and evidence of triggering on non-document events.", "evidenceIds": [ "extend-docs-12", "extend-docs-19", "extend-docs-37", "extend-docs-43" ] }, { "productId": "extend", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Extend documents workflows, webhooks, and async processing, but no evidence of a scheduling mechanism (cron-like triggers, recurring runs) for workflows or jobs. Missing for 10: any mention of scheduled/recurring workflow triggers, cron configuration, or time-based automation.", "evidenceIds": [ "extend-docs-19", "extend-docs-12", "extend-docs-13", "extend-docs-43" ] }, { "productId": "extend", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Extend documents concrete versioning for workflows (\"one editable draft and any number of immutable deployed versions\"), processors (\"the thing you version and iterate on in Extend Studio\"), and evaluation sets scoring specific processor versions — this covers version and review well. However, there is no explicit evidence of a rollback mechanism (reverting a deployed workflow/processor to a prior version) or of review workflows like diffs/approval gates beyond the Composer & Review Agent mention. missing for 10: explicit rollback/revert capability, diff/audit view between versions, independent confirmation of these versioning features in practice.", "evidenceIds": [ "extend-docs-40", "extend-docs-43", "extend-docs-41", "extend-docs-42", "extend-docs-39", "extend-docs-20" ] }, { "productId": "extend", "storyId": "batch-throughput", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Extend documents async processing for all endpoints and a credit-based pricing model, which supports high-volume batch workflows, but there is no evidence of documented rate limits, throughput benchmarks, or guidance for processing millions of pages at scale. Missing for 10: explicit rate-limit documentation, throughput/SLA numbers, and batch-scale case studies or guarantees.", "evidenceIds": [ "extend-docs-13", "extend-docs-37", "extend-docs-15", "extend-docs-24" ] }, { "productId": "extend", "storyId": "citations-provenance", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly describe per-field citations pointing back to the exact spot on the page, confidence scores for trust/routing decisions, and multifile citations/file provenance, directly matching the provenance story (page/location + source snippet). missing for 10: no explicit mention of bounding-box coordinates format, and no independent/hands-on verification of citation accuracy beyond vendor docs.", "evidenceIds": [ "extend-docs-35", "extend-docs-34", "extend-docs-36", "extend-docs-11" ] }, { "productId": "extend", "storyId": "classification-splitting", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Extend has dedicated Classification and Splitting overview docs plus async processing that applies to every processing endpoint (extract, classify, split, parse), and supports file upload with automatic conversion, indicating a workflow where a single multi-document upload can be classified/split into per-document outputs. Community feedback confirms satisfaction with extraction accuracy over a year of production use. Missing for 10: an explicit end-to-end worked example combining classification+splitting on one upload with per-document result output shown in a single doc, and independent hands-on verification of this exact multi-doc splitting workflow.", "evidenceIds": [ "extend-docs-8", "extend-docs-9", "extend-docs-37", "extend-docs-32", "extend-docs-33", "extend-comm-1" ] }, { "productId": "extend", "storyId": "complex-layout-parsing", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Extend markets parsing as high-accuracy for 'hardest documents' with features like agentic OCR, table/checkbox/handwriting detection, and citations pointing to exact page locations, and one community user praises extraction accuracy over a year of use. However, no evidence specifically addresses multi-column layouts, headers/footers, or footnote handling/ordering. Missing for 10: explicit documentation or examples showing correct handling of multi-column reading order, header/footer exclusion, and footnote placement; independent benchmark or hands-on test of these specific layout challenges.", "evidenceIds": [ "extend-docs-1", "extend-docs-22", "extend-docs-23", "extend-docs-35", "extend-comm-1" ] }, { "productId": "extend", "storyId": "confidence-scores-hitl", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Extend has a dedicated Confidence Scores doc explicitly stating per-field confidence values let users 'trust high-confidence values automatically and route the rest to human review,' plus a Composer & Review Agent product feature and citations pointing to exact source locations for verification. This directly matches the calibrated confidence + human-in-the-loop review story. Missing for 10: independent/hands-on evidence validating that confidence scores are well-calibrated in practice and more detail on the review workflow UI itself.", "evidenceIds": [ "extend-docs-11", "extend-docs-34", "extend-docs-35", "extend-docs-20" ] }, { "productId": "extend", "storyId": "document-data-compliance", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Extend's docs explicitly document SOC 2 Type II alignment, HIPAA (with BAA add-on) and GDPR compliance, plus configurable zero data retention (including for AI subprocessors) at the workspace level, directly matching the data-engineer's need to process contracts and medical records securely. Missing for 10: independent third-party audit confirmation or hands-on customer verification of ZDR/HIPAA in production use beyond vendor docs.", "evidenceIds": [ "extend-supp-1", "extend-supp-2", "extend-docs-16", "extend-docs-17", "extend-docs-21", "extend-docs-47" ] }, { "productId": "extend", "storyId": "figure-chart-handling", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Extend documents citations/provenance pointing extracted field values back to exact page locations and detects tables/images/handwriting, but evidence never explicitly describes VLM-generated summaries of figures/charts or image crop extraction with bounding-box positions for visual elements specifically — citations are framed around extracted field values, not figure/chart description. missing for 10: explicit VLM-based figure/chart summarization, image crop extraction as an output artifact, and confirmation that citations/positions apply to figures/charts (not just text fields).", "evidenceIds": [ "extend-docs-22", "extend-docs-26", "extend-docs-35", "extend-docs-36", "extend-docs-11" ] }, { "productId": "extend", "storyId": "file-format-breadth", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Extend documents a single API surface (extract, classify, split, parse, edit) that ingests uploaded files and auto-converts Word/PowerPoint to PDF, and maintains a dedicated 'Supported File Types' doc, showing a unified pipeline for mixed document types. However, the evidence never explicitly confirms XLSX, HTML, or email format support, or shows a worked example spanning all these formats without extra plumbing. Missing for 10: explicit confirmation of XLSX/HTML/email support, and hands-on proof of one-call handling across all listed formats.", "evidenceIds": [ "extend-docs-14", "extend-docs-32", "extend-docs-33", "extend-docs-37" ] }, { "productId": "extend", "storyId": "handwriting-recognition", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Extend explicitly advertises handwriting detection ('Detect tables, checkboxes, images, and handwriting') and 'Agentic OCR' as pricing features, plus a documented Confidence Scores mechanism that lets low-confidence extractions route to human review. However, there is no direct documentation showing handwritten field values are specifically flagged with lower confidence or examples of handwriting-specific extraction accuracy/behavior. Missing for 10: dedicated documentation or examples of handwriting-specific extraction accuracy, confidence flagging tied specifically to handwritten content, and independent/hands-on verification of handwriting OCR quality.", "evidenceIds": [ "extend-docs-22", "extend-docs-23", "extend-docs-11", "extend-docs-34", "extend-docs-35" ] }, { "productId": "extend", "storyId": "large-document-handling", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Extend explicitly advertises 2,000+ page support, async processing for all endpoints (to avoid timeouts on long jobs), and automatic Word/PPT-to-PDF conversion, plus a test environment guide—strong first-party evidence targeting large-document reliability. However, there's no explicit mention of multi-gigabyte file size limits, chunking/truncation safeguards for extremely large files, or independent/hands-on confirmation that thousand-page or multi-GB documents process without silent truncation. missing for 10: independent verification of large-file handling, explicit multi-GB file size limits/support, evidence of no silent truncation on very large inputs.", "evidenceIds": [ "extend-docs-24", "extend-docs-37", "extend-docs-33", "extend-docs-38", "extend-docs-13" ] }, { "productId": "extend", "storyId": "llm-ready-markdown", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Extend's Parse feature outputs structured chunks plus a metadata object (extend-docs-3, extend-docs-9), suggesting some design toward LLM/RAG-ready output, but there is no explicit documentation claiming markdown/JSON output has repeated headers or page furniture stripped for clean LLM consumption. missing for 10: explicit claim/documentation of noise removal (headers, footers, page furniture) in parse output, and independent verification of clean output quality.", "evidenceIds": [ "extend-docs-3", "extend-docs-9", "extend-docs-37" ] }, { "productId": "extend", "storyId": "low-latency-sync-mode", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Docs confirm every processing endpoint supports both sync and async modes (extend-docs-37) and pricing page lists a 'Fast mode' option (extend-docs-25), implying a fast synchronous path exists, but no evidence quantifies actual latency (e.g., 'returns in X seconds') for either mode. Missing for 10: documented latency benchmarks per mode, explicit performance numbers for sync/fast mode, independent confirmation of response times.", "evidenceIds": [ "extend-docs-37", "extend-docs-13", "extend-docs-25" ] }, { "productId": "extend", "storyId": "multilingual-documents", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses multilingual or CJK/RTL document parsing fidelity; all citations cover general parsing, extraction, SDKs, compliance, and pricing without mentioning language coverage.", "evidenceIds": [] }, { "productId": "extend", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Extend documents a broad API/SDK/CLI/MCP surface covering the same core objects seen in its UI (processors, workflows with versioning, evaluation sets, extraction/classification/splitting, file upload) suggesting close UI-API parity, but there is no explicit statement of full parity and UI-centric tools like Studio's interactive testing and the Composer & Review Agent (human-in-the-loop) are not shown to have a documented equivalent API path. missing for 10: explicit parity claim/documentation, API equivalents for Studio interactive iteration and Composer/Review Agent workflows, independent confirmation of full coverage.", "evidenceIds": [ "extend-docs-5", "extend-docs-6", "extend-docs-7", "extend-docs-29", "extend-docs-30", "extend-docs-39", "extend-docs-40", "extend-docs-41", "extend-docs-43", "extend-docs-20" ] }, { "productId": "extend", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Extend is a SaaS document-processing platform holding workflows, processors, evaluation sets and extracted data, so data-portability/export is a fair question, but the evidence pack contains no mention of a bulk data-export feature, open-format export of processed data/configs, or account-closure data dump — only retention/ZDR policies which describe deletion, not export.", "evidenceIds": [ "extend-supp-2", "extend-docs-17" ] }, { "productId": "extend", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Extend is a closed, commercial SaaS document-processing platform with paid tiers, credits, and proprietary API/CLI/SDK access; there is no mention of any open-source license or public source code repository. Open-source licensing is not a fair axis for this kind of hosted proprietary product offering.", "evidenceIds": [] }, { "productId": "extend", "storyId": "openness-self-host", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Extend is a closed SaaS platform; the only evidence bearing on self-hosting is a pricing-page bullet mentioning 'BYOC (your VPC) and Hybrid deployments' as an enterprise add-on, with no documentation of installation, source availability, or a true self-managed deployment path. Missing for 10: open-source or downloadable core product, self-hosting setup docs, infrastructure requirements, and independent confirmation that customers can run it fully outside Extend's cloud.", "evidenceIds": [ "extend-docs-48" ] }, { "productId": "extend", "storyId": "playground-before-code", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Extend references a web-based \"Studio\" for iterating on processors (extend-docs-18, extend-docs-40) which implies a no-code interface for testing parse/extract, but no evidence describes a drag-and-drop upload flow or explicitly shows results appearing before any code is written. Missing for 10: explicit description or screenshot of drag-and-drop file upload in Studio, confirmation that parse/extract results appear instantly without any coding, and independent/hands-on confirmation of this playground UX.", "evidenceIds": [ "extend-docs-18", "extend-docs-40", "extend-docs-39" ] }, { "productId": "extend", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence covers compliance (SOC2, HIPAA, GDPR), data retention/ZDR, and BYOC/Hybrid deployment options, but nowhere mentions region selection or data residency choice for storage location. missing for 10: explicit region/residency selection controls, documentation naming specific data center regions, evidence of a residency configuration option.", "evidenceIds": [ "extend-supp-1", "extend-supp-2", "extend-docs-48" ] }, { "productId": "extend", "storyId": "privacy-no-training", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Extend documents zero data retention (ZDR) options including for AI subprocessors, configurable at workspace level, plus SOC2/HIPAA/GDPR compliance controls, which addresses training-data concerns for enterprise customers. However, there's no explicit statement that customer data is never used to train Extend's own models by default, nor independent verification of this policy. Missing for 10: explicit default no-training-on-customer-data policy statement, independent audit/confirmation, and clarity on whether ZDR is opt-in/paid-tier only.", "evidenceIds": [ "extend-docs-21", "extend-supp-2", "extend-docs-16", "extend-supp-1" ] }, { "productId": "extend", "storyId": "privacy-retention-controls", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Extend's official docs explicitly describe configurable automatic data-retention policies and zero data retention (ZDR) options, settable at the workspace level, plus GDPR-related compliance documentation; the pricing page also advertises 'Zero data retention' as a feature tier. This directly addresses retention and deletion control for AI-native users. Missing for 10: no independent/hands-on confirmation that deletion requests are actually honored, and no explicit self-service deletion API/UI documented beyond retention policy configuration.", "evidenceIds": [ "extend-supp-2", "extend-docs-21", "extend-docs-16", "extend-docs-17" ] }, { "productId": "extend", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Extend documents data retention/ZDR and compliance policies, but there is no mention of telemetry or usage-tracking opt-out settings for the product itself (e.g., CLI/SDK analytics telemetry). missing for 10: any documentation of telemetry collection or an opt-out/disable-tracking mechanism.", "evidenceIds": [] }, { "productId": "extend", "storyId": "rag-ready-chunking", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Extend's Parse API outputs both `chunks` and a `metadata` object, and there's a dedicated Splitting feature for breaking documents into semantic segments, which aligns with the RAG-chunking need. However, there's no explicit documentation framing this as embedding-ready or optimized for RAG pipelines, nor independent corroboration of chunk quality for downstream embedding use. Missing for 10: explicit embedding-ready format specs, RAG-specific use-case documentation, and independent/hands-on validation of chunk semantic quality.", "evidenceIds": [ "extend-docs-3", "extend-docs-9", "extend-docs-35" ] }, { "productId": "extend", "storyId": "reading-order-hierarchy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack confirms Extend parses documents and extracts structured data (chunks, metadata) but contains no specific claim or demonstration that parsed output preserves heading hierarchy, section structure, or reading order for downstream LLM consumption.", "evidenceIds": [ "extend-docs-3", "extend-docs-22" ] }, { "productId": "extend", "storyId": "scanned-doc-ocr", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Extend markets itself as handling 'your hardest documents with unmatched accuracy' with agentic OCR, handwriting/table/checkbox detection, and confidence-based routing to human review, and a year-long HN user reports being happy with extraction accuracy. However no evidence specifically addresses skewed pages, stamps, or low-quality scan handling — these document conditions are never explicitly mentioned in docs or community feedback. Missing for 10: explicit documentation or benchmarks on skew correction, stamp/seal recognition, and degraded scan quality performance, plus independent hands-on testing on such documents.", "evidenceIds": [ "extend-docs-1", "extend-docs-22", "extend-docs-23", "extend-docs-34", "extend-docs-35", "extend-comm-1" ] }, { "productId": "extend", "storyId": "schema-driven-extraction", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Extend has dedicated docs for defining an Extraction Schema, returns structured fields with per-field confidence scores and citations pointing to source location, and supports sync/async extraction endpoints; community feedback corroborates strong extraction accuracy. Missing for 10: independent third-party benchmarking of schema-conformance accuracy beyond a single HN endorsement.", "evidenceIds": [ "extend-docs-10", "extend-docs-11", "extend-docs-34", "extend-docs-35", "extend-docs-37", "extend-comm-1" ] }, { "productId": "extend", "storyId": "table-structure-extraction", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Extend's pricing page claims it can 'Detect tables, checkboxes, images, and handwriting' and offers Agentic OCR/parsing, suggesting table-aware extraction, but there is no documentation addressing merged cells, nested headers, or multi-page table spans, nor evidence of faithful HTML/markdown structural output. Missing for 10: explicit handling of merged cells, nested headers, multi-page table continuity, and documented HTML/markdown fidelity examples.", "evidenceIds": [ "extend-docs-22", "extend-docs-23" ] }, { "productId": "extend", "storyId": "table-to-structured-rows", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Extend documents schema-based extraction that outputs structured JSON with per-field confidence scores and citations, and explicitly advertises table detection, which supports turning extracted tables into typed data with less manual cleanup; community feedback also praises extraction accuracy. However, there is no explicit documentation of table-specific row/column typing, CSV/DB-ready output formats, or direct database load integrations. Missing for 10: table-to-row schema examples, explicit typed-row/CSV export, and direct DB-loading integration guidance.", "evidenceIds": [ "extend-docs-10", "extend-docs-11", "extend-docs-22", "extend-docs-35", "extend-comm-1" ] }, { "productId": "extend", "storyId": "typed-official-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Extend has a dedicated SDKs docs page and a Python package (pip install extend-ai) referenced, plus full API coverage across parse/extract/split/classify/jobs implied by endpoint docs, suggesting SDKs wrap the full API. However, there's no explicit confirmation of a TypeScript SDK, no code samples showing typed usage or sensible defaults, and no independent/community corroboration of SDK quality or completeness. Missing for 10: explicit TypeScript SDK evidence, typed code examples demonstrating parse/extract/jobs coverage, and independent developer confirmation of SDK ergonomics/defaults.", "evidenceIds": [ "extend-docs-7", "extend-docs-2", "extend-docs-29", "extend-docs-37" ] }, { "productId": "extend", "storyId": "vpc-self-host-deployment", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Pricing page explicitly lists 'BYOC (your VPC) and Hybrid deployments' as an offering, indicating self-hosted/VPC deployment options exist, but there is no dedicated documentation detailing setup, architecture, or infrastructure requirements for such deployments, and no independent/hands-on confirmation. missing for 10: detailed self-hosting/VPC deployment docs, architecture guides, independent confirmation of BYOC actually working, clarity on which components run in customer VPC vs SaaS.", "evidenceIds": [ "extend-docs-48" ] }, { "productId": "llamaparse", "storyId": "accuracy-benchmarks-evals", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no vendor-published reproducible accuracy benchmarks (docs describe features, formats, and getting-started SDK usage, but no benchmark methodology, dataset, or published accuracy numbers) and no first-party guidance on running independent evals. A community report (comm-6) even claims a self-run baseline outperformed a previously reported LlamaParse pipeline number, but this is a single community claim referencing an external benchmark, not a vendor-published reproducible benchmark suite, so it cannot establish 'disputed' status. missing for 10: vendor-published benchmark methodology/dataset, published accuracy metrics, eval harness or reference scripts for ML engineers to reproduce results, any first-party corroboration of benchmark claims.", "evidenceIds": [ "llamaparse-comm-6" ] }, { "productId": "llamaparse", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "An llms.txt file is confirmed live via probe (HTTP 200), and LlamaIndex also documents an explicit 'for-agents' page with an MCP server and agent-oriented docs tools (search_docs, grep_docs, read_doc) designed for agents to consume documentation programmatically. missing for 10: independent/community confirmation that agents actually use llms.txt successfully in practice, and the docs-md alternate format probe returned 404 showing incomplete agent-doc surface.", "evidenceIds": [ "llamaparse-probe-1", "llamaparse-docs-26", "llamaparse-docs-27", "llamaparse-docs-21", "llamaparse-probe-4", "llamaparse-probe-2" ] }, { "productId": "llamaparse", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "LlamaParse/LlamaCloud is documented as a document parsing, extraction, classification, and indexing pipeline driven by user-defined schemas and rules, not a product that surfaces its own AI-generated insights or suggestions about the user's data inside a UI. Evidence covers extraction, classification, splitting, and RAG-indexing capabilities, but none describe an insights/suggestion feature (e.g., auto-generated summaries, anomaly flags, or recommendations) exposed to the user. missing for 10: any documented insights/suggestions UI or feature, evidence of autonomous analysis surfaced to users, independent confirmation of such a capability.", "evidenceIds": [ "llamaparse-docs-3", "llamaparse-docs-14", "llamaparse-docs-33", "llamaparse-docs-20" ] }, { "productId": "llamaparse", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "LlamaParse supports asynchronous job processing with webhook notifications instead of polling, which enables integration into automated pipelines that run without manual monitoring, and MCP tool exposure lets agents trigger parsing autonomously as part of larger workflows. However, there's no evidence of native scheduling, cron-like triggers, or persistent background jobs that run independently of an external caller invoking them. Missing for 10: scheduled/triggered automation, autonomous recurring job execution, evidence of agents independently initiating parse jobs on a schedule.", "evidenceIds": [ "llamaparse-docs-6", "llamaparse-docs-8", "llamaparse-docs-16", "llamaparse-probe-4" ] }, { "productId": "llamaparse", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "LlamaParse's evidence describes it as a document parsing/extraction API (Parse, Extract, Classify, Split, Index) callable via SDKs, CLI, REST, or exposed to external agents via an MCP server — but there is no mention of a built-in AI assistant inside the product itself that a user could converse with or delegate tasks to. The MCP endpoint (llamaparse-docs-8/16/27) lets external agents call LlamaParse's tools, which is the reverse direction of this story.", "evidenceIds": [] }, { "productId": "llamaparse", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "LlamaParse offers a REST API, CLI, and SDKs (Python/TypeScript/Go/Java) suitable for scripted/CI use, plus webhooks for async job notification instead of polling, which supports headless automation workflows. Missing for 10: explicit CI/CD examples (GitHub Actions, Docker recipes), documented non-interactive auth/service-account flow for pipelines, and independent hands-on confirmation of headless CI usage.", "evidenceIds": [ "llamaparse-docs-1", "llamaparse-docs-6", "llamaparse-docs-11" ] }, { "productId": "llamaparse", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "medium", "rationale": "LlamaParse is a document-parsing/extraction service that exposes its own capabilities as an MCP server (mcp.llamaindex.ai/mcp) for other agents to call; it is not itself an agent host that plugs in and orchestrates external MCP servers' tools. The evidence pack shows only the server-provider role (docs-8, docs-16, docs-21, docs-27, probe-4), which is the wrong axis for this story.", "evidenceIds": [] }, { "productId": "llamaparse", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "First-party docs explicitly describe an official MCP endpoint (https://mcp.llamaindex.ai/mcp) that exposes Parse, Classify, Extract, Split, and Index as callable tools for any MCP client, and a dedicated docs page details this server. Missing for 10: independent/hands-on confirmation that the MCP server works as documented and clarity on how it relates to the separate docs-search MCP tools (search_docs/grep_docs/read_doc) mentioned elsewhere.", "evidenceIds": [ "llamaparse-docs-8", "llamaparse-docs-16", "llamaparse-probe-4", "llamaparse-docs-27" ] }, { "productId": "llamaparse", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "LlamaParse supports several natural-language-driven controls: Classify uses natural-language rules, Extract can generate schemas from a prompt, and Parse output can be steered with custom natural-language prompts (e.g., focusing only on prices in a receipt). It also exposes an MCP endpoint so agents can invoke Parse/Classify/Extract/Split/Index via tool calls, which is agent-friendly but not itself a natural-language command interface. Missing for 10: a unified conversational/chat interface for operating the whole platform, and independent hands-on evidence confirming natural-language commands reliably control parsing behavior end-to-end.", "evidenceIds": [ "llamaparse-docs-4", "llamaparse-docs-33", "llamaparse-docs-35", "llamaparse-docs-8", "llamaparse-docs-16" ] }, { "productId": "llamaparse", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs mention the CLI as one of several supported interfaces to run parse jobs ('whether you prefer Python, TypeScript, Go, Java, the CLI, the REST API, or the Web UI') but no dedicated CLI reference, install instructions, or command examples are provided in the evidence pack. Missing for 10: CLI installation/usage docs, command examples, independent/community confirmation of CLI usage.", "evidenceIds": [ "llamaparse-docs-1" ] }, { "productId": "llamaparse", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "LlamaParse offers a documented REST API plus SDKs (Python, TypeScript, Go, Java) and a CLI, with example code snippets for calling the parse endpoint, and webhook support for async job notification. Independent probes could not find a public OpenAPI spec, which slightly weakens strict machine-readability of the API. missing for 10: publicly discoverable OpenAPI/swagger spec, independent third-party confirmation of API robustness.", "evidenceIds": [ "llamaparse-docs-1", "llamaparse-docs-11", "llamaparse-docs-6", "llamaparse-docs-20", "llamaparse-probe-3" ] }, { "productId": "llamaparse", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Docs mention SSO and role-based access controls for managing org/project access (llamaparse-docs-9, llamaparse-docs-18), but there is no evidence of scoped or least-privilege API key/credential issuance specifically for agents (e.g., per-key permission scopes, agent-specific tokens). This is a fair axis for an API platform, so absence of evidence yields 'none' rather than 'na'.", "evidenceIds": [ "llamaparse-docs-9", "llamaparse-docs-18" ] }, { "productId": "llamaparse", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "LlamaParse provides official SDKs across multiple languages (Python, TypeScript, Go, Java) plus CLI and REST API, documented with code examples (e.g., client.parsing.parse), and maintained as a public GitHub package (llama-cloud) with migration guidance. This directly supports building AI-native apps against official first-party SDKs rather than reverse-engineered clients. missing for 10: independent third-party benchmarking of SDK reliability/versioning, and no OpenAPI spec found (probe-3) to corroborate REST completeness", "evidenceIds": [ "llamaparse-docs-1", "llamaparse-docs-11", "llamaparse-gh-1", "llamaparse-docs-20" ] }, { "productId": "llamaparse", "storyId": "agentic-webhooks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs explicitly describe configuring webhook endpoints to be notified when parse jobs complete, fail, or reach other states, directly matching the story. Missing for 10: independent/hands-on confirmation of webhook reliability, payload schema/signature verification details, and community corroboration of real-world webhook usage.", "evidenceIds": [ "llamaparse-docs-6" ] }, { "productId": "llamaparse", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Docs show many static code snippets/examples (Python calls, curl-like usage) but there is no evidence of an interactive, runnable API reference (e.g., Swagger/OpenAPI explorer or live code sandbox); explicit probes for OpenAPI/Swagger endpoints returned 404s, indicating no such interactive reference exists.", "evidenceIds": [ "llamaparse-probe-3", "llamaparse-docs-11", "llamaparse-docs-15", "llamaparse-docs-35" ] }, { "productId": "llamaparse", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "LlamaParse exposes a REST API, but there is no evidence of a downloadable OpenAPI/Swagger spec; explicit probes for common OpenAPI endpoints (openapi.json, swagger.json, etc.) all returned 404.", "evidenceIds": [ "llamaparse-probe-3", "llamaparse-docs-1" ] }, { "productId": "llamaparse", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence describes a dedicated sandbox/test environment separate from production for LlamaParse; free credits and self-hosting/BYOC options are mentioned but do not constitute a documented sandbox mode for testing without touching production data.", "evidenceIds": [] }, { "productId": "llamaparse", "storyId": "api-versioning-policy", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "There is a version parameter in the parse API (version=\"latest\") and a GitHub migration notice telling users to move to llama-cloud>=1.0, showing some versioning/migration practice, but no explicit documented deprecation policy, versioning scheme, or changelog is provided. missing for 10: a published API versioning scheme, a formal deprecation policy/timeline, changelog or release notes, and independent confirmation of stability guarantees.", "evidenceIds": [ "llamaparse-docs-11", "llamaparse-gh-1" ] }, { "productId": "llamaparse", "storyId": "async-jobs-webhooks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly describe async job parsing with status polling replaceable by webhooks ('Instead of continuously polling for status updates, you can configure webhook endpoints to be notified immediately when jobs complete, fail, or reach other states') and the getting-started/parse job flow confirms async job submission. Missing for 10: independent/hands-on confirmation of webhook reliability at scale and detail on retry/backoff semantics for polling.", "evidenceIds": [ "llamaparse-docs-6", "llamaparse-docs-11", "llamaparse-docs-1" ] }, { "productId": "llamaparse", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "LlamaParse exposes a programmatic API/SDK plus webhooks so jobs can be automated at scale, and Classify lets rules be saved and referenced across multiple jobs, suggesting some bulk workflow support. However there is no explicit documentation of a batch/bulk submission endpoint, multi-file job queue, or bulk status/management UI for large sets of items. Missing for 10: explicit batch-submit API, bulk job dashboard/management, and independent evidence of large-scale bulk usage in production.", "evidenceIds": [ "llamaparse-docs-6", "llamaparse-docs-39", "llamaparse-docs-1", "llamaparse-docs-11" ] }, { "productId": "llamaparse", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "LlamaParse supports webhooks to notify on job completion/failure and saved reusable classification rule configs, which are event-trigger-like building blocks, but there is no evidence of a general rules engine where users define arbitrary conditions/triggers that automatically fire actions across the platform. missing for 10: a documented rules/automation engine, conditional trigger definitions beyond job status, and evidence of chained automated actions (not just notification via webhook).", "evidenceIds": [ "llamaparse-docs-6", "llamaparse-docs-39" ] }, { "productId": "llamaparse", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "LlamaParse's evidence covers parsing, extraction, classification, splitting, webhooks for job status, self-hosting, and MCP tool exposure, but nothing describes native scheduling of recurring jobs or workflows (e.g., cron-like triggers or recurring pipeline runs). Webhooks only cover async notification for one-off jobs, not recurrence.", "evidenceIds": [ "llamaparse-docs-6", "llamaparse-docs-20" ] }, { "productId": "llamaparse", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of versioning, review workflows, or rollback for LlamaParse automations/configurations (e.g., parsing jobs, extraction schemas, classification rules). Only reusable configs via ID are mentioned (llamaparse-docs-39), but no version history, diff/review, or rollback capability is documented.", "evidenceIds": [] }, { "productId": "llamaparse", "storyId": "batch-throughput", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "LlamaParse documents async job status via webhooks (avoiding polling) and offers self-hosted/BYOC deployment for scaling infrastructure, which supports high-volume async workflows, but there is no documented rate-limit table, batch-submission API, or throughput/SLA numbers for millions-of-pages workloads. Missing for 10: explicit rate-limit documentation, batch endpoint semantics, throughput benchmarks or SLAs for large-scale ingestion.", "evidenceIds": [ "llamaparse-docs-6", "llamaparse-docs-7", "llamaparse-docs-24", "llamaparse-supp-2" ] }, { "productId": "llamaparse", "storyId": "citations-provenance", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show real provenance features—table extraction with source-page provenance, forms enrichment with bounding boxes and field values, and SEC-filing extraction with citations/verification against source—but these are scattered across specific modes (forms, tables, extract) rather than a universal guarantee that every extracted field always carries page+bbox+snippet. A community question explicitly asks whether page numbers can be returned with markdown output, suggesting this isn't uniformly available across the core Parse output. Missing for 10: a unified schema showing page number + bounding box + snippet attached to every field across all extraction modes, and independent confirmation that this works reliably in practice.", "evidenceIds": [ "llamaparse-docs-15", "llamaparse-docs-22", "llamaparse-docs-32", "llamaparse-comm-7" ] }, { "productId": "llamaparse", "storyId": "classification-splitting", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "LlamaParse's Split API explicitly segments concatenated PDFs into logical document sections based on content categories, and Classify auto-categorizes documents by type, together enabling one-upload, multi-document packet handling with per-document results (llamaparse-docs-5, llamaparse-docs-4, llamaparse-docs-37, llamaparse-docs-38). Missing for 10: independent/hands-on validation of the Split+Classify combined workflow at scale and confirmation of per-document output structure/ordering beyond docs examples.", "evidenceIds": [ "llamaparse-docs-5", "llamaparse-docs-4", "llamaparse-docs-37", "llamaparse-docs-38", "llamaparse-docs-39" ] }, { "productId": "llamaparse", "storyId": "complex-layout-parsing", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Docs claim agentic, layout-aware OCR that turns complex layouts, tables, charts, and handwriting into clean markdown (llamaparse-docs-2, llamaparse-docs-25), and one HN commenter found it markedly better than open-source parsers on complex layouts (llamaparse-comm-1). But other hands-on users report concrete accuracy gaps — table data extracted but not returned in usable tabular format (llamaparse-comm-3) and a benchmark run showing LlamaParse-based pipeline faithfulness (0.667) actually lower than a naive PyPDF baseline (0.980) (llamaparse-comm-6) — directly contradicting the 'clean, correctly ordered' claim for some real-world documents. missing for 10: independent benchmark specifically on multi-column/headers/footers/footnotes ordering, resolution of the conflicting community reports, and reproducible accuracy metrics.", "evidenceIds": [ "llamaparse-docs-2", "llamaparse-docs-25", "llamaparse-comm-1", "llamaparse-comm-3", "llamaparse-comm-6" ] }, { "productId": "llamaparse", "storyId": "confidence-scores-hitl", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of calibrated confidence scores per extracted field or any human-in-the-loop review workflow for low-confidence extractions. Docs mention citation/verification against source documents (llamaparse-docs-32) and bounding boxes for form fields (llamaparse-docs-22), but nothing about confidence scoring or a review queue/UI for flagged low-confidence fields.", "evidenceIds": [ "llamaparse-docs-32", "llamaparse-docs-22", "llamaparse-docs-14" ] }, { "productId": "llamaparse", "storyId": "document-data-compliance", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "First-party docs explicitly claim SOC 2 Type II completion and a HIPAA-compliant pipeline with BAA for Enterprise customers, plus BYOC/self-hosting and regional endpoints for data residency, directly addressing contract/medical-record processing needs. However, zero-retention handling specifically is not explicitly documented, HIPAA is gated behind 'Enterprise' tier, and there is no independent/third-party audit corroboration or hands-on confirmation of these compliance claims. Missing for 10: explicit zero-retention policy documentation, independent verification of SOC2/HIPAA claims, and clarity on whether zero-retention applies outside Enterprise tier.", "evidenceIds": [ "llamaparse-supp-1", "llamaparse-supp-2", "llamaparse-docs-29", "llamaparse-docs-28", "llamaparse-docs-9" ] }, { "productId": "llamaparse", "storyId": "figure-chart-handling", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs confirm layout-aware parsing of charts/images into markdown and table extraction with source-page provenance (docs-15, docs-22, docs-25), and a community request for page numbers alongside markdown (comm-7) suggests this is not fully solved out-of-the-box. There's no explicit evidence of VLM-generated figure/chart summaries or image crop extraction with bounding boxes tied back to page position (only forms/checkbox bounding boxes are documented). missing for 10: explicit VLM-based figure/chart description or summarization feature, documented image-crop extraction with bounding-box/page coordinates for charts/figures specifically, and independent hands-on confirmation of this traceability.", "evidenceIds": [ "llamaparse-docs-15", "llamaparse-docs-22", "llamaparse-docs-25", "llamaparse-comm-7" ] }, { "productId": "llamaparse", "storyId": "file-format-breadth", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly claim one unified API/SDK supporting 130+ file formats across categories (PDF, DOCX, PPTX, XLSX, HTML, images, email implied) via a single 'Parse' product with one API key/SDK, and examples show handling PDFs, Excel, scans, images, and forms without separate per-format code paths. Community feedback corroborates strong parsing on complex/mixed documents but focuses mainly on PDFs, and no independent evidence tests DOCX/PPTX/email specifically. Missing for 10: independent hands-on verification across the full non-PDF format set (DOCX, PPTX, XLSX, email) and explicit example code showing format-agnostic single-call usage.", "evidenceIds": [ "llamaparse-docs-10", "llamaparse-docs-20", "llamaparse-docs-2", "llamaparse-docs-36", "llamaparse-comm-1" ] }, { "productId": "llamaparse", "storyId": "handwriting-recognition", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Docs claim handwriting recognition as part of layout-aware OCR ('turn even the most complex layouts, tables, charts, handwriting, checkboxes, and images into clean markdown') and an enriched forms pass returns field values, checkbox states, and bounding boxes as structured JSON, but there is no documented confidence scoring or uncertainty flagging for handwritten fields/annotations specifically. Missing for 10: explicit confidence/uncertainty scores per extracted field, independent validation of handwriting accuracy, and any mention of flagging low-confidence handwritten extractions.", "evidenceIds": [ "llamaparse-docs-25", "llamaparse-docs-22" ] }, { "productId": "llamaparse", "storyId": "large-document-handling", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs mention async webhooks (avoiding polling/timeout issues) and a 'Cost Optimizer' for 'long mixed-complexity documents,' suggesting some design accommodation for large files, but there is no explicit claim, benchmark, or size/page limit documentation addressing thousand-page or multi-gigabyte files specifically, and no community evidence of large-file reliability. Missing for 10: documented size/page limits, explicit large-document benchmarks or case studies, and independent confirmation of no timeouts/truncation on very large files.", "evidenceIds": [ "llamaparse-docs-6", "llamaparse-docs-15", "llamaparse-docs-23" ] }, { "productId": "llamaparse", "storyId": "llm-ready-markdown", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs strongly market LlamaParse as an agentic, layout-aware parser producing 'clean markdown, text, or JSON' from complex documents (docs-2, docs-12, docs-25) with markdown expansion options (docs-11), and early community feedback praised its handling of complex layouts versus open-source parsers (comm-1, comm-2). However, no evidence specifically addresses stripping of repeated headers/page furniture, and some hands-on community feedback flags real output-quality gaps (tables not returned in tabular form, disputed benchmark numbers) that temper the 'clean for LLM consumption' claim (comm-3, comm-6). Missing for 10: explicit documentation/example of header/footer/page-furniture deduplication, and independent verification of noise-free markdown output.", "evidenceIds": [ "llamaparse-docs-2", "llamaparse-docs-12", "llamaparse-docs-25", "llamaparse-docs-11", "llamaparse-comm-1", "llamaparse-comm-3", "llamaparse-comm-6" ] }, { "productId": "llamaparse", "storyId": "low-latency-sync-mode", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack shows async-style job submission with polling or webhooks (llamaparse-docs-6) and tiered parsing modes (e.g., 'agentic' tier in llamaparse-docs-11), but there is no documentation of a fast synchronous mode returning results in seconds, nor any per-mode latency figures.", "evidenceIds": [] }, { "productId": "llamaparse", "storyId": "multilingual-documents", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses multilingual or CJK/RTL parsing fidelity; docs mention layout-aware OCR, format support, and general parsing quality but nothing about language-specific handling or non-English script accuracy.", "evidenceIds": [] }, { "productId": "llamaparse", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show every core capability (Parse, Extract, Classify, Split, Index) is available via Python/TS/Go/Java SDKs, CLI and REST API alongside the Web UI, and webhooks let API users get async job notifications like the UI dashboard. However, admin/enterprise features (SSO, RBAC, project/org management, Trust Center reports) are described only in UI/dashboard terms with no evidence of equivalent API endpoints, and no OpenAPI spec was discoverable (probe 404s) to confirm full parity. Missing for 10: documented API endpoints for org/SSO/RBAC management, a public OpenAPI/swagger spec, and independent confirmation that all UI-configured settings (e.g., extraction schema builder) are fully API-driven.", "evidenceIds": [ "llamaparse-docs-1", "llamaparse-docs-6", "llamaparse-docs-9", "llamaparse-docs-14", "llamaparse-docs-20", "llamaparse-probe-3" ] }, { "productId": "llamaparse", "storyId": "openness-full-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "LlamaParse's core outputs (markdown, text, JSON) are open, portable formats by design, and BYOC/self-hosting options mean users can run the entire platform on their own infrastructure without vendor lock-in. However, there's no documented explicit 'export all account data' feature or bulk data-portability tool beyond the per-job parse outputs. missing for 10: dedicated account-wide data export/deletion tooling, migration guides for leaving the platform, independent verification of format completeness.", "evidenceIds": [ "llamaparse-docs-2", "llamaparse-docs-25", "llamaparse-docs-7", "llamaparse-docs-24", "llamaparse-docs-28" ] }, { "productId": "llamaparse", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows a GitHub repo (llama_cloud_services) but it is only a client SDK/wrapper for calling the hosted LlamaParse API, not the source of the core parsing engine, and no license terms or open-source claim for the actual product are given. There is no documentation indicating LlamaParse's core service or models are released under an open license.", "evidenceIds": [ "llamaparse-gh-1" ] }, { "productId": "llamaparse", "storyId": "openness-self-host", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs explicitly describe self-hosting/BYOC deployments giving full LlamaCloud capabilities within customer infrastructure, including Kubernetes/Helm deployment on AWS/Azure/GCP, plus enterprise deployment options (managed SaaS, single-tenant, BYOC, self-hosting, regional endpoints). Missing for 10: independent/hands-on confirmation of a self-hosted deployment actually running (no community reports), and details on licensing/open-source availability of the self-hosted code vs. it being a gated enterprise offering.", "evidenceIds": [ "llamaparse-docs-7", "llamaparse-docs-24", "llamaparse-docs-28", "llamaparse-supp-2" ] }, { "productId": "llamaparse", "storyId": "playground-before-code", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm a Web UI option alongside SDKs/CLI (llamaparse-docs-1) and explicitly describe dragging and dropping a document into LlamaParse to extract data (llamaparse-docs-14), and community evidence confirms an actual interface exists that users interacted with directly (llamaparse-comm-3, llamaparse-comm-4). However, evidence doesn't show a dedicated 'playground' branding, screenshots, or hands-on positive confirmation of the full parse-before-code workflow, and one community report calls the interface itself confusing. Missing for 10: explicit playground UI walkthrough/screenshots, independent hands-on confirmation of drag-and-drop parse+extract results without any code, and resolution of the UI confusion complaint.", "evidenceIds": [ "llamaparse-docs-1", "llamaparse-docs-14", "llamaparse-comm-3", "llamaparse-comm-4" ] }, { "productId": "llamaparse", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs state the platform offers 'managed SaaS, single-tenant, BYOC, self-hosting, and regional endpoints' for deployment and data residency, and self-hosting/BYOC explicitly lets customers keep data and infrastructure under their own control (including choice of cloud/region via Kubernetes on AWS/Azure/GCP). However, there is no detail on which specific regions are available for the managed SaaS regional-endpoint option, no UI/API mechanism shown for selecting a region, and no independent confirmation. Missing for 10: concrete list of supported regions/endpoints, documented API/UI control to select a region, and third-party or hands-on confirmation of data residency behavior.", "evidenceIds": [ "llamaparse-supp-2", "llamaparse-docs-7", "llamaparse-docs-28" ] }, { "productId": "llamaparse", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence covers enterprise features like SOC2/HIPAA compliance, SSO/RBAC, and self-hosting/BYOC options, but nowhere states an explicit policy or toggle for preventing customer data from being used to train AI models. Self-hosting implies data stays on the customer's infra, but that is not the same as a documented no-training-use guarantee.", "evidenceIds": [ "llamaparse-docs-7", "llamaparse-docs-9", "llamaparse-supp-1", "llamaparse-supp-2" ] }, { "productId": "llamaparse", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "LlamaParse offers BYOC/self-hosting so customers can keep all data under their own infrastructure control, and cites SOC2 Type II/HIPAA compliance, which implies some data governance posture, but no evidence describes an explicit data retention policy, configurable retention windows, or a deletion API/mechanism for the managed SaaS offering. missing for 10: explicit retention period documentation, a data deletion API/endpoint, and confirmation of deletion behavior for managed (non-BYOC) deployments.", "evidenceIds": [ "llamaparse-docs-7", "llamaparse-docs-24", "llamaparse-supp-1", "llamaparse-supp-2", "llamaparse-docs-28" ] }, { "productId": "llamaparse", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack content addresses telemetry/usage-tracking opt-out settings for LlamaParse; only self-hosting/BYOC and enterprise compliance features (SOC2, HIPAA, SSO) are documented, none of which explicitly describe a telemetry opt-out control.", "evidenceIds": [] }, { "productId": "llamaparse", "storyId": "rag-ready-chunking", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "LlamaParse's Index product offers a hosted vector-search/embedding pipeline (docs-19, docs-30, docs-31) and structured JSON/table extraction with page-level provenance (docs-15, docs-22), suggesting some RAG-ready segmentation, but the evidence never documents explicit semantic chunk boundaries or embedding-ready segment metadata as a first-class output of Parse itself, and a community question about even basic page-number metadata in markdown output (llamaparse-comm-7) went unanswered, indicating gaps in chunk-level metadata support. missing for 10: explicit chunking/boundary strategy documentation, embedding-ready segment schema, confirmed page/metadata tagging in default output.", "evidenceIds": [ "llamaparse-docs-19", "llamaparse-docs-30", "llamaparse-docs-31", "llamaparse-docs-15", "llamaparse-docs-22", "llamaparse-comm-7" ] }, { "productId": "llamaparse", "storyId": "reading-order-hierarchy", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs describe LlamaParse as 'layout-aware OCR' that converts complex layouts, tables, charts, and headings into 'clean markdown,' implying structural/hierarchy preservation, and a community reviewer praised its handling of complex layouts. However, another hands-on user reported that table data wasn't returned in proper tabular structure and was only accessible via Q&A, raising doubts about how faithfully structural elements are preserved, and no evidence explicitly confirms heading-level/section hierarchy or reading-order fidelity beyond general layout claims. Missing for 10: explicit documentation/examples of heading-level and section-order preservation, and independent benchmarking confirming structural fidelity beyond tables.", "evidenceIds": [ "llamaparse-docs-2", "llamaparse-docs-25", "llamaparse-docs-35", "llamaparse-comm-1", "llamaparse-comm-3" ] }, { "productId": "llamaparse", "storyId": "scanned-doc-ocr", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "LlamaParse's docs claim strong OCR for scans/photos, complex layouts, handwriting, checkboxes and images (llamaparse-docs-25, llamaparse-docs-12, llamaparse-docs-2), and community feedback confirms it handles complex layouts better than open-source parsers (llamaparse-comm-1). However, no evidence specifically addresses skewed-page correction, stamp detection, or degraded/low-quality scan handling, and community reports flag inconsistent table extraction and an independent benchmark showing lower faithfulness than claimed (llamaparse-comm-3, llamaparse-comm-6). missing for 10: explicit skew-correction handling, stamp/seal recognition, low-quality scan benchmarks, and independent hands-on validation of OCR accuracy on such edge cases.", "evidenceIds": [ "llamaparse-docs-25", "llamaparse-docs-12", "llamaparse-docs-2", "llamaparse-comm-1", "llamaparse-comm-3", "llamaparse-comm-6" ] }, { "productId": "llamaparse", "storyId": "schema-driven-extraction", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "LlamaExtract (part of LlamaParse/LlamaCloud) explicitly supports defining an extraction schema/configuration and returning structured JSON matching that schema, including SEC filings extraction with citations and prompt-based schema generation. This directly matches the story, but evidence is entirely first-party docs with no independent/hands-on corroboration of schema validation accuracy for LlamaExtract specifically. Missing for 10: independent/community validation of extraction accuracy against schemas, explicit detail on validation/error-handling mechanics.", "evidenceIds": [ "llamaparse-docs-3", "llamaparse-docs-13", "llamaparse-docs-14", "llamaparse-docs-32", "llamaparse-docs-33" ] }, { "productId": "llamaparse", "storyId": "table-structure-extraction", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "LlamaParse docs claim to turn 'complex layouts, tables' into clean markdown and to pull tables into structured formats with page provenance, but a hands-on community report says the tool 'can extract numbers in tables, but the output isn't provided in tabular format—access to the numbers is only available through question-answering,' directly contradicting the claim of faithful structured table output. No evidence specifically addresses merged cells, nested headers, or multi-page spans in either direction. missing for 10: explicit documentation/testing of merged-cell, nested-header, and multi-page table handling, and independent corroboration resolving the contradiction.", "evidenceIds": [ "llamaparse-docs-25", "llamaparse-docs-15", "llamaparse-docs-22", "llamaparse-comm-3" ] }, { "productId": "llamaparse", "storyId": "table-to-structured-rows", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Docs claim strong support for turning tables into structured JSON/DataFrames with schemas (LlamaExtract's schema-based extraction, 'pull every table into pandas DataFrames with source-page provenance', 'table row extraction') that would fit a data-engineer's typed-row/JSON pipeline, but hands-on community feedback contradicts this: a user reports 'it can extract numbers in tables, but the output isn't provided in tabular format—access to the numbers is only available through question-answering,' suggesting real-world table extraction fell short of clean, database-ready structure. missing for 10: recent independent verification that schema-driven table extraction yields clean typed rows without manual cleanup, and resolution of the tabular-output complaint.", "evidenceIds": [ "llamaparse-docs-15", "llamaparse-docs-13", "llamaparse-docs-34", "llamaparse-docs-3", "llamaparse-docs-14", "llamaparse-comm-3" ] }, { "productId": "llamaparse", "storyId": "typed-official-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm official Python and TypeScript SDKs (\"whether you prefer Python, TypeScript, Go, Java, the CLI, the REST API\") and show a Python code sample for parsing with defaults (tier, version, expand), plus a webhook/job-status system, and GitHub shows an actively maintained llama-cloud Python package. However, there's no TypeScript code sample, no explicit documentation of a typed Extract or Jobs SDK surface, and no independent corroboration of SDK typing quality or defaults across both languages. Missing for 10: TypeScript SDK code examples, explicit typed Extract/Jobs API coverage in both SDKs, and third-party validation of SDK completeness.", "evidenceIds": [ "llamaparse-docs-1", "llamaparse-docs-11", "llamaparse-docs-6", "llamaparse-gh-1", "llamaparse-docs-3" ] }, { "productId": "llamaparse", "storyId": "vpc-self-host-deployment", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly describe BYOC/self-hosted deployment of the full LlamaCloud platform (Parse, Extract) on Kubernetes via Helm charts across AWS/Azure/GCP, keeping data and infrastructure under customer control, plus enterprise deployment options including single-tenant and regional endpoints. Missing for 10: independent/hands-on verification of self-hosted deployment success and more detail on operational limitations of self-hosting vs managed SaaS.", "evidenceIds": [ "llamaparse-docs-7", "llamaparse-docs-28", "llamaparse-supp-2", "llamaparse-docs-24" ] }, { "productId": "mistral-document-ai", "storyId": "accuracy-benchmarks-evals", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack items describe published, reproducible accuracy benchmarks (datasets, methodology, or eval scripts) that an ML engineer could rerun; docs focus on features/parameters, and community comments only reference qualitative testing or a vague mention that 'their benchmark results are impressive' without detail or reproducibility.", "evidenceIds": [] }, { "productId": "mistral-document-ai", "storyId": "agentic-agent-docs", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "A probe confirms Mistral's docs site serves a working llms.txt (HTTP 200) listing agent-oriented doc links, so an AI agent could be pointed at it. However, the specific Document AI/document-processing page lacks a markdown variant (404 on overview.md), showing incomplete agent-oriented coverage for this particular product area. Missing for 10: confirmation that document-processing pages themselves are included in llms.txt or have .md equivalents, and any evidence of an agent successfully consuming these docs end-to-end.", "evidenceIds": [ "mistral-document-ai-probe-1", "mistral-document-ai-probe-2" ] }, { "productId": "mistral-document-ai", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Document AI's Document QnA and annotation features explicitly combine OCR with LLM capabilities to let users 'extract information and insights from documents by asking questions in natural language,' plus structured annotation for key insights like vendor/clause/amount extraction. Community evidence corroborates strong OCR/extraction quality, though no independent hands-on evidence specifically validates the QnA/insights feature's quality or proactive 'suggestions' beyond Q&A. Missing for 10: independent corroboration of Document QnA/insight quality, evidence of proactive (non-query-driven) suggestions.", "evidenceIds": [ "mistral-document-ai-docs-12", "mistral-document-ai-docs-26", "mistral-document-ai-docs-13", "mistral-document-ai-docs-9", "mistral-document-ai-docs-11", "mistral-document-ai-comm-6" ] }, { "productId": "mistral-document-ai", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers only synchronous OCR/document-extraction capabilities (text extraction, annotations, Q&A) with no mention of scheduling, triggers, webhooks, or any mechanism for autonomous background automation runs. As an AI-native document processing API, such automation features are plausible but entirely unevidenced here.", "evidenceIds": [] }, { "productId": "mistral-document-ai", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Document AI includes a 'Document QnA' capability that lets users interact with document content in natural language and run multi-document queries/comparisons, which functions as a narrow, document-scoped assistant rather than a general-purpose built-in agent for task delegation. missing for 10: evidence of a broader in-product assistant UI for delegating multi-step tasks beyond document Q&A, agentic workflow/tool-use capabilities, and independent hands-on validation of the QnA feature's reliability.", "evidenceIds": [ "mistral-document-ai-docs-12", "mistral-document-ai-docs-13", "mistral-document-ai-docs-22", "mistral-document-ai-docs-26" ] }, { "productId": "mistral-document-ai", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Document AI is delivered purely as an API with official Python and TypeScript SDKs, and inputs can be passed via URL/Base64/file upload — all of which naturally support headless/CI use. However, there is no explicit documentation of CI/CD examples, GitHub Actions, async batch processing at scale, or automation-specific guidance (and community reports note timeouts on large batch jobs, suggesting scaling caveats for automated pipelines). Missing for 10: explicit CI/automation documentation or examples, guidance on rate limits/batching for large-scale headless runs, and independent confirmation of reliable large-batch automation.", "evidenceIds": [ "mistral-document-ai-docs-28", "mistral-document-ai-docs-15", "mistral-document-ai-comm-4" ] }, { "productId": "mistral-document-ai", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Mistral Document AI is a document processing/OCR API, not an agent or client role that consumes external tools; the evidence pack shows no MCP client/host functionality, and this is a category mismatch rather than a missing feature.", "evidenceIds": [] }, { "productId": "mistral-document-ai", "storyId": "agentic-mcp-server", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Mistral Document AI is a document processing/OCR API service, not itself an agent product; the evidence shows only SDKs and REST/API docs with no mention of an MCP server offering. This axis is more relevant to agent-role products or platforms with ecosystems, but no evidence exists here of any MCP server, so it's best treated as not applicable to this API-service product type.", "evidenceIds": [ "mistral-document-ai-docs-28" ] }, { "productId": "mistral-document-ai", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Document AI's Document QnA feature lets users interact with document content via natural-language queries, combining OCR with LLM capabilities (docs-12, docs-13, docs-22), which is a real but narrow slice of 'natural-language commands.' However, this is limited to querying document content, not operating the broader product (setting OCR parameters, triggering annotation types, configuring table/header extraction) via natural language — those are all done through structured API parameters, not NL commands. Missing for 10: evidence of NL-driven configuration/control of the OCR pipeline itself, any chat/agent interface for invoking the API, and independent corroboration of the QnA feature's reliability.", "evidenceIds": [ "mistral-document-ai-docs-12", "mistral-document-ai-docs-13", "mistral-document-ai-docs-22" ] }, { "productId": "mistral-document-ai", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only official Python/TypeScript SDKs and a REST API for Document AI; there is no mention of an official CLI tool anywhere in the docs or community evidence.", "evidenceIds": [ "mistral-document-ai-docs-28" ] }, { "productId": "mistral-document-ai", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Mistral Document AI is delivered entirely as an API with extensively documented parameters (OCR extraction, table_format, extract_header/footer, include_blocks, confidence_scores_granularity, annotation endpoints, Document QnA) plus official Python and TypeScript SDKs, and community evidence confirms real-world API usage (pricing per page, PDF-to-markdown workflows). Missing for 10: a discoverable formal OpenAPI/swagger spec (probe found 404s at standard locations) and independent third-party API-integration write-ups beyond community OCR quality discussions.", "evidenceIds": [ "mistral-document-ai-docs-1", "mistral-document-ai-docs-4", "mistral-document-ai-docs-5", "mistral-document-ai-docs-15", "mistral-document-ai-docs-23", "mistral-document-ai-docs-28", "mistral-document-ai-comm-1", "mistral-document-ai-probe-3" ] }, { "productId": "mistral-document-ai", "storyId": "agentic-scoped-keys", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Mistral Document AI is an OCR/document-processing API product, not an identity/access-management or agent-orchestration platform; issuing scoped/least-privilege credentials for agents is an IAM concern outside its product category and not something this evidence pack addresses as a feature.", "evidenceIds": [] }, { "productId": "mistral-document-ai", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Document AI's OCR/annotation/QnA features and API endpoint details but contains no mention of official SDKs (Python, JS/TS, etc.) for building against Document AI — this is an applicable axis for an API product but no supporting evidence exists in the pack.", "evidenceIds": [] }, { "productId": "mistral-document-ai", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack of webhook support or event subscription for Document AI; documentation covers OCR, annotations, and Q&A only, with no mention of webhooks or event-driven notifications.", "evidenceIds": [] }, { "productId": "mistral-document-ai", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows extensive static documentation pages describing OCR/annotation parameters and SDKs, but no mention of an interactive API reference (e.g., Swagger/Redoc UI) with runnable/try-it examples. Probes for openapi.json/swagger.json all returned 404, and no evidence of an API playground or live code execution in docs.", "evidenceIds": [ "mistral-document-ai-probe-3", "mistral-document-ai-probe-2", "mistral-document-ai-docs-28" ] }, { "productId": "mistral-document-ai", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of a published OpenAPI/machine-readable spec; docs describe SDKs (Python/TypeScript) but a direct probe of common OpenAPI spec URLs (openapi.json, swagger.json, etc.) returned 404 across all candidates, indicating no downloadable spec is provided.", "evidenceIds": [ "mistral-document-ai-probe-3", "mistral-document-ai-docs-28" ] }, { "productId": "mistral-document-ai", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any sandbox/test environment separate from production for Document AI; docs cover OCR features, annotations, and QnA but nothing about non-production testing modes, free trial isolation, or sandbox API keys. Missing for 10: dedicated sandbox environment or test mode documentation, guidance on isolating test data from production, any mention of sandbox API keys or staging endpoints.", "evidenceIds": [] }, { "productId": "mistral-document-ai", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of API versioning scheme or a documented deprecation policy anywhere in the docs pack; OpenAPI spec probes returned 404 and no changelog/versioning page is cited. missing for 10: versioned API endpoints, deprecation policy documentation, changelog/migration guides.", "evidenceIds": [ "mistral-document-ai-probe-3", "mistral-document-ai-probe-2" ] }, { "productId": "mistral-document-ai", "storyId": "async-jobs-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the docs pack of async job submission, status polling endpoints, or completion webhooks for Document AI OCR/annotation calls; the API appears to be synchronous (request/response), and one community report notes 900-page documents caused a timeout rather than being handled as a background job.", "evidenceIds": [ "mistral-document-ai-comm-4", "mistral-document-ai-docs-1", "mistral-document-ai-probe-3" ] }, { "productId": "mistral-document-ai", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Docs mention multi-document querying/comparison in the QnA feature and OCR can be called repeatedly via API for many files, but there is no documented batch/bulk-upload endpoint, and real-world use shows scaling limits (a 900-page single request timed out and had to be chunked), while other users report successful large-scale digitization of forms with human-review overhead. Missing for 10: an explicit bulk/batch processing API or endpoint, documented rate/size limits for large jobs, and independent evidence of reliable bulk throughput at scale.", "evidenceIds": [ "mistral-document-ai-docs-13", "mistral-document-ai-comm-4", "mistral-document-ai-comm-8" ] }, { "productId": "mistral-document-ai", "storyId": "automation-rules-engine", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Mistral Document AI is a document processing/OCR API (extraction, annotation, QnA), not an automation/workflow platform; defining rule-based triggers on events is outside its product category and belongs to a separate orchestration tool.", "evidenceIds": [] }, { "productId": "mistral-document-ai", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Mistral Document AI is an OCR/document-extraction API, not a workflow orchestration or job-scheduling platform; scheduling recurring jobs is outside its product category and would be handled by an external orchestrator or the user's own infrastructure.", "evidenceIds": [] }, { "productId": "mistral-document-ai", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Mistral Document AI is a document processing/OCR API, not an automation/workflow builder; there is no concept of 'automations' with versioning, review, or rollback in this product category.", "evidenceIds": [] }, { "productId": "mistral-document-ai", "storyId": "batch-throughput", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack documents rate limits, batch/async processing endpoints, or throughput guarantees for high-volume (millions of pages) workloads; the only relevant community data point reports a timeout at 900 pages requiring manual chunking, which is the opposite of predictable large-scale throughput.", "evidenceIds": [ "mistral-document-ai-comm-4" ] }, { "productId": "mistral-document-ai", "storyId": "citations-provenance", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs show strong provenance primitives for raw OCR: per-page blocks array with paragraph-level bounding boxes, structural labels, reading-order content, and confidence scores at page/block/word granularity (docs-4/5/18/19). However, the structured 'document_annotation' (key-field extraction like invoice/receipt fields) is described as returning annotations for the whole document/format without explicit per-field bounding boxes or source snippets tied to those extracted key-value fields—only 'bbox_annotation' ties boxes to charts/figures, not to structured field values, and no evidence confirms every extracted field (e.g., vendor name, amount) carries its own page/bbox/snippet link. Community evidence corroborates OCR/bbox usage exists but doesn't validate field-level citation provenance for structured extraction. Missing for 10: explicit documentation or example showing that fields extracted via document_annotation/JSON schema outputs individually include page number, bounding box, and source text snippet; independent verification of citation accuracy for structured (non-OCR-block) fields.", "evidenceIds": [ "mistral-document-ai-docs-4", "mistral-document-ai-docs-5", "mistral-document-ai-docs-7", "mistral-document-ai-docs-8", "mistral-document-ai-docs-9", "mistral-document-ai-comm-1" ] }, { "productId": "mistral-document-ai", "storyId": "classification-splitting", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers OCR extraction, annotations, and Document QnA (including multi-document queries/comparisons across separate docs), but there is no mention of automatically classifying and splitting a single upload containing multiple distinct documents into per-document results — a distinct IDP capability that is not addressed anywhere in the pack.", "evidenceIds": [] }, { "productId": "mistral-document-ai", "storyId": "complex-layout-parsing", "verdict": "disputed", "quality": 6, "confidence": "medium", "rationale": "Docs describe robust structural parsing (headers/footers, block-level reading order, tables, confidence scores) and community feedback broadly praises accuracy versus competitors on degraded/handwritten text. However, concrete hands-on reports contradict clean complex-layout parsing: one user found the model misclassified an entire page as a single image block, and another found bidirectional (Hebrew) text output had 'no correspondence to the text whatsoever,' both indicating real-world layout/order failures despite vendor claims. Missing for 10: independent benchmark specifically on multi-column/footnote-heavy PDFs, and resolution of the bidi/layout failure reports.", "evidenceIds": [ "mistral-document-ai-docs-4", "mistral-document-ai-docs-3", "mistral-document-ai-comm-1", "mistral-document-ai-comm-2", "mistral-document-ai-comm-3", "mistral-document-ai-comm-5" ] }, { "productId": "mistral-document-ai", "storyId": "confidence-scores-hitl", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Mistral Document AI explicitly documents confidence scores at page, block, or word granularity via the confidence_scores_granularity parameter, satisfying the 'calibrated confidence' half of the story. However, there is no evidence of any built-in human-in-the-loop review workflow, UI, or API path for routing low-confidence fields to reviewers — the only related evidence is a community comment noting users manually review 'a low percentage' of outputs themselves, which is not a product capability. Missing for 10: documented review-queue/workflow feature, API/UI for flagging or routing low-confidence extractions, and evidence of calibration methodology.", "evidenceIds": [ "mistral-document-ai-docs-5", "mistral-document-ai-docs-19", "mistral-document-ai-comm-8" ] }, { "productId": "mistral-document-ai", "storyId": "document-data-compliance", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows generic marketing language about 'compliance-first organizations' and a 'Trust Center' link, with no concrete mention of zero-retention data handling, SOC 2 certification, or HIPAA compliance options for the Document AI product specifically. No documentation confirms data retention policies or named compliance certifications applicable to document processing of contracts/medical records.", "evidenceIds": [ "mistral-document-ai-supp-1" ] }, { "productId": "mistral-document-ai", "storyId": "figure-chart-handling", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show bbox_annotation for charts/figures, include_blocks with paragraph-level bounding boxes and structural labels in reading order, and document_annotation for full-document summaries — directly supporting figure/chart extraction with page-traceable positions. Community confirms real-world extraction of embedded images, but one hands-on report shows a failure mode where an entire page was misclassified as a single image rather than granular figure regions, indicating inconsistent reliability. Missing for 10: explicit example of VLM-generated descriptive summaries (vs. just bbox coordinates), independent benchmarking of figure/chart bounding-box accuracy across document types.", "evidenceIds": [ "mistral-document-ai-docs-4", "mistral-document-ai-docs-7", "mistral-document-ai-docs-8", "mistral-document-ai-docs-18", "mistral-document-ai-docs-20", "mistral-document-ai-comm-2", "mistral-document-ai-comm-3" ] }, { "productId": "mistral-document-ai", "storyId": "file-format-breadth", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs confirm a single OCR/Document AI API handles PDF, PPTX, DOCX, images (PNG/JPEG/AVIF) via document_url/image_url inputs, plus URL/base64/upload ingestion, and community reports corroborate strong PDF/image OCR performance. However, XLSX, HTML, and email formats are never mentioned in the evidence, and one community report notes a full-page misread edge case. Missing for 10: explicit support/evidence for XLSX, HTML, and email formats, and independent confirmation of accuracy across all listed formats.", "evidenceIds": [ "mistral-document-ai-docs-23", "mistral-document-ai-docs-15", "mistral-document-ai-comm-1", "mistral-document-ai-comm-3" ] }, { "productId": "mistral-document-ai", "storyId": "handwriting-recognition", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm OCR works on handwritten sources and that confidence scores are returned at page/block/word granularity, and a community user reports successful handwritten-form digitization with a low percentage requiring human review — matching the 'flagged when uncertain' workflow. However, the product doesn't document an explicit automatic uncertainty-flagging mechanism (developers must threshold the raw confidence scores themselves), and there's no dedicated handwriting accuracy benchmark. Missing for 10: built-in automatic flagging/threshold logic for uncertain fields, and handwriting-specific accuracy/benchmark data beyond anecdotal community reports.", "evidenceIds": [ "mistral-document-ai-docs-14", "mistral-document-ai-docs-5", "mistral-document-ai-docs-19", "mistral-document-ai-comm-8" ] }, { "productId": "mistral-document-ai", "storyId": "large-document-handling", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Docs describe PDF/image ingestion via URL, base64, or upload with no documented size or timeout limits (mistral-document-ai-docs-15, docs-23), implying large documents should be supported, but hands-on community testing found a 900-page document caused a timeout and only succeeded after manually chunking the file (mistral-document-ai-comm-4). This is a concrete contradiction of the 'thousand-page documents process reliably without timeouts' claim rather than mere skepticism. Missing for 10: vendor-documented size/page limits or guaranteed timeout thresholds, evidence of successful multi-gigabyte or thousand-page processing without manual chunking, and any official guidance on handling large-file reliability.", "evidenceIds": [ "mistral-document-ai-comm-4", "mistral-document-ai-docs-15", "mistral-document-ai-docs-23" ] }, { "productId": "mistral-document-ai", "storyId": "llm-ready-markdown", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly support clean, LLM-ready output: OCR returns markdown with configurable table_format, dedicated extract_header/extract_footer parameters that pull page furniture into separate header/footer fields rather than the body text, and include_blocks that returns structural, reading-order content — all directly aimed at producing clean structured output for downstream LLM/RAG use. Community feedback independently praises the 'pdf to markdown' quality as fast and highly accurate compared to competitors, corroborating the docs claims, though some testers report unrelated accuracy issues (image misdetection, multilingual garbling) rather than contradicting the header/footer-stripping mechanism itself. Missing for 10: independent benchmark or hands-on confirmation specifically validating repeated-header deduplication across multi-page documents, and no JSON-schema example showing final 'clean' output structure.", "evidenceIds": [ "mistral-document-ai-docs-3", "mistral-document-ai-docs-4", "mistral-document-ai-docs-17", "mistral-document-ai-docs-18", "mistral-document-ai-comm-1", "mistral-document-ai-comm-2" ] }, { "productId": "mistral-document-ai", "storyId": "low-latency-sync-mode", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack documents OCR/annotation/QnA features and parameters but contains no mention of a synchronous vs asynchronous processing mode, nor any documented latency figures per mode; one community note even mentions timeouts on large batches, suggesting no such fast sync mode is called out. missing for 10: any documentation of sync/async modes, any latency benchmarks or SLAs per mode.", "evidenceIds": [ "mistral-document-ai-comm-4" ] }, { "productId": "mistral-document-ai", "storyId": "multilingual-documents", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Mistral's docs claim strong multilingual OCR across 40+ languages, but a hands-on community test found that on a bidirectional English/Hebrew (RTL) document, 'the Hebrew output had no correspondence to the text whatsoever,' directly contradicting the claim of equal fidelity for non-Latin/RTL scripts. No CJK-specific hands-on evidence (positive or negative) is present either. Missing for 10: CJK-specific accuracy evidence, independent benchmarks confirming RTL/CJK fidelity, and resolution of the reported RTL failure.", "evidenceIds": [ "mistral-document-ai-docs-6", "mistral-document-ai-comm-5" ] }, { "productId": "mistral-document-ai", "storyId": "openness-api-parity", "verdict": "na", "quality": 0, "confidence": "low", "rationale": "Mistral Document AI is presented purely as an API/OCR processing capability (endpoints for OCR, annotation, QnA) with no evidence of a distinct standalone UI/console for document processing whose features would need to be mirrored via API — the evidence pack contains only API docs and community API testing, no UI screenshots or console feature descriptions. Since there's no separate UI surface evidenced, the API-vs-UI parity question doesn't meaningfully apply to this product.", "evidenceIds": [] }, { "productId": "mistral-document-ai", "storyId": "openness-full-export", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Output can be requested in markdown/html table formats and structured JSON with bounding boxes, which is a form of open-format export of extracted data, and on-prem deployment options exist for those wanting full control. However, there is no evidence of a bulk/account-wide data export mechanism, no documentation of exporting configuration, annotations schemas, or usage history, and no explicit 'leave with your data' portability guarantee. missing for 10: bulk account data export tooling, documented data portability/export guarantees, evidence of exporting all user data (not just per-document OCR output) in open formats.", "evidenceIds": [ "mistral-document-ai-docs-2", "mistral-document-ai-docs-16", "mistral-document-ai-docs-4", "mistral-document-ai-supp-1" ] }, { "productId": "mistral-document-ai", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Mistral Document AI is a closed, hosted API/on-prem product for document processing; there is no evidence of any open-source license for its source code, and this is a commercial SaaS/enterprise offering rather than an open-source project, making 'read the source under an open license' a category mismatch for this kind of product.", "evidenceIds": [] }, { "productId": "mistral-document-ai", "storyId": "openness-self-host", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The only evidence is a marketing line on the solutions page claiming 'secure on-premises deployment' for compliance-first organizations, alongside a general mention that Mistral offers enterprise on-prem deployments elsewhere. There is no technical documentation, deployment guide, container/image, or licensing detail describing how to actually self-host the Document AI OCR/annotation/QnA capabilities. Missing for 10: concrete self-host deployment docs/instructions, technical requirements, hands-on confirmation that self-hosted Document AI works equivalently to the hosted API.", "evidenceIds": [ "mistral-document-ai-supp-1" ] }, { "productId": "mistral-document-ai", "storyId": "playground-before-code", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers API parameters, SDKs, and OCR quality discussions, but nothing describes a drag-and-drop web playground for trying document parsing/extraction without writing code; all usage shown is via API/SDK calls.", "evidenceIds": [] }, { "productId": "mistral-document-ai", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The only relevant evidence is a solutions-page claim that Mistral offers secure on-premises deployment for 'compliance-first organizations' and maintains a Trust Center, which implies data residency control via self-hosting but does not document explicit region/data-residency selection within the hosted Document AI API itself. Missing for 10: documented region-selection options in the API/console, explicit EU/US or other data-residency guarantees, and independent confirmation of on-prem deployment working in practice.", "evidenceIds": [ "mistral-document-ai-supp-1" ] }, { "productId": "mistral-document-ai", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers OCR/document-processing features and mentions on-premises deployment options and a Trust Center, but contains no explicit statement about data usage for AI training, opt-out policies, or data retention guarantees for La Plateforme/Document AI API usage. Missing for 10: explicit training-data opt-out policy, retention/data-use terms for API calls, independent confirmation of no-training defaults.", "evidenceIds": [ "mistral-document-ai-supp-1" ] }, { "productId": "mistral-document-ai", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers OCR/extraction features and general enterprise positioning (on-prem deployment, Trust Center) but contains no specifics on data retention policies, deletion controls, or configurable data lifecycle for Document AI processing. missing for 10: documented retention windows, explicit deletion/opt-out controls, API parameters or admin settings for data handling, independent confirmation of retention practices.", "evidenceIds": [ "mistral-document-ai-supp-1" ] }, { "productId": "mistral-document-ai", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry or usage-tracking opt-out controls for Document AI; the only privacy-adjacent mention is a general note about on-prem/self-hosted deployment and a Trust Center link, with no specifics on telemetry settings.", "evidenceIds": [ "mistral-document-ai-supp-1" ] }, { "productId": "mistral-document-ai", "storyId": "rag-ready-chunking", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The OCR block-extraction feature returns paragraph-level bounding boxes, structural block labels, and content in reading order (docs-4/18/24), which provides some semantic segmentation and metadata that could feed a RAG pipeline, and markdown/table output aids readability. However, there is no explicit chunking API, no embedding-ready segment format, no chunk-size/overlap controls, and no mention of RAG-specific metadata (e.g., chunk IDs, semantic boundary detection) — the docs frame this as OCR/annotation output, not a purpose-built RAG chunker. missing for 10: explicit RAG-chunking API/parameters, embedding-ready segment format, semantic boundary detection beyond block/paragraph structure, and independent evidence of RAG use.", "evidenceIds": [ "mistral-document-ai-docs-4", "mistral-document-ai-docs-18", "mistral-document-ai-docs-24", "mistral-document-ai-docs-2", "mistral-document-ai-docs-5" ] }, { "productId": "mistral-document-ai", "storyId": "reading-order-hierarchy", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly describe block extraction with paragraph-level bounding boxes, structural block labels, and content in reading order, plus header/footer separation, table structure preservation (markdown/html), and confidence granularity — directly supporting hierarchy/reading-order preservation for downstream LLM consumption. Community feedback corroborates strong OCR/structure quality generally, though no independent test specifically validates heading/section hierarchy fidelity. Missing for 10: no third-party benchmark or hands-on example confirming heading-level hierarchy (vs. just block/paragraph order) is preserved, and one community report notes occasional structural misclassification (whole page as image).", "evidenceIds": [ "mistral-document-ai-docs-4", "mistral-document-ai-docs-3", "mistral-document-ai-docs-2", "mistral-document-ai-docs-5", "mistral-document-ai-comm-1", "mistral-document-ai-comm-3" ] }, { "productId": "mistral-document-ai", "storyId": "scanned-doc-ocr", "verdict": "disputed", "quality": 6, "confidence": "medium", "rationale": "Docs claim OCR works even on low-quality or handwritten sources (docs-14) and community reports strongly corroborate accuracy on degraded, decades-old, and handwritten scans (comm-6, comm-7, comm-8) — but other hands-on users report concrete failures: one saw an entire page misdetected as a single image block (comm-3), and another found Hebrew/bidi text output had no correspondence to the source at all (comm-5), directly contradicting the 'accurate across quality/language conditions' claim. No evidence specifically addresses skew correction or stamp handling. Missing for 10: explicit skew/rotation handling evidence, stamp-detection evidence, and resolution of the conflicting accuracy reports.", "evidenceIds": [ "mistral-document-ai-docs-14", "mistral-document-ai-comm-6", "mistral-document-ai-comm-7", "mistral-document-ai-comm-8", "mistral-document-ai-comm-3", "mistral-document-ai-comm-5" ] }, { "productId": "mistral-document-ai", "storyId": "schema-driven-extraction", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs describe `document_annotation`/`bbox_annotation` that return structured output based on a user-provided 'annotation format', and give concrete extraction examples (invoices, receipts, contracts) implying schema-guided field extraction. However, the pack never explicitly names 'JSON schema' as the input format nor describes validation/error handling if extraction doesn't conform to the schema, and there is no independent/hands-on evidence testing schema-based structured extraction specifically. missing for 10: explicit JSON Schema input format documentation, validation/error behavior on mismatched schema, independent corroboration of structured-extraction accuracy.", "evidenceIds": [ "mistral-document-ai-docs-8", "mistral-document-ai-docs-21", "mistral-document-ai-docs-9", "mistral-document-ai-docs-10", "mistral-document-ai-docs-11", "mistral-document-ai-docs-7" ] }, { "productId": "mistral-document-ai", "storyId": "table-structure-extraction", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm a table_format parameter that outputs markdown or HTML for extracted tables, and community feedback broadly praises OCR accuracy, but no evidence specifically addresses fidelity for merged cells, nested/complex headers, or tables spanning multiple pages. Missing for 10: documented handling of merged cells, nested header structures, multi-page table stitching, and independent verification of structural fidelity beyond simple format conversion.", "evidenceIds": [ "mistral-document-ai-docs-2", "mistral-document-ai-docs-16", "mistral-document-ai-comm-1", "mistral-document-ai-comm-2" ] }, { "productId": "mistral-document-ai", "storyId": "table-to-structured-rows", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs show table_format outputs (null/markdown/html) and a document_annotation feature that can return structured JSON based on a user-provided schema (useful for invoices, receipts, contracts), which could feed typed rows into a DB, but there's no explicit table-to-typed-JSON/row schema example, and community reports note accuracy issues (whole-page-as-image, multilingual mismatches) and the need for human review of a percentage of extractions, meaning manual cleanup is often still required. Missing for 10: a documented end-to-end example of table→typed JSON/row schema output, and independent evidence that output requires no manual correction.", "evidenceIds": [ "mistral-document-ai-docs-2", "mistral-document-ai-docs-8", "mistral-document-ai-docs-9", "mistral-document-ai-docs-21", "mistral-document-ai-comm-3", "mistral-document-ai-comm-5", "mistral-document-ai-comm-8" ] }, { "productId": "mistral-document-ai", "storyId": "typed-official-sdks", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains only API/feature documentation (OCR parameters, annotations, Q&A) and community feedback on OCR accuracy, but nothing about official Python or TypeScript SDKs, their typed interfaces, or coverage of parse/extract/jobs endpoints — missing for 10: any mention of SDK packages, typed client libraries, installation/import examples, or SDK-specific defaults.", "evidenceIds": [] }, { "productId": "mistral-document-ai", "storyId": "vpc-self-host-deployment", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "A single marketing-oriented citation states Mistral positions Document AI for \"compliance-first organizations requiring secure on-premises deployment\" and offers \"secure deployments,\" but there is no technical documentation, API/on-prem architecture, VPC networking details, or independent/hands-on confirmation that the OCR/annotation stack can actually run fully self-hosted or within a customer VPC. Missing for 10: concrete self-hosted deployment docs/architecture, VPC networking specifics, independent verification of on-prem operation, and confirmation that all document-processing features (annotation, QnA) work outside Mistral's cloud.", "evidenceIds": [ "mistral-document-ai-supp-1" ] }, { "productId": "reducto", "storyId": "accuracy-benchmarks-evals", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any published benchmark methodology, dataset, or reproducible accuracy numbers from Reducto; docs cover product features (parse, extract, split, etc.) but no benchmark suite or eval framework is mentioned. Community discussion even shows an open ask for third-party comparisons and accuracy complaints, but nothing about a vendor-published, reproducible eval an ML engineer could run themselves.", "evidenceIds": [ "reducto-comm-7", "reducto-comm-8", "reducto-comm-9" ] }, { "productId": "reducto", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "reducto-probe-1 confirms a live, working llms.txt at docs.reducto.ai/llms.txt returning HTTP 200 with structured links to docs, and the .md-suffixed doc pages (e.g. reducto-docs-29 through -40) show agent-oriented markdown docs are served alongside an MCP server and CLI for agent consumption. Missing for 10: no independent/community confirmation of an agent successfully consuming llms.txt in practice.", "evidenceIds": [ "reducto-probe-1", "reducto-docs-29", "reducto-docs-32", "reducto-probe-3" ] }, { "productId": "reducto", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Reducto's evidence covers structured document parsing, extraction, classification, and agentic prompting for extraction tasks, but nothing shows the product generating its own insights, summaries, or proactive suggestions from processed data — it only returns what the user's schema/prompt explicitly asks for. No evidence of an insights/recommendations layer inside Reducto Studio or the API.", "evidenceIds": [] }, { "productId": "reducto", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Reducto supports async job processing (run_job returns a job ID) plus Svix webhook callbacks for delivery/retry, and Studio pipelines can be deployed as a callable Pipeline ID chaining classify/parse/extract/edit steps — this enables background, non-blocking processing rather than requiring synchronous polling. However, there's no evidence of autonomous triggers (e.g., scheduled jobs, folder/inbox watchers, event-driven ingestion) that would let an automation run on its own without an explicit API call to kick it off. Missing for 10: scheduled/cron-style automation, autonomous trigger mechanisms (e.g., watch new documents arriving), and independent confirmation that pipelines run unattended in production.", "evidenceIds": [ "reducto-docs-9", "reducto-docs-23", "reducto-docs-36", "reducto-docs-10", "reducto-docs-28", "reducto-docs-39", "reducto-docs-6", "reducto-docs-34" ] }, { "productId": "reducto", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Reducto is a document-processing API/platform (parse, extract, split, classify) — it is not a chat/agent product with a built-in assistant UI to which end-users delegate tasks; its 'agentic' features refer to configurable processing modes and its MCP server exposes tools to external agents (client-side), not a first-party in-product assistant. This axis is a category mismatch for an API/infrastructure product.", "evidenceIds": [] }, { "productId": "reducto", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Reducto offers a documented API/SDK, CLI, async job endpoints, webhooks, and OpenAPI spec — all standard building blocks for headless/CI automation, with docs explicitly showing scriptable calls like client.parse.run() and CLI usage. Missing for 10: no explicit CI/CD pipeline examples (e.g., GitHub Actions), no independent hands-on report confirming headless CI usage.", "evidenceIds": [ "reducto-docs-7", "reducto-docs-31", "reducto-docs-9", "reducto-docs-23", "reducto-docs-36", "reducto-docs-18", "reducto-probe-2", "reducto-probe-4", "reducto-docs-39" ] }, { "productId": "reducto", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Reducto is a document-processing API/platform that ships its own MCP *server* so external agents (Claude, Cursor, etc.) can call its tools — the evidence never shows Reducto acting as an agent host or client that consumes external MCP servers' tools. Being an MCP client/tool-consumer is not a fit axis for this kind of backend document API product.", "evidenceIds": [] }, { "productId": "reducto", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Reducto is a document processing SaaS (not itself an agent), and it publishes an official MCP server that lets agents in Claude Desktop, Claude Code, Codex, Cursor, VS Code, Windsurf, or any MCP client classify, parse, extract, split, and edit documents as part of their reasoning loop. This is documented first-party and confirmed via probe. Missing for 10: independent/hands-on third-party corroboration of the MCP server working in practice.", "evidenceIds": [ "reducto-docs-8", "reducto-docs-24", "reducto-docs-32", "reducto-probe-3" ] }, { "productId": "reducto", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Reducto supports natural-language commands for defining schemas, split sections, classification categories, and edit instructions across Extract, Split, Classify, and Fill/Edit endpoints, and its MCP server lets AI agents invoke these capabilities conversationally as part of a reasoning loop. However, this is scoped to document-processing operations rather than general product operation (e.g., no evidence of natural-language control over account/admin settings, billing, or deployment config), and there's no independent hands-on verification of the MCP/CLI natural-language experience. missing for 10: broader natural-language control beyond document parsing/extraction tasks, independent corroboration of the MCP/agentic natural-language workflow actually working well in practice.", "evidenceIds": [ "reducto-docs-2", "reducto-docs-3", "reducto-docs-4", "reducto-docs-5", "reducto-docs-8", "reducto-docs-24", "reducto-docs-30", "reducto-probe-3" ] }, { "productId": "reducto", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Reducto ships an official CLI documented at docs.reducto.ai/cli, giving direct terminal access to parse, extract, split, classify, and edit — core document capabilities — confirmed by both docs and a probe. Missing for 10: independent/hands-on community corroboration of CLI usage and more detail on CLI-specific agentic scripting workflows.", "evidenceIds": [ "reducto-docs-7", "reducto-docs-31", "reducto-probe-4" ] }, { "productId": "reducto", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Reducto ships a documented public API (OpenAPI spec confirmed at openapi.json) covering parse, extract, split, classify, edit/fill, async job workflows, webhooks, and configuration options, plus SDK code examples throughout the docs and a quickstart. It also has a CLI and MCP server that let agents drive the same capabilities programmatically. Missing for 10: independent hands-on developer confirmation of API ergonomics/completeness beyond first-party docs.", "evidenceIds": [ "reducto-probe-2", "reducto-docs-18", "reducto-docs-21", "reducto-docs-9", "reducto-docs-10", "reducto-docs-7", "reducto-probe-3", "reducto-docs-6" ] }, { "productId": "reducto", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack of scoped/least-privilege API key management, role-based credential issuance, or agent-specific token scoping — only general security/compliance policies (ZDR, SOC2, HIPAA) and enterprise deployment options are documented, none of which address credential scoping for agents.", "evidenceIds": [] }, { "productId": "reducto", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show code examples using a client SDK (e.g. client.parse.run, client.extract, run_job) plus an OpenAPI spec, CLI, and MCP server, indicating official SDK-style access for building integrations. However, there is no explicit documentation naming specific language SDKs (e.g. Python/TypeScript packages), install instructions, or versioning/release notes, and no independent developer corroboration of SDK usability. Missing for 10: named official SDK packages with install/versioning docs, independent hands-on validation of SDK usage.", "evidenceIds": [ "reducto-docs-18", "reducto-docs-21", "reducto-docs-9", "reducto-docs-23", "reducto-probe-2", "reducto-probe-4" ] }, { "productId": "reducto", "storyId": "agentic-webhooks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Reducto documents webhook subscriptions via Svix integration, with cryptographic signing, retries, and a delivery dashboard accessible from Reducto Studio, enabling async job completion notifications for agentic workflows. Missing for 10: independent hands-on confirmation of webhook reliability and no detail on event/payload schema variety.", "evidenceIds": [ "reducto-docs-10", "reducto-docs-28", "reducto-docs-39" ] }, { "productId": "reducto", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Reducto's docs include code snippets (quickstart, chunking, async job examples) and an openapi.json spec, suggesting a structured API reference, but there's no evidence of an interactive, in-browser 'try it now' or runnable-example experience typical of tools like Swagger UI or Postman-style docs. Missing for 10: explicit interactive/try-it API console, evidence of live code execution in docs, independent confirmation of interactivity.", "evidenceIds": [ "reducto-docs-18", "reducto-docs-21", "reducto-docs-23", "reducto-probe-2", "reducto-probe-1" ] }, { "productId": "reducto", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A probe confirms a live OpenAPI spec is downloadable at https://docs.reducto.ai/openapi.json returning HTTP 200 with an 'openapi' key, directly satisfying the machine-readable API spec requirement. Missing for 10: no independent/community confirmation of spec usage or completeness beyond the probe check.", "evidenceIds": [ "reducto-probe-2" ] }, { "productId": "reducto", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Reducto's docs describe Studio (build/test workflows on real documents before deploying as a Pipeline ID) and a quickstart sample PDF, but there is no evidence of a distinct sandbox/test environment, test API keys, or staging mode explicitly separated from production data. Missing for 10: dedicated sandbox/test mode or environment, sample/test data set isolated from production, documentation contrasting sandbox vs production usage.", "evidenceIds": [ "reducto-docs-34", "reducto-docs-18" ] }, { "productId": "reducto", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows an OpenAPI spec and API docs exist, but there is no mention of API versioning scheme or a documented deprecation policy anywhere in the pack. missing for 10: versioning scheme documentation, explicit deprecation policy, changelog/migration guides.", "evidenceIds": [ "reducto-probe-2" ] }, { "productId": "reducto", "storyId": "async-jobs-webhooks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly describe async job submission via run_job() returning a job ID immediately (docs-9, docs-23, docs-36), plus a production-grade Svix webhook system with signed requests, retries, and delivery dashboard (docs-10, docs-39, docs-28) for completion notifications, directly matching the non-blocking async pipeline story. Missing for 10: no explicit documentation of a status-polling endpoint/schema alongside webhooks, and no independent/hands-on confirmation of async+webhook reliability in production.", "evidenceIds": [ "reducto-docs-9", "reducto-docs-23", "reducto-docs-36", "reducto-docs-10", "reducto-docs-39", "reducto-docs-28" ] }, { "productId": "reducto", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Reducto documents a batch queue with discounted pricing for Parse/Extract jobs, async job submission (run_job), a CLI for scripting over many files, and webhooks for tracking completions — all of which support processing many documents in bulk. However, there is no explicit documentation of a single bulk/batch API call accepting a list of many items, nor independent evidence of throughput/scale at high volume (one community report even notes struggles with long/large documents).\n\nmissing for 10: explicit bulk-submission API (array of documents in one call), documented scale/performance benchmarks for large batches, independent confirmation of reliable bulk throughput.", "evidenceIds": [ "reducto-docs-25", "reducto-docs-9", "reducto-docs-23", "reducto-docs-7", "reducto-docs-10", "reducto-comm-7" ] }, { "productId": "reducto", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Reducto supports webhook-based event notifications on job completion (Svix, with retries) and conditional routing/classification that lets downstream steps branch by document type, plus chainable multi-step workflows — these are event/condition-triggered automations. However there is no documented general-purpose 'rules engine' where a user defines arbitrary if-this-then-that triggers/actions; the automation is limited to job-lifecycle webhooks and document-type routing within Reducto's own pipeline. Missing for 10: a user-facing rules/trigger configuration UI or API for arbitrary event-action bindings, examples of rules beyond routing/classification, and independent confirmation of this automation working in production.", "evidenceIds": [ "reducto-docs-4", "reducto-docs-6", "reducto-docs-10", "reducto-docs-39", "reducto-docs-34", "reducto-docs-28" ] }, { "productId": "reducto", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Reducto offers async job execution, webhooks, and pipelines, but there is no evidence of a scheduler or cron-like mechanism for recurring/automated jobs triggered on a time basis. Missing for 10: any documentation of scheduled/recurring job triggers, cron syntax, or periodic workflow execution.", "evidenceIds": [] }, { "productId": "reducto", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Reducto is a document-processing API/platform (parse, extract, split, edit) rather than an automation-builder product with a notion of 'automations' that can be versioned/reviewed/rolled back; this axis is a category error for its product type.", "evidenceIds": [] }, { "productId": "reducto", "storyId": "batch-throughput", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Reducto documents async job APIs (run_job/parse_async), a discounted batch queue, Svix webhooks with retries, and presigned uploads up to 5GB, plus enterprise SLAs up to 99.99% and dedicated GPU/VPC infra — all supporting high-volume, async processing. However, no explicit documented per-minute/per-day rate limits or throughput numbers are cited, and community feedback notes struggles with very long documents (300+ pages) needing alternatives for speed/accuracy at scale. missing for 10: explicit published rate-limit tiers/throughput benchmarks, independent verification of million-page batch performance.", "evidenceIds": [ "reducto-docs-9", "reducto-docs-23", "reducto-docs-25", "reducto-docs-10", "reducto-docs-39", "reducto-docs-16", "reducto-docs-33", "reducto-docs-27", "reducto-comm-7" ] }, { "productId": "reducto", "storyId": "citations-provenance", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Reducto's Extract citations feature documents bounding box coordinates pointing to source text for every extracted field, directly matching the provenance story; page-level references also come from Split. missing for 10: no explicit mention of a 'source snippet' text field alongside bounding box/page, and no independent/hands-on confirmation of citation accuracy in the community evidence.", "evidenceIds": [ "reducto-docs-11", "reducto-docs-22", "reducto-docs-2", "reducto-docs-20" ] }, { "productId": "reducto", "storyId": "classification-splitting", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Reducto documents the building blocks needed for this story — Classify to route document types (reducto-docs-4, reducto-docs-30), Split to locate document/section boundaries by natural-language description (reducto-docs-3, reducto-docs-20), and multi-step Workflows/Studio pipelines that chain classification, parsing and extraction into a single API call or Pipeline ID (reducto-docs-6, reducto-docs-34). However, there is no explicit end-to-end example or case study showing a single multi-document packet upload being auto-split into distinct documents with separate per-document extraction results returned — the pieces exist but the composed 'one upload, per-document output' workflow is not directly demonstrated. Missing for 10: a concrete documented example/tutorial of packet splitting into per-document results, and any independent/community corroboration of this specific workflow.", "evidenceIds": [ "reducto-docs-4", "reducto-docs-30", "reducto-docs-3", "reducto-docs-20", "reducto-docs-6", "reducto-docs-34" ] }, { "productId": "reducto", "storyId": "complex-layout-parsing", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Reducto's Parse endpoint is documented to convert complex documents (multi-column, tables, figures, layout) into structured, ordered JSON, and agentic/chunking configs support handling nuanced layouts [reducto-docs-1, reducto-docs-35, reducto-docs-38]. However, hands-on community reports concretely contradict claims of universally clean, accurate parsing: one user reports Reducto 'struggled with long documents' on 300+ page financial PDFs versus a competing model, another prominent competitor's founder claims others are 'better at just getting data out of PDFs accurately,' and an Ask HN thread seeks alternatives specifically citing accuracy concerns on messy PDFs. missing for 10: independent benchmark data confirming column/header/footer/footnote ordering accuracy, and resolution of the long-document accuracy complaints.", "evidenceIds": [ "reducto-docs-1", "reducto-docs-35", "reducto-docs-38", "reducto-comm-7", "reducto-comm-4", "reducto-comm-9" ] }, { "productId": "reducto", "storyId": "confidence-scores-hitl", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows citations/bounding boxes for traceability and a Studio citation viewer for testing, but no mention anywhere of calibrated confidence scores per field or a structured human-in-the-loop review workflow for low-confidence extractions. Missing for 10: confidence score output, review/approval UI or workflow triggered by confidence thresholds, any documentation of HITL review path.", "evidenceIds": [ "reducto-docs-11", "reducto-docs-22", "reducto-docs-34" ] }, { "productId": "reducto", "storyId": "document-data-compliance", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Reducto documents a Zero Data Retention policy (24h expiry) for Growth+ tiers, completed SOC 2 Type I/II, and a HIPAA-compliant pipeline with BAA available for Growth/Enterprise tiers, plus VPC/on-prem deployment options for sensitive data like contracts and medical records. missing for 10: independent/third-party audit confirmation or customer case study validating these compliance claims in practice, and clarity on whether ZDR/HIPAA apply below Growth tier.", "evidenceIds": [ "reducto-supp-1", "reducto-supp-2", "reducto-docs-26", "reducto-docs-15", "reducto-docs-40" ] }, { "productId": "reducto", "storyId": "figure-chart-handling", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Reducto's Parse endpoint extracts figures/charts as part of structured output (reducto-docs-1), agentic mode explicitly supports 'advanced chart extraction' via custom prompts (reducto-docs-13, reducto-docs-38), and Citations attach bounding-box coordinates pointing back to source text for extracted fields (reducto-docs-11, reducto-docs-22), giving page/position traceability. However there's no explicit documentation of VLM-generated figure summaries or image-crop output format, nor independent confirmation that figure/chart citations specifically resolve to accurate bounding boxes. Missing for 10: explicit VLM figure-summary output examples, documented image-crop artifacts, independent verification of figure-level citation accuracy.", "evidenceIds": [ "reducto-docs-1", "reducto-docs-11", "reducto-docs-13", "reducto-docs-22", "reducto-docs-38" ] }, { "productId": "reducto", "storyId": "file-format-breadth", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Reducto's Parse/Extract/Classify docs describe a single API that ingests \"documents\" generically and Fill/Edit explicitly covers PDF forms and DOCX, but the evidence pack never explicitly confirms PPTX, XLSX, HTML, image, or email ingestion as first-class formats through the same endpoint, so the full \"whole document mix\" claim is only inferred, not documented. Missing for 10: explicit docs/examples confirming PPTX, XLSX, HTML, raw image, and email (.eml/.msg) inputs handled by the same Parse/Extract calls, and any independent/hands-on confirmation of cross-format consistency.", "evidenceIds": [ "reducto-docs-1", "reducto-docs-5", "reducto-docs-30", "reducto-docs-33", "reducto-docs-16" ] }, { "productId": "reducto", "storyId": "handwriting-recognition", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Reducto's docs describe Parse/Extract/Split/Classify with tables, layout, citations (bounding boxes) for extracted values, but nowhere mention handwritten text/annotation recognition or confidence scores flagging uncertain extractions. The axis clearly applies to a document-parsing product, but no evidence supports this specific capability.", "evidenceIds": [] }, { "productId": "reducto", "storyId": "large-document-handling", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Reducto documents async job processing, presigned uploads up to 5GB, chunking controls, and batch queuing that suggest large-file/long-document support (reducto-docs-9, 16, 21, 25), but a concrete hands-on community report states Reducto 'did struggle with long documents' on 300+ page financial documents, prompting the reporter to switch to another model for accuracy/speed (reducto-comm-7). This is a direct, specific counter-example to the reliability claim for thousand-page-scale documents, not just general skepticism. Missing for 10: no vendor benchmark or case study specifically addressing multi-thousand-page reliability, no rebuttal or fix documented for the reported struggle, no independent confirmation of timeout-free processing at that scale.", "evidenceIds": [ "reducto-docs-9", "reducto-docs-16", "reducto-docs-21", "reducto-docs-25", "reducto-comm-7" ] }, { "productId": "reducto", "storyId": "llm-ready-markdown", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Reducto's Parse endpoint produces structured JSON with text, tables, figures, and layout, and offers chunking configs (variable chunking splits at semantic boundaries like headers/tables/figures, sized for embedding models), which directly targets LLM/RAG consumption. However, there is no explicit documentation of markdown output format or specific claims about stripping repeated headers/footers/page furniture noise — the evidence emphasizes structure preservation and chunking rather than noise removal. Missing for 10: explicit markdown output evidence, explicit documentation of header/footer/page-furniture deduplication, and independent/hands-on confirmation of clean LLM-ready output quality.", "evidenceIds": [ "reducto-docs-1", "reducto-docs-12", "reducto-docs-21", "reducto-docs-35", "reducto-docs-14" ] }, { "productId": "reducto", "storyId": "low-latency-sync-mode", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs show a default synchronous call (client.parse.run) alongside an explicit async endpoint (run_job() returning a job ID immediately) implying dual modes, and a founder comment notes a cost/latency tradeoff for agentic 'Deep Extract' vs single-pass extraction, but there is no documented per-mode latency benchmark (e.g., 'seconds') for the synchronous path. Missing for 10: explicit sync-mode latency numbers/SLA, a side-by-side latency comparison table for sync vs async, and independent hands-on confirmation of sub-second/seconds response times.", "evidenceIds": [ "reducto-docs-9", "reducto-docs-18", "reducto-docs-23", "reducto-comm-8" ] }, { "productId": "reducto", "storyId": "multilingual-documents", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses multilingual or CJK/RTL parsing fidelity at all — docs describe parse/extract/split/classify features and English-language sample workflows, but there's no mention of language support, script coverage, or benchmarks for non-English documents. Missing for 10: any mention of CJK/RTL script support, multilingual accuracy benchmarks, or language-specific documentation/testimonials.", "evidenceIds": [] }, { "productId": "reducto", "storyId": "openness-api-parity", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Reducto's core document operations (parse, extract, split, classify, edit, route) are all exposed as API endpoints, CLI commands, and MCP tools, and Studio-built pipelines can be deployed as a Pipeline ID callable from code, showing strong UI/API parity for document workflows. However, there's no explicit confirmation that every UI feature (e.g., webhook dashboard management, account/billing settings) is fully API-accessible, so parity is inferred rather than exhaustively documented. Missing for 10: explicit 1:1 feature-parity statement, evidence that Studio-only features (webhook dashboard, billing) are API-manageable, independent verification of parity claims.", "evidenceIds": [ "reducto-docs-34", "reducto-docs-7", "reducto-docs-8", "reducto-docs-28", "reducto-probe-2", "reducto-probe-4" ] }, { "productId": "reducto", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Reducto's core outputs (Parse, Extract, Split) are all structured JSON, an open format the customer fully owns, and enterprise/on-prem deployments keep data resident in the customer's own VPC — both reduce lock-in. However, there is no explicit account-level 'export all my data and leave' feature, no documented bulk-export or account-deletion/migration-out tool, and the only migration incentive documented is for switching TO Reducto, not away from it. missing for 10: an explicit data-export/account-portability feature, documentation of exporting historical job data/configs, and evidence of migrating away from Reducto.", "evidenceIds": [ "reducto-docs-1", "reducto-docs-2", "reducto-docs-3", "reducto-docs-15", "reducto-docs-26" ] }, { "productId": "reducto", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Reducto is a closed, proprietary SaaS API/platform; evidence shows docs, CLI, MCP server, and API references but nothing about source code being available under any open license. No repository or license file is cited anywhere in the evidence pack.", "evidenceIds": [] }, { "productId": "reducto", "storyId": "openness-self-host", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Reducto offers VPC/on-prem deployment options (dedicated VPC, hybrid VPC, fully hosted SaaS) letting enterprise customers keep data and storage within their own VPC, but ephemeral processing still runs on Reducto's own GPU infrastructure rather than a fully self-hostable, open binary/container a user runs entirely on their own hardware. missing for 10: no evidence of a downloadable/open-source self-host package, no details on hybrid model's compute independence from Reducto's infra, and this option is gated behind enterprise tier rather than open to any AI-native user.", "evidenceIds": [ "reducto-docs-15", "reducto-docs-40" ] }, { "productId": "reducto", "storyId": "playground-before-code", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Reducto Studio is documented as a web interface for testing parse/extract/classification workflows on real documents with a citation viewer before deploying as callable pipelines, which matches the spirit of a no-code playground. However, no evidence explicitly describes a drag-and-drop upload flow, and a community report from launch noted the Studio link (links.reducto.ai/studio) was broken with redirect errors, raising some doubt about polish/accessibility. Missing for 10: explicit drag-and-drop UI description, independent hands-on confirmation the playground works reliably today.", "evidenceIds": [ "reducto-docs-34", "reducto-docs-28", "reducto-comm-2" ] }, { "productId": "reducto", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Reducto's enterprise docs describe deployment options (dedicated VPC, hybrid VPC, fully hosted SaaS) where 'all data and storage reside in the customer's VPC,' giving some control over data location, but this is scoped to Enterprise-tier deployments rather than a self-service region-selection setting for standard AI-native users. Community feedback also notes a DPA (needed for EU residency compliance) is only available on the enterprise plan, reinforcing that residency control is gated behind higher tiers. Missing for 10: explicit region/residency selection UI or API parameter for standard/Growth tiers, documented list of supported regions/data centers, and independent verification of VPC deployment residency guarantees.", "evidenceIds": [ "reducto-docs-15", "reducto-docs-40", "reducto-comm-3" ] }, { "productId": "reducto", "storyId": "privacy-no-training", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Reducto documents a Zero Data Retention policy for Growth+ tiers (API data auto-deleted within 24 hours) and SOC2/HIPAA compliance, and offers VPC/on-prem deployment options that keep data within customer control — all of which support preventing data use for training. However, no evidence explicitly states a commitment not to train models on customer data, ZDR is gated behind paid tiers (not default/free), and a community comment notes DPAs (often tied to no-training commitments) are also enterprise-only. Missing for 10: explicit no-training-on-customer-data clause, ZDR/DPA availability for lower/free tiers, independent audit confirming policy adherence.", "evidenceIds": [ "reducto-supp-1", "reducto-supp-2", "reducto-docs-15", "reducto-docs-40", "reducto-comm-3" ] }, { "productId": "reducto", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Reducto documents a concrete data retention/deletion policy: Zero Data Retention for Growth+ tier customers with API-submitted data auto-expiring/deleted within 24 hours (except persisted results), plus VPC/on-prem deployment options for full data control. However, ZDR is gated behind a paid tier (not default/free), there's no documented self-service deletion API or granular per-record deletion control, and a community comment flags DPA/legal-requirement gaps for lower tiers. Missing for 10: default-tier retention controls, explicit user-triggered deletion API, and independent verification of retention enforcement.", "evidenceIds": [ "reducto-docs-26", "reducto-supp-1", "reducto-docs-15", "reducto-comm-3" ] }, { "productId": "reducto", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence covers data retention (ZDR), SOC2/HIPAA compliance, and deployment options, but there is no mention of a telemetry/usage-tracking opt-out mechanism for the product itself (e.g., CLI/SDK/MCP analytics collection or an opt-out flag).", "evidenceIds": [] }, { "productId": "reducto", "storyId": "rag-ready-chunking", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Reducto's Parse endpoint offers explicit variable chunking configured for embedding models, splitting at semantic boundaries (headers, tables, figures) rather than arbitrary text walls, with chunk_size controls and metadata like bounding boxes/citations attached to output — directly matching the RAG-ready chunking story. Docs show concrete API usage (chunk_mode, chunk_size) and best-practices guidance confirming semantic-boundary-aware chunking designed for embedding pipelines. Missing for 10: independent/hands-on benchmark confirming chunk quality in practice beyond vendor docs.", "evidenceIds": [ "reducto-docs-12", "reducto-docs-21", "reducto-docs-35", "reducto-docs-11", "reducto-docs-14" ] }, { "productId": "reducto", "storyId": "reading-order-hierarchy", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Reducto's Parse endpoint documents structured JSON output preserving text, tables, figures, layout, and formatting, and variable chunking explicitly splits at semantic boundaries like section headers to keep related content together, which speaks directly to preserving hierarchy/reading order. However, there's no explicit documentation of heading-level detection or a hierarchical/nested structure representation (e.g., section-tree output), and one community report notes struggles with long documents (300+ pages), suggesting structure fidelity may degrade at scale. Missing for 10: explicit heading/section hierarchy schema in output, independent benchmark on structure/order fidelity, and resolution of the long-document accuracy concern.", "evidenceIds": [ "reducto-docs-1", "reducto-docs-35", "reducto-docs-12", "reducto-comm-7" ] }, { "productId": "reducto", "storyId": "scanned-doc-ocr", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes Reducto's general Parse/Extract capabilities (structured JSON, tables, layout) but contains no documentation or evidence addressing OCR performance specifically on scanned/photographed documents, skewed pages, stamps, or low-quality scans. Community mentions are limited to accuracy comparisons with Mathpix and struggles with long documents, not scan-quality robustness, so there's no positive evidence this specific capability is delivered.", "evidenceIds": [ "reducto-docs-1", "reducto-docs-29", "reducto-comm-4", "reducto-comm-9" ] }, { "productId": "reducto", "storyId": "schema-driven-extraction", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Reducto's Extract endpoint explicitly takes a user-defined JSON schema and returns values matching that schema, with citations/bounding boxes for validation of source location, documented via docs and quickstart-style API calls. Some community feedback notes accuracy struggles on very long/messy documents, but this doesn't contradict the core schema-driven extraction capability itself. Missing for 10: independent hands-on validation of extracted field accuracy/schema conformance beyond vendor docs, and explicit mention of validation/error-handling for malformed schema responses.", "evidenceIds": [ "reducto-docs-2", "reducto-docs-19", "reducto-docs-11", "reducto-docs-22", "reducto-docs-18", "reducto-comm-7", "reducto-comm-8" ] }, { "productId": "reducto", "storyId": "table-structure-extraction", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Reducto's docs explicitly address table extraction with configurable output formats that represent merged cells and headers, and Parse converts docs into structured JSON with tables/layout preserved; agentic mode also supports custom prompts for complex content. However, there's no explicit mention of nested headers or multi-page table spans being handled, and community feedback flags accuracy struggles on long documents, suggesting real-world caveats. missing for 10: explicit documentation/testing of nested headers and multi-page table spans, independent benchmark confirming fidelity on such complex tables.", "evidenceIds": [ "reducto-docs-14", "reducto-docs-37", "reducto-docs-1", "reducto-docs-38", "reducto-comm-7" ] }, { "productId": "reducto", "storyId": "table-to-structured-rows", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Reducto's Extract endpoint lets engineers define a schema and returns typed structured JSON matching it, and Parse extracts tables in multiple configurable output formats (with citations/bounding boxes for traceability), directly enabling load-ready structured data without manual cleanup. Community feedback corroborates the core table/structured-extraction use case though notes accuracy tradeoffs on long documents, which is a quality caveat rather than a capability failure. Missing for 10: independent benchmark evidence of clean, zero-cleanup DB-ready output at scale, and explicit database-loading integration/examples.", "evidenceIds": [ "reducto-docs-2", "reducto-docs-19", "reducto-docs-14", "reducto-docs-37", "reducto-docs-11", "reducto-docs-22", "reducto-comm-7", "reducto-comm-8" ] }, { "productId": "reducto", "storyId": "typed-official-sdks", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Docs and quickstart show a Python client (client.parse.run, client.parse.run_job(), extract usage) covering parse, extract, and async jobs, suggesting a functional Python SDK with reasonable defaults. However, there is no evidence at all of a TypeScript/JS SDK — all code samples are Python only — so the 'Python and TypeScript' claim is only half-supported. Missing for 10: TypeScript SDK existence/usage examples, explicit documentation of SDK method signatures/defaults across all endpoints (split, classify, edit), and independent developer corroboration of DX quality.", "evidenceIds": [ "reducto-docs-18", "reducto-docs-21", "reducto-docs-9", "reducto-docs-23", "reducto-docs-36", "reducto-probe-2" ] }, { "productId": "reducto", "storyId": "vpc-self-host-deployment", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly describe dedicated VPC, hybrid VPC, and fully hosted deployment options, with data/storage residing in customer VPC and only ephemeral processing on Reducto's GPU infra, directly matching the self-hosted/VPC-for-sensitive-documents requirement. Missing for 10: independent/hands-on verification of an actual on-prem deployment and more detail on air-gapped or fully offline operation beyond the docs' description.", "evidenceIds": [ "reducto-docs-15", "reducto-docs-40", "reducto-supp-2" ] }, { "productId": "unstructured", "storyId": "accuracy-benchmarks-evals", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of published reproducible accuracy benchmarks or eval datasets/scripts that an ML engineer could run to verify parse accuracy claims; docs cover product features (partitioning, chunking, connectors) but nothing about benchmark methodology or reproducibility.", "evidenceIds": [] }, { "productId": "unstructured", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A direct probe confirms llms.txt exists at docs.unstructured.io/llms.txt (HTTP 200) with agent instructions pointing to MCP server, SDK, and API docs, directly satisfying the story. Missing for 10: no evidence of additional agent-oriented doc formats (e.g., openapi.json, which probes show is 404) or third-party confirmation of agents successfully using the llms.txt file.", "evidenceIds": [ "unstructured-probe-1", "unstructured-probe-2" ] }, { "productId": "unstructured", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 2, "confidence": "low", "rationale": "Unstructured's docs use marketing language like 'discover hidden insights' when describing its connectors/data unification, and it does perform AI-driven extraction/enrichment (VLM-based table-to-HTML, structured JSON extraction) on data—loosely gesturing at 'insight generation.' However, there's no evidence of a dashboard, summarization, or recommendation feature that surfaces AI-generated insights or suggestions back to the user inside a product UI; the tool is positioned as an ETL/data pipeline for feeding RAG systems, not as a place where users consume AI-generated insights themselves. Missing for 10: a concrete insights/suggestions UI or feature, evidence of the product proactively surfacing analysis or recommendations to end users, and any hands-on/independent confirmation of such a capability.", "evidenceIds": [ "unstructured-docs-9", "unstructured-docs-29", "unstructured-docs-26", "unstructured-docs-27" ] }, { "productId": "unstructured", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Unstructured's Workflow/Job APIs let users define workflows and programmatically trigger and monitor processing jobs (source-to-destination pipelines) via SDK/REST, which can be automated to run without manual intervention. However, there is no explicit evidence of built-in scheduling, event-based triggers, or persistent background automation (e.g., cron-like recurring runs) that would make the automation fully autonomous rather than externally triggered. missing for 10: evidence of native scheduling/triggers, autonomous recurring execution, and confirmation that jobs run without external orchestration.", "evidenceIds": [ "unstructured-docs-5", "unstructured-docs-7", "unstructured-docs-6", "unstructured-docs-35" ] }, { "productId": "unstructured", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Unstructured documents workflows, APIs, SDKs, and an MCP server that let external AI tools (e.g., Claude, Cursor, coding agents) call into the platform, but there is no evidence of a built-in AI assistant inside the Unstructured product itself that users can delegate tasks to.", "evidenceIds": [ "unstructured-docs-2", "unstructured-docs-22", "unstructured-docs-17" ] }, { "productId": "unstructured", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Unstructured exposes REST API and Python SDK callable via standard clients (cURL, Postman) and can build/trigger/monitor workflow jobs programmatically, enabling headless automation in CI pipelines. Missing for 10: explicit CI/CD documentation or example (e.g., GitHub Actions), and independent third-party confirmation of CI usage.", "evidenceIds": [ "unstructured-docs-4", "unstructured-docs-5", "unstructured-docs-7", "unstructured-docs-23", "unstructured-docs-30" ] }, { "productId": "unstructured", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Unstructured exposes its own Transform MCP server so external AI tools (Claude Code, Cursor, etc.) can call Unstructured's tools — this is Unstructured acting as an MCP server/provider, not as an MCP client that plugs in and uses other servers' tools. No evidence describes Unstructured consuming external MCP servers.", "evidenceIds": [ "unstructured-docs-22", "unstructured-docs-17" ] }, { "productId": "unstructured", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Unstructured documents a dedicated 'Transform MCP server' that lets AI tools/agents (Claude Code, Cursor, Devin, etc.) ingest, partition, enrich, chunk, and embed local files directly through Unstructured, which is a first-party, officially supported MCP integration for connecting agents to the platform. Missing for 10: independent/hands-on verification of the MCP server working end-to-end and deeper API-level documentation beyond the overview page.", "evidenceIds": [ "unstructured-docs-22", "unstructured-docs-17", "unstructured-probe-1" ] }, { "productId": "unstructured", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show a dedicated 'Claude and plain language prompts' quickstart and a Transform MCP server that lets AI tools (Claude Code, Cursor, etc.) instruct Unstructured to ingest and process files via natural language, which supports NL-driven operation. However, the core product remains API/workflow-designer driven, and most capabilities (job APIs, connectors, no-code UI) are not natural-language controlled. Missing for 10: independent/hands-on validation of NL command coverage across the full product, and evidence of NL control beyond the MCP/Claude quickstart path.", "evidenceIds": [ "unstructured-docs-2", "unstructured-docs-22", "unstructured-docs-17" ] }, { "productId": "unstructured", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack documents a Python SDK, REST API, no-code workflow UI, and an MCP server for AI tool integration, but no official CLI tool is mentioned anywhere in the docs or probes.", "evidenceIds": [ "unstructured-docs-4", "unstructured-docs-19", "unstructured-docs-23", "unstructured-docs-30", "unstructured-docs-22" ] }, { "productId": "unstructured", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Unstructured documents a public REST API and Python SDK with a comprehensive API reference covering jobs, workflows, partitioning, chunking, and enrichment, plus quickstarts and standard REST client support (cURL/Postman). missing for 10: a discoverable machine-readable OpenAPI spec (probe found openapi.json/swagger.json all 404) and independent third-party corroboration beyond vendor docs.", "evidenceIds": [ "unstructured-docs-4", "unstructured-docs-5", "unstructured-docs-7", "unstructured-docs-19", "unstructured-docs-23", "unstructured-docs-30", "unstructured-probe-2" ] }, { "productId": "unstructured", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence covers API keys/usage, SDK, MCP server, and IP allowlisting but there is no mention of scoped or least-privilege API credentials, role-based permissions, or per-agent credential issuance for the API.", "evidenceIds": [] }, { "productId": "unstructured", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Unstructured documents an official Python SDK for calling its API to partition, enrich, chunk, and embed files, alongside REST client support, and provides quickstarts specifically for building against it in ~5 minutes. Multiple docs pages (SDK overview, workflow build docs, quickstart) corroborate first-party SDK support with concrete code-level use cases. Missing for 10: independent/hands-on developer corroboration beyond vendor docs, and no publicly discoverable OpenAPI spec was found via probe.", "evidenceIds": [ "unstructured-docs-4", "unstructured-docs-19", "unstructured-docs-23", "unstructured-docs-30", "unstructured-docs-1", "unstructured-docs-7" ] }, { "productId": "unstructured", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item mentions webhooks or event-driven subscriptions; the API is described as job-based (trigger, poll status, download results) rather than push-based notifications. Missing for 10: any documentation of webhook registration, event types, or push notification mechanism.", "evidenceIds": [ "unstructured-docs-5" ] }, { "productId": "unstructured", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Unstructured's docs include an 'api-reference' section with quickstarts that provide runnable Python/cURL/Postman code samples (e.g. docs-18, docs-31, docs-4/19/30), giving developers copy-paste examples to try. However, a direct probe for an OpenAPI/Swagger-style interactive spec returned 404 on all standard paths, indicating no live in-browser 'try it' API explorer exists — only static code snippets in prose docs. Missing for 10: a genuine interactive API console (e.g., Swagger UI/Redoc with live request execution), independent confirmation of an in-browser sandbox, and evidence the reference examples can be executed directly from the docs site rather than copied to an external client.", "evidenceIds": [ "unstructured-docs-4", "unstructured-docs-18", "unstructured-docs-19", "unstructured-docs-30", "unstructured-docs-31", "unstructured-probe-2" ] }, { "productId": "unstructured", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While Unstructured documents a REST API and Python SDK, there is no evidence of a downloadable OpenAPI/Swagger spec; a direct probe for common OpenAPI paths (openapi.json, swagger.json, etc.) returned 404 on all candidates, confirming no machine-readable spec is publicly exposed.", "evidenceIds": [ "unstructured-probe-2", "unstructured-docs-19", "unstructured-docs-30" ] }, { "productId": "unstructured", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack shows free-tier trial pages, dedicated VPC deployment, and security/compliance certifications, but nowhere describes a distinct sandbox/test environment isolated from production data for API testing. Missing for 10: explicit sandbox or test-mode environment, staging API keys, or documented separation between test and production data flows.", "evidenceIds": [ "unstructured-docs-14", "unstructured-docs-15", "unstructured-docs-28" ] }, { "productId": "unstructured", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of API versioning scheme (e.g. /v1/, /v2/) or a documented deprecation policy anywhere in the docs; OpenAPI spec probes also 404. missing for 10: explicit API version numbering, published deprecation/sunset policy, changelog for breaking changes.", "evidenceIds": [ "unstructured-probe-2" ] }, { "productId": "unstructured", "storyId": "async-jobs-webhooks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm job-based async processing with status tracking (trigger runs, monitor status, download results, inspect failed files) via the Job APIs, supporting non-blocking pipelines. However, there is no explicit mention of completion webhooks or callback notifications — polling appears to be the only documented mechanism. missing for 10: documented webhook/callback support on job completion, evidence of async job creation returning immediately with a job ID, and independent/hands-on confirmation of non-blocking behavior at scale.", "evidenceIds": [ "unstructured-docs-5", "unstructured-docs-7" ] }, { "productId": "unstructured", "storyId": "automation-bulk-operations", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Unstructured's workflows/jobs API and connectors (35+ sources, 65+ file types) explicitly support ingesting and processing many files at once via automated pipelines, with job tracking, batch partition/chunk/embed operations, and SDK/REST access for programmatic bulk triggering. missing for 10: no explicit documentation of batch size limits, concurrency controls, or independent benchmarks of large-scale throughput.", "evidenceIds": [ "unstructured-docs-5", "unstructured-docs-7", "unstructured-docs-9", "unstructured-docs-35", "unstructured-docs-36", "unstructured-docs-37", "unstructured-docs-8" ] }, { "productId": "unstructured", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Unstructured's docs describe workflows that can be manually run or triggered via API/job calls, but there is no evidence of a rule-engine or event-based trigger system (e.g., 'run this workflow when a new file lands' or webhook-driven automation) that fires actions automatically on defined events.", "evidenceIds": [] }, { "productId": "unstructured", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes creating, running, and monitoring workflows/jobs via API or no-code UI, but nowhere mentions scheduling or recurring/cron-based execution of these jobs. Absence of evidence for this applicable automation-depth capability means it is not shown to deliver.", "evidenceIds": [ "unstructured-docs-5", "unstructured-docs-7", "unstructured-docs-6" ] }, { "productId": "unstructured", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers workflow creation, job tracking, and running pipelines (create/update/run/delete workflows, job status tracking) but nothing about versioning workflow definitions, reviewing changes/diffs, or rolling back to prior versions of an automation.", "evidenceIds": [] }, { "productId": "unstructured", "storyId": "batch-throughput", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Unstructured documents job/workflow APIs for tracking processing jobs, dedicated VPC instances for scale, and a 10,000 free-page tier, implying batch/production-scale capability, but no evidence pack item specifies actual rate limits, throughput numbers, or SLAs for millions of pages. missing for 10: documented rate limits (requests/sec or pages/hour), throughput benchmarks at millions-of-pages scale, and any independent corroboration of predictable performance at that volume.", "evidenceIds": [ "unstructured-docs-5", "unstructured-docs-7", "unstructured-docs-15", "unstructured-docs-28", "unstructured-docs-14" ] }, { "productId": "unstructured", "storyId": "citations-provenance", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs confirm partitioning outputs 'document elements and metadata' in structured JSON (docs-32) and structured extraction into custom JSON fields (docs-26, docs-34), which implies some per-element metadata, but the evidence pack never explicitly documents page number, bounding box, or source-snippet fields as part of that metadata, nor any citation/verification workflow for agents or humans. missing for 10: explicit documentation of page-number/bounding-box/source-snippet fields in the metadata schema, examples of extracted fields carrying provenance, and any human/agent verification or citation workflow.", "evidenceIds": [ "unstructured-docs-32", "unstructured-docs-26", "unstructured-docs-34" ] }, { "productId": "unstructured", "storyId": "classification-splitting", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers partitioning, chunking, enrichment, and structured JSON extraction, but nothing describes detecting multiple distinct documents within a single uploaded file/packet, classifying each, and returning separate per-document results. Missing for 10: any mention of packet/document splitting, document-boundary detection, or classification of sub-documents within one upload.", "evidenceIds": [ "unstructured-docs-32", "unstructured-docs-24", "unstructured-docs-11" ] }, { "productId": "unstructured", "storyId": "complex-layout-parsing", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Unstructured's docs describe partitioning into structured JSON with auto-routing across Fast/High-Res/VLM strategies, table-to-HTML, and generative OCR enrichment, implying handling of complex layouts, but there is no explicit documentation or evidence addressing multi-column reading order, header/footer suppression, or footnote handling specifically, nor any independent benchmark or hands-on validation of parse accuracy on real-world PDFs. missing for 10: explicit multi-column/column-order handling documentation, header/footer/footnote-specific treatment, independent accuracy benchmarks or user testimonials confirming correct ordering.", "evidenceIds": [ "unstructured-docs-10", "unstructured-docs-32", "unstructured-docs-13", "unstructured-docs-12", "unstructured-docs-24" ] }, { "productId": "unstructured", "storyId": "confidence-scores-hitl", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence describes JSON schema extraction, partitioning, chunking, and enrichment, but nowhere mentions confidence scores on extracted fields nor any human-in-the-loop review workflow for low-confidence outputs. Job monitoring/failure inspection (unstructured-docs-5) covers pipeline failures, not per-field confidence-based review.", "evidenceIds": [] }, { "productId": "unstructured", "storyId": "document-data-compliance", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm SOC 2 Type 2 and HIPAA compliance plus dedicated VPC deployment with full data isolation, but no evidence explicitly describes a 'zero-retention' data handling policy for uploaded documents. missing for 10: explicit zero-retention/data-deletion policy statement, independent audit confirmation, and specifics on how contract/medical record data is purged after processing.", "evidenceIds": [ "unstructured-supp-1", "unstructured-docs-15", "unstructured-docs-28", "unstructured-docs-16" ] }, { "productId": "unstructured", "storyId": "figure-chart-handling", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Unstructured docs confirm image/figure handling via enrichment ('image descriptions', VLM-based generative OCR, VLM-based table-to-HTML) and partitioning into structured JSON elements with metadata, supporting the VLM-summary/image-crop part of the story. However, no evidence pack item explicitly documents that extracted figures/images carry page-number or coordinate metadata that traces them back to the source page location. Missing for 10: explicit documentation of bounding-box/page-position metadata for image/figure elements, and any independent/hands-on confirmation of accurate spatial traceability.", "evidenceIds": [ "unstructured-docs-13", "unstructured-docs-27", "unstructured-docs-36", "unstructured-docs-32" ] }, { "productId": "unstructured", "storyId": "file-format-breadth", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Unstructured's core value proposition is a single API/SDK that partitions 65+ file types (PDF, DOCX, PPTX, XLSX, HTML, images, email implied within office/document formats) into a consistent JSON format, with auto-routing of partitioning strategy per page/type, callable via Python SDK or REST — exactly matching the 'one API, no per-format plumbing' story. Missing for 10: an explicit itemized list confirming email/XLSX support specifically (only referenced generically as '65+ file types') and independent third-party benchmarking of format coverage.", "evidenceIds": [ "unstructured-docs-8", "unstructured-docs-9", "unstructured-docs-10", "unstructured-docs-32", "unstructured-docs-4", "unstructured-docs-23", "unstructured-docs-30" ] }, { "productId": "unstructured", "storyId": "handwriting-recognition", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence covers partitioning, chunking, table-to-HTML, and generative OCR/VLM enrichment, but nothing specifically addresses recognition of handwritten fields/annotations or confidence-flagging for uncertain extractions. This is a fair OCR-category question, but no supporting evidence exists.", "evidenceIds": [] }, { "productId": "unstructured", "storyId": "large-document-handling", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs claim 'resilient partitioning logic and robust visibility' and job tracking that lets you 'inspect any files that failed during processing,' implying some built-in reliability at scale, but there is no explicit evidence of handling thousand-page documents or multi-gigabyte files, no timeout limits documented, and no benchmarks or case studies addressing silent truncation. Missing for 10: documented file-size/page limits, explicit large-document benchmarks, evidence of timeout handling, and independent confirmation of reliability at scale.", "evidenceIds": [ "unstructured-docs-5", "unstructured-docs-8", "unstructured-docs-10" ] }, { "productId": "unstructured", "storyId": "llm-ready-markdown", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Unstructured's docs show partitioning converts raw files into a 'predefined, consistent, expressive, and contextualized JSON format' of document elements and metadata, plus chunking strategies to prepare RAG-ready data, which supports the general goal of clean, LLM-consumable structured output (unstructured-docs-32, unstructured-docs-33, unstructured-docs-9). However, there is no explicit mention of markdown output or of specifically stripping repeated headers/footers/page furniture as the story describes. Missing for 10: explicit markdown export format, explicit documentation of header/footer/page-furniture removal logic, and independent verification of output cleanliness.", "evidenceIds": [ "unstructured-docs-32", "unstructured-docs-33", "unstructured-docs-9", "unstructured-docs-29" ] }, { "productId": "unstructured", "storyId": "low-latency-sync-mode", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence mentions 'about 5 minutes' quickstarts and job/pipeline processing, but there is no documentation of a distinct fast synchronous processing mode with per-mode latency figures (seconds) suitable for interactive apps. missing for 10: explicit synchronous/async mode distinction, documented latency benchmarks per mode, evidence of sub-minute/seconds response times.", "evidenceIds": [ "unstructured-docs-1", "unstructured-docs-31", "unstructured-docs-10" ] }, { "productId": "unstructured", "storyId": "multilingual-documents", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items mention language coverage, CJK, right-to-left scripts, or multilingual OCR fidelity at all; the docs discuss partitioning, chunking, table-to-HTML, and generative OCR generically but never address non-English or multilingual document parsing quality.", "evidenceIds": [] }, { "productId": "unstructured", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show workflows, jobs, partitioning, chunking, and enrichment can all be driven via Python SDK or REST API, and the no-code UI (Pipelines) maps to the same workflow API objects, suggesting parity. However, no explicit statement confirms 100% UI-API parity, and the OpenAPI spec is not publicly resolvable (404s), making it hard to verify full API surface coverage independently. missing for 10: explicit UI/API parity statement, public OpenAPI spec confirming full endpoint coverage, independent verification of parity claims.", "evidenceIds": [ "unstructured-docs-6", "unstructured-docs-7", "unstructured-docs-5", "unstructured-docs-19", "unstructured-probe-2" ] }, { "productId": "unstructured", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Unstructured's core output is a 'consistent JSON format' (open, standard) and users can 'download results' from processing jobs and route data to 35+ destinations, which supports data portability rather than lock-in. However, there is no explicit documentation of an account-level 'export all my data and leave' capability, bulk account data export/deletion tooling, or confirmation of full openness beyond the processed JSON outputs. Missing for 10: explicit account/data export or deletion workflow, documentation of full data portability guarantees, independent confirmation of no proprietary lock-in.", "evidenceIds": [ "unstructured-docs-9", "unstructured-docs-29", "unstructured-docs-32", "unstructured-docs-5" ] }, { "productId": "unstructured", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains only SaaS/API/platform documentation (quickstarts, workflows, connectors, pricing, security) with no mention of a public source repository or open-source license for Unstructured's code.", "evidenceIds": [] }, { "productId": "unstructured", "storyId": "openness-self-host", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs mention deploying Unstructured in your own dedicated instance or VPC with multi-user access and full data isolation, which implies a self-hosted deployment option, but this is framed as an enterprise/paid offering with tailored pricing and support rather than a freely self-hostable open-source core with clear installation instructions. missing for 10: no documentation of a downloadable/open-source self-host package, no setup/installation guide for running the core independently, no confirmation this is free or community-supported rather than a paid enterprise deal.", "evidenceIds": [ "unstructured-docs-15", "unstructured-docs-21", "unstructured-docs-28" ] }, { "productId": "unstructured", "storyId": "playground-before-code", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence describes a no-code workflow designer and quickstart guides requiring Python/API calls, but nothing indicates a simple drag-and-drop web playground for instant parse/extract preview without writing code.", "evidenceIds": [ "unstructured-docs-6", "unstructured-docs-18", "unstructured-docs-31" ] }, { "productId": "unstructured", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Unstructured documents dedicated-instance/VPC deployment options (full data isolation, tailored pricing) and GDPR compliance, which implies some customer control over where data resides, but there is no explicit region-selection feature, multi-region hosting options, or documentation describing how a customer chooses a specific data residency zone. Missing for 10: explicit region/residency selection UI or config, list of supported hosting regions, and independent confirmation that VPC deployment lets customers pick geographic location.", "evidenceIds": [ "unstructured-docs-15", "unstructured-docs-28", "unstructured-supp-1" ] }, { "productId": "unstructured", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence covers data isolation, dedicated VPC deployment, IP allowlisting, and SOC2/GDPR/HIPAA compliance, but nowhere states an explicit policy on whether customer data is or isn't used to train Unstructured's or third-party AI models. Missing for 10: an explicit no-training-on-customer-data policy, an opt-out/opt-in training control, or any documentation addressing AI model training use of ingested data.", "evidenceIds": [ "unstructured-docs-15", "unstructured-docs-28", "unstructured-supp-1", "unstructured-docs-16" ] }, { "productId": "unstructured", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Compliance docs claim GDPR conformance (which implies data deletion rights) and workflows can be deleted via API, plus dedicated VPC deployments offer full data isolation, but there is no explicit documentation of data retention periods, configurable deletion of processed/stored documents, or data lifecycle controls beyond deleting workflow definitions. missing for 10: explicit retention policy settings, documented data deletion/purge mechanisms for processed files, and independent verification of these claims.", "evidenceIds": [ "unstructured-supp-1", "unstructured-docs-7", "unstructured-docs-28", "unstructured-docs-15" ] }, { "productId": "unstructured", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions telemetry, usage tracking, analytics collection, or any opt-out mechanism; security/compliance mentions (SOC2, GDPR, HIPAA, IP allowlisting) do not address telemetry opt-out specifically.", "evidenceIds": [] }, { "productId": "unstructured", "storyId": "rag-ready-chunking", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs directly describe chunking as a core post-partitioning step that produces embedding-sized chunks with multiple semantic strategies (by title, by page, by similarity) plus embedding generation and consistent JSON with metadata, matching the RAG-ready story. Missing for 10: independent/hands-on corroboration of chunk quality and semantic boundary accuracy beyond first-party docs.", "evidenceIds": [ "unstructured-docs-33", "unstructured-docs-11", "unstructured-docs-25", "unstructured-docs-32", "unstructured-docs-35" ] }, { "productId": "unstructured", "storyId": "reading-order-hierarchy", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs describe partitioning as producing structured document elements and metadata in a consistent JSON format, and chunking strategies like 'by title' imply awareness of headings/sections for logical grouping. However, there is no explicit documentation addressing reading-order preservation across multi-column or complex layouts, or how nested section hierarchy (e.g., parent/child heading relationships) is represented in output. missing for 10: explicit reading-order guarantees, documented hierarchy/nesting model for headings and sections, and independent/hands-on validation that structure survives complex layouts.", "evidenceIds": [ "unstructured-docs-32", "unstructured-docs-11", "unstructured-docs-25", "unstructured-docs-24" ] }, { "productId": "unstructured", "storyId": "scanned-doc-ocr", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Unstructured supports 'High Res' and VLM-based partitioning strategies and generative OCR enrichment, which are relevant to scanned/photographed documents, but there is no explicit evidence about handling skewed pages, stamps, or low-quality scans specifically, nor any accuracy benchmarks or independent hands-on validation. missing for 10: evidence of deskewing/rotation correction, stamp/handwriting handling, accuracy metrics on low-quality scans, and independent third-party validation of OCR quality.", "evidenceIds": [ "unstructured-docs-10", "unstructured-docs-13", "unstructured-docs-36" ] }, { "productId": "unstructured", "storyId": "schema-driven-extraction", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly describe a structured data extractor where developers define a target JSON schema up front and Unstructured extracts values into matching fields in consistent JSON format (docs-3, docs-26, docs-34), backed by API/SDK access to run this (docs-4, docs-23). Missing for 10: independent/hands-on validation of extraction accuracy and no direct example showing schema validation errors or edge-case handling.", "evidenceIds": [ "unstructured-docs-3", "unstructured-docs-26", "unstructured-docs-34", "unstructured-docs-4", "unstructured-docs-23" ] }, { "productId": "unstructured", "storyId": "table-structure-extraction", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Unstructured documents a table-to-HTML enrichment feature that converts detected tables into HTML markup using agentic AI or a VLM, which is directly relevant to table structure fidelity, but no evidence addresses handling of merged cells, nested headers, or tables spanning multiple pages — the specific complex-table scenarios in the story are unaddressed. missing for 10: evidence of merged-cell handling, nested header preservation, multi-page table stitching, and any accuracy benchmarks or independent validation of table fidelity.", "evidenceIds": [ "unstructured-docs-12", "unstructured-docs-27" ] }, { "productId": "unstructured", "storyId": "table-to-structured-rows", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Unstructured's docs show two relevant capabilities: table detection with HTML output (unstructured-docs-12/27) and a structured data extractor that lets users define a target JSON schema so extracted values map directly to typed fields (unstructured-docs-3/26/34), plus a general 'consistent JSON format' output (unstructured-docs-9/29/32). This supports turning extracted tables into structured JSON, but there's no worked example specifically converting table content into typed rows/records ready for DB loading, and table output defaults to HTML markup rather than row-level JSON unless combined with the separate schema extractor. missing for 10: an explicit end-to-end example/tutorial mapping detected tables to typed row/JSON output for DB ingestion, and independent/hands-on confirmation that this pipeline avoids manual cleanup.", "evidenceIds": [ "unstructured-docs-12", "unstructured-docs-27", "unstructured-docs-3", "unstructured-docs-26", "unstructured-docs-34", "unstructured-docs-9", "unstructured-docs-32" ] }, { "productId": "unstructured", "storyId": "typed-official-sdks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Extensive docs confirm a well-documented Python SDK covering partition, enrich, chunk, embed, and workflow/job management (unstructured-docs-4, 7, 19, 23, 30, 35, 36), but there is no evidence anywhere in the pack of an official TypeScript/JS SDK - only Python SDK and generic REST clients are mentioned. missing for 10: TypeScript SDK evidence, explicit coverage of jobs API via SDK (vs REST-only), independent developer corroboration of SDK ergonomics", "evidenceIds": [ "unstructured-docs-23", "unstructured-docs-4", "unstructured-docs-19", "unstructured-docs-7", "unstructured-docs-5", "unstructured-docs-30" ] }, { "productId": "unstructured", "storyId": "vpc-self-host-deployment", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm dedicated instance/VPC deployment with full data isolation is offered as a paid tier, plus IP allowlisting and SOC2/GDPR/HIPAA compliance, which supports data-residency needs. However, evidence does not describe a fully self-hosted/on-prem/air-gapped option (only 'dedicated instance or VPC' managed by vendor), and there's no detail on installation architecture, network egress requirements, or how open-source components could be run entirely offline. missing for 10: explicit self-hosted/on-prem deployment guide, air-gapped operation details, independent confirmation of VPC isolation claims.", "evidenceIds": [ "unstructured-docs-15", "unstructured-docs-28", "unstructured-docs-16", "unstructured-docs-21", "unstructured-supp-1" ] } ]