[ { "productId": "airweave", "storyId": "agent-self-managed-memory", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Airweave's documented capability is agentic *search* over synced external data sources (docs-4, docs-5, docs-17) and syncing/connecting data, not an agent-writable memory store. There is no evidence of an agent invoking add/update/delete operations on memories mid-conversation — data flows in via source connections/syncs, not via agent tool calls for memory CRUD.", "evidenceIds": [ "airweave-docs-4", "airweave-docs-5", "airweave-docs-2", "airweave-docs-9" ] }, { "productId": "airweave", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Airweave hosts a working llms.txt at docs.airweave.ai/llms.txt (HTTP 200, confirmed by probe) plus markdown-appendable docs pages explicitly designed for agent consumption, and also publishes official 'skills' for Cursor, Claude Code, Gemini CLI, OpenCode, etc. so agents can self-configure. missing for 10: no independent/community confirmation that an agent successfully used llms.txt end-to-end, and no evidence of additional agent-discovery standards (e.g. skills.json manifest validation).", "evidenceIds": [ "airweave-probe-1", "airweave-docs-7", "airweave-docs-21", "airweave-docs-22" ] }, { "productId": "airweave", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Airweave's documented capabilities center on search/retrieval — vector search, AI-agent tool-calling search, MCP server access — for AI agents to query synced data, not on Airweave itself surfacing proactive AI-generated insights or suggestions to the end user. No evidence shows an insights/suggestions feature or dashboard; missing for 10: any proactive insight-generation capability, any UI or output described as 'suggestions' rather than query-driven retrieval.", "evidenceIds": [ "airweave-docs-4", "airweave-docs-17", "airweave-docs-12" ] }, { "productId": "airweave", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Source connections automatically sync data in the background and webhooks push real-time updates instead of requiring polling, showing some autonomous background operation, but there's no evidence of a general-purpose automation/workflow engine (scheduled tasks, conditional triggers, multi-step actions) within Airweave itself—automation-style triggering is delegated to third-party tools like Pipedream. Missing for 10: native scheduling/trigger configuration UI, evidence of autonomous multi-step task execution, and independent confirmation that background syncs run reliably at scale.", "evidenceIds": [ "airweave-docs-2", "airweave-docs-9", "airweave-docs-15", "airweave-docs-23" ] }, { "productId": "airweave", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Airweave is a data-integration/search backend designed to be queried by external AI agents (via MCP, API, CLI, skills) — it is not itself a product with an embedded assistant UI that end-users delegate tasks to inside the product. The 'agentic search' feature (docs-4, docs-17) is a retrieval strategy, not a built-in assistant, so this axis is a category mismatch for this kind of infrastructure product.", "evidenceIds": [] }, { "productId": "airweave", "storyId": "agentic-headless", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Airweave exposes a full REST API (OpenAPI spec), a CLI, and self-hostable deployment via a shell script, plus webhooks for event-driven automation and Pipedream workflow actions—all of which support running it headlessly or wiring it into automation/CI. However, there is no explicit CI/CD example, GitHub Actions template, or documented headless test-mode workflow in the evidence.\nmissing for 10: explicit CI pipeline example/docs, headless testing/automation guide.", "evidenceIds": [ "airweave-docs-6", "airweave-docs-11", "airweave-docs-14", "airweave-docs-9", "airweave-docs-15", "airweave-docs-23", "airweave-probe-2", "airweave-probe-4" ] }, { "productId": "airweave", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Airweave documents itself as an MCP *server* that lets external AI assistants query its data (airweave-docs-5, airweave-probe-3), and its integrations (auth-providers, connectors) are proprietary, not MCP-based consumption of external servers. There is no evidence Airweave itself acts as an MCP client that plugs in and uses tools from other MCP servers.", "evidenceIds": [ "airweave-docs-5", "airweave-docs-10", "airweave-probe-3" ] }, { "productId": "airweave", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Airweave is not itself an agent but a data-integration/search platform, so shipping an official MCP server is a fair axis; docs and a probe confirm a first-party MCP server implementing the Model Context Protocol so AI assistants/agents can search synced data, plus community confirmation it works well with Cursor. Missing for 10: independent hands-on verification of the MCP server's reliability/performance beyond one community anecdote.", "evidenceIds": [ "airweave-docs-5", "airweave-probe-3", "airweave-comm-4" ] }, { "productId": "airweave", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Airweave supports natural-language operation via its MCP server, CLI, and AI-agent tool-calling search, letting agents in Cursor, Claude Code, etc. issue natural-language queries against synced data, and community feedback confirms it works well with Cursor. Missing for 10: independent hands-on verification of natural-language control beyond search/retrieval (e.g., managing connections or collections via NL) and broader third-party corroboration.", "evidenceIds": [ "airweave-docs-4", "airweave-docs-5", "airweave-docs-6", "airweave-docs-7", "airweave-comm-4", "airweave-probe-3", "airweave-probe-4" ] }, { "productId": "airweave", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Airweave documents an official CLI whose primary use case is searching any collection directly from the terminal, confirmed both in docs and via a dedicated CLI docs page probe. Missing for 10: independent/hands-on community verification of the CLI itself and more detail on its full command surface beyond search.", "evidenceIds": [ "airweave-docs-6", "airweave-probe-4" ] }, { "productId": "airweave", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Airweave exposes a documented OpenAPI spec (openapi.json), an llms.txt for AI-native consumption, a CLI, MCP server, and REST endpoints for collections/search/webhooks, all confirmed via probes and docs. missing for 10: independent hands-on validation of API completeness/stability and public API versioning/changelog details.", "evidenceIds": [ "airweave-probe-2", "airweave-probe-1", "airweave-probe-3", "airweave-probe-4", "airweave-docs-6", "airweave-docs-9" ] }, { "productId": "airweave", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers source connections, auth providers, direct token injection, and MCP/CLI access, but there is no mention of issuing scoped or least-privilege API keys/credentials specifically for agents. missing for 10: any documentation of scoped API key creation, permission/role-based credential issuance, or least-privilege agent access controls.", "evidenceIds": [] }, { "productId": "airweave", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Airweave exposes a documented OpenAPI spec (implying auto-generatable SDKs), an official LlamaIndex tool-spec package (llama-index-tools-airweave) giving agents direct API access, plus CLI and MCP server for agentic use, and integrations like Pipedream actions. However there is no explicit first-party Python/TypeScript SDK documentation beyond the LlamaIndex wrapper, so full 'build against official SDKs' coverage is unclear. missing for 10: dedicated language SDK docs (Python/JS/Go), independent developer confirmation of SDK usage beyond LlamaIndex.", "evidenceIds": [ "airweave-docs-20", "airweave-docs-23", "airweave-probe-2", "airweave-probe-3", "airweave-probe-4" ] }, { "productId": "airweave", "storyId": "agentic-webhooks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Airweave documents a dedicated webhooks system: real-time notifications pushed on events like sync completion, source connection creation, and collection updates, explicitly positioned as an alternative to polling. This directly matches the story of subscribing to events via webhooks. Missing for 10: independent/hands-on confirmation of webhook reliability and detail on signature verification/retry semantics.", "evidenceIds": [ "airweave-docs-9", "airweave-docs-15" ] }, { "productId": "airweave", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Airweave publishes a machine-readable OpenAPI spec (probe-2) and doc pages exist for code samples (airweave-comm-5 mentions code samples on the docs site), suggesting some form of API reference, but there is no direct evidence of an interactive 'try it' console or actually runnable examples within the docs — the community note only complains about static code sample formatting, not interactivity. missing for 10: explicit documentation or screenshot of an interactive API console (e.g., Swagger/Redoc 'Try it' UI), evidence that examples can be executed in-browser, independent confirmation of runnable examples working.", "evidenceIds": [ "airweave-probe-2", "airweave-comm-5" ] }, { "productId": "airweave", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Probe evidence confirms a machine-readable OpenAPI spec is directly downloadable at https://docs.airweave.ai/openapi.json returning HTTP 200 with an 'openapi' key, and the docs also expose an llms.txt for machine-readable navigation. Missing for 10: no independent third-party confirmation of the spec's completeness/versioning beyond the probe check.", "evidenceIds": [ "airweave-probe-2", "airweave-probe-1" ] }, { "productId": "airweave", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of a dedicated sandbox/test environment distinct from production data. Self-hosting (docs-11, docs-14) allows running a local instance, but this is not documented as a sandbox mode for safely testing against non-production data, and no staging/test-environment feature is mentioned anywhere in the evidence.", "evidenceIds": [] }, { "productId": "airweave", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "There's evidence of an OpenAPI spec existing, but nothing about API versioning scheme, version history, or a documented deprecation policy. Missing for 10: versioning scheme documentation, deprecation policy, changelog/migration guides.", "evidenceIds": [ "airweave-probe-2" ] }, { "productId": "airweave", "storyId": "assistant-memory-plugins", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Airweave ships an official MCP server that lets assistants like Claude and Cursor search synced data, plus published skills for Cursor, Claude Code, Gemini CLI, and OpenCode that enable setup/search without re-explaining, all pointing to a single shared 'collection' as the underlying memory. Community evidence corroborates real-world use with Cursor. Missing for 10: explicit first-party ChatGPT connector docs and independent evidence of multiple assistants simultaneously sharing state in practice.", "evidenceIds": [ "airweave-docs-5", "airweave-docs-7", "airweave-docs-21", "airweave-docs-1", "airweave-probe-3", "airweave-comm-4" ] }, { "productId": "airweave", "storyId": "async-batch-processing", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Airweave's source connections 'automatically sync data' in the background, and webhooks notify when 'syncs completing' occur, with docs noting this is an alternative to 'constantly polling the API' — implying an async sync/status mechanism exists. However, there is no explicit documentation of batch/large-scale ingestion controls, job-status endpoints, or throughput guarantees for platform-engineer-scale operations. Missing for 10: explicit batch/async ingestion API docs, sync job status endpoint documentation, scale/throughput benchmarks or SLAs.", "evidenceIds": [ "airweave-docs-2", "airweave-docs-9", "airweave-docs-15" ] }, { "productId": "airweave", "storyId": "automatic-memory-extraction", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Airweave is a data-source sync and search/RAG platform (collections, source connections, vector/agent search over connected apps) — it is not a conversational memory layer that autonomously extracts salient facts from chat and consolidates them in the background. This capability is a different product category (conversational memory systems) and is a category error for Airweave's connector/search architecture.", "evidenceIds": [] }, { "productId": "airweave", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Airweave's core sync and search architecture inherently processes many items at once (source connections auto-sync entire datasets, collections span multiple sources, single queries search across large data sets), but there is no explicit documentation of a bulk API for batch creating/updating/deleting many items or connections in one call. Missing for 10: explicit bulk/batch API endpoints, evidence of bulk item management (not just sync), and any hands-on confirmation of bulk operation performance at scale.", "evidenceIds": [ "airweave-docs-1", "airweave-docs-2", "airweave-docs-12" ] }, { "productId": "airweave", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 3, "confidence": "medium", "rationale": "Airweave offers webhooks that push real-time event notifications (sync completed, connection created, collection updated) rather than a true rule engine where users define conditional triggers that automatically execute actions inside the product; actual automation logic must be built externally (e.g., via Pipedream workflows) that consume these events. missing for 10: a native rule/trigger-condition builder, support for user-defined conditional logic, and evidence of actions being executed automatically by Airweave itself rather than just notifying external systems.", "evidenceIds": [ "airweave-docs-9", "airweave-docs-15", "airweave-docs-23" ] }, { "productId": "airweave", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Airweave's source connections 'automatically sync data' and it pushes webhook updates instead of polling, implying some recurring background sync exists, but there's no documented scheduling interface, cron-like controls, or ability to schedule arbitrary recurring jobs/workflows beyond source sync. Missing for 10: explicit scheduling/cron configuration options, user-defined recurring workflow triggers, and any evidence of customizable sync intervals.", "evidenceIds": [ "airweave-docs-2", "airweave-docs-9", "airweave-docs-15" ] }, { "productId": "airweave", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Airweave is a data-sync/search platform for connecting and querying data sources, not an automation/workflow builder; there is no concept of 'automations' with versioning, review, or rollback in the evidence pack. This story targets workflow-automation tools and is a category error for Airweave's product type.", "evidenceIds": [] }, { "productId": "airweave", "storyId": "business-data-ingestion", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Airweave connects and syncs diverse sources — cloud drives, databases, SaaS tools, knowledge bases — into searchable collections, explicitly going beyond chat transcripts to documents, JSON/business data, and structured app data (docs-2, docs-12, docs-16). Community feedback corroborates strong retrieval across integrations. Missing for 10: independent hands-on verification of ingesting raw JSON/business data specifically (beyond documented app connectors), and detailed schema/normalization docs for non-document structured data.", "evidenceIds": [ "airweave-docs-2", "airweave-docs-12", "airweave-docs-16", "airweave-docs-9", "airweave-comm-1", "airweave-comm-2" ] }, { "productId": "airweave", "storyId": "custom-ontology-schema", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Airweave's evidence describes collections, source connections, vector/agentic search, MCP, CLI, and webhooks, but nothing addresses customizing entity types, edge types, or an ontology/schema for a graph-entity memory model — Airweave appears to use a fixed sync/search architecture rather than an editable knowledge-graph schema. Missing for 10: any mention of entity/edge type definitions, schema customization API, or ontology configuration.", "evidenceIds": [ "airweave-docs-1", "airweave-docs-2", "airweave-docs-4" ] }, { "productId": "airweave", "storyId": "entity-graph-memory", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Airweave's docs describe vector search and an agent that can 'navigate entity hierarchies (parent/child/sibling)', but this is document/entity hierarchy navigation within a RAG pipeline, not a knowledge graph of entities and relationships designed for multi-hop, entity-centric reasoning. There is no mention of a graph database, relationship modeling, or multi-hop query capability.", "evidenceIds": [ "airweave-docs-4", "airweave-docs-17", "airweave-docs-3" ] }, { "productId": "airweave", "storyId": "framework-integrations", "verdict": "partial", "quality": 3, "confidence": "medium", "rationale": "Airweave documents a framework-integration pattern (e.g., LlamaIndex's AirweaveToolSpec, Pipedream actions, and MCP server/CLI/skills for coding agents), showing the product does ship first-party SDK-style hooks into agent tooling — but none of the specific frameworks named in the story (LangChain, LangGraph, CrewAI, Vercel AI SDK) appear anywhere in the evidence pack. missing for 10: documented LangChain integration, LangGraph integration, CrewAI integration, Vercel AI SDK integration.", "evidenceIds": [ "airweave-docs-20", "airweave-docs-23", "airweave-docs-5", "airweave-docs-7" ] }, { "productId": "airweave", "storyId": "hybrid-filtered-recall", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Airweave documents two retrieval modes—fast direct vector search and an agentic iterative search that reads full documents and navigates entity hierarchies—giving developers some steering beyond a single fixed similarity search, but there is no explicit documentation of metadata filters, keyword/hybrid search, or a reranking step. missing for 10: metadata filter parameters, keyword/hybrid search mode, explicit reranking mechanism, independent verification of these controls.", "evidenceIds": [ "airweave-docs-3", "airweave-docs-4", "airweave-docs-17", "airweave-docs-18" ] }, { "productId": "airweave", "storyId": "local-embedded-mode", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Airweave documents self-hosting via docker-compose (`./start.sh`) on macOS/Linux/WSL, showing a local deployment path, but there is no evidence it can run embedded in-process (as a library) or configured against local embedding/LLM models to avoid all cloud dependency — the docs describe it as a service with source connectors, vector DB, and search API rather than an embeddable local-only memory layer. missing for 10: evidence of in-process/embedded mode, evidence of local-model (non-cloud) embedding/LLM configuration, confirmation no external API calls are required once self-hosted.", "evidenceIds": [ "airweave-docs-11", "airweave-docs-14" ] }, { "productId": "airweave", "storyId": "long-term-memory-add-search", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Airweave's documented model is syncing external data sources (Slack, GitHub, Drive, databases) into collections and then searching them via vector/agentic search, not an explicit API for adding conversational memories and retrieving them for session persistence. None of the evidence describes a 'store this conversation turn' or memory-write primitive, or session-context persistence semantics — missing for 10: an add-memory/write API for conversational turns, evidence of session-scoped recall, and any first-party or community confirmation of using Airweave as a conversation memory layer.", "evidenceIds": [ "airweave-docs-1", "airweave-docs-3", "airweave-docs-4", "airweave-docs-12" ] }, { "productId": "airweave", "storyId": "low-latency-retrieval-claims", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Airweave documents a single rough latency figure (\"~0.5sec\" for direct vector search) but provides no p50/p95 breakdown, no methodology, benchmark environment, or measured distributions, and no independent corroboration of this number. Missing for 10: documented p50/p95 percentile targets, benchmark methodology/environment details, and independent verification of the latency claim.", "evidenceIds": [ "airweave-docs-3", "airweave-docs-18" ] }, { "productId": "airweave", "storyId": "memory-access-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of RBAC, policies, ACLs, or audit logging for governing memory/data access in Airweave; docs cover collections, sources, search, sync, and integrations but nothing on access control or auditing.", "evidenceIds": [] }, { "productId": "airweave", "storyId": "memory-deletion-erasure", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of deletion capabilities for memories/entities/users — the pack covers syncing, search, MCP, CLI, webhooks, and connectors, but nothing about deleting or erasing synced data on demand. This is a fair capability for a data-sync/RAG platform to expose (e.g., for GDPR compliance) given it stores user data, so the axis applies, but no evidence supports it.", "evidenceIds": [] }, { "productId": "airweave", "storyId": "memory-expiration-ttl", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Airweave's docs cover data sync, search modes, MCP/CLI integrations, and connectors, but there is no mention of TTL, expiration dates, recency weighting, or any mechanism to decay or expire stale data/memories. Missing for 10: TTL/expiration configuration, recency-based scoring or decay, any lifecycle policy documentation.", "evidenceIds": [] }, { "productId": "airweave", "storyId": "memory-export-portability", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers ingestion (source connections), search (vector/agentic), MCP server, CLI, webhooks, and self-hosting, but nothing describes an export mechanism or machine-readable dump of stored memories/collections for portability. Self-hosting (docker-compose) implies some data ownership but does not demonstrate an explicit export feature. missing for 10: documented export/dump endpoint or CLI command, evidence of a standard export format (e.g., JSON/CSV), any migration or data-portability guide.", "evidenceIds": [ "airweave-docs-14", "airweave-docs-11", "airweave-probe-2" ] }, { "productId": "airweave", "storyId": "memory-update-reconcile", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Airweave is a data-sync/search platform that indexes and syncs source data via connectors and search endpoints; the evidence describes syncing, webhooks, and search strategies but contains no mention of a memory layer that detects contradictions, supersedes, or merges conflicting stored facts. This is an applicable axis for a retrieval/knowledge system, but no evidence shows such conflict-resolution or memory-update logic exists.", "evidenceIds": [ "airweave-docs-2", "airweave-docs-9", "airweave-docs-15" ] }, { "productId": "airweave", "storyId": "multi-language-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item mentions official Python or TypeScript SDKs at all; documentation references an OpenAPI spec, MCP server, CLI, and framework integrations (LlamaIndex, Pipedream) but not dedicated Python/TypeScript client libraries with memory APIs. Missing for 10: any mention of an official Python SDK, an official TypeScript SDK, or parity of memory/search APIs between them.", "evidenceIds": [] }, { "productId": "airweave", "storyId": "multimodal-memory-ingestion", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Airweave's docs describe syncing structured data from apps/databases and cloud drives (Slack, GitHub, Google Drive, Notion) into searchable collections, but no evidence shows direct ingestion of images, PDFs, or arbitrary files as memory inputs, nor multimodal parsing/recall of such files. missing for 10: explicit file/image/PDF upload API, multimodal parsing or OCR capability, and any documentation or example of recalling content from a stored image/PDF.", "evidenceIds": [ "airweave-docs-2", "airweave-docs-8", "airweave-docs-12" ] }, { "productId": "airweave", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Airweave's docs describe collections, source connections, search, and sync management as API/CLI-first concepts, and an OpenAPI spec plus CLI and MCP server are documented, implying most functionality is API-accessible. However, no evidence explicitly confirms full parity between UI and API (e.g., whether every dashboard action like billing, org settings, or Airweave Connect widget configuration is also API-exposed). Missing for 10: an explicit parity statement or audit showing all UI features are API-reachable, and independent confirmation from users exercising the API directly.", "evidenceIds": [ "airweave-probe-2", "airweave-probe-4", "airweave-docs-1", "airweave-docs-2", "airweave-docs-6" ] }, { "productId": "airweave", "storyId": "openness-full-export", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Airweave is open-source and self-hostable (git clone, ./start.sh) and exposes a documented OpenAPI/CLI surface, giving some data portability and avoiding lock-in, but there is no explicit documented bulk-export feature or open-format export tooling for a user's synced data. missing for 10: explicit data export/backup feature, documented open export formats, evidence of a 'leave and take your data' workflow.", "evidenceIds": [ "airweave-docs-11", "airweave-docs-14", "airweave-probe-2", "airweave-probe-4" ] }, { "productId": "airweave", "storyId": "openness-open-license", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "The docs show a public GitHub repo (airweave-ai/airweave) that can be cloned and self-hosted (airweave-docs-11, airweave-docs-14), implying the source is publicly readable, but no evidence specifies an actual open-source license (e.g., MIT/Apache) or license file. Missing for 10: explicit license declaration, confirmation of license terms, and independent corroboration that the repo is fully open (not just source-available for self-hosting).", "evidenceIds": [ "airweave-docs-11", "airweave-docs-14" ] }, { "productId": "airweave", "storyId": "openness-self-host", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly provide a self-host path via git clone and ./start.sh, and note it can run on macOS, Linux, or WSL, confirming the core product can be self-hosted rather than only used as SaaS. Missing for 10: independent hands-on report of a successful self-hosted deployment and more detail on production-grade self-host configuration/scaling.", "evidenceIds": [ "airweave-docs-11", "airweave-docs-14" ] }, { "productId": "airweave", "storyId": "per-user-memory-isolation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Airweave's docs describe collections, source connections, and organizations as organizational units, but nothing in the evidence pack describes per-user/agent/application scoping guarantees or tenant-isolation enforcement in retrieval. No mention of access control, row-level security, or per-tenant query filtering that would prevent cross-tenant leakage.", "evidenceIds": [ "airweave-docs-1", "airweave-docs-2", "airweave-docs-9" ] }, { "productId": "airweave", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "There is no mention anywhere in the evidence pack of data residency, region selection, or self-hosting for compliance/geographic control (the self-host option is framed as a deployment preference, not a residency feature). Missing for 10: any documentation of region choice, data residency guarantees, or compliance-driven storage location controls.", "evidenceIds": [] }, { "productId": "airweave", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses AI-model training data usage policies, opt-out mechanisms, or data-retention/training guarantees for Airweave; the docs focus on data syncing, search, and integrations but never mention training-data controls.", "evidenceIds": [] }, { "productId": "airweave", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence pack covers syncing, search, MCP, and connectors but contains no mention of data retention policies, deletion controls, or user-initiated data purging/export for collections or synced data. This is an applicable axis for a data-sync/RAG platform handling third-party app data, so absence of evidence yields 'none'.", "evidenceIds": [] }, { "productId": "airweave", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions telemetry, usage tracking, analytics collection, or an opt-out mechanism for Airweave; the docs cover collections, search, MCP, CLI, and integrations but never address telemetry practices.", "evidenceIds": [] }, { "productId": "airweave", "storyId": "prompt-ready-context-block", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Airweave's docs describe search endpoints (direct vector search and agentic search) returning results from connected sources, but there is no mention of a token-budgeted or prompt-ready context assembly feature. Missing for 10: any documentation of token-limit controls, context-window formatting, or a single-call 'assemble context' endpoint.", "evidenceIds": [ "airweave-docs-3", "airweave-docs-4", "airweave-docs-12" ] }, { "productId": "airweave", "storyId": "published-memory-benchmarks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No published benchmark results (e.g. LongMemEval, LoCoMo) or quantitative recall-accuracy metrics appear anywhere in the evidence; only a single anecdotal HN comment claims superior 'retrieval accuracy' with no data. missing for 10: published benchmark suite results, comparison methodology, any quantitative recall/accuracy numbers.", "evidenceIds": [ "airweave-comm-1" ] }, { "productId": "airweave", "storyId": "self-host-oss-deployment", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly provide a git clone + ./start.sh workflow for local self-hosted deployment on macOS/Linux/WSL, indicating Docker-based open-source self-hosting is supported. Missing for 10: explicit confirmation of production-grade/cloud self-host guidance (beyond local dev) and independent hands-on verification of the self-hosted deployment succeeding.", "evidenceIds": [ "airweave-docs-11", "airweave-docs-14" ] }, { "productId": "airweave", "storyId": "session-summarization", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Airweave is a data-integration/search platform that syncs external sources (Slack, Drive, databases) and exposes them via search/MCP endpoints; it has no documented feature for storing, summarizing, or resuming an agent's own conversation/session history. This story targets conversational memory/session continuity, which is a different product category than Airweave's data-sync-and-search focus.", "evidenceIds": [] }, { "productId": "airweave", "storyId": "shared-team-memory", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Airweave's docs describe collections, source connections, and search endpoints, but nothing addresses per-user/per-agent memory scoping, shared vs. private data boundaries, or team/tenant isolation. The evidence pack has no mention of access control, multi-tenant permissions, or selective sharing of synced data between agents/users.", "evidenceIds": [] }, { "productId": "airweave", "storyId": "temporal-fact-memory", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence Airweave tracks fact validity intervals or temporal versioning of entities; docs describe sync, search strategies, and entity hierarchy navigation but nothing about time-bound validity or invalidation of facts. Missing for 10: any mention of temporal metadata, valid/invalid time tracking, or versioned fact history.", "evidenceIds": [] }, { "productId": "airweave", "storyId": "transparent-usage-pricing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item documents a published pricing page, free tier, or per-unit rates; the only pricing-related mentions are community complaints that pricing is 'complex to reason about' and 'prohibitive,' with a direct ask for usage-based per-unit pricing that goes unanswered, indicating no clear published rate structure exists for cost projection.", "evidenceIds": [ "airweave-comm-5", "airweave-comm-6" ] }, { "productId": "airweave", "storyId": "voice-realtime-memory", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence mentions LiveKit, Pipecat, ElevenLabs, or any voice pipeline integration; Airweave's documented integrations are limited to MCP, CLI, coding agents, LlamaIndex, and Pipedream. Latency claims (~0.5s vector search) exist but are not tied to any voice/real-time conversational framework. missing for 10: any mention of LiveKit/Pipecat/ElevenLabs, voice pipeline docs, real-time conversation latency benchmarks specific to voice use cases.", "evidenceIds": [ "airweave-docs-3", "airweave-docs-5", "airweave-docs-20", "airweave-docs-23" ] }, { "productId": "cognee", "storyId": "agent-self-managed-memory", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Cognee exposes explicit remember/recall/improve/forget operations as unified memory CRUD primitives, and ships an official MCP server so agents (Claude, Cursor, Cline, Codex, etc.) can invoke these as tools mid-conversation, plus a CLI/API for programmatic use. Missing for 10: independent/hands-on evidence of an agent actually performing add/search/update/delete calls in a live conversation, and no OpenAPI spec was found to confirm full API surface.", "evidenceIds": [ "cognee-docs-1", "cognee-docs-2", "cognee-docs-3", "cognee-docs-4", "cognee-docs-6", "cognee-docs-21", "cognee-docs-34", "cognee-docs-30", "cognee-probe-3" ] }, { "productId": "cognee", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A live llms.txt file is confirmed via direct probe (HTTP 200) at docs.cognee.ai/llms.txt, and the docs site provides extensive agent-oriented markdown documentation (core-concepts, CLI, MCP, API reference pages all served as .md), which an agent could be pointed at directly. missing for 10: no evidence of a dedicated agents.txt/llms-full.txt variant or explicit guidance/examples showing an agent being pointed at llms.txt in practice.", "evidenceIds": [ "cognee-probe-1", "cognee-docs-16", "cognee-docs-5", "cognee-docs-21" ] }, { "productId": "cognee", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Cognee's docs show AI-driven enrichment and insight generation: `.improve()` enriches the knowledge graph with new AI-extracted entities/relations, `cognee-cli report` generates a 'Graph Insight Report' describing what a dataset contains, and `recall()` auto-classifies queries to surface relevant information. These support 'AI-generated insights' but the story's 'suggestions' aspect (e.g., proactive recommendations) isn't directly evidenced, and there's no independent/hands-on validation of insight quality. missing for 10: explicit suggestion/recommendation feature, third-party validation of insight usefulness.", "evidenceIds": [ "cognee-docs-12", "cognee-docs-3", "cognee-docs-19", "cognee-docs-31", "cognee-docs-2", "cognee-docs-37" ] }, { "productId": "cognee", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Cognee documents some autonomous integrations — a Linear agent that answers @mentions and processes delegated issues from memory, and automatic indexing of an entire GitHub org's repos into the code graph — which imply background, event-driven automation without manual invocation. However, there's no documentation of a general-purpose scheduling/trigger system, background job orchestration, or first-party 'automations' feature; most operations (.remember, .recall, .improve) are described as explicit API/CLI calls rather than autonomous background processes. Missing for 10: explicit scheduler/cron or trigger framework, broader examples of autonomous multi-step agent workflows, and independent confirmation the Linear/GitHub integrations run unattended in production.", "evidenceIds": [ "cognee-docs-46", "cognee-docs-45", "cognee-docs-15", "cognee-docs-27" ] }, { "productId": "cognee", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Cognee positions itself as a memory/knowledge-graph backend that other AI assistants (Claude, Cursor, Cline) connect to via MCP, not as a product with its own built-in assistant that users delegate tasks to. The closest hint is a Linear 'agent' integration that answers @mentions from memory, but this is a narrow third-party integration, not a built-in assistant inside Cognee itself, and no evidence shows a native chat/delegation interface within the core product.", "evidenceIds": [ "cognee-docs-46", "cognee-docs-6", "cognee-docs-21" ] }, { "productId": "cognee", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Cognee offers a Python API, a CLI (cognee-cli), and a Docker container that can all be scripted/automated headlessly without a UI, and 'cognee-cli demo' explicitly works with no network access, suggesting CI-friendliness. However there is no explicit CI/CD example, no documented non-interactive flags for auth/config in pipelines, and no independent report of someone running it in CI. missing for 10: explicit CI/CD pipeline example or GitHub Actions integration, documented non-interactive/headless auth flow, independent hands-on confirmation of CI usage.", "evidenceIds": [ "cognee-docs-5", "cognee-docs-22", "cognee-docs-32", "cognee-docs-36", "cognee-docs-44", "cognee-docs-26", "cognee-probe-4" ] }, { "productId": "cognee", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows Cognee shipping its own MCP server so external AI tools (Claude, Cursor, Cline) can call Cognee's memory tools — the reverse direction of the story. There is no evidence that Cognee itself acts as an MCP client that can plug in and use external MCP servers' tools.", "evidenceIds": [ "cognee-docs-6", "cognee-docs-21", "cognee-docs-34", "cognee-probe-3" ] }, { "productId": "cognee", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Cognee ships an official MCP server (cognee-mcp) documented as bringing persistent AI memory via MCP, connecting to Claude, Cursor, Cline, Continue, and Codex, confirmed by both docs and an independent probe. Missing for 10: independent/hands-on user reports validating the MCP server's real-world reliability beyond vendor docs.", "evidenceIds": [ "cognee-docs-6", "cognee-docs-21", "cognee-docs-34", "cognee-probe-3" ] }, { "productId": "cognee", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Cognee's `.recall` operation auto-classifies natural-language queries to pick a retrieval strategy, the CLI lets users 'remember data, enrich memory, and ask questions' from the terminal without writing code, and the MCP server exposes these operations to AI assistants (Claude, Cursor, Cline) so an AI-native user can drive Cognee via natural language through those chat interfaces. This directly supports NL-driven operation of memory/recall workflows. Missing for 10: independent/hands-on confirmation that free-form natural-language commands (beyond structured queries) are parsed correctly, and explicit CLI examples showing raw NL command syntax rather than just query-answering.", "evidenceIds": [ "cognee-docs-2", "cognee-docs-5", "cognee-docs-22", "cognee-docs-32", "cognee-docs-6", "cognee-docs-21", "cognee-docs-34", "cognee-probe-3", "cognee-probe-4" ] }, { "productId": "cognee", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Cognee ships an official `cognee-cli` documented in first-party docs, supporting core memory operations (remember, enrich, ask questions) from the terminal without opening Python, plus cloud push, report generation, and an offline demo mode. This is a genuine, well-documented CLI tailored to its agentic/memory workflows. Missing for 10: independent/hands-on community verification of the CLI's reliability beyond vendor docs.", "evidenceIds": [ "cognee-docs-5", "cognee-docs-22", "cognee-docs-32", "cognee-docs-11", "cognee-docs-12", "cognee-docs-44", "cognee-probe-4" ] }, { "productId": "cognee", "storyId": "agentic-public-api", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Cognee documents a public API surface across multiple layers: the Python SDK (.remember, .recall, .improve, .forget, python-api.md), a REST-style API reachable via the documented Docker container exposing port 8000 with an 'api-reference' section, and a CLI wrapping the same operations — all clearly documented for programmatic/agentic use. However the openapi probe found no machine-readable OpenAPI/swagger spec at expected paths, suggesting the 'documented public API' is prose/markdown rather than a formal spec, and there is no independent hands-on corroboration of API usage. Missing for 10: a discoverable OpenAPI/swagger schema, independent developer corroboration of API stability/usage.", "evidenceIds": [ "cognee-docs-1", "cognee-docs-2", "cognee-docs-4", "cognee-docs-41", "cognee-docs-7", "cognee-docs-26", "cognee-probe-2" ] }, { "productId": "cognee", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows dataset-scoped permissions for multi-user isolation, but there is no mention of scoped/least-privilege API credentials or API keys that an ai-native user could issue to an agent with restricted permissions. missing for 10: API key/credential issuance mechanism, scoping/least-privilege token model, documentation of credential lifecycle management.", "evidenceIds": [ "cognee-docs-9", "cognee-docs-25" ] }, { "productId": "cognee", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Cognee documents a Python SDK with clear operations (.remember, .recall, .improve, .forget) and a CLI, giving AI-native users official building blocks, but the API reference lacks a discoverable OpenAPI/swagger spec (all probed endpoints 404) and there's no independent/hands-on corroboration of SDK usage beyond first-party docs. missing for 10: verifiable OpenAPI/REST SDK spec, independent developer corroboration of SDK reliability, multi-language SDK support beyond Python/CLI.", "evidenceIds": [ "cognee-docs-1", "cognee-docs-2", "cognee-docs-3", "cognee-docs-4", "cognee-docs-22", "cognee-docs-41", "cognee-probe-2", "cognee-probe-4" ] }, { "productId": "cognee", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack mentions webhooks, event subscriptions, or push notifications; Cognee's integrations (Slack, GitHub, Linear) are described as data sources to ingest, not as an event/webhook subscription mechanism for users. This is a fair axis for an agentic memory/integration platform, so absence of evidence yields none.", "evidenceIds": [] }, { "productId": "cognee", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The docs mention an 'api-reference' section but only show static Docker run commands, not an interactive, runnable API console; explicit probes for OpenAPI/Swagger specs all returned 404, indicating no interactive API reference exists. Missing for 10: any interactive/runnable API explorer (e.g., Swagger UI, Postman collection, live code sandbox) and independent confirmation of one working.", "evidenceIds": [ "cognee-docs-7", "cognee-docs-26", "cognee-docs-36", "cognee-probe-2" ] }, { "productId": "cognee", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Cognee has an 'api-reference' section referencing a REST API and Docker deployment, but a direct probe for OpenAPI/Swagger specs at all standard locations (openapi.json, swagger.json, etc.) returned 404s, and no documentation page links to a downloadable machine-readable spec.", "evidenceIds": [ "cognee-probe-2", "cognee-docs-7", "cognee-docs-26", "cognee-docs-36" ] }, { "productId": "cognee", "storyId": "api-sandbox", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Cognee supports local-only operation via Docker and cognee.start_ui() for dev/testing (docs-7, docs-26, docs-10/23), plus a cognee-cli demo mode that runs with no API key and no network access (docs-44), and dataset-scoped permissions/isolation (docs-9/25) that could keep test datasets separate from production ones. However, there's no explicit 'sandbox environment' concept, staging/prod environment separation, or documented guarantee that testing never touches production data stores. missing for 10: explicit sandbox/staging environment feature, documented prod-data isolation guarantee, independent confirmation of safe test workflows.", "evidenceIds": [ "cognee-docs-7", "cognee-docs-26", "cognee-docs-44", "cognee-docs-9", "cognee-docs-10" ] }, { "productId": "cognee", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of API versioning scheme or a documented deprecation policy; the OpenAPI probe returned 404s for all candidate spec locations, and no docs mention version support windows or deprecation timelines. Missing for 10: any versioning scheme documentation, explicit deprecation policy, changelog/migration guides for breaking changes.", "evidenceIds": [ "cognee-probe-2" ] }, { "productId": "cognee", "storyId": "assistant-memory-plugins", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Cognee ships an official MCP server documented to connect Claude, Cursor, Cline, Continue, and Codex to the same persistent memory store, directly matching the 'shared memory across assistants' story. However, ChatGPT is never listed among supported MCP clients, and there is no independent/hands-on evidence confirming multi-tool memory sharing actually works in practice. Missing for 10: explicit ChatGPT/MCP support confirmation, independent verification of cross-tool memory sharing in real use.", "evidenceIds": [ "cognee-docs-6", "cognee-docs-21", "cognee-docs-34", "cognee-probe-3" ] }, { "productId": "cognee", "storyId": "async-batch-processing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack details Cognee's core ingestion (.remember), enrichment (.improve), and CLI/Docker deployment options, but nowhere describes asynchronous or batch ingestion pipelines, job queuing, or a way to check the status of background memory operations. Pipelines are mentioned only in passing (e.g., 'run the UI and all pipelines') without any status/monitoring API or async processing model documented.", "evidenceIds": [ "cognee-docs-16", "cognee-docs-23", "cognee-docs-33", "cognee-docs-40" ] }, { "productId": "cognee", "storyId": "automatic-memory-extraction", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Cognee's ingestion pipeline automatically extracts entities and builds a knowledge graph rather than storing raw text verbatim (docs-16), and `.improve()` is explicitly designed to consolidate short-term session memory into permanent memory in the background of a session (docs-31, docs-19). However, both `.remember` and `.improve` are explicit, user/agent-invoked API calls rather than an autonomous decision process that judges salience on its own — there's no evidence of the system independently deciding what to discard vs. keep beyond structured extraction.\n\nmissing for 10: evidence of autonomous/automatic triggering of consolidation without explicit calls, evidence of selective filtering of non-salient content vs. full extraction of everything ingested.", "evidenceIds": [ "cognee-docs-16", "cognee-docs-31", "cognee-docs-19", "cognee-docs-38" ] }, { "productId": "cognee", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show several bulk-oriented affordances: ingesting multiple text/files/URLs in a single `.remember` call, dataset-level or full-user `.forget` cleanup instead of item-by-item deletion, and GitHub-org-wide code indexing across every repo. However, there's no explicit bulk/batch API for arbitrary large-scale multi-item operations with pagination, concurrency, or throughput guarantees, and no independent evidence of bulk performance at scale. Missing for 10: dedicated batch API documentation, evidence of large-scale bulk throughput/limits, independent hands-on validation of bulk operations.", "evidenceIds": [ "cognee-docs-16", "cognee-docs-18", "cognee-docs-30", "cognee-docs-43", "cognee-docs-45" ] }, { "productId": "cognee", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Cognee's evidence pack shows manual operations (remember, recall, improve, forget) invoked via API, CLI, or MCP calls, but no evidence of a rules/trigger engine that fires actions automatically on events (e.g., webhooks, event listeners, conditional automations). This is a plausible axis for a memory/knowledge-graph platform, but no supporting capability is documented.", "evidenceIds": [] }, { "productId": "cognee", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Cognee is a memory/knowledge-graph engine with operations (remember, recall, improve, forget), a CLI, and MCP integration, but there is no evidence of any scheduler, cron-like feature, or recurring job/workflow automation capability in the docs pack.", "evidenceIds": [] }, { "productId": "cognee", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Cognee is a memory/knowledge-graph engine for AI agents, not a workflow/automation-builder product with versionable automations to review or roll back; the story's axis (automation versioning/rollback) is a category error for this product type.", "evidenceIds": [] }, { "productId": "cognee", "storyId": "business-data-ingestion", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs clearly show Cognee ingests text, files, URLs, JSON, and business data (Slack, Notion, Linear, Google Drive, GitHub) into a persistent knowledge graph via .remember and related pipelines, going well beyond chat transcripts, with ontology support for structured business data typing. Missing for 10: independent hands-on verification of ingestion across formats beyond docs, and no concrete example of JSON-specific ingestion walkthrough.", "evidenceIds": [ "cognee-docs-16", "cognee-docs-1", "cognee-docs-15", "cognee-docs-27", "cognee-docs-45", "cognee-docs-46", "cognee-docs-8" ] }, { "productId": "cognee", "storyId": "custom-ontology-schema", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Cognee documents an optional ontology mechanism (RDF/OWL file) that lets you map entity types and mentions to canonical, domain-specific concepts, which is a real form of schema customization. However, there is no evidence of a direct API for defining custom entity/edge type schemas outside of the ontology file, and community feedback flags the ontology/extraction setup as heavy and adding friction for smaller-scale customization. Missing for 10: documented API for defining custom node/edge types independent of RDF/OWL, examples of domain-specific schema customization, and hands-on validation that ontology customization works smoothly in practice.", "evidenceIds": [ "cognee-docs-8", "cognee-docs-20", "cognee-docs-35", "cognee-docs-39", "cognee-docs-42", "cognee-comm-1" ] }, { "productId": "cognee", "storyId": "entity-graph-memory", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Cognee's core docs describe exactly this capability: `.remember` ingests data and 'extracts entities and builds the knowledge graph' automatically, `.recall` searches memory picking the best retrieval strategy, and ontologies link entity types/mentions to canonical concepts for well-defined entity-centric relationships. This directly supports multi-hop and entity-centric querying over a graph memory store. Missing for 10: independent/hands-on benchmark evidence of actual multi-hop QA accuracy, and the one community mention raises friction/scale concerns (not a functional dispute) rather than confirming multi-hop correctness.", "evidenceIds": [ "cognee-docs-16", "cognee-docs-37", "cognee-docs-17", "cognee-docs-8", "cognee-docs-20", "cognee-comm-1" ] }, { "productId": "cognee", "storyId": "framework-integrations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows Cognee integrates with MCP-compatible tools (Claude, Cursor, Cline, Continue, Codex) and offers a Python API, CLI, and Docker deployment, but there is no mention anywhere of first-party integrations with LangChain, LangGraph, CrewAI, or the Vercel AI SDK specifically.", "evidenceIds": [ "cognee-docs-6", "cognee-docs-21", "cognee-docs-34", "cognee-docs-41" ] }, { "productId": "cognee", "storyId": "hybrid-filtered-recall", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs state that recall() supports specifying a search type (with auto-classification as the default), implying developers can override the single fixed similarity mode, but the evidence never names specific modes like keyword/hybrid search, metadata filters, or reranking controls. Missing for 10: explicit documentation of metadata-filter parameters, named keyword/hybrid search types, and reranking configuration options.", "evidenceIds": [ "cognee-docs-2", "cognee-docs-17", "cognee-docs-29", "cognee-docs-37" ] }, { "productId": "cognee", "storyId": "local-embedded-mode", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs confirm Cognee is open source and can run fully locally: Docker quick-start, `cognee.start_ui()` running the UI/pipelines on-machine with no account, and `cognee-cli demo` working with no API key, no embedding provider, and no network access. This strongly supports local, cloud-free operation, but the evidence doesn't explicitly document configuring local LLMs/embedding models for full production use (beyond the offline demo), nor independent hands-on confirmation of a no-cloud deployment at scale. missing for 10: explicit doc/example wiring Cognee to local LLM/embedding backends for real workloads, and independent verification of fully offline operation beyond the demo command.", "evidenceIds": [ "cognee-docs-14", "cognee-docs-7", "cognee-docs-26", "cognee-docs-36", "cognee-docs-10", "cognee-docs-23", "cognee-docs-33", "cognee-docs-40", "cognee-docs-44" ] }, { "productId": "cognee", "storyId": "long-term-memory-add-search", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Cognee's docs directly evidence the story: `.remember` ingests conversation/session data (including session-scoped writes via session_id), `.recall` performs semantic/classified search over stored memory, and `.improve` explicitly bridges short-term session memory into permanent memory for cross-session persistence. MCP integration also lets AI assistants use this persistent memory across sessions. Missing for 10: independent/hands-on evidence of retrieval quality or recall accuracy in practice (only vendor docs, plus one skeptical community note about setup friction at small scale) and no first-party benchmark of semantic search precision.", "evidenceIds": [ "cognee-docs-1", "cognee-docs-13", "cognee-docs-17", "cognee-docs-31", "cognee-docs-38", "cognee-docs-41", "cognee-comm-1" ] }, { "productId": "cognee", "storyId": "low-latency-retrieval-claims", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item documents any retrieval-latency targets, benchmarks, or measured p50/p95 numbers; docs describe features (recall, sessions, caching) qualitatively as 'fast' without quantified performance data.", "evidenceIds": [] }, { "productId": "cognee", "storyId": "memory-access-controls", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Cognee's multi-user mode provides dataset-scoped permissions and per-dataset storage for access isolation between users/orgs, which is a form of access control, but the docs give no detail on granular roles, policy definitions, or read/write ACL granularity, and there is no mention of audit logging or access history. missing for 10: role/policy definition mechanics, granular read/write ACLs, audit-log/trail evidence, independent corroboration of the permissions system in practice.", "evidenceIds": [ "cognee-docs-9", "cognee-docs-25" ] }, { "productId": "cognee", "storyId": "memory-deletion-erasure", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "The `.forget` operation is documented as Cognee's unified deletion command supporting single data item deletion, per-dataset deletion, and full cleanup of everything the current user can delete, directly matching the platform-engineer's need for granular-to-full erasure. Combined with dataset-scoped permissions/isolation, this supports privacy-driven deletion per user. Missing for 10: independent/hands-on verification that forget() actually purges data from graph+vector+storage layers reliably, and explicit per-entity (vs per-dataset) deletion granularity confirmation.", "evidenceIds": [ "cognee-docs-4", "cognee-docs-18", "cognee-docs-30", "cognee-docs-43", "cognee-docs-9" ] }, { "productId": "cognee", "storyId": "memory-expiration-ttl", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The docs describe sessions as short-term memory caches and .forget as manual deletion, but nowhere do they mention TTL, expiration dates, automatic decay, or recency-weighted scoring that would make stale facts stop surfacing automatically.", "evidenceIds": [ "cognee-docs-13", "cognee-docs-38", "cognee-docs-18", "cognee-docs-30" ] }, { "productId": "cognee", "storyId": "memory-export-portability", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The docs describe rich operations (remember, recall, improve, forget) and a CLI with `push` (upload to Cognee Cloud) and `report` (human-readable Graph Insight Report), but there is no documented export command or API that dumps memory/graph data into a portable machine-readable format (e.g., JSON, GraphML) for migration off Cognee. Being open-source/self-hostable reduces lock-in in principle, but does not itself satisfy an explicit data-export requirement, and a community comment even flags a 'trapped in their service' feeling. Missing for 10: a documented export/dump command or API returning graph/vector data in a standard interchange format, and any independent confirmation that exported data can be re-imported elsewhere.", "evidenceIds": [ "cognee-docs-11", "cognee-docs-12", "cognee-docs-14", "cognee-comm-1", "cognee-probe-2" ] }, { "productId": "cognee", "storyId": "memory-update-reconcile", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Cognee documents `.forget` for deletion and `.improve` for enriching an existing graph after ingestion, plus ontology-based canonicalization to keep entity concepts consistent, which together imply some update/merge capability, but no doc explicitly describes automatic detection or resolution of contradictory facts, versioning, or supersession logic. missing for 10: explicit contradiction-detection/merge-resolution mechanism, before/after example of conflicting facts being reconciled, independent validation of update behavior.", "evidenceIds": [ "cognee-docs-3", "cognee-docs-19", "cognee-docs-31", "cognee-docs-4", "cognee-docs-18", "cognee-docs-30", "cognee-docs-43", "cognee-docs-8" ] }, { "productId": "cognee", "storyId": "multi-language-sdks", "verdict": "partial", "quality": 3, "confidence": "high", "rationale": "The evidence shows a rich, well-documented Python API (.remember, .recall, .improve, .forget, sessions, CLI, Docker) but there is no mention anywhere of a TypeScript/JavaScript SDK or equivalent memory API in another language — only Python and CLI/MCP integrations are documented. missing for 10: TypeScript SDK, cross-language API parity documentation, evidence of any non-Python client library.", "evidenceIds": [ "cognee-docs-1", "cognee-docs-2", "cognee-docs-4", "cognee-docs-41", "cognee-docs-5", "cognee-probe-4" ] }, { "productId": "cognee", "storyId": "multimodal-memory-ingestion", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm ingestion of 'text, files, or URLs' via a single call (docs-16) and that `.remember`/`.recall` handle storage and retrieval of that content later (docs-1, docs-37), which covers the general 'files as memory inputs' claim, but no evidence explicitly confirms image or PDF parsing/support specifically, nor any hands-on example of recalling from a PDF/image. missing for 10: explicit documentation or example of PDF/image ingestion, independent verification that non-text file types are correctly parsed and recalled.", "evidenceIds": [ "cognee-docs-16", "cognee-docs-1", "cognee-docs-37", "cognee-docs-41" ] }, { "productId": "cognee", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Cognee is fundamentally API/library-first: remember, recall, improve, and forget are documented as the core operations, and the CLI/MCP surfaces reuse the same underlying calls, suggesting close parity with what the optional browser UI exposes (upload, search, dataset management). However, there is no OpenAPI/swagger spec (probe-2 shows all candidate paths 404), and no explicit documentation confirming graph-visualization/exploration features in the UI have direct API equivalents. Missing for 10: a published OpenAPI/REST spec, explicit doc statement mapping every UI feature (e.g., graph visualization) to an API call, and independent confirmation of full UI/API parity.", "evidenceIds": [ "cognee-docs-1", "cognee-docs-4", "cognee-docs-19", "cognee-docs-24", "cognee-docs-33", "cognee-probe-2", "cognee-probe-4" ] }, { "productId": "cognee", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Cognee is explicitly open source and can be run fully locally with your own storage backend (docs-14, docs-23, docs-26), meaning your data never has to leave your infrastructure — a strong form of avoiding lock-in. However, there is no documented explicit 'export' command or open-format data dump feature; the closest is a Graph Insight Report (docs-12) which is a descriptive summary, not a full data export, and forget/delete operations (docs-18, docs-30) only remove data rather than export it. Missing for 10: a documented export/dump command producing open-format (e.g., JSON, RDF, CSV) files of the full graph/vector data, and any independent confirmation that self-hosted data is fully portable without proprietary lock-in.", "evidenceIds": [ "cognee-docs-14", "cognee-docs-23", "cognee-docs-26", "cognee-docs-12", "cognee-docs-18", "cognee-docs-30" ] }, { "productId": "cognee", "storyId": "openness-open-license", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Cognee's pricing page explicitly states 'Cognee is open source. Run the full memory engine locally or on your own stack — free, forever,' confirming open-source availability, but no specific license (e.g. MIT/Apache) is named nor is a GitHub repository link or independent corroboration provided in the evidence pack. missing for 10: explicit license type/name, link to source repository, independent confirmation of license terms.", "evidenceIds": [ "cognee-docs-14" ] }, { "productId": "cognee", "storyId": "openness-self-host", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Cognee is explicitly open source and can be self-hosted via Docker (single docker run command) or run entirely locally including UI/pipelines with no account required, and pricing page confirms 'Run the full memory engine locally or on your own stack — free, forever.' Community sentiment notes the knowledge-graph setup can feel heavy at small scale, but does not contradict self-hosting capability itself. Missing for 10: independent hands-on verification of a full self-hosted production deployment beyond docs/quick-start.", "evidenceIds": [ "cognee-docs-7", "cognee-docs-26", "cognee-docs-36", "cognee-docs-14", "cognee-docs-10", "cognee-docs-23", "cognee-comm-1" ] }, { "productId": "cognee", "storyId": "per-user-memory-isolation", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Cognee's docs describe a dedicated multi-user permissions system with dataset-scoped storage and access control, explicitly enabling isolation so multiple users/organizations share one instance without data leakage, and sessions are scoped per (user_id, session_id). This covers user-level tenancy well but doesn't explicitly document agent- or application-level scoping constructs, and there's no independent/hands-on test confirming isolation actually holds under load. Missing for 10: explicit agent/application-scoped isolation docs, independent verification of no cross-tenant leakage, and any audit/pen-test evidence.", "evidenceIds": [ "cognee-docs-9", "cognee-docs-25", "cognee-docs-38", "cognee-docs-13" ] }, { "productId": "cognee", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Cognee is open-source and can be fully self-hosted (Docker, local UI, no account required), giving users complete control over where their data physically resides — a strong proxy for data residency. However, there is no documented region-selection feature within Cognee Cloud itself (e.g., choosing EU vs US data centers), so residency is achieved only via self-hosting, not via a first-class regional-storage option. Missing for 10: explicit multi-region cloud storage/selection controls, documented data-residency/compliance certifications (e.g., GDPR region pinning) for the hosted offering.", "evidenceIds": [ "cognee-docs-14", "cognee-docs-23", "cognee-docs-26", "cognee-docs-33", "cognee-docs-36" ] }, { "productId": "cognee", "storyId": "privacy-no-training", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Cognee is open source and can run entirely locally with no account required (`cognee.start_ui()`, self-hosted Docker), meaning data never has to leave the user's infrastructure, which inherently prevents third-party training use. However, there is no explicit privacy policy statement addressing training-data use, especially for the Cognee Cloud option where data is pushed via `cognee-cli push` to a hosted service. Missing for 10: an explicit no-training-on-user-data policy statement, terms-of-service language for the cloud tier, and independent confirmation of this claim.", "evidenceIds": [ "cognee-docs-14", "cognee-docs-23", "cognee-docs-40", "cognee-docs-11" ] }, { "productId": "cognee", "storyId": "privacy-retention-controls", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Cognee documents a unified `.forget` deletion command supporting single-item deletion, full dataset deletion, and full cleanup of everything a user can delete, giving explicit user-controlled retention/deletion granularity. This is backed by dedicated docs pages with concrete API semantics. Missing for 10: independent/hands-on verification of deletion behavior and no explicit data-retention policy/TTL documentation beyond the deletion API.", "evidenceIds": [ "cognee-docs-4", "cognee-docs-18", "cognee-docs-30", "cognee-docs-43" ] }, { "productId": "cognee", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item addresses telemetry, usage tracking, or an opt-out mechanism; the docs cover ingestion, retrieval, deletion, CLI, MCP, and deployment but never mention analytics/telemetry settings.", "evidenceIds": [] }, { "productId": "cognee", "storyId": "prompt-ready-context-block", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Cognee's `.recall()` operation retrieves relevant memories in a single call and automatically picks the best retrieval strategy, and session/caching docs describe assembling short-term memory for fast retrieval, satisfying the 'one call' and 'relevant memories' parts of the story. However, there is no documentation of token-budget management or explicit prompt-ready formatting of the returned context block. missing for 10: token-budget control/truncation, explicit prompt-formatted output examples.", "evidenceIds": [ "cognee-docs-2", "cognee-docs-17", "cognee-docs-29", "cognee-docs-37", "cognee-docs-13", "cognee-docs-38", "cognee-docs-41" ] }, { "productId": "cognee", "storyId": "published-memory-benchmarks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions benchmark results, LongMemEval, LoCoMo, or any published recall-accuracy metrics; all docs describe API features and operations without quantitative evaluation data.", "evidenceIds": [] }, { "productId": "cognee", "storyId": "self-host-oss-deployment", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly confirm Cognee is open source and can be self-hosted via a single Docker run command, run locally with UI/pipelines with no account required, and pricing page states 'Run the full memory engine locally or on your own stack — free, forever.' This directly satisfies the platform-engineer self-host story. Missing for 10: independent hands-on verification of the Docker deployment working in production, and details on scaling/persistence configuration for self-hosted infra.", "evidenceIds": [ "cognee-docs-7", "cognee-docs-26", "cognee-docs-36", "cognee-docs-14", "cognee-docs-10", "cognee-docs-23" ] }, { "productId": "cognee", "storyId": "session-summarization", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Cognee's docs describe a concrete session-memory model (`(user_id, session_id)` short-term memory storing an ordered list of recent interactions) plus a `.improve` operation explicitly meant to 'bridge short-term session memory into permanent memory' and `.recall` to retrieve from it, which supports an agent resuming a past thread. However the docs describe raw retrieval/graph-enrichment rather than an explicit 'summary' artifact, and there's no independent/hands-on confirmation of this workflow. missing for 10: explicit session-summary generation feature, independent verification that recall/improve actually reconstructs prior conversation context in practice.", "evidenceIds": [ "cognee-docs-13", "cognee-docs-38", "cognee-docs-31", "cognee-docs-37", "cognee-docs-12" ] }, { "productId": "cognee", "storyId": "shared-team-memory", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Cognee's multi-user mode explicitly provides dataset-scoped permissions and per-dataset storage so multiple users/orgs can share one instance while keeping data isolated, which maps directly to team vs private memory separation (cognee-docs-9/25). However, the docs describe isolation/access-control mechanics rather than a concrete worked example of selectively sharing one dataset across multiple agents while another stays private, and there's no independent/hands-on confirmation of this permission model in practice. Missing for 10: a documented example of cross-agent shared dataset access alongside private datasets, and independent verification of the permission system's real-world behavior.", "evidenceIds": [ "cognee-docs-9", "cognee-docs-25" ] }, { "productId": "cognee", "storyId": "temporal-fact-memory", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions temporal validity tracking, fact invalidation timestamps, or any mechanism to distinguish current vs. outdated facts; operations like .remember/.forget/.improve describe ingestion, deletion, and enrichment but not temporal versioning of facts.", "evidenceIds": [] }, { "productId": "cognee", "storyId": "transparent-usage-pricing", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "The pricing page confirms a free/open-source tier (self-host free forever) and lists some paid-tier features (integrations, code indexing), but no evidence shows per-unit or usage-based rates that would let a platform engineer project costs. missing for 10: documented per-unit/usage pricing rates, tiered price points, or a cost calculator.", "evidenceIds": [ "cognee-docs-14", "cognee-docs-27", "cognee-docs-28" ] }, { "productId": "cognee", "storyId": "voice-realtime-memory", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any documented integration with voice pipeline frameworks like LiveKit, Pipecat, or ElevenLabs, nor any mention of real-time/low-latency voice conversation support. Cognee's integrations listed are MCP clients (Claude, Cursor, Cline), data sources (Slack, Notion, Linear, GitHub), and its own CLI/cloud UI — none address voice pipelines.", "evidenceIds": [ "cognee-docs-6", "cognee-docs-15", "cognee-docs-27", "cognee-docs-45", "cognee-docs-46" ] }, { "productId": "letta", "storyId": "agent-self-managed-memory", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Letta's memory system is explicitly designed for agent-driven self-editing: agents can be taught new memories via /remember, search via MemFS keyword/semantic search mods, consolidate/update via background 'dreaming', and audit/prune via /doctor — all invoked by the agent itself mid-conversation, and community evidence corroborates this as Letta's core differentiator ('memory-first', 'fully white box' memory editing). missing for 10: no direct API/tool-call schema example showing explicit add/search/update/delete operations invoked programmatically by the agent (docs describe user-facing slash commands and background dreaming rather than a fully enumerated CRUD tool API), and no independent hands-on test confirming the agent autonomously invokes these without user prompting.", "evidenceIds": [ "letta-docs-6", "letta-docs-26", "letta-docs-40", "letta-docs-10", "letta-docs-27", "letta-docs-28", "letta-docs-41", "letta-comm-5", "letta-comm-10", "letta-comm-11" ] }, { "productId": "letta", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A direct probe confirms Letta publishes an llms.txt file at docs.letta.com/llms.txt returning HTTP 200 with agent-oriented summary content, and Letta's extensive docs site is clearly structured for programmatic/agent consumption (agent-sdk, CLI, MCP docs). Missing for 10: no evidence of llms.txt content depth/comprehensiveness or independent confirmation that agents successfully consume it in practice.", "evidenceIds": [ "letta-probe-1", "letta-docs-1" ] }, { "productId": "letta", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Letta's 'dreaming' feature uses background subagents to review conversation history and consolidate learned lessons into memory without interrupting the user, and /doctor audits memory placement/duplication — both are forms of AI-generated insight derived from the agent's own interaction data. However, these are framed as internal memory-maintenance mechanisms rather than explicit user-facing 'suggestions' or dashboards surfacing insights about the user's data, and there is no independent/hands-on evidence of the quality of these generated insights. Missing for 10: user-facing insight/suggestion UI or reports, independent verification that dreaming produces useful proactive suggestions, and examples of insights derived from arbitrary connected data sources (not just conversation history).", "evidenceIds": [ "letta-docs-27", "letta-docs-3", "letta-docs-10", "letta-docs-6", "letta-comm-5", "letta-docs-28" ] }, { "productId": "letta", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Letta explicitly supports autonomous background automation: scheduled/recurring prompts via 'Scheduled tasks' (letta-docs-69), always-on cloud agents that keep running when your laptop is closed (letta-docs-55, letta-docs-68), Slack integration where mentioned agents respond using their own cloud computer (letta-docs-73, letta-docs-83), and background 'dreaming' subagents that consolidate memory without interrupting active work (letta-docs-27, letta-docs-3). Community posts corroborate the always-on, memory-first agent design (letta-comm-12, letta-comm-5). Missing for 10: independent hands-on verification that scheduled/background automations run reliably in practice, and more detail on failure handling/monitoring of unattended runs.", "evidenceIds": [ "letta-docs-69", "letta-docs-55", "letta-docs-68", "letta-docs-73", "letta-docs-83", "letta-docs-27", "letta-docs-3", "letta-comm-12" ] }, { "productId": "letta", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Letta ships a built-in agent runtime (Letta Code / Letta App Server) to which users can delegate real tasks: subagents for parallelized work, autonomous cloud agents that clone/read/push GitHub repos and open PRs, Slack-integrated always-on agents, and scheduled/recurring task automation, all corroborated by first-party docs and HN threads from Letta's own team describing the 'memory-first,' continually-learning agent design. missing for 10: independent hands-on benchmarks or third-party reviews validating real-world task delegation reliability beyond vendor docs and founder HN comments.", "evidenceIds": [ "letta-docs-51", "letta-docs-79", "letta-docs-72", "letta-docs-84", "letta-docs-68", "letta-docs-73", "letta-docs-69", "letta-comm-12", "letta-comm-5" ] }, { "productId": "letta", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Letta explicitly documents a headless CLI mode built for non-interactive use in scripts, CI/CD pipelines, and UNIX tool composition, plus an --ephemeral one-shot flag for stateless CI runs, backed by a self-hostable App Server and SDK for programmatic/automated invocation. Missing for 10: independent hands-on CI usage report or example CI config demonstrating it in practice.", "evidenceIds": [ "letta-docs-4", "letta-docs-13", "letta-docs-14", "letta-docs-37", "letta-docs-32", "letta-docs-46" ] }, { "productId": "letta", "storyId": "agentic-mcp-client", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Letta's Agent SDK documents MCP client support: MCP tools are discovered from servers supplied via `mcpServers` and proxied by the SDK host to the MCP server, with support for stdio, Streamable HTTP, and legacy SSE transports. This directly confirms the story of plugging in MCP servers so the agent can use their tools, corroborated by community discussion of Letta building MCP-client support. Missing for 10: no independent hands-on report confirming real-world MCP tool usage end-to-end, and no detail on configuration UX/limitations.", "evidenceIds": [ "letta-docs-61", "letta-docs-76", "letta-comm-3" ] }, { "productId": "letta", "storyId": "agentic-mcp-server", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Letta is itself an agent/coding-assistant platform (Letta Code, stateful agents, Agent SDK) — the 'agent as MCP client that connects to MCP servers' role documented in letta-docs-61/76 and letta-comm-3 is the client side, not the product exposing an official MCP server for other agents to connect to. No evidence shows Letta itself running as an MCP server, so this axis is out of category for an agent-type product.", "evidenceIds": [ "letta-docs-61", "letta-docs-76", "letta-comm-3" ] }, { "productId": "letta", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Letta Code and CLI are extensively controlled via natural-language chat plus slash commands (/remember, /init, /fork, /btw, /doctor, /agents) that wrap agent behavior in conversational form, and agents themselves interpret natural-language instructions to manage memory, tools, and subagents. missing for 10: independent hands-on evidence (beyond vendor docs/HN Q&A) confirming reliability of natural-language command interpretation across complex workflows.", "evidenceIds": [ "letta-docs-50", "letta-docs-65", "letta-docs-66", "letta-docs-6", "letta-docs-10", "letta-docs-19", "letta-docs-20", "letta-comm-5", "letta-comm-7" ] }, { "productId": "letta", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Letta ships an official CLI (\"Letta Code\") with rich documentation covering interactive slash commands, headless/non-interactive scripting mode for CI/CD, local/self-hosted mode, and cloud sync — confirmed by first-party docs and a probe hitting the CLI docs page directly. Community evidence (HN threads with Letta employees) corroborates real usage of CLI features like --yolo mode and /init, indicating it's a genuine, actively used tool. Missing for 10: independent hands-on reviews specifically benchmarking the CLI's reliability/UX beyond employee-provided community threads, and no direct evidence of install/version metrics or broad third-party adoption data.", "evidenceIds": [ "letta-docs-4", "letta-docs-13", "letta-docs-31", "letta-docs-50", "letta-docs-9", "letta-probe-3", "letta-comm-5", "letta-comm-7", "letta-comm-12" ] }, { "productId": "letta", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Letta ships a fully documented Agent SDK (TypeScript, with browser/mobile variants) covering agent creation, sessions, streaming, permissions, MCP tool proxying, and an OpenAI-compatible App Server API, all backed by extensive first-party docs and a public GitHub SDK repo. This clearly satisfies driving the product programmatically as an AI-native user via a documented public API. missing for 10: a discoverable formal OpenAPI/swagger spec (probe found only 404s), and independent/hands-on developer corroboration of the SDK's public API stability.", "evidenceIds": [ "letta-docs-42", "letta-docs-43", "letta-docs-44", "letta-docs-59", "letta-docs-60", "letta-docs-61", "letta-docs-76", "letta-docs-77", "letta-docs-23", "letta-gh-1", "letta-probe-2" ] }, { "productId": "letta", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Letta offers some related scoped-access mechanisms — Secrets that let an agent use API keys/tokens without ever seeing the raw values, GitHub integration where agents can only access repositories explicitly granted, and tool/action permission modes (unrestricted/acceptEdits/standard, canUseTool callback) — but there's no documented mechanism for issuing a distinct, scoped/least-privilege API credential specifically for an agent's own access to the Letta API or external services. missing for 10: explicit per-agent API key/credential issuance with configurable scopes, documentation of least-privilege credential lifecycle (rotation/revocation), and independent confirmation of scoped-credential behavior in practice.", "evidenceIds": [ "letta-docs-70", "letta-docs-72", "letta-docs-85", "letta-docs-44", "letta-docs-87" ] }, { "productId": "letta", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Letta ships an official Agent SDK (letta-ai/letta on GitHub, TypeScript) with extensive first-party documentation covering agent creation, sessions, streaming, MCP tool integration, permissions/canUseTool callbacks, cloud/self-host deployment options, and demo apps for browser, Expo/React Native, and desktop clients - a substantial, well-documented SDK surface for AI-native builders. Missing for 10: independent hands-on developer reviews/benchmarks of the SDK itself, and evidence of official SDKs beyond TypeScript (e.g., Python).", "evidenceIds": [ "letta-gh-1", "letta-docs-42", "letta-docs-43", "letta-docs-44", "letta-docs-45", "letta-docs-46", "letta-docs-59", "letta-docs-61", "letta-docs-76", "letta-docs-77", "letta-docs-78" ] }, { "productId": "letta", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers agent SDK streaming, subagents, mods, scheduled tasks, and Slack/GitHub integrations, but there is no mention of a webhook subscription mechanism for events (e.g., agent state changes, task completion) that external systems could subscribe to. Streaming responses (letta-docs-43, letta-docs-60) are pull-based within an active session, not webhook push notifications.", "evidenceIds": [] }, { "productId": "letta", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows extensive SDK documentation (agent-sdk pages) but no interactive API reference or runnable-example explorer; a direct probe for OpenAPI/Swagger endpoints returned 404 on all candidate paths, indicating no such interactive reference exists.", "evidenceIds": [ "letta-probe-2" ] }, { "productId": "letta", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Direct probes for OpenAPI/Swagger specs at all standard paths (docs.letta.com/openapi.json, swagger.json, api/openapi.json, .well-known/openapi.json) returned 404, and no documentation elsewhere in the evidence pack references a downloadable machine-readable API spec.", "evidenceIds": [ "letta-probe-2" ] }, { "productId": "letta", "storyId": "api-sandbox", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Letta offers isolation-oriented options that could support sandbox-style testing: an --ephemeral CLI mode that runs one-shot tasks without persisting any agent, memory, or MemFS state (letta-docs-14), and a managed 'cloud sandbox' for running agents with no local setup (letta-docs-16, letta-docs-33, letta-docs-58). However, there is no explicit documentation of a staging/production environment separation or guarantee that testing won't touch production agent data. Missing for 10: explicit staging-vs-production environment concept, documentation of data isolation guarantees, and any evidence/testimonial confirming this workflow is used for safe testing.", "evidenceIds": [ "letta-docs-14", "letta-docs-16", "letta-docs-33", "letta-docs-58" ] }, { "productId": "letta", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence anywhere in the pack of API versioning schemes or a documented deprecation policy; the openapi.json probe even 404s, indicating no discoverable API spec that would carry version/deprecation info.", "evidenceIds": [ "letta-probe-2" ] }, { "productId": "letta", "storyId": "assistant-memory-plugins", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Letta integrating with Codex, Claude Code, Hermes Agent, OpenClaw for skills (docs-57), and offering an OpenAI-compatible API for tools like Open WebUI plus an ACP adapter for Zed (docs-23), but nothing shows Letta connecting to or sharing memory with off-the-shelf Claude, ChatGPT, or Cursor specifically. Letta's MCP support is client-side (Letta consuming MCP servers, docs-61/76), and community evidence notes only a third-party 'Letta-MCP-server' built by a user, not an official integration exposing Letta memory to these named assistants.", "evidenceIds": [ "letta-docs-57", "letta-docs-23", "letta-docs-61", "letta-docs-76", "letta-comm-3" ] }, { "productId": "letta", "storyId": "async-batch-processing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Letta's docs describe per-agent memory consolidation ('dreaming' background subagents, triggers on steps/compaction) but there is no evidence of a bulk/batch ingestion pipeline for scale data loading nor any API/CLI to check the status of background memory jobs, which is what this platform-engineer story requires.", "evidenceIds": [] }, { "productId": "letta", "storyId": "automatic-memory-extraction", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Letta's 'dreaming' feature explicitly uses background subagents to review conversations, extract salient lessons, and consolidate/update memory without interrupting active work, and /remember and /init let the agent decide where lessons belong in MemFS autonomously. Community corroboration (Charles/Wooders on HN) confirms this memory-first, self-consolidating design and contrasts it with RAG-based approaches, though skeptics questioned scaling. Missing for 10: independent hands-on benchmarking of extraction quality/accuracy and more detail on the automatic (non-explicit) salience-detection heuristics beyond trigger conditions.", "evidenceIds": [ "letta-docs-27", "letta-docs-65", "letta-docs-66", "letta-docs-3", "letta-comm-5", "letta-comm-10", "letta-comm-11" ] }, { "productId": "letta", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Letta offers some multi-item automation primitives — subagents for parallel task delegation ('divide and conquer tasks') and a cross-agent `/search` command that queries across all agents at once — which support limited bulk-style operations, but there is no documented bulk API for batch-creating, editing, importing/exporting, or mass-updating agents, memory blocks, or files. Missing for 10: explicit bulk CRUD/batch APIs, bulk import/export tooling, and evidence of large-scale multi-item operations beyond parallel subagent delegation and cross-agent search.", "evidenceIds": [ "letta-docs-51", "letta-docs-79", "letta-docs-54", "letta-docs-36", "letta-docs-49" ] }, { "productId": "letta", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Letta documents several trigger-based automation mechanisms: scheduled tasks that 'automate prompts to your agent on a recurring schedule or at a specific time' (letta-docs-69), mods that add 'event hooks' taking effect immediately (letta-docs-71), dreaming triggers configurable to fire 'after a number of steps, on context compaction, or never' (letta-docs-3/12), and Slack-connected agents that automatically respond when mentioned (letta-docs-73/83). These collectively show event- and time-based automatic action triggers, though there is no single documented general-purpose 'rules engine' where a user defines arbitrary event→action conditions in one place. Missing for 10: a unified rules/automation API or UI for arbitrary custom event conditions, and independent/hands-on confirmation that these triggers reliably fire as documented.", "evidenceIds": [ "letta-docs-69", "letta-docs-71", "letta-docs-3", "letta-docs-73", "letta-docs-83", "letta-docs-12" ] }, { "productId": "letta", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows always-on hosting, headless/non-interactive CLI mode for CI/CD integration, and stateful agents resumable across sessions, but no documentation of a built-in scheduler, cron-like trigger, or recurring job/workflow feature.", "evidenceIds": [] }, { "productId": "letta", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Letta offers conversation branching via /fork (create alternate paths without altering the original thread) and stores shared memory as Git repositories, implying underlying version history, plus /doctor for auditing memory drift — but there is no documented explicit 'rollback' or 'revert to previous version' command for agents/automations themselves, nor any review/diff UI for automation history. Missing for 10: an explicit versioning/rollback command for agent automations, a diff/review interface for past automation states, and independent confirmation that forked/git-backed memory can be restored to prior versions.", "evidenceIds": [ "letta-docs-5", "letta-docs-19", "letta-docs-35", "letta-docs-21", "letta-docs-29", "letta-docs-10" ] }, { "productId": "letta", "storyId": "business-data-ingestion", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Letta's memory system (MemFS) is file/markdown-based and the SDK lets developers 'pass memory for full control' where 'each entry becomes a Markdown file' — implying arbitrary content beyond chat transcripts can seed agent memory, and /init lets an agent ingest a repository's files. However, there is no explicit documentation of ingesting structured JSON or generic 'business data' formats into memory, nor any first-party example of a document/JSON ingestion pipeline. Missing for 10: explicit JSON/structured-data ingestion support, document-upload workflows, and independent/hands-on evidence of non-chat data being loaded into agent memory.", "evidenceIds": [ "letta-docs-11", "letta-docs-65", "letta-docs-28", "letta-docs-41", "letta-docs-48" ] }, { "productId": "letta", "storyId": "custom-ontology-schema", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Letta's documented memory model is block/file-based (Markdown memory blocks, MemFS, persona/human blocks) rather than a graph model with configurable entity types, edge types, or ontology; no evidence pack item describes schema customization of entities/relationships. missing for 10: any documentation of graph-based memory representation, entity/edge type definitions, or ontology configuration APIs.", "evidenceIds": [ "letta-docs-11", "letta-docs-48", "letta-docs-28", "letta-docs-41" ] }, { "productId": "letta", "storyId": "entity-graph-memory", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Letta's documented memory system is file/text-based (MemFS, memory blocks) with optional keyword/semantic/hybrid search (letta-docs-28, letta-docs-41), not a graph of entities and relationships; a Letta employee explicitly frames its approach as 'primarily text/files based' as an alternative to structured memory graphs (letta-comm-10). No evidence describes entity/relationship modeling or multi-hop graph traversal for memory.", "evidenceIds": [ "letta-docs-28", "letta-docs-41", "letta-comm-10" ] }, { "productId": "letta", "storyId": "framework-integrations", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers Letta's own Agent SDK, CLI, App Server, MCP support, and various demo apps, but contains no mention of LangChain, LangGraph, CrewAI, or Vercel AI SDK integrations anywhere in the docs, GitHub, or community sources.", "evidenceIds": [] }, { "productId": "letta", "storyId": "hybrid-filtered-recall", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Letta docs mention that installing the 'MemFS Search mod' enables keyword search and optional semantic or hybrid search, showing some retrieval-mode flexibility beyond fixed similarity search, but this requires an add-on rather than being built-in. Missing for 10: explicit metadata-filter support, reranking capability, and any evidence this mod is core/first-class rather than an optional plugin.", "evidenceIds": [ "letta-docs-28", "letta-docs-41" ] }, { "productId": "letta", "storyId": "local-embedded-mode", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs clearly describe a local-only mode where the CLI/agent state, memory, and provider connections all stay on-device with no Letta account or cloud backup required, and a self-hosted App Server that lets agents run entirely on infrastructure you control. However, the evidence never confirms running against fully local models (e.g., local LLM inference/embeddings) — 'model-agnostic' language points to swapping API-based providers (Claude, GPT, Gemini) rather than in-process/local-model operation, and other docs emphasize cloud sandboxes and server-hosted agents as the default path. Missing for 10: explicit support/documentation for local model backends (e.g., Ollama/local embeddings), and confirmation that memory/embedding search (MemFS Search) works without any external API calls.", "evidenceIds": [ "letta-docs-31", "letta-docs-74", "letta-docs-30", "letta-docs-15", "letta-docs-75", "letta-docs-28" ] }, { "productId": "letta", "storyId": "long-term-memory-add-search", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Letta clearly supports persistent memory across sessions (MemFS, /remember, stateful agents resumable across sessions/computers) and explicitly documents semantic search as an optional add-on via the 'MemFS Search mod' for keyword/semantic/hybrid search rather than a built-in default. Community commentary confirms the memory system is real and text/file-based but transparency-focused, not natively semantic-first, and some users raise unresolved concerns about memory scaling/self-improvement. missing for 10: first-party evidence that semantic search is enabled by default (not an optional mod), independent hands-on benchmarking of recall/retrieval quality, and clearer documentation of retrieval accuracy at scale.", "evidenceIds": [ "letta-docs-24", "letta-docs-66", "letta-docs-28", "letta-docs-41", "letta-comm-5", "letta-comm-10", "letta-comm-8" ] }, { "productId": "letta", "storyId": "low-latency-retrieval-claims", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions retrieval latency, p50/p95, or any measured/documented performance numbers for search or memory retrieval; documentation covers features like MemFS search mods and dreaming but never quantifies speed. missing for 10: any documented latency targets, benchmark numbers, or measured p50/p95 retrieval performance data.", "evidenceIds": [] }, { "productId": "letta", "storyId": "memory-access-controls", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Letta's pricing docs mention you can 'share agents across your team (and set permissions for access)' for organizational use, implying some access control exists, but there is no documentation of roles, granular ACLs, or an audit log/trail for memory reads/writes. Missing for 10: documented role/policy definitions for memory access, explicit read/write ACLs on memory blocks, and any audit logging or access history feature.", "evidenceIds": [ "letta-docs-87" ] }, { "productId": "letta", "storyId": "memory-deletion-erasure", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers extensive memory creation, editing, auditing (/doctor), and consolidation (/remember, /init, dreaming) but never documents an explicit delete/erase memory command, API, or per-entity/full-wipe capability for privacy compliance. The one related command, /clear, explicitly 'resets the message buffer but keeps learned context' (letta-comm-5), meaning conversation clearing does not erase underlying memory — the opposite of the requested erasure guarantee.", "evidenceIds": [ "letta-comm-5", "letta-docs-10", "letta-docs-66" ] }, { "productId": "letta", "storyId": "memory-expiration-ttl", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes Letta's memory system (dreaming/consolidation, /doctor audits, MemFS search, shared memory) but nowhere documents TTL, expiration dates, or recency-weighted decay causing stale facts to stop surfacing; /doctor flags drift/duplication but is manual, not automatic decay.", "evidenceIds": [ "letta-docs-3", "letta-docs-10", "letta-docs-27", "letta-docs-41" ] }, { "productId": "letta", "storyId": "memory-export-portability", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Letta's memory is stored as plain Markdown files in a Git-backed 'memory repository' (MemFS), and self-hosting keeps all agent state on-device with no proprietary lock-in, which implies inherent portability of memory data. However, there is no explicit documented export/import command, no described export file format spec, or CLI subcommand dedicated to exporting memories for migration — the pack only shows cloud backup/sync features, not a data-portability workflow. missing for 10: an explicit export/import command or documented export format, third-party confirmation that memory can be extracted and reloaded elsewhere, guidance on migrating memory out of Letta Cloud.", "evidenceIds": [ "letta-docs-11", "letta-docs-21", "letta-docs-30", "letta-docs-74", "letta-docs-9" ] }, { "productId": "letta", "storyId": "memory-update-reconcile", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs describe mechanisms that plausibly support updating/merging memory — /remember lets the agent 'decide where the lesson belongs and commit the update to MemFS' (letta-docs-66), 'dreaming' subagents 'consolidate useful lessons and update memory' (letta-docs-27), and /doctor audits for 'duplication' and drift in the memory hierarchy (letta-docs-10) — and Letta staff claim memory is transparent/controllable and addresses context-poisoning concerns (letta-comm-10, letta-comm-11). However, none of this is a concrete demonstration of contradiction detection/supersession logic, and a community skeptic raises unresolved doubts about memory scaling and self-correction (letta-comm-8, letta-comm-9) without a hands-on refutation either way. Missing for 10: explicit documentation or example of the system detecting/resolving conflicting facts, independent hands-on verification that stale memories are actually superseded rather than just appended/deduplicated.", "evidenceIds": [ "letta-docs-66", "letta-docs-27", "letta-docs-10", "letta-comm-10", "letta-comm-11", "letta-comm-8", "letta-comm-9" ] }, { "productId": "letta", "storyId": "multi-language-sdks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The evidence extensively documents a TypeScript/JavaScript Agent SDK (client.createAgent(), sessions, streaming, MCP, browser/mobile support) but no evidence pack item confirms an official Python SDK or its API parity with the TypeScript SDK — despite Letta's backend being Python-based, docs cited here only reference the JS/TS Agent SDK and CLI. missing for 10: explicit official Python SDK documentation/package, confirmation of equivalent memory API surface (createAgent, sessions, streaming, MCP) across Python and TypeScript, and any independent corroboration of Python SDK usage.", "evidenceIds": [ "letta-gh-1", "letta-docs-42", "letta-docs-59", "letta-docs-77", "letta-docs-45", "letta-docs-61", "letta-docs-76" ] }, { "productId": "letta", "storyId": "multimodal-memory-ingestion", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Letta's documented memory system (MemFS, memory blocks, /remember, /init) is explicitly text/Markdown-file based, and 'image attachments' appear only in a demo chat app's UI feature list, not as a supported memory-recall input type for PDFs or images. No evidence shows agents can ingest images or PDFs into memory and recall facts from them later.", "evidenceIds": [ "letta-docs-11", "letta-docs-28", "letta-docs-41", "letta-docs-62" ] }, { "productId": "letta", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Letta's SDK/App Server exposes agent creation, memory, and OpenAI-compatible API access (letta-docs-1, letta-docs-23, letta-docs-32), suggesting an API-first architecture, but the evidence never confirms that CLI/UI-specific features like /doctor memory audits, /pin, /fork, /btw, or /agents browsing have API equivalents, and a direct probe for a public OpenAPI spec returned 404s across all candidate paths (letta-probe-2), undermining a clean 'everything via API' claim. missing for 10: explicit API endpoints/SDK methods mirroring each UI/CLI convenience command, a discoverable OpenAPI spec confirming full surface parity, and independent confirmation of parity.", "evidenceIds": [ "letta-docs-1", "letta-docs-23", "letta-docs-32", "letta-docs-38", "letta-probe-2", "letta-docs-5", "letta-docs-10", "letta-docs-18" ] }, { "productId": "letta", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Letta's architecture strongly favors openness: memory is stored as plain Markdown/MemFS files (letta-docs-11, letta-docs-66), shared memory is a Git repository (letta-docs-21, letta-docs-29), and self-hosting keeps 'all agent state... on-device' with no cloud account required (letta-docs-74, letta-docs-30), reinforced by community claims of a 'fully white box' text-based memory system (letta-comm-10, letta-comm-11). However, there is no explicit documented 'export my data and delete my account' feature or guarantee that cloud-hosted shared-memory git repos can be freely cloned/exported by the user. Missing for 10: an explicit data-export/account-deletion workflow, documentation on extracting cloud-hosted agent state, and independent verification that self-hosted mode fully replaces all cloud-only features.", "evidenceIds": [ "letta-docs-11", "letta-docs-21", "letta-docs-29", "letta-docs-30", "letta-docs-74", "letta-comm-10", "letta-comm-11" ] }, { "productId": "letta", "storyId": "openness-open-license", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Letta's own docs state the agent harness is 'fully open source' and link to a public GitHub repo (letta-ai/letta) for the Agent SDK, giving concrete evidence of source availability. However, the evidence pack never names the actual license (e.g., Apache/MIT) or confirms the full product (App Server, CLI, memory subsystems) is under that same open license rather than just the harness. Missing for 10: explicit license identification/file, confirmation that all core components (not just the agent harness) are open-sourced, and independent corroboration of license terms.", "evidenceIds": [ "letta-probe-1", "letta-gh-1", "letta-docs-30" ] }, { "productId": "letta", "storyId": "openness-self-host", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Letta's docs explicitly describe self-hosting the core product: running the Letta App Server on infrastructure you control, local CLI mode that stores state on-device without requiring a Letta account, and running agents entirely on infra you control (letta-docs-2, letta-docs-15, letta-docs-30, letta-docs-31, letta-docs-74). This is first-party documentation of a genuine self-host mode, not just cloud-hosted convenience.\n\nmissing for 10: independent/hands-on verification that self-hosted deployment works as documented, and more detail on infra/ops requirements for self-hosting at scale.", "evidenceIds": [ "letta-docs-2", "letta-docs-15", "letta-docs-30", "letta-docs-31", "letta-docs-74", "letta-docs-38" ] }, { "productId": "letta", "storyId": "per-user-memory-isolation", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Letta's core model is per-agent stateful memory (each agent has its own memory blocks/files, docs-42, docs-48), and cross-agent sharing is opt-in via explicit 'shared-memory' repos (docs-8, docs-21, docs-29) rather than automatic leakage, plus team/org permission controls exist (docs-87: 'share agents across your team and set permissions for access'). However there is no documentation of explicit multi-tenant isolation guarantees (per-user/per-application namespacing, access-control enforcement details, or audit of leakage prevention) beyond these general mechanisms. Missing for 10: explicit tenant/user isolation model, documented ACL/RBAC enforcement for memory retrieval, and any independent verification that scoping prevents cross-tenant leakage.", "evidenceIds": [ "letta-docs-42", "letta-docs-48", "letta-docs-8", "letta-docs-21", "letta-docs-29", "letta-docs-87" ] }, { "productId": "letta", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Letta offers self-hosting where 'agents can run entirely on infrastructure you control' and local mode where 'all agent state ... stays on-device' (letta-docs-30, letta-docs-74), giving users a way to control where data physically lives. However, there is no documented region/residency selection feature for the managed Letta Cloud offering itself (e.g., choosing EU vs US data centers). Missing for 10: explicit cloud region/residency picker, compliance certifications (GDPR/data-locality), and any documentation of where Letta Cloud stores data by default.", "evidenceIds": [ "letta-docs-30", "letta-docs-74", "letta-docs-15", "letta-docs-31" ] }, { "productId": "letta", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item addresses data-training opt-out, a training-data policy, or any privacy controls preventing use of user data for model training; self-hosting is mentioned but not tied to a training-data guarantee.", "evidenceIds": [] }, { "productId": "letta", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Letta gives users some retention control: local mode stores state on-device without a cloud login (letta-docs-31), self-hosting keeps 'all agent state, including messages, memory, and provider connections... on-device, and no Letta account is required' (letta-docs-74, letta-docs-30), and --ephemeral mode runs a one-shot task 'without creating, resuming, or persisting an agent, memory blocks, or MemFS' (letta-docs-14). However, there is no documented explicit 'delete agent/data' command, data-retention policy for Letta Cloud, or export/erase workflow. Missing for 10: explicit deletion/erasure command or API, documented cloud data-retention/purge policy, independent confirmation that deletion actually removes cloud-stored data.", "evidenceIds": [ "letta-docs-14", "letta-docs-31", "letta-docs-74", "letta-docs-30" ] }, { "productId": "letta", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item addresses telemetry, usage analytics, or an opt-out mechanism; while local self-hosted mode is documented, nothing confirms or denies data collection/telemetry settings.", "evidenceIds": [] }, { "productId": "letta", "storyId": "prompt-ready-context-block", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Letta's docs describe memory blocks, MemFS, /doctor token audits, and search mods, but there is no evidence of a single API/CLI call that returns a token-budgeted, prompt-ready assembled context block for developer consumption outside the agent's own turn — memory in Letta is managed internally as part of the agent's system prompt, not exposed as a discrete retrieval endpoint.", "evidenceIds": [ "letta-docs-10", "letta-docs-11", "letta-docs-28", "letta-docs-41", "letta-docs-48" ] }, { "productId": "letta", "storyId": "published-memory-benchmarks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains extensive documentation of Letta's memory architecture (MemFS, dreaming, shared memory, etc.) and community discussion of memory quality skepticism/responses, but nowhere cites any published benchmark results (LongMemEval, LoCoMo, or similar) backing recall-accuracy claims. Since memory-recall quality is a core claimed feature of Letta, this axis clearly applies, but no benchmark evidence exists in the pack.", "evidenceIds": [ "letta-comm-8", "letta-comm-9", "letta-comm-10", "letta-comm-11", "letta-docs-27", "letta-docs-28" ] }, { "productId": "letta", "storyId": "self-host-oss-deployment", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Letta explicitly documents a self-hosting path: 'Letta agents can run entirely on infrastructure you control' via the self-hosted App Server, with a local mode that stores state on-device and requires no cloud account, and the agent harness is described as fully open source. However, the evidence pack contains no explicit Docker instructions, container image, or hands-on/community confirmation that self-hosting works in practice. Missing for 10: Docker/container deployment specifics, independent verification of the self-host setup succeeding.", "evidenceIds": [ "letta-docs-30", "letta-docs-15", "letta-docs-32", "letta-docs-38", "letta-docs-74", "letta-docs-31", "letta-probe-1" ] }, { "productId": "letta", "storyId": "session-summarization", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Letta's core architecture is built around persistent, stateful agents that can be resumed across sessions/computers, with explicit APIs (resumeSession, agent IDs) and memory consolidation ('dreaming') that summarizes past conversations into durable memory so agents pick up where they left off. Community/founder commentary (letta-comm-5, letta-comm-12) corroborates the 'memory-first, no more forgetting' design intent.\n\nmissing for 10: independent hands-on verification of summary/context fidelity over long horizons, and quantitative benchmarks of resume/summarization quality beyond vendor docs.", "evidenceIds": [ "letta-docs-24", "letta-docs-59", "letta-docs-27", "letta-docs-5", "letta-docs-54", "letta-docs-65", "letta-docs-66", "letta-comm-5", "letta-comm-12" ] }, { "productId": "letta", "storyId": "shared-team-memory", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Letta ships a dedicated shared-memory feature (`letta shared-memory create`, Git-backed shared repositories giving multiple agents common context) alongside per-agent private memory blocks, and pricing docs explicitly mention team-level agent sharing with configurable permissions. This directly matches sharing selected memory across agents/users while keeping other memory private. Missing for 10: independent/hands-on corroboration of the privacy boundary enforcement and finer-grained detail on how access permissions are scoped per user vs team.", "evidenceIds": [ "letta-docs-8", "letta-docs-21", "letta-docs-29", "letta-docs-87" ] }, { "productId": "letta", "storyId": "temporal-fact-memory", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Letta's documented memory model is file/block-based (MemFS, memory blocks, dreaming/consolidation) with no evidence of tracking fact validity intervals, timestamps of when facts became true/false, or any temporal-graph reasoning distinguishing current vs outdated facts. The axis is applicable to a memory-focused product, but nothing in the evidence pack shows Letta supports temporal fact versioning.", "evidenceIds": [] }, { "productId": "letta", "storyId": "transparent-usage-pricing", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Letta's docs confirm a published pricing page with a free BYOK tier (letta-docs-56), a capped free stateful-agent tier (letta-docs-36) and a usage-based developer plan (letta-docs-49), which addresses the free-tier and general cost-model part of the story. However, no explicit per-unit rate table (e.g., $/agent-hour, $/GB memory, $/token) is shown in the evidence, so a platform engineer cannot precisely project memory costs. Missing for 10: concrete per-unit pricing figures/rate card, and independent confirmation of actual billing granularity for memory usage.", "evidenceIds": [ "letta-docs-36", "letta-docs-49", "letta-docs-56", "letta-docs-87" ] }, { "productId": "letta", "storyId": "voice-realtime-memory", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no documentation or mention of LiveKit, Pipecat, ElevenLabs, or any real-time voice pipeline integration. The closest evidence (letta-comm-4) is a forum reply suggesting a DIY postgres-backed memory setup for a hypothetical voice use case, not a documented, low-latency SDK integration for live conversation.", "evidenceIds": [ "letta-comm-4" ] }, { "productId": "mem0", "storyId": "agent-self-managed-memory", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Mem0 documents an MCP server that hands agents add/search/update memory tools they can invoke themselves, a CLI supporting add/search/list/update/delete, and a flow for a coding agent to self-provision an account and start storing memories mid-session — directly matching the story of self-directed, tool-invoked memory management. Missing for 10: independent/hands-on verification that agents reliably invoke delete/update mid-conversation in practice, and explicit API-level delete examples beyond CLI mentions.", "evidenceIds": [ "mem0-docs-2", "mem0-docs-14", "mem0-docs-34", "mem0-docs-40", "mem0-docs-45", "mem0-probe-3", "mem0-docs-43" ] }, { "productId": "mem0", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A probe confirms docs.mem0.ai/llms.txt returns HTTP 200 with a proper agent-oriented summary of Mem0, and the docs also expose an OpenAPI spec, MCP server, and CLI that an agent can consume directly. Missing for 10: independent/community confirmation that an agent successfully used llms.txt in practice.", "evidenceIds": [ "mem0-probe-1", "mem0-probe-2", "mem0-probe-3", "mem0-probe-4" ] }, { "productId": "mem0", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Mem0 automatically extracts facts from conversations and builds a graph linking people, places, and concepts, and reranks results by semantic relevance — these are forms of AI-generated structuring of raw data, but there is no documented feature that proactively surfaces 'insights' or 'suggestions' to the end user (e.g., a dashboard summary or recommendation engine); the product is positioned as memory storage/retrieval infrastructure for agents rather than an insight-generation tool. missing for 10: explicit insights/suggestions surfacing feature, evidence of proactive recommendations, independent corroboration that graph connections are presented as user-facing insights.", "evidenceIds": [ "mem0-docs-26", "mem0-docs-38", "mem0-docs-57", "mem0-docs-10", "mem0-docs-29" ] }, { "productId": "mem0", "storyId": "agentic-autonomous-automation", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Mem0 is a memory-layer API/service for storing and retrieving facts for LLM agents; it does not offer a workflow/automation-scheduling capability where users configure tasks to run autonomously in the background. Webhooks (event notifications) are the closest feature but they are outbound notifications tied to memory CRUD events, not user-configured autonomous automations, so this axis is a category mismatch for the product type.", "evidenceIds": [] }, { "productId": "mem0", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Mem0 is a memory-layer infrastructure product (API/SDK/MCP server) for other agents to use, not itself an AI assistant with a task-delegation UI; this axis is a category error for its product type.", "evidenceIds": [] }, { "productId": "mem0", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Mem0 is fundamentally API/SDK-first and documents non-interactive setup: a terminal-only account/API-key flow explicitly designed for coding agents with no email or dashboard, a CLI for add/search/list/update/delete, and a self-hostable Docker/REST stack with API keys and audit logs — all of which are naturally scriptable in CI. missing for 10: an explicit CI/CD pipeline example or GitHub Actions integration doc, and independent (non-vendor) confirmation of headless CI usage.", "evidenceIds": [ "mem0-docs-3", "mem0-docs-40", "mem0-docs-45", "mem0-docs-17", "mem0-docs-14", "mem0-docs-34", "mem0-docs-21", "mem0-docs-22", "mem0-docs-36", "mem0-probe-2" ] }, { "productId": "mem0", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Mem0 is a memory-layer backend that itself exposes an MCP server for agents to consume (docs-2, docs-18, probe-3) rather than an agentic client that consumes other MCP servers' tools; plugging external MCP servers into Mem0 so it can use their tools is a category mismatch for this product's role.", "evidenceIds": [ "mem0-docs-2", "mem0-docs-18", "mem0-probe-3" ] }, { "productId": "mem0", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Mem0 documents an official hosted MCP server that exposes memory tools (add/search/update) to any agent, with a one-command connection setup, corroborated by a probe hit confirming the docs page exists. Missing for 10: independent/hands-on third-party verification of the MCP server working in practice beyond vendor docs.", "evidenceIds": [ "mem0-docs-2", "mem0-docs-18", "mem0-docs-27", "mem0-probe-3" ] }, { "productId": "mem0", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Mem0 supports natural-language queries for its 'search' operation and exposes memory tools via MCP so an agent can decide in natural language when to save/retrieve/update memories, which covers the core NL-driven interaction pattern. However, other operations (add, update, delete, CLI commands) are structured API/CLI calls rather than free-form natural-language commands, so full conversational control of the product isn't evidenced. Missing for 10: a unified NL command interface covering all memory operations (not just search), and independent hands-on evidence of agents operating purely via natural language.", "evidenceIds": [ "mem0-docs-7", "mem0-docs-48", "mem0-docs-2", "mem0-docs-34", "mem0-docs-59" ] }, { "productId": "mem0", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Mem0 documents an official CLI that lets users add, search, list, update, and delete memories directly from the terminal, explicitly for both humans and AI agents, plus a related agent-signup flow via terminal commands. missing for 10: independent/hands-on verification of the CLI beyond vendor docs, and more detail on CLI command coverage/versioning.", "evidenceIds": [ "mem0-docs-14", "mem0-docs-34", "mem0-docs-53", "mem0-docs-59", "mem0-docs-45", "mem0-probe-4" ] }, { "productId": "mem0", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Mem0 provides a well-documented REST/SDK API (openapi.json confirmed live, quickstart with Python/JS SDKs, add/search operations), plus a CLI and MCP server enabling agents to programmatically create accounts and drive memory operations without human intervention. Coverage spans platform and self-hosted API surfaces with concrete request/response examples. Missing for 10: independent third-party benchmark or hands-on verification of API robustness beyond vendor docs.", "evidenceIds": [ "mem0-docs-1", "mem0-docs-25", "mem0-docs-35", "mem0-docs-40", "mem0-docs-45", "mem0-probe-2", "mem0-probe-3", "mem0-probe-4", "mem0-docs-14" ] }, { "productId": "mem0", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Mem0 supports per-user/agent API keys and lets an agent self-provision its own account and key via CLI, plus entity-scoped memory (user_id/agent_id/app_id/session_id) to isolate data access, which gives some least-privilege-like scoping. However there is no documented fine-grained permission model (e.g., read-only vs write, scope restrictions per key) beyond per-user key issuance and an audit log. Missing for 10: explicit permission/scope levels on API keys, revocation/rotation controls, and independent verification of least-privilege enforcement.", "evidenceIds": [ "mem0-docs-3", "mem0-docs-40", "mem0-docs-45", "mem0-docs-5", "mem0-docs-22", "mem0-docs-9", "mem0-docs-30", "mem0-docs-28" ] }, { "productId": "mem0", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Mem0 provides official Python/JavaScript SDKs with quickstart docs, an OpenAPI-backed REST API, and documented setup guides for 22+ frameworks (LangChain, CrewAI, LlamaIndex, Vercel AI SDK), all core to AI-native agentic workflows. Missing for 10: independent hands-on developer reviews specifically validating SDK ergonomics/reliability beyond vendor docs.", "evidenceIds": [ "mem0-docs-1", "mem0-docs-15", "mem0-docs-25", "mem0-docs-35", "mem0-docs-44", "mem0-probe-2" ] }, { "productId": "mem0", "storyId": "agentic-webhooks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Mem0 documents webhooks explicitly, letting users configure HTTP POST callbacks for memory created/updated/deleted/categorized events, which is exactly a webhook subscription mechanism for agentic event-driven workflows. Missing for 10: no independent/hands-on corroboration of webhook reliability and no detail on payload schema or retry/security guarantees.", "evidenceIds": [ "mem0-docs-13", "mem0-docs-20", "mem0-docs-33", "mem0-docs-52" ] }, { "productId": "mem0", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "An OpenAPI spec is exposed (mem0-probe-2) and quickstart docs include copyable Python/JS code snippets for add/search calls (mem0-docs-25, mem0-docs-35), which is consistent with an API reference, but there is no evidence of an interactive, in-browser 'try it now' console or runnable-example sandbox tied to that OpenAPI spec. Missing for 10: explicit interactive API explorer/playground UI, evidence of live request execution from docs, and independent confirmation that examples are runnable rather than just illustrative code blocks.", "evidenceIds": [ "mem0-probe-2", "mem0-docs-25", "mem0-docs-35", "mem0-docs-44" ] }, { "productId": "mem0", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A direct probe confirms https://docs.mem0.ai/openapi.json returns HTTP 200 and contains an 'openapi' key, i.e. a machine-readable OpenAPI spec is downloadable, and the API is documented elsewhere for developers. Missing for 10: no explicit vendor-side documentation page linking/describing the spec's versioning or completeness beyond the raw probe.", "evidenceIds": [ "mem0-probe-2", "mem0-docs-1", "mem0-docs-25" ] }, { "productId": "mem0", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items mention a sandbox, staging, or test-mode environment distinct from production for Mem0's Platform API; self-hosting (mem0-docs-4, mem0-docs-21) offers infrastructure control but is not described as a sandbox/test environment feature. This is a fair capability to expect from an API-based memory platform, but nothing in the docs or community evidence documents it.", "evidenceIds": [] }, { "productId": "mem0", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack shows quickstart docs, an OpenAPI spec, and various feature docs, but nothing about API versioning scheme or a documented deprecation policy. No changelog, version headers, or deprecation notices are mentioned anywhere in the pack.", "evidenceIds": [] }, { "productId": "mem0", "storyId": "assistant-memory-plugins", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Mem0 documents an official MCP server that lets any MCP-compatible client (Claude, Cursor, etc.) connect to the same hosted memory store, plus explicit plugins for Claude Code, Cursor, and Codex, and a one-command way to point multiple clients at the hosted server, all backed by user/agent/app-scoped memory so different tools share the same persistent memory. Missing for 10: independent hands-on confirmation that ChatGPT specifically integrates via MCP/plugin (only Claude/Cursor/Codex are named) and no third-party report validating cross-tool memory sharing in practice.", "evidenceIds": [ "mem0-docs-2", "mem0-docs-16", "mem0-docs-18", "mem0-docs-27", "mem0-docs-30", "mem0-probe-3" ] }, { "productId": "mem0", "storyId": "async-batch-processing", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack shows single add/search operations, CLI, webhooks, and quickstart flows, but nothing about async/batch ingestion APIs or a way to poll/check status of background memory operations. This is a fair capability for a memory platform at scale, so absence of evidence yields 'none' rather than 'na'.", "evidenceIds": [ "mem0-docs-6", "mem0-docs-35", "mem0-docs-43", "mem0-docs-14", "mem0-docs-13" ] }, { "productId": "mem0", "storyId": "automatic-memory-extraction", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs show Mem0 automatically extracts individual facts from raw conversation and organizes them into a graph without user-defined schema (mem0-docs-26, mem0-docs-8, mem0-docs-38, mem0-docs-42), and the search/retrieval flow lets the system decide what's salient (mem0-docs-7, mem0-docs-43). However, evidence of true background 'self-improving' consolidation (deduping, merging over time) is thin, and one independent report notes Mem0 doesn't implicitly learn behavioral patterns beyond stored facts (mem0-comm-7), tempering the 'decides for itself' framing. Missing for 10: explicit documentation/evidence of background consolidation jobs or memory merging over time, and independent hands-on validation of extraction quality.", "evidenceIds": [ "mem0-docs-26", "mem0-docs-6", "mem0-docs-8", "mem0-docs-38", "mem0-docs-42", "mem0-docs-43", "mem0-comm-7" ] }, { "productId": "mem0", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Mem0's CLI and API document per-item add/search/list/update/delete, and the Memory Export feature lets users pull structured exports of memories at once, which is the closest evidence to a bulk operation, but there is no documented batch-add, bulk-delete, or multi-item transactional endpoint. missing for 10: explicit batch/bulk add or delete API, documented multi-item transaction support, evidence of performance/testing at scale for bulk operations.", "evidenceIds": [ "mem0-docs-12", "mem0-docs-32", "mem0-docs-34", "mem0-docs-14" ] }, { "productId": "mem0", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Mem0's webhooks let external systems receive real-time HTTP POST notifications when memories are created, updated, deleted, or categorized, which is the closest thing to event-driven automation — but this is a fixed notification hook, not a user-defined 'rules engine' that lets AI-native users specify custom conditions/actions to trigger. Missing for 10: no evidence of a rules/conditions builder, no support for arbitrary trigger logic beyond CRUD events, and no in-product action execution (only outbound POSTs for external systems to act on).", "evidenceIds": [ "mem0-docs-13", "mem0-docs-20", "mem0-docs-33", "mem0-docs-52" ] }, { "productId": "mem0", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Mem0 is a memory layer/API for LLM agents, not a workflow/job orchestration or scheduling product; no evidence pack content relates to recurring jobs or scheduled workflows, and this capability is outside its product category.", "evidenceIds": [] }, { "productId": "mem0", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Mem0 is a memory layer for LLM agents, not an automation/workflow platform with version-controlled automations to review or roll back; this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "mem0", "storyId": "business-data-ingestion", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All evidence describes Mem0's add/search API in terms of conversation messages (client.add(messages, user_id=...)) and fact extraction from chat turns; there is no mention of ingesting documents, PDFs, JSON payloads, or arbitrary business data as a memory source. Memory Export uses Pydantic schemas for output, not input ingestion of external structured data.", "evidenceIds": [ "mem0-docs-6", "mem0-docs-26", "mem0-docs-35", "mem0-docs-43", "mem0-docs-47", "mem0-docs-12" ] }, { "productId": "mem0", "storyId": "custom-ontology-schema", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Mem0's docs explicitly state Graph Memory is fully automatic and built 'without you defining any schema' (mem0-docs-42), with no external graph DB, connection strings, or relations field to configure (mem0-docs-23, mem0-docs-24). There is no evidence of any API, config, or ontology mechanism letting an ml-engineer define custom entity or edge types.", "evidenceIds": [ "mem0-docs-42", "mem0-docs-23", "mem0-docs-24", "mem0-docs-38" ] }, { "productId": "mem0", "storyId": "entity-graph-memory", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Mem0 Platform explicitly builds a native graph linking entities, places, and concepts across memories with no external graph database required, directly supporting multi-hop and entity-centric queries like what do we know about Alice. Docs describe automatic graph construction without schema definition, replacing earlier Neo4j-based integration. missing for 10: independent or hands-on verification of graph memory multi-hop retrieval accuracy, and no benchmark showing entity-relationship correctness", "evidenceIds": [ "mem0-docs-8", "mem0-docs-23", "mem0-docs-24", "mem0-docs-29", "mem0-docs-38", "mem0-docs-39", "mem0-docs-42", "mem0-docs-57" ] }, { "productId": "mem0", "storyId": "framework-integrations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs reference setup guides covering 22 tools including LangChain, CrewAI, LlamaIndex, and the Vercel AI SDK, which directly supports the story, but the evidence pack only gives a top-level index reference rather than detailed per-framework integration docs or independent confirmation that these integrations work as advertised. Missing for 10: concrete per-framework code samples/docs excerpts (e.g. LangGraph-specific), independent/hands-on verification of the integrations, and any community confirmation of successful use with these specific frameworks.", "evidenceIds": [ "mem0-docs-15" ] }, { "productId": "mem0", "storyId": "hybrid-filtered-recall", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Mem0 docs confirm metadata-style filtering (user/agent/app/session scoping) and a dedicated reranking feature ('Advanced memory search with intelligent reranking') alongside 'semantic and filtered search capabilities', showing retrieval can be steered beyond plain similarity search. However, there is no documented keyword or hybrid (lexical+vector) search mode, and no independent benchmark validating reranking quality. Missing for 10: explicit keyword/hybrid search mode, third-party evidence of retrieval-tuning effectiveness.", "evidenceIds": [ "mem0-docs-10", "mem0-docs-31", "mem0-docs-49", "mem0-docs-56", "mem0-docs-9", "mem0-docs-30" ] }, { "productId": "mem0", "storyId": "local-embedded-mode", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Mem0's open-source docs clearly support self-hosting the memory engine (Docker REST server, dashboard, own infrastructure) rather than relying on the Platform SaaS, per mem0-docs-4/21/22/46/28/54. However, none of the evidence confirms an embedded in-process mode or explicit support for local embedding/LLM backends (e.g., Ollama) that would eliminate all cloud calls — the quickstart and core examples default to hosted API keys and cloud model calls. Missing for 10: explicit documentation of local/offline model backends, confirmation that vector store and embedder can run fully in-process without any external API calls, and independent verification of a no-cloud-dependency deployment.", "evidenceIds": [ "mem0-docs-4", "mem0-docs-21", "mem0-docs-22", "mem0-docs-46", "mem0-docs-28", "mem0-docs-54", "mem0-docs-5" ] }, { "productId": "mem0", "storyId": "long-term-memory-add-search", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs clearly describe the core add/search workflow (client.add, search by natural language, scoping by user/session) that persists memories across sessions, and this is corroborated by community reports of using it in production for exactly this purpose. Missing for 10: independent benchmarking of recall/semantic-search quality and long-term persistence beyond vendor docs.", "evidenceIds": [ "mem0-docs-6", "mem0-docs-7", "mem0-docs-26", "mem0-docs-35", "mem0-docs-43", "mem0-docs-56", "mem0-comm-1", "mem0-comm-2", "mem0-comm-7" ] }, { "productId": "mem0", "storyId": "low-latency-retrieval-claims", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items mention latency numbers, p50/p95 metrics, or any documented performance/speed targets for retrieval; docs focus on features (search, graph memory, reranking) but never quantify speed.", "evidenceIds": [] }, { "productId": "mem0", "storyId": "memory-access-controls", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Mem0 documents per-user API keys and a request audit log in the self-hosted bundle, plus entity-scoped memory (user/agent/app/session) that could support basic access separation, but there is no evidence of role-based access control, granular permission policies, or ACLs governing who can read/write specific memories. missing for 10: RBAC/permission policies, ACL enforcement on read/write, admin console for managing roles, independent verification of audit log completeness.", "evidenceIds": [ "mem0-docs-5", "mem0-docs-22", "mem0-docs-28", "mem0-docs-30", "mem0-docs-36" ] }, { "productId": "mem0", "storyId": "memory-deletion-erasure", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Mem0 documents a CLI and API that can add, search, list, update, and delete memories, and its entity-scoped memory model (user_id/agent_id/app_id) lets memories be scoped so a target audience can be identified for deletion. However, the evidence never explicitly documents a bulk 'delete all memories for a user_id' or full-erasure/right-to-be-forgotten endpoint distinct from per-memory delete, and expiration is explicitly called out as non-deletion ('Nothing is deleted'). Missing for 10: explicit bulk/per-entity erasure API or docs (e.g., delete_all by user_id), compliance-oriented erasure guarantees, and independent confirmation that full erasure actually removes underlying data/embeddings.", "evidenceIds": [ "mem0-docs-14", "mem0-docs-34", "mem0-docs-53", "mem0-docs-59", "mem0-docs-30", "mem0-docs-9", "mem0-docs-19", "mem0-docs-50" ] }, { "productId": "mem0", "storyId": "memory-expiration-ttl", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Mem0 documents a first-class expiration_date feature that stops memories from surfacing once the date passes without deleting them, working on both Platform and Open Source (mem0-docs-11, mem0-docs-19, mem0-docs-37, mem0-docs-50, mem0-docs-58). This directly satisfies the 'expiration date' part of the story. Missing for 10: explicit TTL (duration-based) configuration syntax, recency-weighting/decay scoring in search ranking, and independent/hands-on corroboration beyond vendor docs.", "evidenceIds": [ "mem0-docs-11", "mem0-docs-19", "mem0-docs-37", "mem0-docs-50", "mem0-docs-58" ] }, { "productId": "mem0", "storyId": "memory-export-portability", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Mem0 documents a dedicated Memory Export feature that creates structured exports of memories using customizable Pydantic schemas, and separately offers a self-hosted open-source deployment giving full ownership of the data and stack, both directly addressing the portability/lock-in concern. Missing for 10: no independent/hands-on verification of export fidelity or completeness, and no documented bulk import/migration tooling to move exported data between Platform and self-hosted stores.", "evidenceIds": [ "mem0-docs-12", "mem0-docs-32", "mem0-docs-51", "mem0-docs-4", "mem0-docs-21", "mem0-docs-46" ] }, { "productId": "mem0", "storyId": "memory-update-reconcile", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs confirm an 'update' operation exists (CLI and API can update/delete memories) and webhooks fire on update events, implying the system does modify stored memories over time, but there is no explicit documentation describing automatic contradiction detection, superseding, or merging logic when new facts conflict with old ones. missing for 10: explicit description of conflict/contradiction detection, merge algorithm details, and independent hands-on evidence that Mem0 correctly resolves contradictory facts.", "evidenceIds": [ "mem0-docs-14", "mem0-docs-34", "mem0-docs-53", "mem0-docs-59", "mem0-docs-20", "mem0-docs-33", "mem0-docs-26" ] }, { "productId": "mem0", "storyId": "multi-language-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs indicate both Python and JavaScript/TypeScript SDKs exist and expose the same core operations (add, search) via a common quickstart flow, and code snippets show `client.add()` usage. However, there is no dedicated documentation confirming feature parity between the two SDKs, no independent/hands-on verification of the TypeScript SDK, and most of the pack is Python-centric examples. Missing for 10: explicit TypeScript SDK reference/API docs, confirmation of full method parity across languages, and independent developer corroboration of TS SDK usage.", "evidenceIds": [ "mem0-docs-25", "mem0-docs-35", "mem0-docs-44", "mem0-docs-15" ] }, { "productId": "mem0", "storyId": "multimodal-memory-ingestion", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "All evidence describes Mem0 storing and retrieving text-based conversational facts (add/search operations, graph memory, expiration, etc.); nothing in the docs or community evidence mentions ingesting images, PDFs, or other file types as memory inputs.", "evidenceIds": [ "mem0-docs-6", "mem0-docs-26", "mem0-docs-35", "mem0-docs-43", "mem0-docs-47" ] }, { "productId": "mem0", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Mem0 is API/CLI/MCP-first: docs show a full REST API (OpenAPI spec), a CLI that can add/search/list/update/delete memories, and an MCP server exposing memory tools to agents, suggesting core operations (add, search, update, delete, scope by user/agent, webhooks, export, expiration) are all reachable via API rather than only through the dashboard. However, the dashboard is described as offering audit logs and API-key management, and there's no explicit evidence enumerating every UI-only feature and confirming full parity, so full API/UI equivalence isn't directly demonstrated. Missing for 10: an explicit comparison or docs statement confirming every dashboard feature (e.g., audit log viewing, key management, graph visualization) is also exposed via API, and independent confirmation of parity.", "evidenceIds": [ "mem0-docs-14", "mem0-docs-34", "mem0-probe-2", "mem0-probe-3", "mem0-probe-4", "mem0-docs-5", "mem0-docs-28" ] }, { "productId": "mem0", "storyId": "openness-full-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Mem0 offers a Memory Export feature using customizable Pydantic schemas, a CLI to list/export memories, and self-hosted open-source deployment giving full data ownership, which together support exporting and leaving with your data. However, the export feature is schema-based/structured rather than a documented fully-open standard format, and there's no explicit bulk 'export all data and delete account' workflow or independent confirmation of export completeness. missing for 10: evidence of a full bulk export in a standard open format (e.g., JSON/CSV dump of entire account), confirmation of data portability across the graph/vector layers, and independent/hands-on verification that exports are complete and truly open.", "evidenceIds": [ "mem0-docs-12", "mem0-docs-32", "mem0-docs-51", "mem0-docs-14", "mem0-docs-59", "mem0-docs-4", "mem0-docs-21", "mem0-docs-46" ] }, { "productId": "mem0", "storyId": "openness-open-license", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs confirm a genuine 'Mem0 Open Source' offering that runs on your own infrastructure and gives you 'the stack, the data, and every component,' implying source availability, but no evidence pack item names the actual license (e.g., Apache/MIT) or links to a public repository for verification. missing for 10: explicit license name, link to source repository, independent confirmation of license terms.", "evidenceIds": [ "mem0-docs-4", "mem0-docs-21", "mem0-docs-46" ] }, { "productId": "mem0", "storyId": "openness-self-host", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Mem0 documents a distinct Open Source self-hosted bundle that runs the same memory engine as the Platform, deployable via Docker with REST API, dashboard, per-user API keys, and audit log, giving full ownership of stack and data. Missing for 10: independent/hands-on verification of self-hosting (all evidence is first-party docs) and details on feature parity limits (e.g., graph memory) between Platform and self-hosted version.", "evidenceIds": [ "mem0-docs-4", "mem0-docs-5", "mem0-docs-21", "mem0-docs-22", "mem0-docs-28", "mem0-docs-36", "mem0-docs-46", "mem0-docs-54" ] }, { "productId": "mem0", "storyId": "per-user-memory-isolation", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs explicitly describe entity-scoped memory with user_id, agent_id, app_id, and session scoping so memories 'land exactly where they belong' and are separated across users/agents/apps, matching the tenancy story directly (mem0-docs-9, mem0-docs-30, mem0-docs-35). Missing for 10: independent/hands-on verification that isolation is enforced at retrieval time (no cross-tenant leakage tested), and no detail on access-control enforcement (e.g., can a request with wrong user_id still retrieve another user's memories) beyond first-party docs.", "evidenceIds": [ "mem0-docs-9", "mem0-docs-30", "mem0-docs-35" ] }, { "productId": "mem0", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Mem0 does not document any explicit region/residency selection for its hosted Platform, but the Open Source self-hosted option lets users run 'on your own infrastructure' and 'own the stack, the data, and every component,' which indirectly lets a user choose where data lives by choosing their own hosting location. Missing for 10: explicit region-selection controls in the hosted Platform, documented data-residency guarantees, and any compliance/geo-location settings.", "evidenceIds": [ "mem0-docs-4", "mem0-docs-21", "mem0-docs-46", "mem0-docs-22" ] }, { "productId": "mem0", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence anywhere in the pack addresses opting out of AI-model-training use of data, data-training policies, or contractual/privacy commitments about training; the docs focus entirely on memory storage/retrieval features. This is a fair privacy-posture question for a data-storing SaaS product, but nothing in the evidence confirms or denies such a control exists, so it defaults to none.", "evidenceIds": [] }, { "productId": "mem0", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Mem0 documents CLI/API delete and update operations, expiration_date to stop surfacing memories, and self-hosted deployments giving 'full control over your infrastructure and data', plus audit logs for tracking changes — all supporting retention/deletion control. However, expiration explicitly states 'nothing is deleted' (soft suppression, not erasure), and there's no documented hard-delete/right-to-be-forgotten workflow, data export-then-purge guarantee, or retention policy enforcement (e.g., GDPR compliance statements). missing for 10: explicit hard-delete/purge guarantees, compliance-grade retention policy documentation, independent verification that deletion is permanent.", "evidenceIds": [ "mem0-docs-14", "mem0-docs-34", "mem0-docs-59", "mem0-docs-19", "mem0-docs-50", "mem0-docs-37", "mem0-docs-21", "mem0-docs-22", "mem0-docs-28" ] }, { "productId": "mem0", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack discusses telemetry, usage tracking, or any opt-out/privacy configuration setting for Mem0; the docs cover memory features, self-hosting, MCP, and CLI but never mention telemetry controls.", "evidenceIds": [] }, { "productId": "mem0", "storyId": "prompt-ready-context-block", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Mem0's search operation retrieves and reranks relevant memories in a single call (mem0-docs-7, mem0-docs-10, mem0-docs-56), and quickstart flow shows add/search used to fetch context before the next model call (mem0-docs-43). However, there is no evidence of token-budget control, truncation, or an explicit prompt-ready formatted context block being assembled — search returns raw memory results, not a pre-packaged prompt string sized to a token limit. Missing for 10: token-budget parameter or context-length control, explicit prompt-template/context-block formatting output, and any independent confirmation of this packaging behavior.", "evidenceIds": [ "mem0-docs-7", "mem0-docs-10", "mem0-docs-43", "mem0-docs-56", "mem0-docs-49" ] }, { "productId": "mem0", "storyId": "published-memory-benchmarks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any published benchmark results (LongMemEval, LoCoMo, or similar) or recall-accuracy metrics anywhere in the docs or community sources; the pack only covers feature descriptions and setup guides. missing for 10: any benchmark citation, LongMemEval/LoCoMo results, accuracy/recall metrics, third-party evaluation.", "evidenceIds": [] }, { "productId": "mem0", "storyId": "self-host-oss-deployment", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Mem0 docs explicitly describe an open-source self-hosted bundle ('same memory engine as the Platform, running on your own infrastructure') delivered as a Docker stack with REST API, dashboard, per-user API keys, and audit log, giving platform engineers full infra control. Missing for 10: independent/hands-on confirmation of the Docker deployment working in practice and details on infra requirements (e.g., DB/vector store provisioning) beyond first-party docs.", "evidenceIds": [ "mem0-docs-4", "mem0-docs-21", "mem0-docs-22", "mem0-docs-5", "mem0-docs-28", "mem0-docs-36", "mem0-docs-46", "mem0-docs-54" ] }, { "productId": "mem0", "storyId": "session-summarization", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Mem0's core design explicitly targets cross-session continuity — memories persist and are scoped by session/user/agent, and search retrieves the relevant facts before the next model call so an agent can resume context (mem0-docs-30, mem0-docs-43, mem0-probe-1). However, Mem0 stores discrete extracted facts rather than producing an actual 'summary' of a past thread/session, so the story's specific 'summary' framing is only approximated by fact retrieval, not a dedicated summarization feature. Missing for 10: an explicit session/thread summarization API or feature, and independent hands-on evidence that retrieved facts effectively reconstruct 'where we left off' in practice.", "evidenceIds": [ "mem0-docs-30", "mem0-docs-43", "mem0-docs-55", "mem0-docs-7", "mem0-probe-1" ] }, { "productId": "mem0", "storyId": "shared-team-memory", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Mem0's entity-scoped memory lets you scope by user_id, agent_id, app_id, and session, which can be used to segregate private memory per user while sharing memory under a common agent_id or app_id — a workable pattern for group/team memory. However, the docs never explicitly describe a 'team' or 'group' memory concept, permission model, or access-control rules distinguishing private vs shared visibility across users/agents. Missing for 10: explicit team/group memory feature, role-based access control or sharing permissions, and any documentation of enforcing privacy boundaries between scoped entities.", "evidenceIds": [ "mem0-docs-9", "mem0-docs-30" ] }, { "productId": "mem0", "storyId": "temporal-fact-memory", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Mem0 supports manual expiration_date so a memory stops surfacing after a set date, and it can update existing facts, but there is no documented capability to automatically detect when a fact becomes invalid/outdated (e.g., contradiction detection, temporal versioning, or 'valid from/until' metadata) — the expiration mechanism is a manual TTL, not temporal reasoning. Missing for 10: automatic invalidation of superseded facts, tracking validity windows for graph relations, and any evidence of reasoning about fact recency versus outdatedness.", "evidenceIds": [ "mem0-docs-11", "mem0-docs-19", "mem0-docs-37", "mem0-docs-41", "mem0-docs-50", "mem0-docs-58", "mem0-docs-24" ] }, { "productId": "mem0", "storyId": "transparent-usage-pricing", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions pricing, free tier, or per-unit rates anywhere in the docs, community, or probes; all citations concern product features (memory ops, MCP, CLI, graph memory) rather than pricing plans.", "evidenceIds": [] }, { "productId": "mem0", "storyId": "voice-realtime-memory", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack lists integrations for LangChain, CrewAI, LlamaIndex, Vercel AI SDK, and coding-agent harnesses (Claude Code, Cursor, Codex), but never mentions LiveKit, Pipecat, ElevenLabs, or any real-time voice pipeline integration or latency guarantees for live conversation use.", "evidenceIds": [ "mem0-docs-15", "mem0-docs-16" ] }, { "productId": "supermemory", "storyId": "agent-self-managed-memory", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Supermemory exposes API/SDK operations (add, search hybrid, update via customId, delete via container tag) and an official MCP server that lets an agent invoke these tools mid-conversation without pre-configured tool names ('Ask naturally... selects the right Supermemory tool'). This directly matches the story of an agent managing its own memory via tool/API calls. missing for 10: no independent hands-on evidence of an agent autonomously invoking delete via MCP, and no explicit example combining add/search/update/delete in one live agent session.", "evidenceIds": [ "supermemory-docs-10", "supermemory-docs-24", "supermemory-docs-27", "supermemory-docs-28", "supermemory-docs-30", "supermemory-docs-11", "supermemory-docs-35", "supermemory-docs-19", "supermemory-probe-3" ] }, { "productId": "supermemory", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A probe confirms a working llms.txt endpoint (HTTP 200) at docs.supermemory.ai, and first-party docs describe agent-oriented resources: a remote MCP server for agents to search Supermemory documentation while implementing integrations, an npx supermemory setup CLI that detects the project and launches the integration flow, and an installable skill so agents use real endpoints instead of hallucinating APIs. Missing for 10: independent/hands-on report of an agent successfully using llms.txt or the docs-MCP end-to-end, and no discoverable OpenAPI spec (probe found 404s on all candidate paths).", "evidenceIds": [ "supermemory-probe-1", "supermemory-docs-5", "supermemory-docs-6", "supermemory-docs-17", "supermemory-docs-46", "supermemory-probe-3" ] }, { "productId": "supermemory", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Supermemory automatically extracts facts, builds a living knowledge graph, and maintains auto-updating user profiles from ingested data — a form of AI-generated insight derived from raw content (supermemory-docs-14, supermemory-docs-15, supermemory-docs-33, supermemory-docs-40). However, there's no evidence of proactive 'suggestions' (e.g., recommended actions, next-step prompts) surfaced to the user — the product is positioned as memory infrastructure/API for retrieval rather than an insights/suggestions dashboard. Missing for 10: evidence of proactive suggestion generation, a user-facing insights UI, or explicit 'recommendation' features beyond fact/profile extraction.", "evidenceIds": [ "supermemory-docs-14", "supermemory-docs-15", "supermemory-docs-33", "supermemory-docs-40", "supermemory-docs-23" ] }, { "productId": "supermemory", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Supermemory offers background automation only in the narrow sense of connector auto-sync via real-time webhooks (Google Drive, Gmail, Notion, OneDrive, GitHub) and automatically-maintained user profiles that update as users interact — both run autonomously without manual triggering. But there's no evidence of a general-purpose automation/workflow builder, scheduled agent tasks, or triggers-and-actions system that an AI-native user could configure to run arbitrary autonomous background jobs. Missing for 10: a documented automation/workflow engine, scheduling capabilities beyond data-sync connectors, and any evidence of user-configurable autonomous background tasks.", "evidenceIds": [ "supermemory-gh-1", "supermemory-gh-2", "supermemory-docs-40", "supermemory-docs-33", "supermemory-docs-43" ] }, { "productId": "supermemory", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Supermemory is positioned as a memory infrastructure/API layer that other AI assistants connect to (via MCP, SDKs, connectors) rather than as a product with its own built-in AI assistant for task delegation; 'Ask naturally' (supermemory-docs-35) describes external assistants querying Supermemory's tools, not a native in-app agent. No evidence describes a built-in assistant inside Supermemory itself that a user can delegate tasks to.", "evidenceIds": [] }, { "productId": "supermemory", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Supermemory is API-first: official TypeScript/Python SDKs and a documented REST API let content be added and searched entirely programmatically (docs-24, docs-26, docs-27), and the self-hosted binary explicitly boots with no Docker/config, suitable for automated/CI environments (docs-7, docs-31, docs-44). A CLI (npx supermemory setup) also supports scripted agent setup (docs-6). missing for 10: explicit CI/automation examples or docs, independent confirmation of unattended API-key usage, and clarity on whether initial key minting can be fully non-interactive", "evidenceIds": [ "supermemory-docs-26", "supermemory-docs-24", "supermemory-docs-27", "supermemory-docs-7", "supermemory-docs-31", "supermemory-docs-44", "supermemory-docs-6", "supermemory-docs-20" ] }, { "productId": "supermemory", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Supermemory is a memory-storage/retrieval API and platform, not an agent or tool-orchestration hub; the evidence shows it *publishes* an MCP server so other AI assistants can call Supermemory's own tools, which is the inverse of this story (product consuming external MCP servers' tools). There is no indication Supermemory acts as an MCP client that ingests arbitrary external MCP servers and uses their tools, making this axis a category mismatch for this product type.", "evidenceIds": [ "supermemory-docs-5", "supermemory-docs-11", "supermemory-docs-35", "supermemory-probe-3" ] }, { "productId": "supermemory", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Supermemory is not itself an agent but a memory service, so shipping an official MCP server is a fair axis; docs confirm a Supermemory MCP server that gives 'every MCP-compatible assistant a shared memory layer' with natural-language tool selection, plus a separate docs-search MCP for integration, and a probe confirms the MCP docs page exists. missing for 10: no independent/hands-on confirmation of the MCP server working end-to-end with a real agent, and no explicit setup/connection instructions cited in the pack.", "evidenceIds": [ "supermemory-docs-11", "supermemory-docs-35", "supermemory-docs-5", "supermemory-docs-18", "supermemory-docs-46", "supermemory-probe-3" ] }, { "productId": "supermemory", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Supermemory's MCP docs explicitly describe natural-language operation: 'Ask naturally. The assistant selects the right Supermemory tool without requiring code or tool names,' and the MCP layer lets any MCP-compatible assistant collaborate using shared context. This directly matches the AI-native natural-language command story. Missing for 10: independent/hands-on confirmation of the natural-language MCP experience and broader examples of the range of commands supported beyond search/recall.", "evidenceIds": [ "supermemory-docs-35", "supermemory-docs-11", "supermemory-docs-36", "supermemory-probe-3" ] }, { "productId": "supermemory", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs mention an official CLI command (`npx supermemory setup`) that detects a project and launches the integration flow, and reference 'CLI, skill, and docs MCP' as part of agent setup, showing a CLI exists for AI-native/agentic workflows. However, evidence is thin — no dedicated CLI reference page, no list of full command set, and no independent/hands-on confirmation of its use. Missing for 10: a full CLI command reference, examples beyond the single setup command, independent user confirmation of the CLI in practice.", "evidenceIds": [ "supermemory-docs-6", "supermemory-docs-46", "supermemory-docs-17" ] }, { "productId": "supermemory", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Supermemory documents a full public API (add/search/memory-operations, containerTag scoping, filtering) plus official TypeScript and Python client libraries wrapping it, and an API reference overview — clearly enabling AI-native users to drive the product programmatically. Missing for 10: a discoverable OpenAPI/swagger spec (probe found only 404s) and independent hands-on confirmation of API robustness beyond docs.", "evidenceIds": [ "supermemory-docs-26", "supermemory-docs-10", "supermemory-docs-24", "supermemory-docs-27", "supermemory-docs-2", "supermemory-docs-39", "supermemory-probe-2" ] }, { "productId": "supermemory", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows container-tag based data isolation (namespacing memories per user/tenant) and mention that API keys are minted somewhere other than the consumer app, but nothing describes issuing scoped or least-privilege API credentials/tokens (e.g., read-only vs write, per-agent permission scopes) for agents. OAuth credential replacement (docs-47) is about branding the OAuth app name, not permission scoping.", "evidenceIds": [ "supermemory-docs-4", "supermemory-docs-20", "supermemory-docs-38", "supermemory-docs-47" ] }, { "productId": "supermemory", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly reference official TypeScript and Python SDKs ('npm install supermemory', 'pip install supermemory') that wrap the full API, with extensive documented usage examples (add, search, container tags) across the docs, indicating a mature developer-facing SDK surface. Missing for 10: independent hands-on developer corroboration of the SDKs themselves (community evidence only discusses unrelated ingestion/auth complaints) and no visible changelog/versioning evidence for SDK maturity.", "evidenceIds": [ "supermemory-docs-26", "supermemory-docs-24", "supermemory-docs-27", "supermemory-docs-37", "supermemory-probe-1" ] }, { "productId": "supermemory", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The only webhook references describe internal connector sync (e.g., Google Drive/Gmail/Notion changes triggering Supermemory's own ingestion pipeline via 'real-time webhooks'), not an outbound webhook subscription API for end users to receive event notifications. No docs describe registering a webhook endpoint or subscribing to events like memory-created, sync-complete, etc.", "evidenceIds": [ "supermemory-gh-1", "supermemory-gh-2", "supermemory-docs-43" ] }, { "productId": "supermemory", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only a static API-reference overview page and SDK code snippets (client.add, client.search), with no interactive 'try-it' console or runnable sandbox; a probe explicitly checked for an OpenAPI/Swagger spec (which typically powers interactive references) and found all candidate URLs returning 404.", "evidenceIds": [ "supermemory-docs-26", "supermemory-probe-2" ] }, { "productId": "supermemory", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe for machine-readable API specs at standard OpenAPI/Swagger paths returned 404 on all candidates, and no evidence pack item links to a downloadable OpenAPI/JSON spec despite an 'API reference' doc existing.", "evidenceIds": [ "supermemory-probe-2", "supermemory-docs-26" ] }, { "productId": "supermemory", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes container-tag based data isolation between end-users/tenants, self-hosting, and a MemoryBench benchmarking tool, but none of this documents an actual sandbox/staging environment (e.g., a test API key, non-production project, or explicit test mode) that a developer could use to try things without touching production data.", "evidenceIds": [] }, { "productId": "supermemory", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of API versioning scheme (e.g., v1/v2 paths) or any documented deprecation policy; OpenAPI spec probes returned 404 and no changelog/deprecation docs appear in the pack.", "evidenceIds": [ "supermemory-probe-2" ] }, { "productId": "supermemory", "storyId": "assistant-memory-plugins", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Supermemory ships an official MCP server explicitly described as giving 'every MCP-compatible assistant a shared memory layer,' which covers Claude, ChatGPT, Cursor and other MCP clients, plus container-tag scoping to keep the shared memory user-specific and natural-language tool selection without code. Missing for 10: explicit named walkthroughs/screenshots for each specific assistant (Claude, ChatGPT, Cursor) connecting simultaneously, and independent hands-on confirmation that cross-tool memory sharing works smoothly in practice.", "evidenceIds": [ "supermemory-docs-11", "supermemory-docs-35", "supermemory-docs-36", "supermemory-docs-38", "supermemory-probe-3", "supermemory-docs-25" ] }, { "productId": "supermemory", "storyId": "async-batch-processing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Docs describe ingestion mechanics (raw content ingestion, customId dedup, connectors with webhooks, automatic extraction) but there is no mention of a batch/async ingestion API, job IDs, or an endpoint/mechanism to check the status of background memory-processing operations — the evidence pack contains no support for this platform-engineer-facing capability. missing for 10: batch ingestion API, async job/task status endpoint, documentation of processing queue or webhook completion callbacks.", "evidenceIds": [ "supermemory-docs-1", "supermemory-docs-30", "supermemory-docs-43", "supermemory-gh-1" ] }, { "productId": "supermemory", "storyId": "automatic-memory-extraction", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs clearly describe automatic extraction of salient facts from raw content (docs-1, docs-41, docs-42) and background consolidation into an evolving knowledge graph and auto-maintained user profiles (docs-14, docs-15, docs-33, docs-40) with no manual tagging required. However, the vendor's own comparison page admits a real consolidation gap — no temporal truth resolution, so outdated and current facts both remain 'relevant' (docs-48) — and there is no independent/hands-on evidence corroborating extraction quality. missing for 10: independent corroboration of extraction accuracy, resolution of the documented temporal-truth/consolidation gap.", "evidenceIds": [ "supermemory-docs-1", "supermemory-docs-14", "supermemory-docs-15", "supermemory-docs-33", "supermemory-docs-40", "supermemory-docs-41", "supermemory-docs-42", "supermemory-docs-48" ] }, { "productId": "supermemory", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Evidence shows container-tag scoped deletion of all memories in a tenant at once, connector auto-sync ingesting many documents in bulk, and metadata filtering across memories, which together imply some bulk-style operations, but there is no documented dedicated bulk API (e.g., batch add/update/delete of many arbitrary items in one call) or independent confirmation of true bulk throughput. missing for 10: explicit bulk/batch endpoint documentation, evidence of bulk operations beyond tag-scoped delete and connector sync, independent/hands-on validation of bulk performance.", "evidenceIds": [ "supermemory-docs-12", "supermemory-docs-34", "supermemory-docs-43", "supermemory-gh-1", "supermemory-docs-39", "supermemory-docs-30" ] }, { "productId": "supermemory", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Supermemory is a memory/storage layer for AI apps — it supports connectors, auto-sync, and memory extraction, but there is no evidence of a rules/automation engine where users define 'if event X then action Y' triggers. Ingestion and connector syncing are automatic pipelines, not user-defined conditional automation.", "evidenceIds": [] }, { "productId": "supermemory", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Supermemory is a memory/context API and storage layer, not a workflow/job scheduling or automation orchestration product; there's no scheduler or recurring-job concept in its evidence, and asking a memory API to run cron-like workflows is a category mismatch.", "evidenceIds": [] }, { "productId": "supermemory", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Supermemory is a memory/context API for AI agents, not an automation-builder product with workflows to version, review, or roll back; there is no concept of 'automations' in this product's evidence at all — this axis targets automation platforms, which is a category error here.", "evidenceIds": [] }, { "productId": "supermemory", "storyId": "business-data-ingestion", "verdict": "disputed", "quality": 6, "confidence": "medium", "rationale": "First-party docs strongly document ingesting raw documents, files, URLs, JSON-like content, code, PDFs, images, and connectors (Google Drive, Notion, OneDrive, Gmail, GitHub) beyond chat transcripts, with customId-based document updates and content-type-specific extraction. However, a hands-on community report describes a concrete failure — a large-content link being rejected with a 'content too large' error — directly contradicting the 'just upload, it just works' ingestion claim for real-world documents. Missing for 10: resolution/acknowledgment of the size-limit failure, and independent corroboration of large-scale JSON/business-data ingestion succeeding in practice.", "evidenceIds": [ "supermemory-docs-1", "supermemory-docs-41", "supermemory-docs-43", "supermemory-gh-3", "supermemory-docs-30", "supermemory-comm-1" ] }, { "productId": "supermemory", "storyId": "custom-ontology-schema", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Supermemory documents automatic knowledge-graph construction, buckets for profile categorization, and metadata filtering, but no evidence anywhere describes defining custom entity types, edge/relationship types, or an ontology to fit a domain schema — extraction is described as fully automatic ('Supermemory builds a living knowledge graph... automatically').", "evidenceIds": [ "supermemory-docs-14", "supermemory-docs-27", "supermemory-docs-49", "supermemory-docs-39", "supermemory-docs-41" ] }, { "productId": "supermemory", "storyId": "entity-graph-memory", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs explicitly describe a 'living knowledge graph of facts on top of other facts' with graph traversal as one of three retrieval modes, and show entity-centric queries (e.g. 'where does Alex work?' with relatedMemories) demonstrating multi-hop entity resolution across facts like 'Alex mentioned he just started at Stripe'. missing for 10: independent/hands-on validation of multi-hop accuracy, and deeper documentation of explicit entity/relationship schema or graph query language beyond the relatedMemories flag.", "evidenceIds": [ "supermemory-docs-14", "supermemory-docs-23", "supermemory-docs-27", "supermemory-docs-37", "supermemory-docs-48" ] }, { "productId": "supermemory", "storyId": "framework-integrations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents official TypeScript/Python SDKs, an MCP server, a CLI setup flow, and third-party connectors (Google Drive, Gmail, Notion, OneDrive, GitHub) but contains no mention of first-party integrations for LangChain, LangGraph, CrewAI, or the Vercel AI SDK specifically. Since a memory-layer product plausibly could ship such framework adapters, absence of evidence here is 'none' rather than 'na'.", "evidenceIds": [ "supermemory-docs-26", "supermemory-docs-46", "supermemory-docs-11" ] }, { "productId": "supermemory", "storyId": "hybrid-filtered-recall", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly document hybrid search mode (searchMode: \"hybrid\" combining memories and document chunks), metadata filtering by custom properties like category/status/date, and container-tag scoping as retrieval controls beyond plain similarity search. Multiple retrieval modes (document RAG, memory graph traversal, user profile) are also documented as selectable. missing for 10: no explicit mention of a dedicated reranking feature/parameter, and no independent/hands-on corroboration of these filtering/hybrid modes working as described.", "evidenceIds": [ "supermemory-docs-2", "supermemory-docs-28", "supermemory-docs-39", "supermemory-docs-23", "supermemory-docs-38", "supermemory-docs-27" ] }, { "productId": "supermemory", "storyId": "local-embedded-mode", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs explicitly describe a self-hosting option running as a single self-contained binary with local graph engine, local embeddings, and local LLM, claiming data never leaves the building and no Docker/config needed. However, there's no independent/hands-on corroboration of fully offline operation, no detail on which local models are supported or how embedding/LLM swapping works, and self-hosting is positioned as a paid-tier feature (SOC2/HIPAA self-hosted option) rather than a fully documented open local dev path. missing for 10: independent verification of the self-hosted binary running fully offline, concrete local-model configuration docs, and evidence this isn't gated behind enterprise pricing.", "evidenceIds": [ "supermemory-docs-7", "supermemory-docs-8", "supermemory-docs-16", "supermemory-docs-31", "supermemory-docs-44", "supermemory-docs-52" ] }, { "productId": "supermemory", "storyId": "long-term-memory-add-search", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs show a complete workflow: add raw content or direct memories (client.add), semantic/hybrid search (client.search with searchMode:hybrid), container tags to scope and isolate memory per user/session, and automatic extraction/indexing of conversations and documents — directly matching the story of persisting and recalling context across sessions. Missing for 10: independent hands-on validation of recall quality/accuracy, and one HN complaint noting ingestion size limits on large linked content suggests some real-world friction in the add-memory pipeline.", "evidenceIds": [ "supermemory-docs-1", "supermemory-docs-2", "supermemory-docs-10", "supermemory-docs-24", "supermemory-docs-27", "supermemory-docs-38", "supermemory-comm-1" ] }, { "productId": "supermemory", "storyId": "low-latency-retrieval-claims", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack cites any documented latency targets or measured p50/p95 numbers for retrieval; docs discuss features like hybrid search, self-hosting, and MemoryBench for benchmarking against competitors, but no concrete performance figures are given. Missing for 10: published latency SLAs/targets, measured p50/p95 benchmark numbers, or any performance dashboard/report.", "evidenceIds": [] }, { "productId": "supermemory", "storyId": "memory-access-controls", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Supermemory provides container-tag based tenant isolation (hard namespace boundaries) and SOC2/HIPAA BAA compliance claims, which partially address data segregation and enterprise trust, but there is no documented role-based access control, granular read/write policies, or audit-log capability for tracking who accessed or modified memories. Missing for 10: explicit RBAC/ACL system, audit trail/logging of access events, and evidence of enforcement beyond tag-scoped isolation.", "evidenceIds": [ "supermemory-docs-4", "supermemory-docs-38", "supermemory-docs-12", "supermemory-docs-52" ] }, { "productId": "supermemory", "storyId": "memory-deletion-erasure", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs explicitly describe deleting a user's entire memory scope via container tags ('When the user requests deletion, delete that container's content via the API / console workflow'), which covers per-entity/full erasure for privacy compliance. However, evidence never documents a single-memory delete endpoint/method, nor any confirmation flow, audit trail, or independent verification that deletion actually purges data (e.g., from backups/embeddings). missing for 10: single-memory-level delete API/example, confirmation of hard-delete semantics (vs soft-delete), independent/hands-on evidence of successful deletion, and audit/compliance documentation beyond container-tag erasure.", "evidenceIds": [ "supermemory-docs-12", "supermemory-docs-19", "supermemory-docs-25", "supermemory-docs-34", "supermemory-docs-38" ] }, { "productId": "supermemory", "storyId": "memory-expiration-ttl", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No documentation describes TTL, expiration dates, or recency-weighted decay; the only related mention (isStatic flag) is about marking facts permanent vs. default, not about expiring stale ones. Moreover the product's own comparison doc admits 'No temporal truth (\"loved Adidas\" then \"switched to Puma\", both stay equally relevant)', indicating stale facts are not down-weighted or expired.", "evidenceIds": [ "supermemory-docs-29", "supermemory-docs-48" ] }, { "productId": "supermemory", "storyId": "memory-export-portability", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents ingestion, search, self-hosting, and API clients, but never mentions a bulk export endpoint, data-dump command, or documented export format for memories — nothing addresses portability of existing data out of the platform. Self-hosting (supermemory-docs-7/8/16/44) is a deployment option, not an export mechanism for data already stored in the hosted service.", "evidenceIds": [] }, { "productId": "supermemory", "storyId": "memory-update-reconcile", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Supermemory documents a customId mechanism that lets you resend content and have it 'intelligently process only what's new' (supermemory-docs-30), and memory-graph docs show new facts (e.g., new employer) being added on top of old ones (supermemory-docs-27, supermemory-docs-37), suggesting some update/graph-merge behavior. But the product's own comparison page explicitly admits it lacks 'temporal truth' — contradictory facts like 'loved Adidas' then 'switched to Puma' both remain equally relevant rather than the newer one superseding the older (supermemory-docs-48) — a direct, first-party admission that conflicting memories are not reliably resolved. missing for 10: no documented mechanism for detecting/resolving direct factual contradictions (vs. just adding new facts or updating same-ID documents), no independent/community verification of merge or supersession behavior in practice.", "evidenceIds": [ "supermemory-docs-30", "supermemory-docs-48", "supermemory-docs-27", "supermemory-docs-37" ] }, { "productId": "supermemory", "storyId": "multi-language-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs explicitly confirm official TypeScript (npm install supermemory) and Python (pip install supermemory) SDKs wrapping the same API, with code examples shown mainly in TS/JS (client.add, client.search). There's no direct evidence of Python-specific code samples or confirmation the Python SDK has full feature parity (e.g., container tags, hybrid search, profiles) documented equivalently. Missing for 10: Python code examples demonstrating equivalent memory operations, explicit parity statement across SDKs, independent developer corroboration of Python SDK usage.", "evidenceIds": [ "supermemory-docs-26", "supermemory-docs-24", "supermemory-docs-27", "supermemory-docs-28" ] }, { "productId": "supermemory", "storyId": "multimodal-memory-ingestion", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "GitHub docs explicitly list multi-modal extractors for PDFs and images (OCR) with 'upload and it works,' and docs describe raw file ingestion plus hybrid search/recall across document chunks and memories. Missing for 10: independent hands-on verification specifically for image/PDF recall (the only community evidence found concerns large link content failing, not files/images).", "evidenceIds": [ "supermemory-gh-3", "supermemory-docs-1", "supermemory-docs-41", "supermemory-docs-2", "supermemory-docs-28", "supermemory-docs-42" ] }, { "productId": "supermemory", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show a comprehensive REST API (add/search/memory operations, container tags, user profiles, connectors) with official TS/Python clients, and note that app.supermemory.ai is just the consumer UI on the same engine (implying API-first parity). However, some workflows (e.g. deletion) are described as done 'via the API / console workf[low]', suggesting console-only steps exist, and a live probe found no discoverable OpenAPI spec, undercutting full API self-service. missing for 10: explicit statement/evidence that every UI action (connector setup, buckets, profile management, deletion) is fully API-equivalent, and a public OpenAPI/schema reference confirming complete API surface.", "evidenceIds": [ "supermemory-docs-26", "supermemory-docs-20", "supermemory-docs-12", "supermemory-docs-43", "supermemory-probe-2" ] }, { "productId": "supermemory", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The docs describe ingestion, search, self-hosting, and API access, but no citation documents a bulk data-export feature or open-format export tool that would let a user extract all stored memories and leave the platform. Self-hosting (docs-7/16/31/44) shows you can run your own instance, but that is not the same as exporting existing hosted data in an open format.", "evidenceIds": [ "supermemory-docs-7", "supermemory-docs-16", "supermemory-docs-26", "supermemory-probe-2" ] }, { "productId": "supermemory", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "There is a public GitHub repository (supermemoryai/supermemory) referenced, and self-hosting docs describe a distributable binary, but no evidence pack item states an open-source license, and the self-hosting binary is described as a closed compiled artifact rather than published source. Missing for 10: any explicit license file/name, evidence the GitHub repo contains the actual product source (vs. docs/SDK), and confirmation the self-hosted binary corresponds to inspectable open-licensed code.", "evidenceIds": [ "supermemory-gh-1", "supermemory-docs-7", "supermemory-docs-44" ] }, { "productId": "supermemory", "storyId": "openness-self-host", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly document a self-hosting mode running the same memory engine as a single self-contained binary with local graph engine, embeddings, and LLM, requiring no Docker or config, and no data leaving the building. Missing for 10: independent hands-on verification of the self-host binary working, and details on feature parity/limitations vs hosted version.", "evidenceIds": [ "supermemory-docs-7", "supermemory-docs-8", "supermemory-docs-16", "supermemory-docs-31", "supermemory-docs-44", "supermemory-docs-52" ] }, { "productId": "supermemory", "storyId": "per-user-memory-isolation", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs clearly document container tags as hard isolation boundaries per user/agent/app, with explicit statements that tag-scoped searches never return cross-tenant memories and code examples showing containerTag usage for scoping and deletion. missing for 10: independent/hands-on verification of isolation guarantees beyond vendor docs, and no detail on cross-tenant leakage testing or edge cases (e.g., shared profiles/buckets interacting with tags).", "evidenceIds": [ "supermemory-docs-4", "supermemory-docs-38", "supermemory-docs-24", "supermemory-docs-27", "supermemory-docs-37", "supermemory-docs-12", "supermemory-docs-19", "supermemory-docs-25", "supermemory-docs-34" ] }, { "productId": "supermemory", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Supermemory offers a self-hosted binary option (\"data never leaves the building\") and lists SOC2/HIPAA BAA/self-hosted as compliance options, which lets an org control the physical location of data by running it on their own infrastructure, but there is no documented feature to choose a specific region/data-residency setting within the hosted SaaS offering itself. missing for 10: explicit region/residency selection controls in the hosted product, documentation of where hosted data is stored geographically, and any multi-region deployment options.", "evidenceIds": [ "supermemory-docs-7", "supermemory-docs-16", "supermemory-docs-52" ] }, { "productId": "supermemory", "storyId": "privacy-no-training", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Supermemory offers a self-hosted deployment where 'your data never leaves the building' with local embeddings/LLM (supermemory-docs-8, supermemory-docs-16), which implicitly prevents data from reaching any third-party model training pipeline, and it advertises SOC2/HIPAA compliance (supermemory-docs-52). However, there is no explicit policy statement for the hosted/cloud platform about not training on customer data or an opt-out toggle. missing for 10: explicit no-training/data-use policy for the hosted SaaS offering, independent verification of self-host data isolation claims.", "evidenceIds": [ "supermemory-docs-8", "supermemory-docs-16", "supermemory-docs-44", "supermemory-docs-52" ] }, { "productId": "supermemory", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs explicitly document a deletion workflow scoped by container tags (delete a user's/tenant's content via API/console) plus self-hosting for full data control and SOC2/HIPAA compliance options, giving users a documented path to control retention and deletion. However, there's no evidence of granular per-memory deletion controls, retention policy settings (auto-expiry/TTL), or an end-user-facing UI for self-service deletion — the workflow described is developer/console-mediated only. Missing for 10: user-facing self-service deletion UI, configurable retention/expiry policies, and independent confirmation that deletion actually purges data from backups/indexes.", "evidenceIds": [ "supermemory-docs-12", "supermemory-docs-19", "supermemory-docs-25", "supermemory-docs-34", "supermemory-docs-52", "supermemory-docs-7", "supermemory-docs-8" ] }, { "productId": "supermemory", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item addresses telemetry opt-out or usage-tracking controls for Supermemory; docs cover security/compliance (SOC 2, HIPAA) and self-hosting but never mention a telemetry toggle or analytics opt-out mechanism.", "evidenceIds": [] }, { "productId": "supermemory", "storyId": "prompt-ready-context-block", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Supermemory offers a single-call hybrid search that returns relevant memories/document chunks and related memories, and user profiles that are 'always ready' without a search call, which covers assembling relevant context in one call — but no docs mention a token-budget parameter or explicit prompt-sized context-block output. missing for 10: explicit token-budget/max-token control on retrieval, evidence of a single call returning a pre-formatted prompt-ready block combining profile+graph+documents together.", "evidenceIds": [ "supermemory-docs-2", "supermemory-docs-28", "supermemory-docs-3", "supermemory-docs-33", "supermemory-docs-23", "supermemory-docs-27" ] }, { "productId": "supermemory", "storyId": "published-memory-benchmarks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Supermemory documents a 'MemoryBench' tool that lets users run their own benchmarks against Mem0/Zep/Supermemory, but the evidence contains no published results on standard benchmarks like LongMemEval or LoCoMo, nor any recall-accuracy numbers backing its claims.", "evidenceIds": [ "supermemory-docs-9", "supermemory-docs-22" ] }, { "productId": "supermemory", "storyId": "self-host-oss-deployment", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Docs confirm a genuine self-host mode — 'runs on your own hardware... single self-contained binary' with local embeddings/LLM — but explicitly state 'No Docker' and describe a closed single-binary distribution rather than open-source code the engineer builds/deploys via Docker. The GitHub repo evidence in the pack only documents connectors/extractors, not that the self-host binary itself is open-source or containerized. missing for 10: Docker/container deployment path, confirmation the self-hosted binary is built from open-source code (license/repo), and independent hands-on verification of self-host setup.", "evidenceIds": [ "supermemory-docs-7", "supermemory-docs-16", "supermemory-docs-31", "supermemory-docs-44", "supermemory-docs-52" ] }, { "productId": "supermemory", "storyId": "session-summarization", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Supermemory ingests raw conversations and auto-extracts memories, then exposes them via hybrid search, memory graph traversal, and always-ready user profiles that update as users interact — this gives an agent a way to recall prior conversation context and continue it (docs-1, docs-2, docs-3, docs-15, docs-23, docs-33). However there is no explicit 'session/thread summary' artifact or transcript-resume feature documented, and no hands-on/independent evidence of an agent actually resuming a specific thread using this data. Missing for 10: a dedicated session-summary or thread-resume API/feature, and independent verification that agents successfully pick up prior conversations using this context.", "evidenceIds": [ "supermemory-docs-1", "supermemory-docs-2", "supermemory-docs-3", "supermemory-docs-15", "supermemory-docs-23", "supermemory-docs-33", "supermemory-docs-30" ] }, { "productId": "supermemory", "storyId": "shared-team-memory", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs describe container tags as hard-boundary namespaces for per-user/tenant isolation (private memory) and separately describe 'spaces' and the Supermemory MCP as a shared memory layer letting teams collaborate on the same authorized context, which together map to the private-vs-shared story. However there's no explicit example of selectively promoting specific memories from a private container into a shared team space, nor documented fine-grained ACL/role controls for partial sharing. missing for 10: worked example of mixed private+shared memory for the same agent/user, explicit role-based access control docs, independent/hands-on verification of isolation guarantees.", "evidenceIds": [ "supermemory-docs-4", "supermemory-docs-38", "supermemory-docs-36", "supermemory-docs-11", "supermemory-docs-19" ] }, { "productId": "supermemory", "storyId": "temporal-fact-memory", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Supermemory's own comparison docs explicitly admit the product lacks temporal truth tracking, giving the example that 'loved Adidas' then 'switched to Puma' both remain equally relevant rather than one being marked outdated — the opposite of the story's requirement. No other evidence describes validity windows, fact invalidation, or temporal versioning of memories.", "evidenceIds": [ "supermemory-docs-48" ] }, { "productId": "supermemory", "storyId": "transparent-usage-pricing", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "A dedicated pricing.md page is referenced with some billing mechanics (unique-content-only billing, prompt-cache discount, plan tiers like 'Max plan' gating connectors, SOC2/HIPAA/self-host options), but no evidence pack citation actually shows a free-tier allotment or explicit per-unit dollar rates. Missing for 10: explicit free-tier limits, published per-unit/usage pricing figures, and any independent corroboration of the pricing page's numbers.", "evidenceIds": [ "supermemory-docs-13", "supermemory-docs-21", "supermemory-docs-50", "supermemory-docs-52" ] }, { "productId": "supermemory", "storyId": "voice-realtime-memory", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack shows generic SDKs, connectors (Drive, Gmail, Notion, etc.), MCP integration, and self-hosting, but nowhere mentions LiveKit, Pipecat, ElevenLabs, or any real-time voice pipeline integration, latency benchmarks, or streaming APIs suited for live conversation. No documented voice-pipeline integration exists in the pack.", "evidenceIds": [] }, { "productId": "zep", "storyId": "agent-self-managed-memory", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Zep exposes API/SDK calls the agent (or an MCP-connected agent) can invoke itself to add memories (thread.add_messages), search the graph (semantic+BM25, scope=auto), and delete edges/nodes (graph.edge.delete), and the Memory MCP Server is confirmed live and functioning for end-user agent memory access (zep-docs-6, zep-probe-2, zep-probe-rt-2). However, no explicit 'update' operation on existing memories is documented (only add/delete), and the MCP server is gated behind OAuth per the runtime probe, meaning autonomous mid-conversation self-service isn't fully frictionless. Missing for 10: an explicit update-memory API/tool, and clearer evidence the agent (not just external MCP clients) autonomously triggers add/search/delete without human-configured auth setup.", "evidenceIds": [ "zep-docs-6", "zep-docs-20", "zep-docs-21", "zep-docs-9", "zep-docs-10", "zep-probe-2", "zep-probe-rt-2" ] }, { "productId": "zep", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Zep hosts an llms.txt file at help.getzep.com/llms.txt (confirmed HTTP 200 via direct probe) with a clear summary of the product, and also runs a docs MCP server that completed a live keyless JSON-RPC handshake, giving agents multiple direct machine-readable entry points into the docs. Missing for 10: no independent/community confirmation that agents actually use these entry points successfully in practice.", "evidenceIds": [ "zep-probe-1", "zep-probe-rt-1", "zep-docs-6" ] }, { "productId": "zep", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Zep does generate AI-derived artifacts from ingested data — thread summaries (natural-language summaries incrementally updated by Zep) and entity/relation extraction into the Context Graph — which qualify as AI-generated insights from user data. However, there is no evidence of proactive 'suggestions' or recommendation-style outputs, nor any user-facing dashboard/UI surfacing these insights to an end user; the artifacts are primarily consumed programmatically by the calling agent/app rather than presented as insights inside a product experience. Missing for 10: evidence of suggestion/recommendation generation, and evidence of a UI or interface that surfaces these AI-generated insights directly to a human/ai-native user.", "evidenceIds": [ "zep-docs-8", "zep-docs-27", "zep-docs-33", "zep-docs-9", "zep-docs-10" ] }, { "productId": "zep", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "zep", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "zep", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Zep is fundamentally API/SDK-driven (thread.add_messages, graph.edge.delete calls) and ships a full-access CLI (zepctl) for managing projects, users, threads, and data from the terminal — both of which are inherently scriptable/headless and usable in CI pipelines. However, there is no explicit documentation or example of CI/CD integration, automated testing, or headless deployment guidance. Missing for 10: explicit CI/CD pipeline examples, headless automation docs, non-interactive auth/service-account setup guidance for CI.", "evidenceIds": [ "zep-docs-7", "zep-docs-23", "zep-docs-37", "zep-docs-20", "zep-docs-21", "zep-probe-3" ] }, { "productId": "zep", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Zep is a memory infrastructure/platform product, not an agentic client that consumes tools. All MCP evidence shows Zep in the server role — it exposes Memory MCP and Graphiti MCP servers so that other clients (Claude, Cursor, ChatGPT, etc.) can pull from it — not evidence that Zep itself plugs in external MCP servers to use their tools. This story's axis (product-as-MCP-client) is a category error for a memory backend whose purpose is to be a consumed service, not a tool-using agent.", "evidenceIds": [ "zep-docs-6", "zep-docs-16", "zep-docs-32", "zep-probe-rt-2" ] }, { "productId": "zep", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Zep documents and runtime-verifies an official Memory MCP server that lets end users connect Claude, ChatGPT, Cursor, Codex and other MCP clients to their agent memory, with a live OAuth-gated endpoint confirmed at api.getzep.com/mcp. missing for 10: independent third-party (non-vendor) hands-on review of the MCP client connection experience beyond Zep's own docs/probe.", "evidenceIds": [ "zep-docs-6", "zep-probe-2", "zep-probe-rt-2", "zep-docs-25" ] }, { "productId": "zep", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Zep exposes an MCP server so agents/AI clients can invoke memory operations via natural-language-driven tool calls, and the Memory MCP endpoint is confirmed live/IdP-gated in a runtime probe, which supports natural-language operation through an AI assistant. However, the primary administrative surface is zepctl, a traditional structured CLI, not a natural-language command interface, and there is no documented natural-language command parser or chat-driven admin console. missing for 10: evidence of natural-language command parsing/execution outside MCP-client pass-through, first-party NL command examples, independent hands-on confirmation of NL-driven operation.", "evidenceIds": [ "zep-docs-6", "zep-probe-rt-2", "zep-docs-7", "zep-docs-23" ] }, { "productId": "zep", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Zep documents zepctl, an official CLI providing full administrative access to Zep projects (managing users, threads, Context Graphs, and data operations from the terminal), independently confirmed via probe at the documented URL. missing for 10: no independent/community hands-on usage reports of zepctl, and no detail on installation/distribution channels.", "evidenceIds": [ "zep-docs-7", "zep-docs-23", "zep-docs-37", "zep-probe-3" ] }, { "productId": "zep", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Zep documents a full REST/SDK API (thread.add_messages, graph.edge.delete, get_user_context, etc.) with quick-start guides, plus a CLI (zepctl) providing full programmatic/administrative access, and runtime probes confirm live endpoints (MCP server, docs API) behaving as documented. This is clearly a documented, agent-drivable public API surface with concrete code examples and independent runtime verification. Missing for 10: no independent third-party API reference/benchmark beyond vendor docs and probes.", "evidenceIds": [ "zep-docs-1", "zep-docs-7", "zep-docs-11", "zep-docs-20", "zep-docs-21", "zep-probe-3", "zep-probe-rt-2", "zep-docs-23" ] }, { "productId": "zep", "storyId": "agentic-scoped-keys", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Zep explicitly documents ABAC policies attached to API keys and UserGroups to enable least-privilege access for agents and Memory MCP users (zep-docs-25, zep-docs-38, zep-docs-31, zep-docs-12), and this is corroborated at runtime by the Memory MCP endpoint being IdP-gated with OAuth-protected-resource metadata rather than open access (zep-probe-rt-2). Missing for 10: no worked example showing granular scope definitions (e.g. per-thread or per-action policy syntax) or independent third-party audit of the ABAC enforcement.", "evidenceIds": [ "zep-docs-25", "zep-docs-38", "zep-docs-31", "zep-docs-12", "zep-probe-rt-2" ] }, { "productId": "zep", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show official SDK method calls (client.graph.edge.delete, thread.add_messages, three-line quick start) and a real pip-installable OSS engine (graphiti-core) confirmed via probe, indicating an official Python SDK/client exists and works. However, there's no explicit documentation of the full SDK ecosystem (e.g., multiple language SDKs, GitHub repo for the client library itself) or independent developer corroboration of building against these SDKs. Missing for 10: explicit multi-language SDK listing, dedicated SDK GitHub repo evidence, third-party developer confirmation of SDK integration.", "evidenceIds": [ "zep-docs-1", "zep-docs-18", "zep-docs-20", "zep-docs-21", "zep-docs-26", "zep-probe-rt-3" ] }, { "productId": "zep", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions webhooks or event subscription mechanisms; Zep's docs cover MCP servers, CLI, SDKs, and API access but nothing about outbound event notifications or webhook subscriptions.", "evidenceIds": [] }, { "productId": "zep", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Zep's docs pages show static code snippets (e.g., zep-docs-21, zep-docs-29) but there is no evidence of an interactive, runnable API reference (e.g., embedded sandbox, 'try it' console, Postman/Swagger integration) anywhere in the evidence pack.", "evidenceIds": [] }, { "productId": "zep", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Zep is an API-first service with SDKs, a CLI (zepctl), and MCP servers, so a downloadable OpenAPI spec would be a natural artifact — but no evidence pack item mentions an OpenAPI/Swagger spec, API reference export, or machine-readable schema file being available for download.", "evidenceIds": [] }, { "productId": "zep", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of a dedicated sandbox/staging environment, test API keys, or non-production project mode for Zep's memory platform; the only related items are deployment options (Cloud vs VPC) and the standalone open-source Graphiti library, neither of which document a sandbox for testing against production-like data safely.", "evidenceIds": [] }, { "productId": "zep", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers Zep's features (memory, graph, MCP, CLI) but contains no mention of API versioning scheme or a documented deprecation policy for breaking changes. This is a fair axis for an API-first product, but no evidence supports it.", "evidenceIds": [] }, { "productId": "zep", "storyId": "assistant-memory-plugins", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Zep documents an official Memory MCP Server explicitly enabling Claude, ChatGPT, Claude Code, Codex, Cursor, and others to connect to a shared end-user memory graph, with a runtime probe confirming the endpoint is live and IdP-gated as documented. This directly satisfies the story of connecting multiple off-the-shelf assistants to shared memory; missing for 10: independent hands-on multi-assistant test showing simultaneous cross-tool memory sharing in practice.", "evidenceIds": [ "zep-docs-6", "zep-docs-25", "zep-docs-38", "zep-probe-2", "zep-probe-rt-2" ] }, { "productId": "zep", "storyId": "async-batch-processing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes single-message ingestion (thread.add_messages) and various data types, but nowhere documents a batch/async ingestion API or a way to poll status of background memory-processing jobs. Missing for 10: batch ingestion endpoint, async job submission, job status/polling API.", "evidenceIds": [] }, { "productId": "zep", "storyId": "automatic-memory-extraction", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Zep automatically ingests raw chat messages into a temporal knowledge graph (zep-docs-20, zep-gh-1/2/3), extracting facts/entities/relations without manual tagging, and separately generates and incrementally updates natural-language thread summaries in the background (zep-docs-8). Community commentary confirms this is a deliberate design evolution from naive LLM fact-extraction toward a more robust temporal graph approach for salience and consolidation (zep-comm-1). Missing for 10: independent benchmark or hands-on evidence of extraction accuracy/salience quality, and detail on exactly how 'importance' is scored versus simply ingesting everything.", "evidenceIds": [ "zep-docs-20", "zep-docs-8", "zep-gh-1", "zep-gh-2", "zep-comm-1", "zep-docs-27" ] }, { "productId": "zep", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Zep supports some bulk-style operations — deleting a User removes all associated threads/artifacts in one API call, and the zepctl CLI gives scriptable full access to manage users, threads, and graphs from the terminal — but there is no documented batch/bulk create-ingest API for processing many items in a single call. missing for 10: explicit bulk ingest/create endpoints, batch size limits or throughput guidance, and independent evidence of large-scale bulk operations being exercised in practice.", "evidenceIds": [ "zep-docs-5", "zep-docs-35", "zep-docs-7", "zep-docs-23" ] }, { "productId": "zep", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Zep is a memory/context-graph layer with search, retrieval, MCP access, and governance policies, but no evidence describes a rules engine or event-trigger mechanism that automatically fires actions on defined conditions/events. Governance features (ABAC/RBAC) restrict access rather than trigger automated actions.", "evidenceIds": [] }, { "productId": "zep", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Zep is a memory/context-graph layer for AI agents, not a workflow/job orchestration or scheduling product; there is no concept of recurring jobs or cron-like automation in its evidence, and this axis is a category error for its product type.", "evidenceIds": [] }, { "productId": "zep", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Zep is an agent-memory / context-graph infrastructure product, not an automation-builder or workflow tool; it has no concept of 'automations' to version, review, or roll back. The evidence pack only covers memory graph versioning of facts/data (temporal knowledge graph), not automation workflows, making this a category mismatch rather than a missing feature.", "evidenceIds": [] }, { "productId": "zep", "storyId": "business-data-ingestion", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly state Zep ingests JSON, text, business data, and documents into the Context Graph, not just chat messages (zep-docs-2, zep-docs-27), with API examples for graph operations and custom entity/edge types for structuring business data. Community context confirms the underlying Graphiti engine tracks facts/entities beyond raw transcripts. Missing for 10: no hands-on independent verification of ingesting large documents or complex JSON schemas at scale, and no detail on supported document formats (PDF, etc.).", "evidenceIds": [ "zep-docs-2", "zep-docs-27", "zep-docs-33", "zep-docs-20", "zep-gh-1" ] }, { "productId": "zep", "storyId": "custom-ontology-schema", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Zep explicitly supports Custom Entity/Edge Types via Pydantic-like classes to customize creation/retrieval of entities and relations in the Context Graph (zep-docs-3, zep-docs-33), and the underlying Graphiti engine documents support for both prescribed and learned ontology (zep-gh-1). This directly matches the ML-engineer story of customizing entity/edge types to fit a domain. Missing for 10: no independent hands-on example showing a full custom ontology in production use, and no detail on limits/complexity of ontology customization beyond entity/edge type classes.", "evidenceIds": [ "zep-docs-3", "zep-docs-33", "zep-gh-1" ] }, { "productId": "zep", "storyId": "entity-graph-memory", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Zep explicitly builds a Context Graph (temporal knowledge graph via Graphiti) from chat/business data, with entities and relationships, custom entity/edge types, edge/node deletion, and hybrid semantic+BM25 search across nodes/edges/episodes enabling multi-hop and entity-centric retrieval; the underlying Graphiti engine is verified installable and open-source with independent GitHub corroboration. Missing for 10: no explicit hands-on multi-hop query benchmark or third-party validation of entity-centric answer accuracy beyond vendor docs.", "evidenceIds": [ "zep-docs-2", "zep-docs-3", "zep-docs-9", "zep-docs-10", "zep-docs-13", "zep-docs-27", "zep-gh-1", "zep-gh-2", "zep-probe-rt-3" ] }, { "productId": "zep", "storyId": "framework-integrations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Zep docs explicitly reference 'agent frameworks' integrations (zep-docs-14, zep-docs-24) that persist and retrieve conversation data, and community/HN threads confirm real-world adoption, but the evidence pack never names or links specific first-party integration docs for LangChain, LangGraph, CrewAI, or Vercel AI SDK—only generic 'supported frameworks' language. missing for 10: named/linked docs pages for LangChain, LangGraph, CrewAI, or Vercel AI SDK integrations, and independent confirmation these integrations work as advertised.", "evidenceIds": [ "zep-docs-14", "zep-docs-24", "zep-docs-1", "zep-docs-18" ] }, { "productId": "zep", "storyId": "hybrid-filtered-recall", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Zep's docs confirm hybrid retrieval (semantic similarity + BM25 full-text) and a configurable 'scope' that composes across edges/nodes/episodes/summaries, showing retrieval is not a single fixed similarity search. However, there is no documented support for explicit metadata-based filtering or a dedicated reranking step/model in the search API. Missing for 10: metadata filter parameters in search calls, explicit reranking option/model, independent confirmation of these controls in practice.", "evidenceIds": [ "zep-docs-9", "zep-docs-22", "zep-docs-10" ] }, { "productId": "zep", "storyId": "local-embedded-mode", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Zep's open-source engine Graphiti can be pip-installed and imported with no API key and used to build/query a Context Graph 'locally' in-process, giving a genuine local path, but the main Zep memory service (the one with threads, users, MCP server, zepctl) is documented as deployed via Cloud or 'Your VPC', not as an embeddable local-only runtime, and there is no evidence of configuring it against local LLMs/embedding models. Missing for 10: explicit support for local/self-hosted LLM or embedding models, confirmation that the full Zep memory layer (not just Graphiti) can run fully offline/in-process, and any docs on air-gapped or no-cloud-dependency operation.", "evidenceIds": [ "zep-docs-15", "zep-probe-rt-3", "zep-docs-17", "zep-gh-2" ] }, { "productId": "zep", "storyId": "long-term-memory-add-search", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Zep's docs directly cover the story: adding messages persists them into a user-level knowledge graph (thread.add_messages), retrieval via semantic + BM25 hybrid graph search with sub-200ms context blocks, and cross-session persistence via Threads/Users tied to a Context Graph. This is corroborated by runtime probes confirming the memory endpoints are live and by community discussion of the underlying Graphiti engine's design rationale. Missing for 10: independent hands-on benchmarking of recall quality/accuracy across long sessions beyond vendor-stated latency claims.", "evidenceIds": [ "zep-docs-20", "zep-docs-9", "zep-docs-22", "zep-docs-10", "zep-docs-34", "zep-docs-18", "zep-comm-1", "zep-probe-rt-2" ] }, { "productId": "zep", "storyId": "low-latency-retrieval-claims", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Zep's docs make an explicit performance claim ('sub-200ms context retrieval regardless of graph size or number of graphs') and reference it in the quick-start guide, giving platform engineers a concrete latency target to plan against. However, there is no documented p50/p95 breakdown, benchmark methodology, or dataset/scale disclosure, and no independent hands-on measurement corroborates the figure. Missing for 10: percentile (p50/p95) breakdowns, benchmark methodology/conditions, and independent/third-party latency verification.", "evidenceIds": [ "zep-docs-34", "zep-docs-28", "zep-docs-18", "zep-docs-11", "zep-docs-26" ] }, { "productId": "zep", "storyId": "memory-access-controls", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Zep's governance docs describe RBAC for dashboard users and ABAC policies scoped to API keys and UserGroups, letting admins limit what agents and Memory MCP users can read/write (zep-docs-12, zep-docs-25, zep-docs-31, zep-docs-38). However, there is no documented audit logging/trail capability to verify who accessed what memory. Missing for 10: audit log/trail evidence, granular per-resource ACLs beyond API-key/UserGroup scope, and independent verification of policy enforcement.", "evidenceIds": [ "zep-docs-12", "zep-docs-25", "zep-docs-31", "zep-docs-38" ] }, { "productId": "zep", "storyId": "memory-deletion-erasure", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Zep documents multiple granularities of deletion: single edge deletion via client.graph.edge.delete (with cascading edge deletion when a node is deleted), and full user erasure via a single API call that deletes all threads and graph data, explicitly framed as satisfying Right To Be Forgotten/privacy regulation requirements. This covers single-memory, per-entity (node/edge), and full-erasure use cases directly relevant to a platform-engineer's data-lifecycle needs. Missing for 10: no independent/hands-on confirmation of deletion behavior beyond first-party docs, and no explicit documented API for bulk per-entity-type deletion across a graph (only node/edge-level and full-user-level are shown).", "evidenceIds": [ "zep-docs-5", "zep-docs-35", "zep-docs-13", "zep-docs-21", "zep-docs-29" ] }, { "productId": "zep", "storyId": "memory-expiration-ttl", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Zep's underlying Graphiti engine is a temporal knowledge graph that tracks how facts change over time, which implies automatic invalidation of superseded facts as new information arrives — a decay-like mechanism rather than explicit TTL. However, no evidence describes explicit TTL settings, expiration dates, or recency-weighted retrieval; only manual node/edge deletion APIs are documented for removing stale data. Missing for 10: explicit TTL/expiration-date configuration, documented recency-weighting in search/ranking, and confirmation that fact invalidation actually suppresses surfacing of stale facts in retrieval.", "evidenceIds": [ "zep-gh-1", "zep-gh-2", "zep-docs-13", "zep-docs-21", "zep-docs-9" ] }, { "productId": "zep", "storyId": "memory-export-portability", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Zep exposes a graph API (JSON data model, node/edge delete by UUID) and a zepctl CLI described as having 'full access... to manage users, threads, Context Graphs, and data operations from the terminal,' which implies some programmatic read/write access to graph data, and the underlying Graphiti engine is open-source and pip-installable, reducing lock-in. However, no evidence explicitly documents a bulk 'export' command, a defined export file format, or a documented data-portability workflow for moving a full Context Graph out of Zep. Missing for 10: an explicit export/dump feature or format, documentation confirming zepctl or API supports full-graph export, and independent verification of successful export/import round-trips.", "evidenceIds": [ "zep-docs-7", "zep-docs-23", "zep-docs-2", "zep-docs-21", "zep-docs-29", "zep-probe-rt-3", "zep-docs-15" ] }, { "productId": "zep", "storyId": "memory-update-reconcile", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Zep's Graphiti-based temporal knowledge graph explicitly tracks how facts change over time and maintains provenance, and Zep's own team describes moving away from naive fact-extraction to a temporal graph specifically to handle contradiction/reconciliation issues, with edge/node deletion APIs supporting explicit correction. missing for 10: no explicit documented API or hands-on example showing automatic detection/supersession of a contradicting fact (e.g., 'address changed from X to Y' auto-invalidating the old edge) or independent benchmark validating merge/supersede accuracy.", "evidenceIds": [ "zep-gh-1", "zep-gh-2", "zep-comm-1", "zep-docs-21", "zep-docs-13", "zep-docs-9" ] }, { "productId": "zep", "storyId": "multi-language-sdks", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Code examples in docs (e.g. client.graph.edge.delete, thread.add_messages()) show a Python-style SDK, but the evidence pack never mentions a TypeScript/JavaScript SDK or confirms parity between two language SDKs. Missing for 10: explicit TypeScript SDK docs/install instructions, side-by-side API parity confirmation, and any independent corroboration of both SDKs.", "evidenceIds": [ "zep-docs-21", "zep-docs-29", "zep-docs-20", "zep-docs-26", "zep-docs-30" ] }, { "productId": "zep", "storyId": "multimodal-memory-ingestion", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Zep's documentation explicitly enumerates ingestible data types as JSON, text, and message only (zep-docs-2), with no mention of images, PDFs, or file uploads anywhere in the evidence pack. No feature, API, or integration is described for storing or recalling from binary/file-based memory inputs.", "evidenceIds": [ "zep-docs-2", "zep-docs-27" ] }, { "productId": "zep", "storyId": "openness-api-parity", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Zep is fundamentally API/SDK-driven (thread.add_messages, graph.edge.delete, etc.) and zepctl/CLI is documented as providing 'full access to Zep, enabling you to manage users, threads, Context Graphs, and data operations from the terminal,' implying strong API/CLI parity with the dashboard UI. However, there is no explicit confirmation that every dashboard-only feature (e.g., RBAC/ABAC policy configuration, analytics views) is fully API-exposed. Missing for 10: explicit parity statement or docs enumerating dashboard-only features and confirming API equivalents, independent verification of full CLI/API coverage.", "evidenceIds": [ "zep-docs-7", "zep-docs-23", "zep-docs-37", "zep-docs-31", "zep-docs-38", "zep-docs-20", "zep-docs-21" ] }, { "productId": "zep", "storyId": "openness-full-export", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Zep exposes API/CLI access to graph data (zepctl, client.graph.edge.delete, thread/user APIs) and its underlying Graphiti engine is open-source and locally installable, giving some data portability, but there is no documented bulk 'export all my data' feature or standard export format. missing for 10: explicit data-export/portability feature or format documentation, evidence of a full account-data export tool, independent confirmation of successful full export.", "evidenceIds": [ "zep-docs-7", "zep-docs-23", "zep-docs-37", "zep-docs-21", "zep-docs-29", "zep-probe-rt-3", "zep-docs-5" ] }, { "productId": "zep", "storyId": "openness-open-license", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Zep's underlying temporal knowledge-graph engine, Graphiti, is confirmed open-source on GitHub (30.6k stars) and independently verified as pip-installable with no API key required, satisfying an open-license read for the core Context Graph technology. However, the full Zep product itself (the hosted memory service, zepctl CLI, Memory MCP server) is offered as a commercial Cloud/VPC deployment with no evidence of its own source being under an open license. Missing for 10: confirmation that Zep's actual service/CLI code (not just the Graphiti dependency) is open-licensed, and any repo/license file for Zep itself.", "evidenceIds": [ "zep-gh-1", "zep-gh-2", "zep-gh-3", "zep-docs-15", "zep-probe-rt-3", "zep-docs-17" ] }, { "productId": "zep", "storyId": "openness-self-host", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Zep offers a 'Your VPC' deployment option alongside Cloud (zep-docs-17), suggesting self-hosting is possible, and the underlying Graphiti temporal knowledge-graph engine that powers Zep's Context Graphs is genuinely open-source and pip-installable with no API key (zep-probe-rt-3, zep-gh-1/2/3). However, the full Zep product (users/threads API, zepctl, governance, Memory MCP server) is only ever described as a commercial Cloud/VPC deployment, not as an openly licensed self-hostable package — VPC deployment likely requires an enterprise contract rather than free self-hosting. Missing for 10: explicit open-source license or self-host instructions for the full Zep service (not just the Graphiti sub-component), pricing/terms for VPC self-hosting, and any community confirmation of successfully self-hosting core Zep.", "evidenceIds": [ "zep-docs-17", "zep-probe-rt-3", "zep-gh-1", "zep-gh-2", "zep-docs-23" ] }, { "productId": "zep", "storyId": "per-user-memory-isolation", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Zep documents user-scoped Context Graphs (each user gets their own knowledge graph, created explicitly per app user) and thread-level isolation, plus ABAC/RBAC policies that limit which context each agent, API key, UserGroup, or Memory MCP user can reach — directly supporting tenant isolation across users/agents. Deletion is also scoped per user (RTBF), reinforcing that one user's data is a separable unit that can be walled off or purged without affecting others.\n\nmissing for 10: explicit documentation of 'application'-level tenancy/namespace isolation beyond User/UserGroup scoping, and independent/hands-on verification that cross-tenant retrieval leakage cannot occur.", "evidenceIds": [ "zep-docs-19", "zep-docs-20", "zep-docs-5", "zep-docs-35", "zep-docs-12", "zep-docs-25", "zep-docs-31", "zep-docs-38" ] }, { "productId": "zep", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Zep's pricing page states 'Deploy anywhere: Cloud, Your VPC' (zep-docs-17), implying customers can self-host in their own VPC and thus control where data resides, and the open-source Graphiti engine can be run entirely locally (zep-docs-15). However, there is no documentation of a region-selection feature within Zep Cloud itself or explicit data-residency guarantees for hosted customers. Missing for 10: explicit region/residency options in the managed cloud offering, documented list of supported regions, and any compliance certification tied to region choice.", "evidenceIds": [ "zep-docs-17", "zep-docs-15" ] }, { "productId": "zep", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers RTBF/deletion, access control, and MCP/CLI tooling, but contains no statement about Zep's or its LLM providers' use of customer data for model training, nor any opt-out/no-training guarantee. Since Zep processes conversational data through LLMs, this is a fair privacy-posture question, but no evidence addresses it.", "evidenceIds": [] }, { "productId": "zep", "storyId": "privacy-retention-controls", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Zep documents user-level deletion (single API call deletes all threads/artifacts) explicitly framed for Right To Be Forgotten/privacy compliance, plus granular deletion of individual edges/nodes and cascading edge deletion. This is corroborated by concrete API examples (client.graph.edge.delete) and admin CLI (zepctl) for data operations. Missing for 10: no documented data-retention policy/TTL configuration or independent third-party audit of deletion completeness.", "evidenceIds": [ "zep-docs-5", "zep-docs-35", "zep-docs-13", "zep-docs-21", "zep-docs-29", "zep-docs-7" ] }, { "productId": "zep", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry/usage-tracking opt-out settings for Zep itself; the closest topics are RTBF/user-deletion and access control, which do not cover product telemetry opt-out.", "evidenceIds": [] }, { "productId": "zep", "storyId": "prompt-ready-context-block", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Zep's docs describe a single-call 'Context Block' — via `scope=\"auto\"` graph search or returned directly from `thread.add_messages()` — that composes relevant edges, nodes, episodes, and summaries into one prompt-ready block, with sub-200ms retrieval regardless of graph size, and prior HN commentary confirms the long-term-state/token-count design intent. missing for 10: explicit documentation of a configurable token budget/limit parameter and independent hands-on benchmarking of the assembled context block's quality.", "evidenceIds": [ "zep-docs-10", "zep-docs-11", "zep-docs-26", "zep-docs-30", "zep-docs-34", "zep-docs-18", "zep-comm-4" ] }, { "productId": "zep", "storyId": "published-memory-benchmarks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of published benchmark results such as LongMemEval or LoCoMo, nor any recall-accuracy benchmark comparison; it only covers performance latency claims (sub-200ms retrieval) and general architecture/feature docs. missing for 10: any citation to LongMemEval/LoCoMo results, any third-party or first-party benchmark study, or recall-accuracy metrics.", "evidenceIds": [ "zep-docs-34", "zep-docs-28" ] }, { "productId": "zep", "storyId": "self-host-oss-deployment", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Zep explicitly offers a 'Your VPC' deployment option alongside cloud (zep-docs-17), and its underlying open-source engine Graphiti is pip-installable and runs locally with no API key (zep-probe-rt-3), showing some self-hostable OSS components exist. However, there is no Docker Compose file, self-hosting guide, or explicit instructions for standing up the full commercial 'memory layer' (threads/users/MCP server) on infrastructure a platform engineer controls — the evidence conflates the OSS Graphiti library with the hosted Zep service. Missing for 10: explicit Docker/self-host deployment docs for the full Zep memory service, confirmation that the main product (not just Graphiti) is open-source, and any hands-on report of a successful self-hosted deployment.", "evidenceIds": [ "zep-docs-17", "zep-probe-rt-3", "zep-gh-1" ] }, { "productId": "zep", "storyId": "session-summarization", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Zep documents thread summaries that are auto-generated and incrementally updated per thread, plus a Context Block retrievable via thread.add_messages()/get_user_context() combining summaries, edges, nodes, and episodes so an agent can resume with full prior context. This directly supports resuming past sessions/threads with sub-200ms retrieval performance documented. Missing for 10: independent hands-on verification of summary quality/accuracy over long conversations beyond vendor docs.", "evidenceIds": [ "zep-docs-8", "zep-docs-10", "zep-docs-11", "zep-docs-26", "zep-docs-30", "zep-docs-34", "zep-docs-20" ] }, { "productId": "zep", "storyId": "shared-team-memory", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Zep's governance model provides ABAC policies attached to API keys and UserGroups to limit which context/actions each agent or MCP user can reach, implying some ability to scope shared vs. private access, and user-level Context Graphs are the default privacy boundary (zep-docs-12, zep-docs-25, zep-docs-31, zep-docs-38). However, there is no explicit documentation of a mechanism for selectively sharing specific memory items across multiple users/agents as a 'team or group memory' feature—only access-control policies, not a shared-graph or memory-merging capability. Missing for 10: explicit team/group memory graph construct, documented API for cross-user memory sharing, and any example/tutorial showing selective sharing while isolating private data.", "evidenceIds": [ "zep-docs-12", "zep-docs-25", "zep-docs-31", "zep-docs-38", "zep-docs-19", "zep-docs-20" ] }, { "productId": "zep", "storyId": "temporal-fact-memory", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Zep's Context Graph is built on Graphiti, explicitly described as a temporal knowledge graph that 'tracks how facts change over time' and maintains provenance, directly matching the ml-engineer's need to distinguish current vs outdated facts; this is corroborated by GitHub docs, the Zep team's own explanation of moving from static fact extraction to a temporal graph to fix hallucination/staleness issues, and a runtime probe confirming the open-source Graphiti engine is real and installable. Missing for 10: explicit documentation of valid_at/invalid_at timestamp fields or an API example showing querying facts as of a point in time, and independent hands-on verification of temporal query accuracy.", "evidenceIds": [ "zep-gh-1", "zep-gh-2", "zep-gh-3", "zep-comm-1", "zep-probe-rt-3", "zep-docs-15" ] }, { "productId": "zep", "storyId": "transparent-usage-pricing", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The only pricing-related evidence (zep-docs-17) shows a 'Deploy anywhere / Cloud / Your VPC' snippet from the pricing page but contains no mention of a free tier, per-unit rates, or any cost figures that would let a platform engineer project costs. Missing for 10: published free-tier details, per-unit/usage pricing rates, and any cost calculator or tier breakdown.", "evidenceIds": [ "zep-docs-17" ] }, { "productId": "zep", "storyId": "voice-realtime-memory", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack shows generic 'agent framework' memory integrations, an MCP server, and a CLI, but contains no mention of LiveKit, Pipecat, ElevenLabs, or any real-time voice pipeline integration or documentation. This is an applicable axis for a memory SDK, but no supporting evidence exists.", "evidenceIds": [] } ]