[ { "productId": "gecko-security", "storyId": "agent-runs-scan-headless", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Gecko exposes a v1 API and webhooks that can trigger scans, pull machine-readable vulnerability data, triage findings, and file tickets in Jira/Linear/ClickUp/Shortcut, and an MCP/AI-tools connector for agents to do the same — satisfying the 'headless, parse, file issues' intent via API rather than a literal CLI. However, no dedicated CLI binary/command is documented anywhere in the evidence, and community reports raise concrete concerns about false positives (100% confidence findings that weren't real, missed real issues) undermining 'file issues for real findings'. Missing for 10: an actual CLI tool/subcommand documented, and independent confirmation that AI-driven ticket filing acts only on validated/real findings rather than the false positives community testers observed.", "evidenceIds": [ "gecko-security-docs-9", "gecko-security-docs-10", "gecko-security-docs-6", "gecko-security-docs-24", "gecko-security-comm-3", "gecko-security-comm-5" ] }, { "productId": "gecko-security", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Direct probes confirm Gecko serves both llms.txt (HTTP 200) and a docs.md aggregate at the docs root, plus an OpenAPI spec and documented MCP/AI-tool integration for agents to consume — exactly the agent-oriented docs surface the story asks for. Missing for 10: no independent/community confirmation that an agent actually consumed llms.txt successfully in practice.", "evidenceIds": [ "gecko-security-probe-1", "gecko-security-probe-2", "gecko-security-probe-3", "gecko-security-docs-9" ] }, { "productId": "gecko-security", "storyId": "agentic-ai-insights", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Gecko generates AI insights (severity, confidence score, call chain, PoC, suggested patch, AI-written repo wiki, endpoint map, PR review summaries) directly from scanned code data, per docs-2/17/18/19. However, hands-on community reports show these AI-generated findings can be inaccurate — 100%-confidence 'vulnerabilities' that weren't real, missed real issues a competing tool caught, and a trivial/irrelevant finding on another scan — directly contradicting the reliability of the insights. Missing for 10: independent validation of insight accuracy at scale, resolution of the false-positive/false-negative reports.", "evidenceIds": [ "gecko-security-docs-2", "gecko-security-docs-17", "gecko-security-docs-18", "gecko-security-docs-19", "gecko-security-comm-3", "gecko-security-comm-4", "gecko-security-comm-5" ] }, { "productId": "gecko-security", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Gecko documents autonomous background workflows: scheduled scans via the API ('manage scan schedules'), automatic PR-triggered scans with pass/fail checks, auto-fix PR generation and re-verification, ticket creation, and webhook events pushed to external systems as scans run — all without manual intervention once configured. This directly matches the 'automations that run autonomously in the background' story for a security-scanning product. missing for 10: independent/hands-on confirmation that scheduled scans and auto-fix loops run reliably unattended over time, and more detail on configuring recurring schedules beyond a brief API mention.", "evidenceIds": [ "gecko-security-docs-10", "gecko-security-docs-24", "gecko-security-docs-5", "gecko-security-docs-3", "gecko-security-docs-4", "gecko-security-docs-6", "gecko-security-docs-30" ] }, { "productId": "gecko-security", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Gecko is framed as an 'AI security engineer' that autonomously scans, triages, patches, and files tickets, so users can delegate security tasks (request fix, ticket creation, PR fixes) to its built-in AI (gecko-security-docs-3, gecko-security-docs-6, gecko-security-docs-15, gecko-security-probe-1). However, this is a fixed pipeline of security actions rather than a general-purpose in-product assistant you can converse with or direct on arbitrary tasks; community reports also raise doubts about the AI's accuracy in these delegated tasks (gecko-security-comm-3, gecko-security-comm-5). Missing for 10: an explicit conversational/task-delegation assistant UI, evidence of open-ended task delegation beyond the scan/fix/ticket workflow, and independent confirmation the delegated AI reliably completes tasks.", "evidenceIds": [ "gecko-security-docs-3", "gecko-security-docs-6", "gecko-security-docs-15", "gecko-security-probe-1", "gecko-security-comm-3", "gecko-security-comm-5" ] }, { "productId": "gecko-security", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Gecko exposes a v1 API to trigger scans and read/triage results, dedicated CI/CD-labeled API keys, PR-check integration with configurable fail-on-severity thresholds that post pass/fail status back to GitHub/GitLab, and webhooks for automation pipelines — all enabling headless operation in CI. Missing for 10: independent/hands-on verification that CI pipelines actually work end-to-end, and no explicit example CI configs (e.g., GitHub Actions YAML) are shown.", "evidenceIds": [ "gecko-security-docs-10", "gecko-security-docs-25", "gecko-security-docs-5", "gecko-security-docs-29", "gecko-security-docs-24", "gecko-security-probe-3" ] }, { "productId": "gecko-security", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All evidence describes Gecko exposing its own data via an MCP server so external AI tools/agents can call into Gecko (docs-9, probe-4) — the reverse relationship. There is no evidence that Gecko itself can act as an MCP client and plug in external MCP servers to use their tools.", "evidenceIds": [ "gecko-security-docs-9", "gecko-security-probe-4" ] }, { "productId": "gecko-security", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Gecko documents an official MCP/AI-tools integration (docs-9, probe-4) letting a connected AI agent list repositories/scans, pull full vulnerability evidence, triage findings, and file tickets, and the docs.md/llms.txt endpoints confirm agent-facing documentation exists. Missing for 10: independent or hands-on confirmation that the MCP server works as described, and details on setup/auth flow for connecting an external agent.", "evidenceIds": [ "gecko-security-docs-9", "gecko-security-probe-4", "gecko-security-probe-2" ] }, { "productId": "gecko-security", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Gecko documents an official MCP/AI-tools integration letting a connected AI assistant list repos/scans, pull vulnerability evidence, triage findings, and file tickets via natural language, plus a full v1 API and OpenAPI spec (docs-9, docs-10, probe-3, probe-4). However, there's no hands-on or independent evidence confirming the natural-language workflow actually works well end-to-end — community reports focus on scan accuracy issues rather than the NL/agent interface itself. Missing for 10: independent/hands-on verification of the MCP natural-language workflow, and detail on command coverage/limitations.", "evidenceIds": [ "gecko-security-docs-9", "gecko-security-docs-10", "gecko-security-probe-3", "gecko-security-probe-4" ] }, { "productId": "gecko-security", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of an official CLI; Gecko offers a GitHub App, REST API, webhooks, and an MCP server for AI tools, but nothing in the docs, changelog, or probes mentions a CLI tool for AI-native workflows.", "evidenceIds": [] }, { "productId": "gecko-security", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Gecko documents a public v1 API (trigger scans, read scan/repo/vulnerability data, triage findings, manage schedules, rotate tokens, webhooks) with a live OpenAPI spec confirmed by probe, plus API key management and per-request logging. missing for 10: independent third-party developer confirmation of API usage/reliability beyond docs and probes, and more detail on API rate limits/versioning stability.", "evidenceIds": [ "gecko-security-docs-10", "gecko-security-docs-11", "gecko-security-docs-24", "gecko-security-docs-25", "gecko-security-probe-3" ] }, { "productId": "gecko-security", "storyId": "agentic-scoped-keys", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Gecko documents API keys with named scopes, GitHub App 'scoped, auto-rotating access' and GitLab token rotation (docs-11, docs-25, docs-26, docs-12), which nominally support least-privilege credential issuance for connected tools/agents. However, community hands-on reports concretely contradict the least-privilege claim: the GitHub OAuth flow requests broad 'act on my behalf' and profile/email write permissions rather than fine-grained scopes, prompting explicit user pushback (comm-1, comm-7). Missing for 10: documented per-agent scope configuration, evidence that API keys/tokens can be restricted to specific actions/resources, and resolution of the OAuth over-permissioning complaint.", "evidenceIds": [ "gecko-security-docs-26", "gecko-security-docs-25", "gecko-security-docs-11", "gecko-security-docs-12", "gecko-security-comm-1", "gecko-security-comm-7" ] }, { "productId": "gecko-security", "storyId": "agentic-sdks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Gecko documents a v1 REST API with OpenAPI spec, API keys, and webhooks that developers can build against, but there is no evidence of official client SDK libraries (e.g., Python/JS packages) beyond the raw API/OpenAPI reference. missing for 10: dedicated SDK libraries/language clients, independent developer corroboration of building against them.", "evidenceIds": [ "gecko-security-docs-10", "gecko-security-docs-11", "gecko-security-probe-3" ] }, { "productId": "gecko-security", "storyId": "agentic-webhooks", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly describe webhook subscriptions for scan/finding lifecycle events (scan fail, critical finding) and the API reference lists webhook events as a first-class capability alongside the API. This directly matches the story's ask for event subscription via webhooks. Missing for 10: independent/hands-on confirmation that webhook delivery works reliably and no detail on payload schema/retry semantics.", "evidenceIds": [ "gecko-security-docs-24", "gecko-security-docs-10" ] }, { "productId": "gecko-security", "storyId": "ai-assisted-triage", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Docs describe confidence scores, severity, call chains, and PR-introduced vs pre-existing separation, plus an AI-tools API explicitly for triaging findings (gecko-security-docs-2, -9, -28), which maps to explaining/deduplicating/flagging false positives. However, hands-on community reports directly contradict the false-positive flagging claim: the tool assigned 100% confidence to two non-existent vulnerabilities and flagged an implausible 'RCE' in a CTF IRC bot script as high severity, while missing a real issue found by another scanner (gecko-security-comm-3, gecko-security-comm-5, gecko-security-comm-4). Missing for 10: independent validation that confidence scoring reliably suppresses false positives, and resolution of the reported inaccurate high-confidence findings.", "evidenceIds": [ "gecko-security-docs-2", "gecko-security-docs-9", "gecko-security-docs-28", "gecko-security-comm-3", "gecko-security-comm-5", "gecko-security-comm-4" ] }, { "productId": "gecko-security", "storyId": "ai-remediation-pr", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs describe a complete AI-drafted remediation PR workflow: 'Request fix' opens a gecko/* PR with the patch applied, fixes can commit directly to the PR's head branch, and Gecko rechecks and marks 'Fix verified' once merged and the vulnerability is resolved — squarely matching the ai-native review-and-merge story. Community reports raise concerns about finding accuracy/false positives, but do not contradict that the PR-drafting/remediation mechanism itself works as documented. Missing for 10: independent hands-on confirmation that the auto-generated patches are high quality and mergeable without heavy rework.", "evidenceIds": [ "gecko-security-docs-3", "gecko-security-docs-15", "gecko-security-docs-16", "gecko-security-docs-30", "gecko-security-docs-19", "gecko-security-comm-3" ] }, { "productId": "gecko-security", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While Gecko exposes an OpenAPI spec (gecko-security-probe-3) and documents API capabilities (gecko-security-docs-10), there is no evidence of an interactive API reference UI with runnable/try-it examples for developers or AI agents to explore.", "evidenceIds": [ "gecko-security-probe-3", "gecko-security-docs-10" ] }, { "productId": "gecko-security", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A probe confirms a live OpenAPI spec at gecko.security/openapi.json (HTTP 200, contains 'openapi' key), and docs describe a full v1 API for scans, vulnerabilities, webhooks, and integrations. Missing for 10: no independent/community confirmation of the spec's completeness or usage beyond the vendor's own probe.", "evidenceIds": [ "gecko-security-probe-3", "gecko-security-docs-10" ] }, { "productId": "gecko-security", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence describes scanning source repos/ZIP uploads and hybrid AWS deployment (scanner runs in customer's own account), but nothing explicitly describes a dedicated sandbox environment for safely testing exploits or PoCs without touching production data. missing for 10: explicit sandbox/isolated test environment description, confirmation that PoC execution or exploit verification never touches live production systems or data.", "evidenceIds": [ "gecko-security-docs-7", "gecko-security-docs-13", "gecko-security-docs-21" ] }, { "productId": "gecko-security", "storyId": "api-versioning-policy", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Gecko documents a versioned v1 API with an OpenAPI spec and a release/channel/version resolution endpoint (digest-pinned image refs), showing some notion of API versioning, but there is no documented deprecation policy, version sunset timeline, or migration guidance for API consumers. missing for 10: explicit deprecation policy, version support lifecycle/EOL dates, migration guides between API versions.", "evidenceIds": [ "gecko-security-docs-10", "gecko-security-docs-14", "gecko-security-docs-31", "gecko-security-probe-3" ] }, { "productId": "gecko-security", "storyId": "autofix-findings", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs describe a full remediation loop: suggested patches with 'Request fix' opening a gecko/* PR, ability to commit fixes directly to an existing PR's head branch, and automatic re-scan marking 'Fix verified' once resolved — this goes beyond read-only reporting into applied/automatic fixes. missing for 10: independent/hands-on confirmation that generated fixes actually work correctly (community evidence focuses on detection false positives, not fix application quality).", "evidenceIds": [ "gecko-security-docs-3", "gecko-security-docs-4", "gecko-security-docs-30", "gecko-security-docs-19", "gecko-security-docs-15", "gecko-security-docs-16" ] }, { "productId": "gecko-security", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The v1 API and AI-tool integration let a user programmatically list repositories/scans, pull vulnerability data, triage findings, and trigger scans across many items, which supports scripted bulk-style automation, but there is no documented bulk-fix, bulk-triage, or multi-select UI action for operating on many findings/repos simultaneously. missing for 10: a dedicated bulk-action endpoint or UI (e.g., bulk fix/triage/dismiss across many findings at once), independent evidence of bulk workflows in practice.", "evidenceIds": [ "gecko-security-docs-9", "gecko-security-docs-10", "gecko-security-docs-24" ] }, { "productId": "gecko-security", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Gecko supports some event-driven automation—webhooks that fire on scan completion/finding changes, configurable fail-on-severity thresholds for PR checks, and automatic ticket/ PR creation on findings—but these are fixed, product-defined triggers rather than a user-configurable rules engine where arbitrary conditions map to arbitrary actions. missing for 10: a general-purpose rule/condition builder, support for chaining multiple custom actions per event, and evidence of user-defined logic beyond severity thresholds.", "evidenceIds": [ "gecko-security-docs-24", "gecko-security-docs-29", "gecko-security-docs-6", "gecko-security-docs-3" ] }, { "productId": "gecko-security", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "The v1 API reference states it lets you 'manage scan schedules,' implying recurring scan jobs can be configured programmatically, but there is no dedicated doc page, UI walkthrough, or example showing how to set cadence, or evidence this extends to broader 'workflows' beyond scanning. missing for 10: dedicated scheduling documentation/UI, example of setting recurring cadence, evidence of scheduling beyond scans (e.g., recurring triage/ticket workflows), independent confirmation it works as claimed.", "evidenceIds": [ "gecko-security-docs-10" ] }, { "productId": "gecko-security", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Gecko Security is a vulnerability-scanning/security tool, not an automation-building or workflow tool; the evidence shows PR creation, fix verification, and ticket filing but nothing about versioning, reviewing, or rolling back 'automations' (i.e., the scanning/triage workflows themselves). No changelog history, version control, or rollback mechanism for the automations it runs is documented.", "evidenceIds": [ "gecko-security-docs-3", "gecko-security-docs-4", "gecko-security-docs-6" ] }, { "productId": "gecko-security", "storyId": "baseline-suppression", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show PR scans separate pre-existing findings from new ones so reviewers focus on what changed (docs-28), and the API supports 'triage findings' (docs-10), implying some noise-reduction workflow. However, there's no explicit documentation of a false-positive marking/ignore mechanism or a persistent baseline snapshot feature, and community reports flag inaccurate high-confidence findings (comm-3) without confirming a suppression workflow exists. missing for 10: explicit 'mark as false positive'/dismiss UI, documented baseline snapshotting across scans, independent confirmation that dismissed findings stay suppressed on rescans.", "evidenceIds": [ "gecko-security-docs-28", "gecko-security-docs-10", "gecko-security-docs-6", "gecko-security-comm-3" ] }, { "productId": "gecko-security", "storyId": "ci-pipeline-gate", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs describe PR-triggered scans that post a pass/fail check to GitHub/GitLab, with a configurable fail-on-severity threshold and separation of new vs pre-existing findings, which together enable failing a PR build on new findings. This matches the CI/PR-gating story closely, though it's implemented as a GitHub/GitLab check status rather than an explicit CLI exit code and lacks independent hands-on confirmation of build-failing behavior. Missing for 10: independent verification that the pass/fail check actually blocks merges/builds in practice, and explicit mention of CLI exit-code semantics for non-GitHub CI runners.", "evidenceIds": [ "gecko-security-docs-5", "gecko-security-docs-28", "gecko-security-docs-29", "gecko-security-docs-1" ] }, { "productId": "gecko-security", "storyId": "container-image-scan", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Gecko's documented capability is AI-driven source-code (SAST) analysis across languages like TypeScript, Python, Go, Java, etc., producing call-chain/PoC findings and PR fixes — no evidence describes scanning container images for OS-package (e.g., CVE feeds on apt/rpm packages) or layered image vulnerabilities. The only container-adjacent mention is a changelog entry resolving a version to a digest-pinned image_ref, which is about release artifact resolution, not vulnerability scanning of container layers.", "evidenceIds": [ "gecko-security-docs-20", "gecko-security-docs-14", "gecko-security-docs-31" ] }, { "productId": "gecko-security", "storyId": "cross-file-dataflow", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Docs explicitly claim 'compiler-accurate' cross-file analysis and 'full source-to-sink call chain' reporting, which directly describes taint-style detection across file/function boundaries (gecko-security-docs-2, gecko-security-docs-20). However, independent hands-on reports show concrete failures of this claim: 100%-confidence findings that were not real vulnerabilities, missed a real known issue flagged by another tool, and an inaccurate/irrelevant finding on a scanned project (gecko-security-comm-3, gecko-security-comm-4, gecko-security-comm-5). Missing for 10: independent verification of true-positive cross-file taint chains, resolution of the reported false positive/negative cases, and reproducible benchmark data.", "evidenceIds": [ "gecko-security-docs-2", "gecko-security-docs-20", "gecko-security-comm-3", "gecko-security-comm-4", "gecko-security-comm-5" ] }, { "productId": "gecko-security", "storyId": "custom-sast-rules", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Gecko Security is an AI-driven vulnerability scanner using compiler-accurate cross-file analysis, not a rule-based SAST engine; there is no evidence of a custom rule-writing syntax (e.g., YAML/DSL rules like Semgrep) or any mechanism to define and share such rules across repositories.", "evidenceIds": [ "gecko-security-docs-20", "gecko-security-docs-17" ] }, { "productId": "gecko-security", "storyId": "custom-secret-rules", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Gecko Security is a vulnerability-scanning tool focused on code security review (SAST-style findings, PR checks, fixes), and none of the evidence mentions custom secret/token pattern detection rules or any mechanism for defining organization-specific secret formats.", "evidenceIds": [] }, { "productId": "gecko-security", "storyId": "dependency-vulnerability-scan", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Gecko's documented capability is AI-driven source-code (SAST) analysis — call-chain tracing, endpoint mapping, PR-level code review — with no mention of scanning package manifests, lockfiles, or third-party dependency CVEs. Since Gecko is a security scanner, dependency/SCA scanning is a fair axis to ask about, but no evidence shows it exists.", "evidenceIds": [] }, { "productId": "gecko-security", "storyId": "detect-secrets-in-repo", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Gecko's documented capability is AI-driven vulnerability discovery (source-to-sink call chains, CVSS scoring, PoCs, patches) for application code, not secret/credential scanning or git-history scanning. Missing for 10: any mention of secret/credential detection, hardcoded key scanning, or scanning git commit history for leaked secrets.", "evidenceIds": [ "gecko-security-docs-2", "gecko-security-docs-17", "gecko-security-docs-20", "gecko-security-probe-1" ] }, { "productId": "gecko-security", "storyId": "fix-upgrade-guidance", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Gecko's documented capability is source-to-sink code vulnerability detection (SAST-style) with a generic 'Request fix' PR mechanism for findings it detects in application code; there is no evidence of dependency/package manifest scanning, CVE-to-package matching, or specific upgrade-path recommendations for vulnerable dependencies (SCA). Missing for 10: any mention of software composition analysis, dependency manifest parsing, package version resolution, or upgrade-path guidance tied to third-party library CVEs.", "evidenceIds": [ "gecko-security-docs-2", "gecko-security-docs-3", "gecko-security-docs-15", "gecko-security-docs-20" ] }, { "productId": "gecko-security", "storyId": "guardrails-for-agent-code", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Gecko documents an MCP/AI-tool integration that lets an agent list repos/scans, pull vulnerability evidence, triage findings, and file tickets, plus PR/CI checks that fail builds above a severity threshold — together enabling a coding-agent workflow to scan AI-generated code before merge. However, independent hands-on reports raise real accuracy concerns (false positives at 100% confidence, missed real bugs, UI/scan bugs), which undercuts confidence that the scanning is reliable enough for a pre-merge gate. Missing for 10: independent corroboration that the MCP/agent wiring works smoothly in practice, and evidence addressing the accuracy complaints specifically in an agent-driven pre-commit/pre-merge context.", "evidenceIds": [ "gecko-security-docs-9", "gecko-security-probe-4", "gecko-security-docs-5", "gecko-security-docs-29", "gecko-security-comm-3", "gecko-security-comm-4", "gecko-security-comm-5" ] }, { "productId": "gecko-security", "storyId": "iac-misconfig-scan", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Gecko's documented scanning scope is source-code vulnerability analysis (TypeScript, JavaScript, Python, Go, Java, Scala, C#, Rust) with call-chain/PoC findings; no evidence anywhere mentions Terraform, Kubernetes manifests, or IaC misconfiguration scanning. The axis is plausible for a security-scanning product, but no capability is evidenced, so this is 'none' rather than 'na'.", "evidenceIds": [ "gecko-security-docs-20", "gecko-security-docs-17", "gecko-security-docs-18" ] }, { "productId": "gecko-security", "storyId": "language-coverage", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs list a defined set of 'compiler-accurate' languages (TypeScript, JavaScript, Python, Go, Java, Scala, C#, Rust) supporting cross-file analysis, but there is no mention of package/dependency-ecosystem scanning (npm, pip, Maven, Cargo, etc.) or of broader language coverage (C/C++, Ruby, PHP, Kotlin, Swift). Community testing on a C project (unsupported language) found nothing, reinforcing that coverage is limited to the stated language list rather than 'all major languages and ecosystems.' Missing for 10: explicit package/dependency-ecosystem scanning support, broader language list, and independent verification of coverage breadth.", "evidenceIds": [ "gecko-security-docs-20", "gecko-security-comm-4" ] }, { "productId": "gecko-security", "storyId": "license-compliance-scan", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Gecko is positioned as an AI vulnerability/exploit scanner (SAST-style call-chain analysis, patch suggestions) with no mention anywhere in docs, changelog, or API reference of dependency license detection, SBOM generation, or license policy enforcement.", "evidenceIds": [] }, { "productId": "gecko-security", "storyId": "machine-readable-output", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Gecko's v1 API lets developers read scan, repository, and vulnerability data programmatically (implying JSON responses) and an OpenAPI spec is published, but there is no explicit documentation of a SARIF export format or dedicated 'JSON report' output for CI pipelines. Missing for 10: explicit SARIF format support, documented JSON report export command, and any CI-integration example showing machine-readable output being consumed downstream.", "evidenceIds": [ "gecko-security-docs-10", "gecko-security-probe-3", "gecko-security-docs-24" ] }, { "productId": "gecko-security", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Gecko exposes a documented v1 API (and OpenAPI spec) covering scans, repository/vulnerability data, triage, scheduling, token rotation and webhooks, plus an AI-tools/MCP-style integration that can list repos, pull evidence, triage, and file tickets — a substantial subset of UI functionality. However, there's no explicit claim or evidence that every UI feature (e.g., repository wiki generation, endpoint map, PR review summaries, initial GitHub App connection setup) is API-accessible, so full UI/API parity isn't demonstrated. Missing for 10: explicit parity statement or evidence that wiki/endpoint-map/PR-review generation and account/connection setup are API-triggerable, independent confirmation of API completeness.", "evidenceIds": [ "gecko-security-docs-10", "gecko-security-docs-9", "gecko-security-probe-3", "gecko-security-docs-17", "gecko-security-docs-18", "gecko-security-docs-19" ] }, { "productId": "gecko-security", "storyId": "openness-full-export", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The v1 API lets users read scan, repository, and vulnerability data programmatically (JSON via OpenAPI), which provides a technical path to export findings data, but there is no documented bulk 'export all data' feature, no mention of open-format guarantees for full account data, and no offboarding/exit workflow. missing for 10: explicit full-data export tooling, documented open-format (e.g. CSV/JSON) export guarantee, and account-closure/data-portability policy.", "evidenceIds": [ "gecko-security-docs-10", "gecko-security-probe-3" ] }, { "productId": "gecko-security", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence that Gecko Security's source code is published or licensed openly; it is a proprietary SaaS/security scanning product with documented APIs and hybrid deployment, but nothing indicates its codebase is open source.", "evidenceIds": [] }, { "productId": "gecko-security", "storyId": "openness-self-host", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm a 'hybrid' deployment where the scanner, S3 storage, and AI inference all run inside the customer's AWS account, and the pricing page lists an 'On-prem / self-hosted / private cloud' tier — both indicating genuine self-hosting options exist. However, the same hybrid docs state Gecko still hosts the dashboard, database, and scan orchestration, so the core control plane is not actually self-hostable in the standard offering, and no architecture/setup docs for the full on-prem tier are provided. Missing for 10: detailed on-prem deployment guide/architecture for the fully self-hosted tier, and confirmation that dashboard/orchestration can also run on customer infrastructure.", "evidenceIds": [ "gecko-security-docs-13", "gecko-security-docs-21", "gecko-security-docs-23" ] }, { "productId": "gecko-security", "storyId": "pr-comments-scm", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs describe a GitHub/GitLab App integration that posts pass/fail checks and PR security review summaries, separates new vs pre-existing findings, supports fail-on-severity thresholds, and can auto-commit fixes to the PR — a fairly complete PR-check/comment workflow (gecko-security-docs-5, -19, -28, -29, -30). Missing for 10: independent/hands-on confirmation that PR comments/checks actually render as described (community evidence only discusses finding accuracy, not the PR-check UX itself).", "evidenceIds": [ "gecko-security-docs-5", "gecko-security-docs-19", "gecko-security-docs-28", "gecko-security-docs-29", "gecko-security-docs-30" ] }, { "productId": "gecko-security", "storyId": "pre-commit-hook", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Gecko's documented workflow is GitHub App/PR-based scanning with pass/fail PR checks, not a local pre-commit git hook that blocks commits before they enter history — no evidence of a pre-commit hook, CLI hook script, or local git-hook integration exists in the evidence pack.", "evidenceIds": [] }, { "productId": "gecko-security", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Gecko offers a hybrid deployment where the scanner, S3 storage, and AI inference run inside the customer's own AWS account, and an on-prem/self-hosted/private cloud pricing tier, both of which let customers effectively control where their data resides. However, there's no explicit UI/setting for choosing a region for the Gecko-hosted dashboard/database components, and no documentation of specific supported regions or compliance certifications tied to residency. Missing for 10: explicit region-selection controls for Gecko-hosted components, list of supported regions, and residency/compliance documentation (e.g., GDPR/data-locality guarantees).", "evidenceIds": [ "gecko-security-docs-13", "gecko-security-docs-21", "gecko-security-docs-23" ] }, { "productId": "gecko-security", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data-training opt-out, AI training data policies, or any privacy controls governing whether user code/data is used to train AI models; Gecko's docs focus on scanning, deployment, and integrations, not this privacy axis.", "evidenceIds": [] }, { "productId": "gecko-security", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers scanning, PR fixes, integrations, hybrid deployment, and API/webhooks, but there is no mention of data retention policies, data deletion controls, or user ability to purge stored findings/scan data. Hybrid deployment keeps scanner/AI inference in the customer's AWS account, but this does not address retention/deletion of dashboard-hosted data. missing for 10: explicit data retention policy documentation, a deletion/purge mechanism or API, and any account/data-export or GDPR-style deletion workflow.", "evidenceIds": [ "gecko-security-docs-13", "gecko-security-docs-21" ] }, { "productId": "gecko-security", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack of a telemetry/usage-tracking opt-out setting, privacy policy control, or data-collection toggle; the docs focus on scanning, PR fixes, API keys, and deployment but never mention telemetry opt-out.", "evidenceIds": [] }, { "productId": "gecko-security", "storyId": "reachability-prioritization", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Gecko's docs describe source-to-sink call-chain analysis and reachability for vulnerabilities found in first-party code, but there is no evidence it ingests dependency/SCA alerts (e.g., known-CVE package advisories) or prioritizes them by reachability — it's positioned as a SAST-style code vulnerability scanner, not a dependency/SCA tool.", "evidenceIds": [ "gecko-security-docs-2", "gecko-security-docs-20", "gecko-security-docs-17", "gecko-security-docs-18" ] }, { "productId": "gecko-security", "storyId": "sast-code-scanning", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Gecko's docs describe compiler-accurate, cross-file static analysis with source-to-sink call chains, CVSS severity, and confidence scores that would encompass classes like SQLi/XSS (docs-2, docs-20), and it supports multiple languages. However, independent hands-on testing on Hacker News reports both missed real vulnerabilities and high-confidence false positives (comm-3, comm-4, comm-5), directly contradicting the accuracy implied by the docs. Missing for 10: independent benchmark data on injection/XSS detection rates, and resolution of the false-positive/false-negative reports.", "evidenceIds": [ "gecko-security-docs-2", "gecko-security-docs-20", "gecko-security-comm-3", "gecko-security-comm-4", "gecko-security-comm-5" ] }, { "productId": "gecko-security", "storyId": "sbom-generation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Gecko Security is an AI vulnerability scanner focused on code review, exploit findings, and auto-fix PRs; no evidence anywhere mentions generating an SBOM in SPDX or CycloneDX format for a project or container image.", "evidenceIds": [] }, { "productId": "gecko-security", "storyId": "scan-beyond-git", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gecko Security is a code-vulnerability/SAST scanner focused on repositories (Git, ZIP uploads, PR scanning) and application security analysis, not a secrets-detection tool for non-git artifacts like S3 buckets, Docker images, or CI logs. Nothing in the evidence pack mentions secret scanning of cloud storage, container images, or CI log artifacts — this is outside the product's category of static/dynamic vulnerability analysis on source code.", "evidenceIds": [] }, { "productId": "gecko-security", "storyId": "severity-filtering", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm a 'fail-on severity' threshold for PR checks (e.g., fail on High and above) and that findings carry both severity and confidence scores, supporting build-gating by severity. However, there's no documented mechanism to gate/filter specifically by confidence score, only severity threshold is described as configurable. Missing for 10: explicit confidence-based filtering/gating controls, and independent verification that severity gating reliably suppresses low-value noise.", "evidenceIds": [ "gecko-security-docs-2", "gecko-security-docs-29", "gecko-security-docs-28" ] }, { "productId": "gecko-security", "storyId": "verify-live-secrets", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gecko Security is a vulnerability/SAST scanner focused on code-flow analysis (source-to-sink, patches, PR checks); the evidence pack shows no secret-scanning or live-credential-verification capability at all — this axis belongs to a secrets-detection product category, not this AST/vulnerability scanner.", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "agent-runs-scan-headless", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Gitleaks CLI supports headless scanning of repos/dirs/stdin and can output baseline reports like findings.json, giving a machine-readable artifact an agent could parse; gitleaks-action also automates PR-based alerting. However there's no explicit documentation of a structured JSON/SARIF report flag, and no evidence of automated issue-filing (only PR comments/alerts to accounts, not issue tracker integration). Missing for 10: explicit machine-readable output format docs (e.g. --report-format json/sarif), and evidence of automated issue creation for real findings.", "evidenceIds": [ "gitleaks-gh-9", "gitleaks-gh-10", "gitleaks-gh-11", "gitleaks-gh-13", "gitleaks-docs-1", "gitleaks-docs-2", "gitleaks-docs-3" ] }, { "productId": "gitleaks", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe found no llms.txt file (404) and no agent-oriented docs/openapi endpoints, so there is no evidence Gitleaks provides machine-readable docs for AI agents.", "evidenceIds": [ "gitleaks-probe-1", "gitleaks-probe-2" ] }, { "productId": "gitleaks", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a rule-based secret-detection scanner with no evidence of AI-generated insights, summaries, or suggestions; probes for llms.txt and API/OpenAPI endpoints returned 404, and all documented features are deterministic regex/rule-based scanning, alerting, and CI integration.", "evidenceIds": [ "gitleaks-probe-1", "gitleaks-probe-2", "gitleaks-gh-5", "gitleaks-gh-17" ] }, { "productId": "gitleaks", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Gitleaks-action lets teams wire secret scanning to run automatically in the background on every push/PR (and pre-commit hooks trigger locally) without manual intervention, satisfying the 'runs autonomously' spirit of the story. Missing for 10: no evidence of scheduled/cron-based or event-driven automation beyond git hooks/CI triggers, and no AI-agent-specific orchestration or self-triggering capability.", "evidenceIds": [ "gitleaks-docs-1", "gitleaks-docs-5", "gitleaks-gh-4", "gitleaks-docs-7" ] }, { "productId": "gitleaks", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a secret-scanning CLI/CI tool, not an assistant-style product; there is no built-in AI assistant to delegate tasks to, and this is a category error for this kind of tool rather than a missing feature.", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Gitleaks is a CLI that scans repos, directories, or stdin, is installable via Homebrew/Docker/Go, and has an official GitHub Action (gitleaks-action) for automated CI scanning on PRs/commits with exit-code-based pass/fail reporting, making it well-suited for headless CI automation. Missing for 10: evidence of non-GitHub CI integrations (GitLab CI, Jenkins, CircleCI) and JSON/SARIF output examples for machine-readable pipeline consumption.", "evidenceIds": [ "gitleaks-gh-1", "gitleaks-gh-2", "gitleaks-gh-3", "gitleaks-gh-8", "gitleaks-docs-1", "gitleaks-docs-5", "gitleaks-gh-12" ] }, { "productId": "gitleaks", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a secret-scanning CLI/action, not an AI agent or MCP-capable client; no evidence relates to MCP servers or tool-plugging, so this agenticness axis is a category error for this product.", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "agentic-mcp-server", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a secret-scanning CLI/CI tool, not an agent, and there is no evidence of an official MCP server offering; this is a category mismatch as the product is a scanning tool rather than an agentic client or platform designed for agent connectivity.", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "agentic-nl-commands", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a CLI secret-scanning tool operated via fixed subcommands and flags, not through natural-language instructions; no evidence of NL interface, and the category (a static analysis scanner) does not conceivably support conversational/agentic command input as a core product axis.", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Gitleaks ships a first-party CLI (git, dir, stdin subcommands) that is well documented with usage examples and is the primary interface for the tool, making it fully scriptable/automatable for agentic workflows. Missing for 10: no explicit agent/LLM-oriented docs (e.g., llms.txt returns 404) or structured JSON-output guidance tailored to AI consumers.", "evidenceIds": [ "gitleaks-gh-1", "gitleaks-gh-2", "gitleaks-gh-3", "gitleaks-gh-17", "gitleaks-probe-3" ] }, { "productId": "gitleaks", "storyId": "agentic-public-api", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a CLI/scanner tool with no documented public API for programmatic/agentic access; probes confirm no llms.txt and no OpenAPI/swagger endpoint exist, and the CLI/GitHub Action docs don't constitute a public API.", "evidenceIds": [ "gitleaks-probe-1", "gitleaks-probe-2", "gitleaks-probe-3" ] }, { "productId": "gitleaks", "storyId": "agentic-scoped-keys", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a secret-detection CLI/scanner; it has no API or credential-issuing surface for agents, so scoped API credential provisioning is a category error for this product.", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "agentic-sdks", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a secret-scanning CLI/action, not an SDK-providing platform for building AI-native integrations; there's no evidence of, or plausible category fit for, official SDKs. This is a category mismatch rather than a missing capability.", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a CLI secret-scanning tool with CI integrations (GitHub Action, PR comments, alerts) but no evidence of a webhook subscription mechanism for events; probes for API/openapi specs returned 404s, and no webhook docs exist.", "evidenceIds": [ "gitleaks-probe-1", "gitleaks-probe-2", "gitleaks-docs-1", "gitleaks-docs-3" ] }, { "productId": "gitleaks", "storyId": "ai-assisted-triage", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any AI-based triage, explanation, deduplication, or false-positive flagging feature; Gitleaks only offers manual allowlists, baselines, and composite rules for reducing noise, none of which use AI.", "evidenceIds": [ "gitleaks-gh-20", "gitleaks-gh-6", "gitleaks-gh-13", "gitleaks-probe-1", "gitleaks-probe-2" ] }, { "productId": "gitleaks", "storyId": "ai-remediation-pr", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a secret-detection scanner (CLI/CI action) that flags leaked secrets; it has no evidence of AI-driven remediation PR drafting, which is a code-fix generation capability outside its category of tools like Copilot Autofix or Dependabot.", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a CLI secret-scanning tool with no interactive API reference; probes explicitly show no llms.txt and no OpenAPI/swagger spec found on gitleaks.io, and evidence only documents CLI commands, not a runnable API playground.", "evidenceIds": [ "gitleaks-probe-1", "gitleaks-probe-2" ] }, { "productId": "gitleaks", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a CLI secret-scanning tool with no API/web service, and direct probes confirm no OpenAPI or machine-readable spec exists at any expected location.", "evidenceIds": [ "gitleaks-probe-1", "gitleaks-probe-2" ] }, { "productId": "gitleaks", "storyId": "api-sandbox", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a secret-scanning CLI/CI tool, not an agentic AI service with sandbox/production environments to test against; the sandbox-vs-production distinction is a category error for this product type.", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a CLI/CI secret-scanning tool with no evidence of a public/versioned API; probes for OpenAPI spec and llms.txt both returned 404s, and there's no mention of an API deprecation policy anywhere in the evidence.", "evidenceIds": [ "gitleaks-probe-1", "gitleaks-probe-2" ] }, { "productId": "gitleaks", "storyId": "autofix-findings", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Gitleaks is purely a secret-detection tool that reports findings (via CLI, git hooks, or gitleaks-action PR comments); no evidence anywhere in the pack of suggested or automatic remediation/fixes for detected secrets—only detection, alerting, allowlisting, and baseline suppression are described.", "evidenceIds": [ "gitleaks-gh-17", "gitleaks-gh-20", "gitleaks-docs-1", "gitleaks-gh-13" ] }, { "productId": "gitleaks", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Gitleaks natively scans across many items at once — entire directories/file trees via the `dir` command and full git commit history via `git log -p` with the `git` command, and gitleaks-action can scan all PRs/commits automatically in CI. However, there's no evidence of a broader batch/programmatic API for bulk operations beyond directory/repo scanning (e.g., processing many repos, bulk config management, or structured bulk output for agentic pipelines). Missing for 10: multi-repo/bulk-target orchestration, programmatic batch API beyond CLI file/dir args, and evidence of AI-native bulk workflows.", "evidenceIds": [ "gitleaks-gh-1", "gitleaks-gh-9", "gitleaks-gh-2", "gitleaks-gh-10", "gitleaks-docs-5", "gitleaks-docs-1" ] }, { "productId": "gitleaks", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Gitleaks supports custom detection rules (regex-based) and can trigger automatic actions on events—pre-commit hook blocks commits containing secrets, and gitleaks-action runs on GitHub push/PR events to alert accounts and comment on PRs. However this is limited to secret-scanning triggers tied to git/CI events, not a general-purpose rule/action automation engine. Missing for 10: evidence of arbitrary event types or custom action definitions beyond GitHub-specific alerting/commenting, and no AI-native/agentic trigger framework.", "evidenceIds": [ "gitleaks-gh-4", "gitleaks-gh-5", "gitleaks-docs-1", "gitleaks-docs-3", "gitleaks-docs-2", "gitleaks-gh-12" ] }, { "productId": "gitleaks", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Gitleaks integrates with git hooks and GitHub Actions triggered by commits/PRs, but nothing describes scheduling recurring/cron-based scans or workflows as a distinct capability. The axis applies (CI security tools commonly support scheduled scans) but no evidence confirms this for Gitleaks.", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a secret-detection scanner, not an automation/workflow builder; it has no concept of 'automations' to version, review, or roll back — this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "baseline-suppression", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Gitleaks explicitly supports baseline files via --baseline-path so only new findings appear in reports, and allowlists let users suppress known false positives per-rule. This directly matches the triage-noise story of ignoring existing/false-positive findings so only new issues alert. Missing for 10: no evidence of a centralized/dashboard-based suppression workflow or independent third-party validation of baseline accuracy at scale.", "evidenceIds": [ "gitleaks-gh-6", "gitleaks-gh-13", "gitleaks-gh-16", "gitleaks-gh-20" ] }, { "productId": "gitleaks", "storyId": "ci-pipeline-gate", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Gitleaks-Action enables scanning on GitHub PR workflows with alerts/comments, and the CLI's detect command exits non-zero on findings (shown by failed commit hook example), which is the mechanism CI systems use to fail builds. missing for 10: explicit documentation of CI-specific exit code semantics (e.g., non-zero exit on new findings vs baseline) and non-GitHub CI examples (GitLab/Jenkins) beyond the GitHub Action.", "evidenceIds": [ "gitleaks-docs-1", "gitleaks-docs-5", "gitleaks-gh-12", "gitleaks-gh-13", "gitleaks-gh-16" ] }, { "productId": "gitleaks", "storyId": "container-image-scan", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gitleaks is purely a secret-detection tool for git repos, files, and stdin; it has no functionality for scanning container images for OS-package or application vulnerabilities, which is entirely outside its product category (that's the domain of tools like Trivy/Grype).", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "cross-file-dataflow", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a regex/entropy-based secret-detection tool, not a SAST/taint-analysis engine; it has no concept of data-flow tracking across files or function boundaries. This story targets a different product category (SAST tools), so the axis does not apply.", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "custom-sast-rules", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Gitleaks documents a TOML-based config format for writing custom secret-detection rules, including composite rules and allowlists, and this config can be centralized as a `gitleaks.toml` or referenced via `GITLEAKS_CONFIG` so it can be reused across repos (e.g. gitleaks-action auto-detecting it). This is more of a config-file sharing mechanism than a purpose-built 'rule sharing' feature, and there's no independent corroboration of ease-of-use or a rule marketplace/registry. Missing for 10: independent hands-on evidence of rule syntax simplicity, evidence of a shared rule registry/marketplace, and confirmation of seamless cross-repo rule versioning.", "evidenceIds": [ "gitleaks-gh-5", "gitleaks-gh-7", "gitleaks-gh-14", "gitleaks-gh-20", "gitleaks-docs-11", "gitleaks-docs-13" ] }, { "productId": "gitleaks", "storyId": "custom-secret-rules", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Gitleaks documents a configuration format for writing custom secret-detection rules, plus advanced features like composite rules and allowlists to reduce false positives, directly enabling org-specific token/secret pattern definitions. Missing for 10: no independent hands-on example of a custom rule being validated in production, and no dedicated deep-dive docs beyond the config-format mention.", "evidenceIds": [ "gitleaks-gh-5", "gitleaks-gh-7", "gitleaks-gh-14", "gitleaks-gh-20", "gitleaks-gh-21" ] }, { "productId": "gitleaks", "storyId": "dependency-vulnerability-scan", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a secret-detection tool (scanning for hardcoded passwords, API keys, tokens) and has no dependency/SCA vulnerability scanning capability; this axis belongs to a different product category (SCA tools like Dependabot/Snyk).", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "detect-secrets-in-repo", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Gitleaks' `git` command explicitly scans full git history via `git log -p`, and `dir`/`stdin` commands cover directory/file/stdin scanning for secrets like passwords, API keys, and tokens, with hands-on example output showing detection working. missing for 10: no independent third-party benchmark of detection accuracy across large real-world repos.", "evidenceIds": [ "gitleaks-gh-9", "gitleaks-gh-1", "gitleaks-gh-17", "gitleaks-gh-12", "gitleaks-gh-10" ] }, { "productId": "gitleaks", "storyId": "fix-upgrade-guidance", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a secrets-detection tool that scans for hardcoded credentials; it does not analyze software dependencies for known vulnerabilities (SCA) and has no concept of dependency upgrade paths or fix PRs. This story belongs to the dependency-SCA product category, not a secrets scanner.", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "guardrails-for-agent-code", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Gitleaks offers generic mechanisms (pre-commit hook script, CLI `dir`/`git`/`stdin` scanning, CI action) that a user could wire into an AI coding agent's workflow to scan generated code before it lands, but there is no documented agent-specific integration (no MCP server, no Claude Code/Cursor plugin, no explicit guidance for AI-native pipelines). missing for 10: dedicated AI-agent integration docs, MCP/agent-hook examples, evidence of use specifically for AI-generated code review.", "evidenceIds": [ "gitleaks-gh-4", "gitleaks-gh-11", "gitleaks-gh-10", "gitleaks-docs-1" ] }, { "productId": "gitleaks", "storyId": "iac-misconfig-scan", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a secrets-detection tool focused on finding hardcoded credentials in code, not an IaC misconfiguration scanner (e.g., checking Terraform/Kubernetes for insecure settings); this is a wrong axis for this product category.", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "language-coverage", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Gitleaks scans git repos, directories/files, and stdin using regex-based pattern matching, which is inherently language-agnostic and works on any text-based source file or config regardless of the language or package ecosystem in use. However, there is no explicit evidence of ecosystem-specific coverage claims, testing across specific language/package manager formats, or documentation asserting broad multi-language validation. missing for 10: explicit language/ecosystem coverage claims, evidence of testing across specific ecosystems (npm, pip, maven, etc.), independent corroboration of coverage breadth.", "evidenceIds": [ "gitleaks-gh-1", "gitleaks-gh-2", "gitleaks-gh-3", "gitleaks-gh-17", "gitleaks-gh-10", "gitleaks-gh-11" ] }, { "productId": "gitleaks", "storyId": "license-compliance-scan", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a secrets-detection tool focused on scanning for hardcoded credentials; it has no SBOM/dependency inventory or license-scanning capability, so license policy enforcement is outside its product category.", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "machine-readable-output", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Evidence shows Gitleaks writes findings to a JSON report file (e.g., findings.json via --baseline-path) confirming JSON output support, but there is no mention in the evidence pack of SARIF or other machine-readable formats. missing for 10: explicit SARIF output support, documentation of --report-format flag or other structured formats, independent confirmation of downstream tooling integration.", "evidenceIds": [ "gitleaks-gh-13", "gitleaks-gh-16" ] }, { "productId": "gitleaks", "storyId": "openness-api-parity", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a CLI/GitHub Action tool with no web UI or API surface to compare against; probes confirm no OpenAPI/llms.txt exposed, and all documented interaction is via CLI commands or CI action config, not a UI/API pair.", "evidenceIds": [ "gitleaks-probe-1", "gitleaks-probe-2", "gitleaks-probe-3" ] }, { "productId": "gitleaks", "storyId": "openness-full-export", "verdict": "na", "quality": 0, "confidence": "medium", "rationale": "Gitleaks is a local/CLI secret-scanning tool with no proprietary data store or SaaS account holding user data — findings and configs are already local, open-format files (JSON/TOML), so there is no vendor lock-in or 'export and leave' scenario to evaluate; this axis is a category error for this kind of tool.", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "openness-open-license", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Gitleaks' source code is openly hosted and directly referenced across many citations (e.g. gitleaks-gh-1, gitleaks-probe-3), confirming it is a publicly readable GitHub repository that anyone, including AI-native users, can inspect. However, the evidence pack never explicitly cites a license file or license name (e.g. MIT), so full confirmation of an 'open license' designation is inferred rather than directly documented. Missing for 10: an explicit license statement/citation (e.g. LICENSE file content) and independent corroboration of the license terms.", "evidenceIds": [ "gitleaks-gh-1", "gitleaks-probe-3", "gitleaks-gh-8" ] }, { "productId": "gitleaks", "storyId": "openness-self-host", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Gitleaks is a fully open-source CLI (installable via Homebrew, Docker, or Go) that runs entirely locally against git repos, directories, or stdin, requiring no cloud service — a clear self-hostable core product. Missing for 10: no explicit self-hosting deployment guide (e.g., server/container orchestration docs) beyond basic install instructions.", "evidenceIds": [ "gitleaks-gh-8", "gitleaks-gh-1", "gitleaks-gh-2", "gitleaks-gh-3", "gitleaks-gh-17" ] }, { "productId": "gitleaks", "storyId": "pr-comments-scm", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Gitleaks-Action explicitly integrates into GitHub workflows and uses the GitHub API to post PR comments/reviews when secrets are detected, and it can also fail CI checks on findings; it's a well-documented first-party GitHub Action. Missing for 10: GitLab-specific MR comment/check integration is not evidenced (only GitHub is covered), and no independent/hands-on confirmation of the PR-comment behavior beyond vendor docs.", "evidenceIds": [ "gitleaks-docs-1", "gitleaks-docs-2", "gitleaks-docs-6", "gitleaks-docs-9", "gitleaks-docs-3", "gitleaks-docs-4" ] }, { "productId": "gitleaks", "storyId": "pre-commit-hook", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Gitleaks documents running as a pre-commit hook via the example pre-commit.py script copied into .git/hooks, and demonstrates it blocking a commit containing a secret (\"git commit ... Detect hardcoded secrets.................................................Failed\"), confirming secrets are caught before entering history. Missing for 10: no mention of official pre-commit-framework (.pre-commit-hooks.yaml) integration or independent hands-on confirmation beyond the vendor repo.", "evidenceIds": [ "gitleaks-gh-4", "gitleaks-gh-12", "gitleaks-gh-15" ] }, { "productId": "gitleaks", "storyId": "privacy-data-residency", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a local/CI CLI tool that scans repos and files on the user's own infrastructure; it does not store or host user data in any cloud service, so data residency/region selection is not a meaningful axis for this product type.", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a secret-scanning CLI/CI tool with no AI model training or data-usage policy dimension; preventing AI-training use of user data is not an applicable axis for this product category.", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "privacy-retention-controls", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a local/CI scanning CLI (and GitHub Action) that does not collect or retain user data on vendor servers; there is no data-retention/deletion surface to control, so this privacy-posture axis does not apply to this product category.", "evidenceIds": [ "gitleaks-gh-1", "gitleaks-gh-2", "gitleaks-gh-3", "gitleaks-docs-1" ] }, { "productId": "gitleaks", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry collection or an opt-out mechanism for Gitleaks or gitleaks-action; the docs cover scanning commands, CI integration, and config but never mention data collection or privacy controls.", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "reachability-prioritization", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a secrets-detection tool that scans git history, files, and streams for hardcoded credentials; it has no dependency/SCA vulnerability scanning or code-reachability analysis capability. Reachability-based prioritization of dependency alerts is a different product category (SCA tooling) and is a category error for this product.", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "sast-code-scanning", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gitleaks is exclusively a secrets-detection tool (passwords, API keys, tokens); it does not perform SAST for vulnerability classes like injection or XSS, which is an entirely different product category.", "evidenceIds": [ "gitleaks-gh-17" ] }, { "productId": "gitleaks", "storyId": "sbom-generation", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Gitleaks is a secret-scanning tool; nothing in its evidence relates to SBOM generation (SPDX/CycloneDX), which is outside its product category entirely.", "evidenceIds": [] }, { "productId": "gitleaks", "storyId": "scan-beyond-git", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Gitleaks supports scanning git repos, directories/files, and stdin, but there is no evidence of native support for scanning S3 buckets, Docker images, or CI logs as distinct source types; stdin streaming could theoretically pipe in arbitrary data but no documentation or examples confirm scanning these non-git sources.", "evidenceIds": [ "gitleaks-gh-1", "gitleaks-gh-2", "gitleaks-gh-3", "gitleaks-gh-17" ] }, { "productId": "gitleaks", "storyId": "severity-filtering", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Gitleaks evidence shows allowlists to suppress false positives and baseline diffing to ignore old findings, but there is no mention of a severity or confidence scoring/classification system, nor any flag to gate builds based on such levels. This axis is a fair ask for a secret-scanning tool but the evidence pack shows no severity/confidence triage mechanism.", "evidenceIds": [ "gitleaks-gh-20", "gitleaks-gh-6", "gitleaks-gh-13" ] }, { "productId": "gitleaks", "storyId": "verify-live-secrets", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack indicates Gitleaks or gitleaks-action performs live verification of detected secrets against the issuing service (e.g., checking if an API key is still active); its capabilities are limited to detection, allowlisting, baselines, and PR alerting.", "evidenceIds": [] }, { "productId": "semgrep", "storyId": "agent-runs-scan-headless", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Semgrep's CLI clearly supports headless, unauthenticated scanning (`semgrep ci`/`semgrep scan`, no login required, --dry-run) and machine-readable output via SARIF export, which an agent could parse. Community evidence (semgrep-comm-14) shows this pattern used in practice — SARIF output plus LLM triage to surface real findings for human review at Microsoft. Docs also mention 'send alerts/notifications or create tickets to track findings' for Semgrep Code, but this is vague and tied to the platform tier rather than a demonstrated CLI-to-issue-tracker pipeline. Missing for 10: explicit documented JSON/SARIF-to-ticket automation workflow, first-party GitHub/Jira issue-filing integration for CLI findings, and independent hands-on confirmation that filed issues correspond only to 'real' (non-false-positive) findings given community complaints about false positives (semgrep-comm-11).", "evidenceIds": [ "semgrep-docs-19", "semgrep-docs-27", "semgrep-docs-33", "semgrep-docs-34", "semgrep-docs-3", "semgrep-docs-20", "semgrep-docs-22", "semgrep-docs-30", "semgrep-comm-14" ] }, { "productId": "semgrep", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Semgrep hosts a working llms.txt at docs.semgrep.dev/llms.txt (HTTP 200, confirmed by probe) plus per-page .md agent-friendly docs (e.g. semgrep-docs-49, semgrep-docs-50), showing an agent can be pointed directly at machine-readable docs. missing for 10: independent/community confirmation that agents actually consume this llms.txt successfully, and no evidence of a broader agent-oriented docs index beyond the single llms.txt file.", "evidenceIds": [ "semgrep-probe-1", "semgrep-docs-49", "semgrep-docs-50", "semgrep-probe-4" ] }, { "productId": "semgrep", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Semgrep Code's 'AI-Powered Detection' and 'Semgrep Assistant (AI)' features are documented as generating insights (identifying complex business-logic flaws like IDORs, reducing false-positive noise by ~20%, combining AI reasoning with rule-based analysis for triage/remediation), which matches the story of in-product AI-generated insights. However, independent corroboration is thin — the one relevant community datapoint (semgrep-comm-14) describes a third party building their own external LLM triage layer on top of Semgrep's SARIF output rather than validating Semgrep Assistant itself, and another (semgrep-comm-13) notes an LLM hallucinating semgrep syntax when asked to help write rules, which is tangential. missing for 10: independent/hands-on validation of Semgrep Assistant's own AI suggestions, more detail on scope/accuracy of AI-Powered Detection, and confirmation these AI features are broadly available rather than gated/beta.", "evidenceIds": [ "semgrep-docs-6", "semgrep-gh-3", "semgrep-docs-48", "semgrep-docs-7", "semgrep-comm-14" ] }, { "productId": "semgrep", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Semgrep supports automation via CI triggers on push/PR events, pre-commit hooks, and bulk 'Managed Scans' that run without manual invocation, and it has an AI Assistant/AI-Powered Detection layer for triage — providing real background automation. However, this is rule-based scan automation, not an autonomous agent that independently plans/executes multi-step actions; there's no evidence of scheduled/cron-like agentic orchestration beyond scan-on-event triggers. Missing for 10: evidence of autonomous multi-step agentic decision-making, scheduling/orchestration beyond CI-triggered scans, and independent confirmation that AI-driven remediation runs unattended.", "evidenceIds": [ "semgrep-docs-25", "semgrep-docs-28", "semgrep-docs-15", "semgrep-docs-16", "semgrep-docs-46", "semgrep-docs-6", "semgrep-gh-3", "semgrep-docs-48" ] }, { "productId": "semgrep", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Semgrep ships 'Semgrep Assistant', a built-in AI feature that performs AI-powered detection of complex logic flaws, contextual post-processing to reduce noise, and combines AI reasoning with rule-based analysis for detection/triage/remediation, which functions as a task-delegable AI assistant within the product. However, this is scoped to security triage/detection only, not a general-purpose conversational assistant, and there's no independent hands-on account confirming quality of delegation. Missing for 10: evidence of a general chat/agent interface for arbitrary task delegation, independent/hands-on validation of Assistant's outputs, and detail on how users interact with/invoke it beyond marketing copy.", "evidenceIds": [ "semgrep-docs-6", "semgrep-gh-3", "semgrep-docs-47", "semgrep-docs-48" ] }, { "productId": "semgrep", "storyId": "agentic-headless", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Semgrep has a well-documented CLI (`semgrep ci`, `semgrep scan`) designed for headless/CI use, with dedicated CI provider integrations, pre-commit hooks, SARIF export, no-login/dry-run modes, and an API for programmatic access to findings—clearly supporting automated, non-interactive pipelines. Independent evidence (Zulip, Microsoft SARIF+LLM triage pipeline) confirms real-world CI/automation usage at scale. Missing for 10: no first-party benchmark of CI runtime performance at scale beyond anecdotal community complaints about slowness.", "evidenceIds": [ "semgrep-docs-19", "semgrep-docs-14", "semgrep-docs-25", "semgrep-docs-45", "semgrep-docs-16", "semgrep-docs-33", "semgrep-docs-34", "semgrep-docs-20", "semgrep-docs-59", "semgrep-comm-14", "semgrep-comm-7" ] }, { "productId": "semgrep", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Semgrep exposes itself as an MCP server (semgrep-probe-3) so other agents can call its scanning tools, but there is no evidence that Semgrep itself can consume/plug in external MCP servers to use their tools — no client-side MCP integration is documented anywhere in the docs, CLI reference, or platform API pages.", "evidenceIds": [ "semgrep-probe-3", "semgrep-docs-59", "semgrep-docs-17" ] }, { "productId": "semgrep", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 6, "confidence": "medium", "rationale": "Semgrep is not itself an agent, so a first-party MCP server for connecting external agents is a fair axis, and evidence confirms one exists at docs.semgrep.dev/mcp. However, the evidence pack only shows the URL's existence via a probe with no detail on the tools/capabilities exposed or independent corroboration. Missing for 10: detailed documentation of what MCP tools/resources are exposed, setup instructions, and independent/hands-on confirmation of usage.", "evidenceIds": [ "semgrep-probe-3" ] }, { "productId": "semgrep", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Semgrep is fundamentally a CLI/rule-based SAST tool; there's no documented natural-language command interface for the core product. The only relevant hook is a documented MCP server (semgrep-probe-3) which would let an AI agent invoke Semgrep via natural language, plus marketing mentions of 'AI reasoning' combined with rule-based detection (semgrep-docs-47/48), but no detail on what commands/actions are exposed or how they map from NL input.\nmissing for 10: documented list of MCP tools/actions, evidence of actual natural-language command usage, hands-on confirmation that AI-native NL control works end-to-end.", "evidenceIds": [ "semgrep-probe-3", "semgrep-docs-47", "semgrep-docs-48", "semgrep-gh-3" ] }, { "productId": "semgrep", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Semgrep ships a well-documented, installable official CLI (pipx/uv install, `semgrep scan`/`semgrep ci`, SARIF export, autofix, dry-run, no-login mode) that is fully scriptable and thus usable by AI agents, and this is corroborated by community usage reports and a dedicated probe confirming the CLI docs page. Missing for 10: explicit documentation of machine-readable (e.g., JSON) output tailored for programmatic/agent consumption and independent evidence of AI agents specifically driving the CLI.", "evidenceIds": [ "semgrep-docs-26", "semgrep-docs-27", "semgrep-docs-19", "semgrep-docs-33", "semgrep-docs-34", "semgrep-docs-20", "semgrep-probe-4", "semgrep-comm-7" ] }, { "productId": "semgrep", "storyId": "agentic-public-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Semgrep AppSec Platform documents a public API for listing deployments, findings, and projects, and the CLI itself is scriptable for automation, but there is no OpenAPI/swagger spec discoverable (probe returned 404s on all candidate paths) and no independent/community corroboration of programmatic API usage. missing for 10: discoverable OpenAPI/schema documentation, broader API surface details (write operations, rate limits), and independent evidence of third-party integrations built against the API.", "evidenceIds": [ "semgrep-docs-17", "semgrep-docs-59", "semgrep-probe-2" ] }, { "productId": "semgrep", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Semgrep AppSec Platform has an API for listing deployments/findings/projects, but there is no mention of scoped or least-privilege API tokens/credentials designed for agent use, nor any granular permission/role system for API keys.", "evidenceIds": [ "semgrep-docs-17", "semgrep-docs-59" ] }, { "productId": "semgrep", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows Semgrep offers a CLI, a REST API for the AppSec Platform, and an MCP server, but nowhere is an official client SDK (e.g., Python/JS/Go library for programmatic integration) documented or referenced.", "evidenceIds": [ "semgrep-docs-17", "semgrep-docs-59", "semgrep-probe-2", "semgrep-probe-4" ] }, { "productId": "semgrep", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Semgrep provides a REST API for querying findings/deployments and CI-integration on push/PR events, but nothing describes a webhook subscription mechanism for events; no docs mention webhook endpoints or event-driven notifications to external systems.", "evidenceIds": [ "semgrep-docs-17", "semgrep-docs-59", "semgrep-docs-25" ] }, { "productId": "semgrep", "storyId": "ai-assisted-triage", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Semgrep ships a native AI feature (Semgrep Assistant) explicitly described as 'contextual post-processing of findings...to further reduce noise by ~20%', and marketing docs claim combining 'AI reasoning with rule-based analysis for detection, triage, and remediation,' plus AI-powered detection for complex logic flaws — directly supporting AI-assisted false-positive reduction and triage. However, deduplication specifically is never mentioned, and the strongest independent evidence of AI triage in practice comes from a third-party tool built on Semgrep's SARIF output plus external LLMs (used at Microsoft) rather than the native Assistant, suggesting the first-party AI-triage feature is thin on independent corroboration. Missing for 10: explicit deduplication capability, deeper documentation of Assistant's explanation/false-positive-flagging behavior, independent hands-on evidence validating the native AI triage feature itself.", "evidenceIds": [ "semgrep-gh-3", "semgrep-docs-6", "semgrep-docs-48", "semgrep-comm-14" ] }, { "productId": "semgrep", "storyId": "ai-remediation-pr", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Semgrep offers rule-defined autofix (--autofix), AI-powered detection/triage/remediation messaging ('Combine AI reasoning with rule-based analysis for detection, triage, and remediation'), and GitHub PR comment automation, but there is no documented evidence of an end-to-end AI-drafted pull request workflow for remediation that a user reviews and merges. missing for 10: evidence of automatic PR/MR creation with AI-generated code fixes, a documented workflow showing AI drafting a remediation diff as a mergeable PR, and independent/hands-on confirmation of this remediation-PR feature working.", "evidenceIds": [ "semgrep-docs-38", "semgrep-docs-48", "semgrep-docs-57", "semgrep-docs-10" ] }, { "productId": "semgrep", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Semgrep documents an AppSec Platform API (list deployments, findings, projects) but there is no evidence of an interactive API reference with runnable/try-it examples; the openapi.json/swagger probes returned 404 on all candidate paths, indicating no interactive spec-based docs.", "evidenceIds": [ "semgrep-docs-17", "semgrep-docs-59", "semgrep-probe-2" ] }, { "productId": "semgrep", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Semgrep documents a REST API for the AppSec Platform (semgrep-docs-17/59) but a direct probe for machine-readable OpenAPI/Swagger specs at standard paths returned 404 everywhere (semgrep-probe-2), and no evidence of a published API spec exists elsewhere in the pack.", "evidenceIds": [ "semgrep-probe-2", "semgrep-docs-17", "semgrep-docs-59" ] }, { "productId": "semgrep", "storyId": "api-sandbox", "verdict": "na", "quality": 0, "confidence": "medium", "rationale": "Semgrep is a static code analysis/SAST tool that scans source repositories and dependency manifests, not a runtime agent that interacts with live production systems or data; the sandbox-vs-production distinction is not a meaningful axis for this kind of product.", "evidenceIds": [] }, { "productId": "semgrep", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Semgrep documents an AppSec Platform API (semgrep-docs-17/59) but no evidence pack item mentions API versioning scheme or a documented deprecation policy; an OpenAPI spec probe returned 404s (semgrep-probe-2), suggesting no formal API contract is published. missing for 10: versioning scheme documentation, deprecation policy/notice process, changelog or migration guides for API changes.", "evidenceIds": [ "semgrep-docs-17", "semgrep-docs-59", "semgrep-probe-2" ] }, { "productId": "semgrep", "storyId": "autofix-findings", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Semgrep supports rule-defined autofix via the --autofix flag, letting developers apply suggested fixes directly to files (semgrep-docs-38, semgrep-docs-53), and AppSec Platform mentions 'remediation' and 'fix AI-generated code' (semgrep-docs-45, semgrep-docs-47). However, autofix depends on individual rules defining a fix and is not universal across all findings, and there's no independent/community evidence confirming real-world reliability of autofix. missing for 10: community/hands-on validation of autofix accuracy, evidence of coverage breadth (what % of rules ship with fixes), and any first-party numbers on fix acceptance rates.", "evidenceIds": [ "semgrep-docs-38", "semgrep-docs-53", "semgrep-docs-45", "semgrep-docs-47", "semgrep-docs-48" ] }, { "productId": "semgrep", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Semgrep supports bulk-style operations such as scanning an entire codebase/monorepo in one command, cross-file/interfile analysis across many files, and 'Managed Scans' explicitly designed to 'bulk onboard and scan your repositories without requiring changes to your CI.' The AppSec Platform API also lets you programmatically list deployments/projects/findings across an organization. However, there is no documented bulk-action tooling (e.g., bulk triage, bulk rule application, batch API calls) beyond onboarding/scanning, so the story is only partially covered. Missing for 10: explicit bulk API endpoints for mass finding triage/remediation, evidence of batch operations beyond scanning/onboarding, and independent confirmation of large-scale bulk workflows in practice.", "evidenceIds": [ "semgrep-docs-15", "semgrep-docs-1", "semgrep-docs-49", "semgrep-docs-17", "semgrep-docs-59", "semgrep-docs-2" ] }, { "productId": "semgrep", "storyId": "automation-rules-engine", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Semgrep rules can be configured to run automatically on events (push, PR/MR, pre-commit) and trigger downstream actions such as PR comments, alerts/notifications, ticket creation, and autofix application — all documented as first-class features (semgrep-docs-25/28/16/46/22/30/38/53/57). This directly matches the story of rules triggering automated actions on events.\nmissing for 10: independent/hands-on evidence confirming these automated trigger-action pipelines (e.g., ticket creation, alerting) work reliably in production beyond vendor docs.", "evidenceIds": [ "semgrep-docs-25", "semgrep-docs-28", "semgrep-docs-16", "semgrep-docs-46", "semgrep-docs-22", "semgrep-docs-30", "semgrep-docs-38", "semgrep-docs-53", "semgrep-docs-57" ] }, { "productId": "semgrep", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Semgrep supports automated scanning triggered by CI events (push, PR/MR) and offers Managed Scans for bulk repo scanning without CI changes, which implies some recurring automation, but there is no explicit documentation of a scheduling/cron mechanism or workflow orchestration for recurring jobs as an AI-native automation feature. missing for 10: explicit scheduling/cron support, documented recurring job configuration, and evidence of AI-native workflow orchestration beyond CI trigger events.", "evidenceIds": [ "semgrep-docs-15", "semgrep-docs-25", "semgrep-docs-28", "semgrep-docs-14" ] }, { "productId": "semgrep", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Semgrep's evidence covers rule creation, testing, private-rule publishing, and CI integration, but nothing describes version history, review workflows, or rollback of rules/policies within Semgrep itself (rules are just YAML files presumably managed via external git, not a documented Semgrep versioning feature). Missing for 10: any documentation of rule/version history, approval/review workflow for rule changes, or a rollback mechanism in the Semgrep AppSec Platform.", "evidenceIds": [ "semgrep-docs-36", "semgrep-docs-37", "semgrep-docs-4" ] }, { "productId": "semgrep", "storyId": "baseline-suppression", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs mention a 'Triage findings' capability (alerts/tickets for tracking findings) and Semgrep Assistant's contextual post-processing to reduce noise by ~20%, implying some findings-management workflow, but there is no explicit evidence of baseline scanning (e.g., baseline-commit diffing), ignore-comments (nosemgrep), or a documented false-positive suppression/triage-state workflow. missing for 10: explicit baseline-commit/diff scanning documentation, .semgrepignore or inline nosemgrep suppression evidence, and a documented 'mark as false positive / ignored' triage state in the platform.", "evidenceIds": [ "semgrep-docs-22", "semgrep-docs-30", "semgrep-gh-3" ] }, { "productId": "semgrep", "storyId": "ci-pipeline-gate", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly cover running `semgrep ci` in CI pipelines triggered on push/PR events, with dedicated CI provider guides, SARIF export, and exit-code based build behavior implied by 'ci' command design for gating; community evidence corroborates real-world CI usage. missing for 10: explicit documented mention of exit code semantics for failing builds on new findings (implied but not directly quoted).", "evidenceIds": [ "semgrep-docs-19", "semgrep-docs-14", "semgrep-docs-25", "semgrep-docs-45", "semgrep-docs-20", "semgrep-comm-7" ] }, { "productId": "semgrep", "storyId": "container-image-scan", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers Semgrep Code (SAST), Secrets, and Supply Chain (dependency/OSS SCA) scanning of source repositories, CI pipelines, and manifests, but contains no mention of scanning container images or detecting OS-package vulnerabilities inside image layers — a capability offered by dedicated container scanners. Since Semgrep explicitly compares itself to AppSec competitors (Checkmarx, Snyk) that do offer this, the axis is a fair comparison point but no evidence shows Semgrep delivers it.", "evidenceIds": [ "semgrep-docs-29", "semgrep-docs-12", "semgrep-docs-13", "semgrep-docs-8" ] }, { "productId": "semgrep", "storyId": "cross-file-dataflow", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Semgrep explicitly documents taint tracking/taint mode for catching injection-style vulnerabilities (semgrep-docs-41, semgrep-docs-58) and separately documents cross-file (interfile) analysis via Semgrep Code (semgrep-docs-2, semgrep-docs-32) plus interprocedural/interfile constant propagation (semgrep-docs-42), together supporting taint flows across files and function boundaries. Missing for 10: no independent/hands-on evidence specifically validating cross-file taint tracking accuracy (community evidence only covers simpler pattern-matching bypasses, not taint-mode cross-file cases).", "evidenceIds": [ "semgrep-docs-41", "semgrep-docs-58", "semgrep-docs-2", "semgrep-docs-32", "semgrep-docs-42" ] }, { "productId": "semgrep", "storyId": "custom-sast-rules", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Semgrep's docs extensively cover writing custom rules with a simple YAML-based pattern syntax, testing them, using taint tracking, and sharing rules across repos via the Registry and private rules for teams; community evidence (HN) confirms security engineers actually write custom rules to catch codebase-specific antipatterns and find the syntax readable/flexible across languages. Missing for 10: independent evidence specifically validating cross-repo sharing workflows at scale, and community reports note real limitations (bypassable rules, parser errors, performance issues) that slightly temper confidence in robustness.", "evidenceIds": [ "semgrep-docs-4", "semgrep-docs-5", "semgrep-docs-21", "semgrep-docs-36", "semgrep-docs-37", "semgrep-docs-41", "semgrep-docs-58", "semgrep-comm-1", "semgrep-comm-6" ] }, { "productId": "semgrep", "storyId": "custom-secret-rules", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Semgrep's generic rule-writing engine is well documented (custom rules, pattern matching, Editor for testing) and community evidence confirms security engineers write custom rules for org-specific patterns, which extends naturally to matching internal token/secret formats via pattern or regex-based rules. However, there is no explicit documentation showing how to author custom rules specifically within the Semgrep Secrets product (e.g., custom secret validators or format definitions) — the evidence only shows built-in secret detection (docs-8/9/10) and generic custom rule capability (docs-4/5/21), not a documented workflow for adding custom secret-format rules to Semgrep Secrets. Missing for 10: explicit docs/examples for authoring custom Semgrep Secrets rule types (validators, custom regex-based secret detectors), and independent confirmation of doing so for secrets specifically.", "evidenceIds": [ "semgrep-docs-4", "semgrep-docs-5", "semgrep-docs-21", "semgrep-docs-8", "semgrep-docs-9", "semgrep-comm-1" ] }, { "productId": "semgrep", "storyId": "dependency-vulnerability-scan", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Semgrep Supply Chain is explicitly documented as an SCA tool that scans manifests/lockfiles, resolves dependency inventories, flags reachable vulnerabilities, detects malicious packages, and generates SBOMs (semgrep-docs-11,12,13,23,29,55). This directly matches the story of scanning dependency manifests/lockfiles for known vulnerabilities. Missing for 10: independent/hands-on community validation of SCA accuracy or usability (community evidence only covers SAST rule-writing, not the SCA/dependency-scanning feature), and one community comment (semgrep-comm-12) notes SCA isn't open-source and may require uploading data, a caveat not fully addressed in docs.", "evidenceIds": [ "semgrep-docs-11", "semgrep-docs-12", "semgrep-docs-13", "semgrep-docs-23", "semgrep-docs-29", "semgrep-docs-44", "semgrep-docs-55", "semgrep-comm-12" ] }, { "productId": "semgrep", "storyId": "detect-secrets-in-repo", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Semgrep Secrets is documented to scan code for exposed API keys, passwords, and credentials and validate them (semgrep-docs-8, semgrep-docs-9, semgrep-docs-43), and pre-commit hooks aim to prevent secrets from entering git history (semgrep-docs-16/46), but this only prevents future leaks — none of the evidence shows Semgrep actually scanning the full historical git commit log/diffs for previously committed secrets (a capability tools like gitleaks/trufflehog specialize in). missing for 10: explicit documentation or feature for scanning past git commits/history (not just current working tree), evidence of retroactive history-wide secret detection, independent confirmation this works in practice.", "evidenceIds": [ "semgrep-docs-8", "semgrep-docs-9", "semgrep-docs-43", "semgrep-docs-16", "semgrep-docs-46" ] }, { "productId": "semgrep", "storyId": "fix-upgrade-guidance", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Semgrep Supply Chain (SCA) evidence covers dependency vulnerability detection, reachability analysis, SBOM generation, and malicious dependency detection, but nothing in the evidence pack describes generating concrete upgrade paths or automated fix pull requests for vulnerable dependencies. The --autofix flag ([semgrep-docs-38],[semgrep-docs-53]) applies to Semgrep Code's rule-defined fixes for SAST findings, not dependency remediation.", "evidenceIds": [ "semgrep-docs-11", "semgrep-docs-12", "semgrep-docs-13", "semgrep-docs-23", "semgrep-docs-29", "semgrep-docs-38" ] }, { "productId": "semgrep", "storyId": "guardrails-for-agent-code", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Semgrep ships an official MCP server (docs.semgrep.dev/mcp) letting coding agents call the scanner directly, plus first-party messaging on scanning/fixing AI-generated code the moment it's written and combining AI reasoning with rule-based detection, and a pre-commit hook to catch issues before they land in Git history. Missing for 10: independent/hands-on validation that the MCP integration reliably works inside real agent workflows, and detail on latency or false-positive behavior when used inline with AI code generation.", "evidenceIds": [ "semgrep-probe-3", "semgrep-docs-47", "semgrep-docs-48", "semgrep-docs-16", "semgrep-docs-46" ] }, { "productId": "semgrep", "storyId": "iac-misconfig-scan", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Semgrep's docs mention that rules can be written to 'scan configuration files' as one use case (docs-21/35), which implies some IaC coverage, but the evidence pack never explicitly names Terraform, Kubernetes, Dockerfile, or IaC misconfiguration scanning as a supported capability. missing for 10: explicit documentation of Terraform/Kubernetes/Dockerfile rule packs, dedicated IaC scanning mode or product tier, and any community/hands-on confirmation of IaC misconfiguration detection.", "evidenceIds": [ "semgrep-docs-21", "semgrep-docs-35", "semgrep-docs-52" ] }, { "productId": "semgrep", "storyId": "language-coverage", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs and community evidence show Semgrep scanning across many languages (Python, JS/TS, Ruby, YAML, HTML, CSS) and separately covering open-source package ecosystems via Semgrep Supply Chain (docs-11,12,13,23,29,44,55), plus a generic-pattern fallback for unsupported languages (docs-39). However there's no explicit doc listing 'all major languages/ecosystems' coverage, and a hands-on report shows Semgrep's parser failing on valid TypeScript syntax (comm-3), plus performance complaints scanning large multi-language codebases (comm-2, comm-7) that temper the breadth claim. missing for 10: an explicit enumerated language/ecosystem coverage matrix, resolution of the TypeScript parser bug, and independent benchmarking across the full claimed language set.", "evidenceIds": [ "semgrep-docs-11", "semgrep-docs-12", "semgrep-docs-13", "semgrep-docs-23", "semgrep-docs-29", "semgrep-docs-39", "semgrep-comm-3", "semgrep-comm-5", "semgrep-comm-2", "semgrep-comm-7" ] }, { "productId": "semgrep", "storyId": "license-compliance-scan", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Semgrep Supply Chain evidence focuses on SCA vulnerability detection, reachability analysis, malicious dependency detection, and SBOM generation, but nothing in the pack mentions license detection or license-policy enforcement for dependencies.", "evidenceIds": [ "semgrep-docs-12", "semgrep-docs-13", "semgrep-docs-23", "semgrep-docs-29" ] }, { "productId": "semgrep", "storyId": "machine-readable-output", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly document exporting SARIF output via `semgrep ci --sarif --sarif-output=semgrep.sarif`, and community evidence shows real-world use of Semgrep's SARIF output for downstream tooling (e.g., piping into LLM triage at Microsoft). JSON output is also a standard supported format implied by the CLI's general reporting capabilities, though not separately documented in this pack. missing for 10: explicit docs snippet for --json flag usage.", "evidenceIds": [ "semgrep-docs-3", "semgrep-docs-20", "semgrep-comm-14" ] }, { "productId": "semgrep", "storyId": "openness-api-parity", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Semgrep AppSec Platform ships an API for listing deployments, projects and findings (semgrep-docs-17/59), and CLI/CI features (SARIF export, autofix, rule testing) give some parity with UI actions, but there is no evidence of a comprehensive OpenAPI spec (probe finds 404s at all openapi.json paths) or that triage actions, rule creation/management, policy configuration, or dashboard-only workflows are all exposed via API. missing for 10: documented OpenAPI/API reference covering full UI functionality, API endpoints for rule authoring/publishing and triage actions, independent confirmation that all UI capabilities are API-accessible.", "evidenceIds": [ "semgrep-docs-17", "semgrep-docs-59", "semgrep-probe-2", "semgrep-docs-22", "semgrep-docs-36" ] }, { "productId": "semgrep", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Semgrep supports open-format export of scan results (SARIF) and a platform API to gather findings/projects, plus local CLI scanning that never uploads code by default (dry-run, no login required), which supports data portability and avoiding lock-in. However there's no documented comprehensive 'export all your data' feature covering full platform history, configs, and org data in one open bundle. Missing for 10: explicit bulk/account-level data export documentation, confirmation of open formats beyond SARIF (e.g., JSON schema docs), and independent verification of a full data portability workflow.", "evidenceIds": [ "semgrep-docs-3", "semgrep-docs-20", "semgrep-docs-33", "semgrep-docs-34", "semgrep-docs-59", "semgrep-gh-2" ] }, { "productId": "semgrep", "storyId": "openness-open-license", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The core Semgrep engine is hosted on a public GitHub repo (semgrep-gh-1/2/3), showing source code is publicly readable, but no evidence pack item states an explicit open-source license (e.g., LGPL) or points to a LICENSE file. Community evidence also notes that a commercial component, Semgrep Supply Chain, is explicitly 'not open source' (semgrep-comm-12), meaning the broader product is not uniformly open-licensed. missing for 10: explicit license text/citation, confirmation covering all product components, independent verification of license terms.", "evidenceIds": [ "semgrep-gh-1", "semgrep-gh-2", "semgrep-gh-3", "semgrep-comm-12" ] }, { "productId": "semgrep", "storyId": "openness-self-host", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Semgrep's core CLI/engine is open source and runs entirely on local infrastructure by default, never uploading code, and works without an account or login, which is effectively self-hosting the core scanning product (semgrep-gh-2, semgrep-docs-33/34/50/51/27). Missing for 10: explicit documentation of self-hosting the full AppSec Platform/server components, and independent hands-on confirmation of a self-hosted enterprise deployment.", "evidenceIds": [ "semgrep-gh-2", "semgrep-docs-33", "semgrep-docs-34", "semgrep-docs-50", "semgrep-docs-51", "semgrep-docs-27" ] }, { "productId": "semgrep", "storyId": "pr-comments-scm", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs confirm Semgrep integrates into CI at push/PR/MR events, posts PR/MR comments for Secrets findings, and offers a GitHub integration for automating PR review comments, with findings sent to the AppSec Platform after scans. missing for 10: explicit first-party documentation of GitHub/GitLab PR status checks (not just comments) for Semgrep Code/Supply Chain findings, and independent hands-on confirmation of the PR-comment UX beyond docs.", "evidenceIds": [ "semgrep-docs-10", "semgrep-docs-25", "semgrep-docs-28", "semgrep-docs-45", "semgrep-docs-57", "semgrep-docs-14" ] }, { "productId": "semgrep", "storyId": "pre-commit-hook", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Semgrep has explicit first-party documentation for integrating with the pre-commit framework, stating it runs semgrep on commit specifically to prevent secrets and security issues from leaking into git history, matching the story verbatim. Missing for 10: no independent/hands-on community confirmation specifically of the pre-commit hook workflow (community evidence covers CLI/CI use generally, not pre-commit hook experience).", "evidenceIds": [ "semgrep-docs-16", "semgrep-docs-46", "semgrep-docs-56" ] }, { "productId": "semgrep", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence discusses local scanning (data never uploaded by default) and CLI dry-run options, but there is no mention of data residency, regional hosting options, or ability to choose storage location for the Semgrep AppSec Platform's cloud data. missing for 10: explicit region/residency selection controls, documentation of where cloud-stored findings/data reside, any enterprise data-residency options.", "evidenceIds": [ "semgrep-gh-2", "semgrep-docs-33" ] }, { "productId": "semgrep", "storyId": "privacy-no-training", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Semgrep's docs and GitHub note that scanning runs locally by default and code is never uploaded, and the --dry-run flag guarantees no upload to the Semgrep web app, which operationally limits data exposure that could feed model training. However, there is no explicit privacy policy or statement addressing whether data sent to Semgrep's cloud/AI features (e.g., AI-Powered Detection, Semgrep Assistant) is used for AI training, nor any opt-out mechanism specific to AI training. Missing for 10: explicit AI-training data-use policy, opt-out toggle for AI features, and independent confirmation of this guarantee.", "evidenceIds": [ "semgrep-gh-2", "semgrep-docs-33", "semgrep-docs-51", "semgrep-docs-34" ] }, { "productId": "semgrep", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs show privacy-preserving defaults (local-only scanning, --dry-run to avoid upload, no login required), giving users some control over whether data leaves their machine, but there is no evidence of explicit data retention periods, deletion mechanisms, or account/data-erasure controls for data that is uploaded to the Semgrep AppSec Platform. missing for 10: documented retention policy, explicit deletion/erasure controls for platform-stored findings, GDPR/CCPA data handling commitments.", "evidenceIds": [ "semgrep-gh-2", "semgrep-docs-33", "semgrep-docs-51", "semgrep-docs-34" ] }, { "productId": "semgrep", "storyId": "privacy-telemetry-optout", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Semgrep's docs and GitHub README indicate privacy-conscious defaults: code is analyzed locally and never uploaded by default, no login is required to run a scan, and the --dry-run flag explicitly prevents scan results from being uploaded to the Semgrep web app. However, there's no explicit documentation of a dedicated 'telemetry/usage tracking' opt-out setting (e.g., an env var or flag specifically for metrics/telemetry) beyond the scan-upload behavior. missing for 10: explicit telemetry/usage-metrics opt-out flag or setting, independent confirmation of what usage data (if any) is collected outside of code content.", "evidenceIds": [ "semgrep-gh-2", "semgrep-docs-33", "semgrep-docs-34", "semgrep-docs-51" ] }, { "productId": "semgrep", "storyId": "reachability-prioritization", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Semgrep Supply Chain explicitly implements reachability analysis, flagging findings as reachable when a matching code pattern exists in the codebase, which directly supports prioritizing dependency alerts by reachability. However, evidence lacks detail on how reachability is surfaced in triage workflows (e.g., filtering/sorting UI, PR gating specifically by reachability) and no independent/hands-on verification of reachability accuracy is present. missing for 10: evidence of UI/workflow for filtering or sorting alerts by reachability status, independent validation of reachability detection accuracy, detail on false negative/positive rates for reachability analysis.", "evidenceIds": [ "semgrep-docs-11", "semgrep-docs-24", "semgrep-docs-29", "semgrep-docs-13" ] }, { "productId": "semgrep", "storyId": "sast-code-scanning", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Semgrep explicitly documents SAST scanning with taint tracking designed to catch injection and XSS bugs (semgrep-docs-41, semgrep-docs-58), backed by a CLI scan workflow (semgrep-docs-19, semgrep-docs-27) and corroborated by community use for security lint rules including injection detection (semgrep-comm-1, semgrep-comm-7). Missing for 10: independent benchmark data on vulnerability-class detection rates, and community reports show simple evasions (e.g., function aliasing) can bypass naive rules (semgrep-comm-6, semgrep-comm-10), indicating real-world efficacy caveats.", "evidenceIds": [ "semgrep-docs-41", "semgrep-docs-58", "semgrep-docs-19", "semgrep-docs-27", "semgrep-comm-1", "semgrep-comm-7", "semgrep-comm-6", "semgrep-comm-10" ] }, { "productId": "semgrep", "storyId": "sbom-generation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Semgrep Supply Chain docs explicitly state it can 'generate reports and software bills of materials (SBOM)' as part of dependency inventory, confirming the capability exists, but this is gated behind the paid Supply Chain/AppSec Platform tier and no evidence specifies SPDX/CycloneDX format support, CLI flags, or container image SBOM generation. missing for 10: confirmation of specific SPDX/CycloneDX output formats, CLI command/flag documentation for SBOM export, container/image scanning SBOM support, and independent/hands-on corroboration of SBOM generation working as described.", "evidenceIds": [ "semgrep-docs-12", "semgrep-docs-13", "semgrep-docs-29", "semgrep-docs-44" ] }, { "productId": "semgrep", "storyId": "scan-beyond-git", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Semgrep Secrets is documented to scan code/repositories for exposed credentials, but nothing in the evidence pack shows support for scanning non-git sources like S3 buckets, Docker images, or CI logs — all scanning references are to codebases, folders, monorepos, or CI pipelines running against git-based projects.", "evidenceIds": [ "semgrep-docs-8", "semgrep-docs-9", "semgrep-docs-43", "semgrep-docs-54" ] }, { "productId": "semgrep", "storyId": "severity-filtering", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes general triage/noise features (AI post-processing to reduce noise, ticketing/alerts, reachability analysis) but never mentions severity levels, confidence scores, or a mechanism to filter/gate CI builds based on them. Missing for 10: explicit severity/confidence filtering flags (e.g., --severity), CI gating configuration based on severity/confidence thresholds, and documentation tying these to build pass/fail behavior.", "evidenceIds": [] }, { "productId": "semgrep", "storyId": "verify-live-secrets", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Semgrep Secrets explicitly performs validity checks by testing detected secrets against the issuing web service, distinguishing 'valid' (live, actively in use) secrets from invalid ones, and surfaces this in PR/MR comments to prioritize live secrets. Missing for 10: independent/hands-on corroboration of validation accuracy and broader coverage details across secret types/providers.", "evidenceIds": [ "semgrep-docs-9", "semgrep-docs-8", "semgrep-docs-10", "semgrep-docs-43" ] }, { "productId": "snyk", "storyId": "agent-runs-scan-headless", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The Snyk CLI is confirmed to run headlessly in CI/CD pipelines (snyk-gh-3, snyk-gh-7) and produces a terminal report of vulnerabilities with severity, path, and fix guidance (snyk-gh-4, snyk-gh-6), and Jira/Slack integrations exist for tracking issues (snyk-docs-6, snyk-intdir-1). However, the pack never documents a machine-readable (JSON/SARIF) output flag for CLI scans, nor an automated pipeline that parses CLI results and files issues for real findings specifically — Jira export appears to be a platform-level feature rather than a CLI-driven headless workflow. missing for 10: documented CLI JSON/SARIF output flag, evidence of an agent parsing that output and auto-filing issues end-to-end.", "evidenceIds": [ "snyk-gh-3", "snyk-gh-7", "snyk-gh-4", "snyk-gh-6", "snyk-docs-6", "snyk-probe-5" ] }, { "productId": "snyk", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Snyk publishes an llms.txt file confirmed live via direct probe (HTTP 200) with structured docs navigation, and also maintains extensive agent-oriented documentation (MCP server setup, Snyk Studio, remediation agent) that an AI agent could be pointed at. missing for 10: no independent/community corroboration of agents actually consuming llms.txt in practice, and no evidence of additional agent-friendly formats (e.g., structured API schemas, since openapi.json probes 404).", "evidenceIds": [ "snyk-probe-1", "snyk-docs-10", "snyk-docs-28", "snyk-probe-3", "snyk-probe-2" ] }, { "productId": "snyk", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Snyk provides AI-generated insights directly in-product: DeepCode AI-powered risk scoring/prioritization, actionable fix advice, an LLM-driven Remediation Agent that scans, generates fix plans enriched with Snyk security intelligence, applies fixes, and verifies results, plus natural-language querying via MCP/Snyk Studio. missing for 10: independent/hands-on validation of insight quality and accuracy (community evidence is largely about unrelated PR/dependency issues, not AI-insight quality), and no data on how these insights surface for non-code data sources beyond code/deps/IaC.", "evidenceIds": [ "snyk-docs-12", "snyk-docs-14", "snyk-docs-29", "snyk-docs-36", "snyk-docs-13", "snyk-docs-23", "snyk-docs-10", "snyk-docs-11", "snyk-probe-6" ] }, { "productId": "snyk", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Snyk supports autonomous background operation via CI/CD pipeline scanning (snyk-gh-3) and an LLM-driven Remediation Agent that scans, generates a fix plan, applies fixes and verifies results without manual step-by-step intervention (snyk-docs-12, snyk-docs-29). However, the CLI's flagship 'agentic' fix mode is explicitly described as a 'human-in-the-loop interactive' experience rather than fully autonomous (snyk-docs-13), and community reports describe Snyk's older automated PR/remediation feature as noisy and low-value, leading users to disable it (snyk-comm-6, snyk-comm-11) — a concrete real-world caveat on unattended automation quality. Missing for 10: independent verification that the newer Remediation Agent runs unattended in production pipelines at scale, and confirmation that automated PR/fix quality issues raised by users have been resolved.", "evidenceIds": [ "snyk-docs-12", "snyk-docs-13", "snyk-docs-23", "snyk-docs-29", "snyk-docs-30", "snyk-gh-3", "snyk-comm-6", "snyk-comm-11" ] }, { "productId": "snyk", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Snyk ships a documented built-in AI 'Remediation Agent' that scans a project, generates and applies a fix plan, and verifies results directly inside the coding assistant or CLI (snyk-docs-12/13/23/29/30), backed by DeepCode AI-driven autofixes (snyk-docs-14/15/36) and a local MCP server confirmed to actually start and respond via handshake (snyk-probe-6). This goes beyond simple scanning to actual task delegation to an in-product AI agent. Missing for 10: independent/hands-on evidence validating the remediation agent's real-world fix quality/success rate (only vendor-claimed 85% accuracy), and no third-party review of the delegation UX itself.", "evidenceIds": [ "snyk-docs-12", "snyk-docs-13", "snyk-docs-23", "snyk-docs-29", "snyk-docs-30", "snyk-docs-28", "snyk-probe-6", "snyk-docs-14", "snyk-docs-36" ] }, { "productId": "snyk", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Snyk CLI explicitly supports running in CI/CD pipelines (snyk-gh-3, snyk-gh-7) with test/code test commands producing terminal reports (snyk-gh-4, snyk-gh-5, snyk-gh-6), and Snyk Code/Open Source docs confirm scanning integrates across CI/CD pipelines (snyk-docs-18, snyk-docs-26, snyk-docs-33). missing for 10: no independent/community corroboration of CI usage specifically (only vendor docs/GitHub), and no explicit example of a CI YAML config or automation-friendly non-interactive flags/output formats (e.g., JSON output for pipelines) in the evidence.", "evidenceIds": [ "snyk-gh-3", "snyk-gh-7", "snyk-gh-4", "snyk-gh-5", "snyk-gh-6", "snyk-docs-18", "snyk-docs-26", "snyk-docs-33" ] }, { "productId": "snyk", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All MCP-related evidence describes Snyk shipping its own local MCP server (snyk-docs-28, snyk-probe-3, snyk-probe-6) so other agents/IDEs can call Snyk's tools — this is Snyk acting as an MCP tool provider, not as a client that plugs in external MCP servers to use their tools. There is no evidence that Snyk's Remediation Agent, Snyk Studio, or CLI can consume/connect to third-party MCP servers.", "evidenceIds": [ "snyk-docs-28", "snyk-probe-3", "snyk-probe-6", "snyk-docs-12", "snyk-docs-35" ] }, { "productId": "snyk", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Snyk documents an official local MCP server (Snyk Studio/MCP server running via the CLI) that lets coding agents connect and be prompted with natural language to scan for security issues, and a hands-on probe confirms the CLI actually responds to a JSON-RPC MCP handshake exposing tools/prompts/resources. missing for 10: independent third-party corroboration of MCP client interoperability across multiple agents, and clearer documentation on authentication requirements for the MCP server itself.", "evidenceIds": [ "snyk-docs-10", "snyk-docs-11", "snyk-docs-28", "snyk-probe-3", "snyk-probe-6" ] }, { "productId": "snyk", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Snyk documents an MCP server and 'Snyk Studio' agentic workflow where users prompt coding agents in natural language (e.g., 'scan my directory for security issues'), plus slash-style commands (/snyk-fix, /snyk-batch-fix) and an LLM-driven Remediation Agent that plans and applies fixes autonomously. A hands-on probe confirms the MCP server actually starts and responds to JSON-RPC initialize, corroborating the documented natural-language/agentic workflow beyond marketing claims. Missing for 10: broader independent (non-Snyk) user reports specifically validating natural-language command accuracy/reliability in real projects.", "evidenceIds": [ "snyk-docs-11", "snyk-docs-35", "snyk-docs-23", "snyk-docs-30", "snyk-docs-28", "snyk-docs-12", "snyk-docs-13", "snyk-probe-3", "snyk-probe-6" ] }, { "productId": "snyk", "storyId": "agentic-official-cli", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Snyk ships a well-documented official CLI (snyk-gh-1/3/7, snyk-probe-4/7) supporting local, IDE, and CI/CD workflows, plus explicit AI-native/agentic capabilities: `snyk fix --agentic` for human-in-the-loop remediation, `/snyk-fix` and `/snyk-batch-fix` slash commands for coding assistants, and a built-in MCP server (`snyk mcp -t stdio`) verified live via handshake probe. This directly satisfies the ai-native CLI story with both docs and hands-on confirmation. missing for 10: independent/community corroboration specifically praising the CLI's agentic features (community evidence here is generic and mixed, not focused on this), and probe-5 shows every scan requires prior authentication (no keyless/unauthenticated use).", "evidenceIds": [ "snyk-gh-1", "snyk-gh-3", "snyk-gh-7", "snyk-probe-4", "snyk-probe-6", "snyk-probe-7", "snyk-docs-13", "snyk-docs-23", "snyk-docs-28", "snyk-docs-30" ] }, { "productId": "snyk", "storyId": "agentic-public-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows CLI and MCP-server access but no documented public REST/GraphQL API; a direct probe for OpenAPI/swagger specs on docs.snyk.io returned 404 for all candidate paths, indicating no discoverable public API documentation.", "evidenceIds": [ "snyk-probe-2" ] }, { "productId": "snyk", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence describes scoped or least-privilege API credentials/tokens for AI agents; the only authentication evidence shows the CLI/MCP server requiring a standard Snyk account/API token with no mention of granular scopes or agent-specific permission levels. missing for 10: any documentation of scoped API tokens, role/permission-limited credentials, or agent-specific least-privilege auth mechanisms.", "evidenceIds": [ "snyk-probe-5", "snyk-probe-6" ] }, { "productId": "snyk", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Snyk's CLI, MCP server, and various product scanners, but contains no mention of an official SDK (e.g., Python, Node, Java client libraries) for programmatic integration, and a probe for an OpenAPI/swagger spec returned 404s, indicating no discoverable official API SDK.", "evidenceIds": [ "snyk-probe-2", "snyk-gh-1", "snyk-docs-28" ] }, { "productId": "snyk", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items mention webhooks or event subscription mechanisms; Snyk's integrations directory references Jira, Slack, GitHub/GitLab brokers, but nothing about outbound webhook events for AI-native consumption.", "evidenceIds": [] }, { "productId": "snyk", "storyId": "ai-assisted-triage", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Snyk documents AI-driven risk scoring (reachability, exploit maturity, popularity) and a Remediation Agent that scans, explains, and applies fixes, which supports explaining/prioritizing findings, but there is no explicit evidence of AI-based deduplication of findings or automated false-positive flagging — only manual 'ignore' suppression is documented. Community feedback (snyk-comm-10) also notes that legacy alerts remain largely 'noise' without context-aware triage, tempering confidence in how well false positives are actually filtered. Missing for 10: explicit deduplication feature, explicit AI false-positive detection/flagging, and independent hands-on confirmation that noise is meaningfully reduced.", "evidenceIds": [ "snyk-docs-4", "snyk-docs-36", "snyk-docs-12", "snyk-docs-5", "snyk-comm-10" ] }, { "productId": "snyk", "storyId": "ai-remediation-pr", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Snyk documents both a rule-based fix-PR workflow for Open Source vulnerabilities (snyk-docs-21) and a newer LLM-driven 'Remediation Agent' that scans, generates a fix plan, and applies fixes via CLI/coding assistant (snyk-docs-12, snyk-docs-29, snyk-docs-30), which aligns with 'AI drafts a remediation PR for review.' However, hands-on community feedback specifically about Snyk's automated PR feature reports it as poor quality — generating dozens/hundreds of noisy PRs rather than meaningful rewrites, to the point the user disabled it in favor of Dependabot (snyk-comm-6) — directly contradicting the smooth 'draft-then-review-and-merge' experience implied by the story. Missing for 10: first-party or independent evidence the Remediation Agent itself opens a git PR (docs show it applying fixes locally/in-IDE, not confirmed PR creation), and corroboration that the AI-drafted PR quality is trustworthy enough for simple review-and-merge.", "evidenceIds": [ "snyk-docs-21", "snyk-docs-12", "snyk-docs-29", "snyk-docs-30", "snyk-comm-6" ] }, { "productId": "snyk", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an interactive API reference with runnable examples; the evidence pack includes an explicit probe showing all candidate OpenAPI/Swagger spec URLs return 404, and no docs reference an API explorer or 'try it' console.", "evidenceIds": [ "snyk-probe-2" ] }, { "productId": "snyk", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "A probe explicitly checked for an OpenAPI/swagger spec at Snyk's docs domain and found all candidate paths (openapi.json, swagger.json, etc.) returning 404 — no machine-readable API spec was found. No other evidence in the pack shows a downloadable OpenAPI/Swagger file for Snyk's API.", "evidenceIds": [ "snyk-probe-2" ] }, { "productId": "snyk", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence describes a dedicated sandbox/test environment isolated from production data; Snyk's CLI, remediation agent, and MCP server all operate directly on the user's local project or repository, and no docs mention an explicit non-production sandbox mode for AI-native agent testing.", "evidenceIds": [] }, { "productId": "snyk", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation of a versioned API or a deprecation policy; in fact the probe for OpenAPI/swagger specs at docs.snyk.io returned 404 on all candidate paths, and no other artifact (CLI, MCP server, docs) references API versioning or deprecation guarantees.", "evidenceIds": [ "snyk-probe-2" ] }, { "productId": "snyk", "storyId": "autofix-findings", "verdict": "disputed", "quality": 6, "confidence": "medium", "rationale": "Snyk's docs document multiple concrete fix-application paths beyond reporting: PR-based fix workflows for open-source (snyk-docs-21), suggested-fix ignore/config (snyk-docs-5), IaC 'fix advice... make changes directly to code' (snyk-docs-34), and newer agentic tooling — Remediation Agent that 'applies the fixes and verifies the result', `snyk fix --agentic`, and `/snyk-fix`/`/snyk-batch-fix` slash commands (snyk-docs-12,13,23,29,30) plus DeepCode AI's '85%-accurate security autofixes' (snyk-docs-14). However, independent hands-on community feedback specifically on the PR-based auto-fix workflow contradicts vendor claims of quality: a developer reports being 'very disappointed with their PR tools' to the point of disabling automated PRs, saying Dependabot is '100x better to work with' (snyk-comm-6), and another calls the appsec/PR tooling 'not worth it' after a demo (snyk-comm-12). missing for 10: independent verification that the newer Remediation Agent/agentic autofix and SAST/IaC suggested-fix mechanisms (as opposed to the older PR-fix flow) actually work reliably in practice, and resolution of the direct community contradiction on fix-PR quality.", "evidenceIds": [ "snyk-docs-21", "snyk-docs-5", "snyk-docs-34", "snyk-docs-12", "snyk-docs-13", "snyk-docs-23", "snyk-docs-29", "snyk-docs-30", "snyk-docs-14", "snyk-comm-6", "snyk-comm-12" ] }, { "productId": "snyk", "storyId": "automation-bulk-operations", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Snyk explicitly documents a `/snyk-batch-fix` slash command for coding assistants to fix multiple vulnerabilities in one operation, and the Remediation Agent scans, generates a fix plan, and applies fixes across a whole project rather than one issue at a time. Independent community evidence corroborates bulk behavior in practice (Snyk generating 'dozens or hundreds of PRs' for dependency updates), showing the batch capability actually operates at scale, even if some users found the volume unwieldy. Missing for 10: a documented bulk/batch REST API for programmatic multi-item operations, and clearer scale limits or first-party benchmarks for large-scale automation.", "evidenceIds": [ "snyk-docs-23", "snyk-docs-12", "snyk-docs-29", "snyk-comm-6" ] }, { "productId": "snyk", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Snyk supports some automated, event-driven actions (auto-generated fix PRs on new vulnerabilities, CI/CD pipeline gating, hooks-based Studio integration for agentic environments, and policy management mentioned in pricing docs) but there is no documented general-purpose rule engine letting an AI-native user define custom triggers/conditions to fire arbitrary actions. Community feedback (snyk-comm-6, snyk-comm-11) even notes the automated PR behavior is inflexible and noisy rather than configurable. missing for 10: explicit custom rule/condition builder, documentation of webhook or event-subscription API for arbitrary triggers, evidence of user-configurable automation beyond default PRs/hooks.", "evidenceIds": [ "snyk-docs-10", "snyk-docs-21", "snyk-docs-32", "snyk-comm-6", "snyk-comm-11" ] }, { "productId": "snyk", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Snyk supports scanning via CLI, CI/CD pipelines, and IDE integrations, but there is no evidence of a native scheduler or built-in mechanism for recurring/cron-style jobs or workflows; recurring scans would need to be built externally via CI/CD cron triggers, which is not documented as a Snyk feature.", "evidenceIds": [] }, { "productId": "snyk", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Snyk is a security scanning/remediation tool; there is no evidence of versioning, reviewing, or rolling back 'automations' (e.g., saved workflows, agent configurations, or fix scripts) — the closest features are PR-based fixes and a remediation agent, but no version history, review/audit trail, or rollback mechanism for automations is documented.", "evidenceIds": [] }, { "productId": "snyk", "storyId": "baseline-suppression", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Snyk docs mention configuring Snyk to ignore suggested fixes to suppress specific warnings (snyk-docs-5) and 'Policy management' as a platform capability (snyk-docs-32), which map to ignoring/baselining findings so only new issues surface. However, there's no detailed evidence of a dedicated baseline/false-positive triage workflow, ignore-policy scope (temporary vs permanent, org-wide), or CLI/API commands for it. missing for 10: explicit baseline scan feature, false-positive marking UI/API, ignore policy scoping and audit trail, independent confirmation of noise reduction effectiveness.", "evidenceIds": [ "snyk-docs-5", "snyk-docs-32" ] }, { "productId": "snyk", "storyId": "ci-pipeline-gate", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Evidence confirms Snyk CLI is explicitly designed to run in CI/CD pipelines (snyk-gh-3, snyk-gh-7) and that `snyk test`/`snyk code test` reports vulnerabilities found (snyk-gh-4, snyk-gh-6), which is the standard mechanism CI systems use to gate builds. However, the pack has no explicit documentation of PR-trigger configuration or exit-code semantics for failing builds, and the probe shows the CLI requires authentication for every scan (snyk-probe-5), adding an operational caveat for CI setup. Missing for 10: explicit docs on PR-triggered CI workflows, documented exit-code behavior (e.g., non-zero on new findings), and any GitHub Actions/PR-check integration guide.", "evidenceIds": [ "snyk-gh-3", "snyk-gh-7", "snyk-gh-4", "snyk-gh-6", "snyk-probe-5" ] }, { "productId": "snyk", "storyId": "container-image-scan", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Snyk explicitly ships 'Snyk Container' for scanning container images and is described platform-wide as scanning 'container images' alongside code and dependencies, and one community comment even notes hearing good things about the container security tooling specifically. missing for 10: explicit doc language distinguishing OS-package vs application-layer vulnerability detection within images, and any hands-on/independent verification of a container scan (the probes only exercise `snyk test`/MCP, not `snyk container test`).", "evidenceIds": [ "snyk-docs-7", "snyk-docs-25", "snyk-comm-12" ] }, { "productId": "snyk", "storyId": "cross-file-dataflow", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Snyk Code's docs explicitly describe SAST scanning with 'source to sink' step-by-step flow visualization, which is textbook taint-style vulnerability detection tracing data flow through code paths (snyk-docs-2, snyk-docs-3, snyk-docs-18/26/33). This is corroborated by the CLI's snyk code test command for source code scanning (snyk-gh-5). missing for 10: explicit documentation stating flows are tracked across separate files/modules and function boundaries specifically (vs. within a single file), and independent hands-on validation of cross-file taint accuracy", "evidenceIds": [ "snyk-docs-2", "snyk-docs-3", "snyk-docs-18", "snyk-docs-33", "snyk-gh-5" ] }, { "productId": "snyk", "storyId": "custom-sast-rules", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Only one thin mention (writing custom queries with DeepCode AI logic and autocomplete) suggests any custom rule authoring, but there is no detail on the rule syntax, no evidence of how rules are structured or shared across repositories, and no mention of a rule-sharing/policy mechanism analogous to Semgrep-style custom SAST rules. Missing for 10: documented custom rule syntax/DSL, examples of custom rules, and a clear cross-repo sharing/config mechanism.", "evidenceIds": [ "snyk-docs-15", "snyk-docs-36" ] }, { "productId": "snyk", "storyId": "custom-secret-rules", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Snyk's SAST, SCA, container, IaC, and agentic remediation features but contains no mention of secret detection or custom secret/token pattern rules; the story requires a capability not evidenced anywhere in the pack.", "evidenceIds": [] }, { "productId": "snyk", "storyId": "dependency-vulnerability-scan", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Snyk Open Source is documented extensively as a dependency/manifest/lockfile scanner for known vulnerabilities, including indirect dependencies, with CLI (`snyk test`) and CI/IDE integration, actionable fix advice, and PR-based remediation (snyk-docs-1,19,20,21,27; snyk-gh-1,2,4,6). Hands-on probes confirm the CLI scan functionality actually works end-to-end (reporting severity, path, fix guidance) and community reports corroborate real-world usage, though some users note noisy/low-value alerts and mismatches with npm audit.\n\nmissing for 10: independent benchmark data on false-positive rate, and confirmation of coverage across all lockfile formats (e.g., poetry.lock, Gemfile.lock) beyond npm examples.", "evidenceIds": [ "snyk-docs-1", "snyk-docs-19", "snyk-docs-20", "snyk-docs-21", "snyk-docs-27", "snyk-gh-1", "snyk-gh-2", "snyk-gh-4", "snyk-gh-6", "snyk-comm-4", "snyk-comm-7", "snyk-comm-9", "snyk-probe-5" ] }, { "productId": "snyk", "storyId": "detect-secrets-in-repo", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers Snyk Open Source (dependency vulnerabilities), Snyk Code (SAST), Snyk Container, and Snyk IaC, but contains no mention of secret/credential detection or git-history scanning for hardcoded secrets. This is a fair capability to expect from an AppSec scanning platform, but no evidence shows Snyk offers it.", "evidenceIds": [ "snyk-docs-25", "snyk-docs-2", "snyk-docs-1", "snyk-docs-7", "snyk-docs-8" ] }, { "productId": "snyk", "storyId": "fix-upgrade-guidance", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Snyk docs claim actionable fix advice and automated PR-based remediation workflows (snyk-docs-21), reinforced by CLI fix guidance (snyk-gh-6) and newer agentic 'snyk fix --agentic'/Remediation Agent features (snyk-docs-12, snyk-docs-13, snyk-docs-29). However, hands-on community reports concretely contradict the automated-PR experience: users describe disappointment with the PR tool generating dozens/hundreds of noisy PRs instead of proper rewrites, missing whole version bumps, and disabling the feature entirely in favor of Dependabot (snyk-comm-6, snyk-comm-11). missing for 10: independent verification that the newer agentic/Remediation Agent fix-PR flow resolves these historically reported PR-quality issues, and quantitative data on fix-PR success/merge rates.", "evidenceIds": [ "snyk-docs-21", "snyk-docs-12", "snyk-docs-13", "snyk-gh-6", "snyk-comm-6", "snyk-comm-11" ] }, { "productId": "snyk", "storyId": "guardrails-for-agent-code", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Snyk documents and a hands-on probe confirm a local MCP server (snyk mcp -t stdio) that plugs into coding agents/ADEs, plus hooks-based secure-at-inception workflows, natural-language scan prompts, and slash-commands (/snyk-fix, /snyk-batch-fix) for scanning and remediating AI-generated code before it lands. This is corroborated independently by a local probe showing the MCP server actually starts and responds to JSON-RPC initialize. Missing for 10: independent/community verification specifically of the agentic MCP workflow's real-world reliability (community evidence found is about the CLI/PR features generally, not the MCP-agent integration).", "evidenceIds": [ "snyk-docs-10", "snyk-docs-11", "snyk-docs-28", "snyk-docs-23", "snyk-docs-30", "snyk-probe-3", "snyk-probe-6" ] }, { "productId": "snyk", "storyId": "iac-misconfig-scan", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Snyk IaC explicitly documents scanning Terraform, CloudFormation, Kubernetes, and Azure ARM configs for misconfigurations across IDE, SCM, CLI, and CI workflows, plus drift detection for manually-created cloud resources. This is corroborated by CLI docs and the general Snyk platform overview listing cloud configuration scanning as a core capability. Missing for 10: independent hands-on validation of IaC scan accuracy/false-positive rates and no direct probe confirming `snyk iac test` output.", "evidenceIds": [ "snyk-docs-8", "snyk-docs-9", "snyk-docs-22", "snyk-docs-34", "snyk-docs-25" ] }, { "productId": "snyk", "storyId": "language-coverage", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Snyk documents dedicated scanners spanning open-source dependencies (with indirect/transitive dependency detection), SAST (Snyk Code) across IDEs/repos/CI, container images, and IaC configs (Terraform, CloudFormation, Kubernetes, ARM), plus a CLI usable locally, in IDEs, and in CI/CD — collectively covering the major ecosystems a typical org would need. Community feedback raises accuracy/noise concerns (e.g., mismatched results vs npm audit, PR spam) but does not concretely dispute the breadth of language/ecosystem coverage itself. Missing for 10: an explicit enumerated list of supported languages/package managers and independent verification of coverage completeness across less-common ecosystems.", "evidenceIds": [ "snyk-docs-1", "snyk-docs-19", "snyk-docs-2", "snyk-docs-18", "snyk-docs-7", "snyk-docs-8", "snyk-docs-22", "snyk-gh-1", "snyk-gh-3", "snyk-comm-7", "snyk-comm-9" ] }, { "productId": "snyk", "storyId": "license-compliance-scan", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Snyk Open Source explicitly scans dependency licenses for compliance against known licenses (snyk-docs-20), and the platform advertises general 'Policy management' features (snyk-docs-32), implying license policy enforcement capability. However, the evidence never details concrete license-policy enforcement mechanics (e.g., failing builds/PRs on license violations, custom license policy rules) and lacks independent/hands-on corroboration of this specific workflow. Missing for 10: explicit enforcement workflow details (build/PR gating on license violations), documented custom license policy configuration, and independent confirmation of license-policy enforcement in practice.", "evidenceIds": [ "snyk-docs-20", "snyk-docs-32", "snyk-docs-27" ] }, { "productId": "snyk", "storyId": "machine-readable-output", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack documents Snyk CLI's terminal reports, CI/CD integration, and various scan types, but never mentions JSON or SARIF output formats or any machine-readable export capability for downstream tooling. This is a fair and applicable axis for a security scanner, but no evidence confirms it.", "evidenceIds": [ "snyk-gh-4", "snyk-gh-6", "snyk-gh-3" ] }, { "productId": "snyk", "storyId": "openness-api-parity", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation of a public Snyk API offering feature parity with the UI/CLI; the only probe for an OpenAPI spec returned 404s at all candidate locations, and all other evidence focuses on CLI, IDE, MCP, and UI-based workflows rather than a comprehensive API surface for AI-native/programmatic use.", "evidenceIds": [ "snyk-probe-2", "snyk-gh-1", "snyk-gh-3" ] }, { "productId": "snyk", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Snyk can export individual issues to Jira and CLI scan reports print to terminal, but there is no documentation of a comprehensive data export feature (projects, scan history, findings) in open/portable formats, nor any account-deletion/data-portability guidance for users wanting to leave the platform.", "evidenceIds": [ "snyk-docs-6", "snyk-gh-4", "snyk-gh-6" ] }, { "productId": "snyk", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Snyk is a proprietary SaaS security platform; only the CLI wrapper is on GitHub (snyk-gh-1..7) with no license text or claim of an open license provided in the evidence, and the core scanning engine, Snyk Code, DeepCode AI, and platform backend remain closed-source. No evidence pack item states an open-source license for any Snyk component, so the story is unmet. missing for 10: explicit open-source license (e.g., MIT/Apache) for the CLI repo, open-sourcing of core scanning/backend code, any statement of source availability for Snyk Code/DeepCode AI.", "evidenceIds": [ "snyk-gh-1", "snyk-gh-3", "snyk-gh-7" ] }, { "productId": "snyk", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack of a self-hosted/on-prem deployment option for Snyk's core scanning platform; instead, hands-on probe evidence shows the opposite — `snyk test` fails with a 401 Authentication error unless connected to Snyk's cloud, and the CLI/MCP server function as clients to a hosted backend, with only a 'Broker' for connecting on-prem SCM to Snyk's cloud (not a self-hosted core).", "evidenceIds": [ "snyk-probe-5", "snyk-probe-6", "snyk-intdir-3", "snyk-intdir-4" ] }, { "productId": "snyk", "storyId": "pr-comments-scm", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm Snyk integrates scanning results into 'repositories' and 'source code managers' and supports GitHub/GitLab via Snyk Broker (snyk-intdir-3, snyk-intdir-4), plus CI/CD pipeline integration (snyk-docs-18, snyk-gh-3) and PR-based fix workflows (snyk-docs-21). However, no evidence explicitly describes PR comments or GitHub/GitLab check-run surfaces for findings—only generic 'integrations' and 'fix PR' language. Missing for 10: explicit documentation or screenshot of PR comment/check UI, independent confirmation of check-status behavior, and GitLab-specific PR comment support.", "evidenceIds": [ "snyk-docs-18", "snyk-docs-21", "snyk-docs-26", "snyk-docs-31", "snyk-gh-3", "snyk-intdir-3", "snyk-intdir-4" ] }, { "productId": "snyk", "storyId": "pre-commit-hook", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Snyk CLI can run locally, in IDEs, and in CI/CD pipelines, but there is no mention of a pre-commit hook integration, and Snyk's product line (SAST, SCA, container, IaC) does not include secrets scanning at all. Missing for 10: any documentation of a pre-commit hook workflow, and any secrets-detection capability.", "evidenceIds": [ "snyk-gh-3", "snyk-gh-1", "snyk-docs-2" ] }, { "productId": "snyk", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data residency, regional storage options, or data location controls for Snyk's platform or AI features.", "evidenceIds": [] }, { "productId": "snyk", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation of an opt-out or data-usage policy specifically preventing customer code/data from being used to train Snyk's AI models (DeepCode AI, Remediation Agent, MCP-based agents). The only related evidence is a general licensing clause granting Snyk broad rights to uploaded content (snyk-comm-13), which if anything raises concern rather than confirming a training opt-out. Since Snyk processes source code via AI features, this axis clearly applies, but no capability or setting addressing AI-training data use is evidenced.", "evidenceIds": [ "snyk-comm-13" ] }, { "productId": "snyk", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item documents Snyk offering configurable data-retention windows or a user-initiated deletion mechanism for scanned code/data; the only tangential evidence is a community citation noting Snyk's terms grant it a broad, perpetual license to uploaded source code, which runs counter to user control over retention rather than supporting it. missing for 10: documented retention policy/settings, data deletion request workflow, DPA/GDPR deletion tooling, any first-party privacy-controls page.", "evidenceIds": [ "snyk-comm-13" ] }, { "productId": "snyk", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item discusses a telemetry/usage-tracking opt-out mechanism for Snyk CLI, IDE plugins, or MCP server; scans require authentication with a Snyk account (per snyk-probe-5), implying data is sent to Snyk servers, but no opt-out or privacy controls are documented.", "evidenceIds": [] }, { "productId": "snyk", "storyId": "reachability-prioritization", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Snyk's DeepCode AI documentation explicitly states it supports 'context-aware risk scoring, assessing the popularity of the affected package and the reachability of vulnerable code' (snyk-docs-36), directly addressing reachability-based prioritization, and Snyk Code offers risk scoring incorporating prevalence and ease of fix (snyk-docs-4). However, this reachability capability is described only briefly and tied to DeepCode AI/Code rather than clearly extended to Snyk Open Source (dependency/SCA) alerts specifically, and there is no independent or hands-on evidence confirming reachability analysis works reliably in practice for dependency vulnerabilities. missing for 10: explicit documentation that reachability analysis applies to Snyk Open Source (SCA) alerts specifically (not just SAST/DeepCode), independent/hands-on verification of reachability accuracy, and detail on how reachability signals surface in the alert triage UI/workflow.", "evidenceIds": [ "snyk-docs-36", "snyk-docs-4", "snyk-docs-19" ] }, { "productId": "snyk", "storyId": "sast-code-scanning", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Snyk Code is explicitly documented as a developer-first SAST solution that scans source code in real-time across IDEs, repos, and CI/CD, with source-to-sink flow visualization and CLI support (`snyk code test`) confirming actual static analysis capability. While injection/XSS are common SAST vulnerability classes, the evidence doesn't explicitly name these specific classes as detected, and community feedback focuses mostly on Open Source/dependency scanning rather than Snyk Code's SAST accuracy for these specific vuln types. Missing for 10: explicit mention of injection/XSS as detected vulnerability classes, independent hands-on verification of SAST rule coverage/accuracy for these specific classes.", "evidenceIds": [ "snyk-docs-2", "snyk-docs-3", "snyk-docs-18", "snyk-docs-26", "snyk-docs-33", "snyk-gh-5" ] }, { "productId": "snyk", "storyId": "sbom-generation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of SBOM generation, SPDX, or CycloneDX anywhere across Snyk's docs, CLI, or MCP materials — only vulnerability scanning, fixing, and prioritization features are documented. Missing for 10: any documentation or CLI command output referencing SBOM export, SPDX format, or CycloneDX format.", "evidenceIds": [] }, { "productId": "snyk", "storyId": "scan-beyond-git", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Snyk's evidence pack covers SAST, SCA, container/image vulnerability scanning, IaC scanning, and AI-code-assist security, but there is no mention anywhere of secret-detection scanning of non-git sources such as S3 buckets, standalone Docker images for embedded secrets, or CI logs. The container scanning focus is vulnerabilities, not leaked secrets, so this axis is unaddressed.", "evidenceIds": [] }, { "productId": "snyk", "storyId": "severity-filtering", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show severity reporting per issue (snyk-gh-6), risk-based prioritization scoring (snyk-docs-4, snyk-docs-17, snyk-docs-36), policy management (snyk-docs-32), and the ability to ignore/suppress specific findings (snyk-docs-5), which together support filtering low-value noise. However, there is no explicit documentation of a CI/CD severity-threshold gate flag or confidence-level filtering mechanism in the evidence pack. Missing for 10: explicit severity-threshold CLI/CI gating docs, confidence-score filtering mechanism, independent hands-on verification of gating behavior.", "evidenceIds": [ "snyk-gh-6", "snyk-docs-4", "snyk-docs-17", "snyk-docs-36", "snyk-docs-32", "snyk-docs-5" ] }, { "productId": "snyk", "storyId": "verify-live-secrets", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers Snyk's SAST, SCA, container, IaC, and agentic remediation capabilities, but there is no mention of secret detection at all, let alone verifying detected secrets against issuing services to determine liveness. No product feature for secrets scanning or validation is documented anywhere in this pack.", "evidenceIds": [] }, { "productId": "trivy", "storyId": "agent-runs-scan-headless", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Trivy is a CLI-first scanner explicitly designed for headless CI use, with documented commands (trivy image/fs) and structured output formats like SBOM (CycloneDX/SPDX), and community reports confirm it's 'super easy to integrate in every ci/cd pipeline' and easy to script automation around. However, the evidence pack lacks explicit documentation of JSON/table output flags for machine parsing, and there's no evidence of built-in issue-filing/ticketing integration (that would require external glue code an agent would write). missing for 10: explicit --format json output docs, native issue-tracker filing integration, independent validation of an agent successfully parsing results end-to-end.", "evidenceIds": [ "trivy-docs-1", "trivy-docs-2", "trivy-docs-6", "trivy-gh-1", "trivy-gh-2", "trivy-comm-11", "trivy-comm-14" ] }, { "productId": "trivy", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Probes confirm no llms.txt exists (404) and no agent-oriented docs format is available; while an MCP server exists (separate axis), there's no evidence of llms.txt or agent-native documentation being served.", "evidenceIds": [ "trivy-probe-1", "trivy-probe-2" ] }, { "productId": "trivy", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Trivy ships an official MCP server (trivy-mcp) that lets AI tools query scan results via natural language ('Ask questions about security issues in natural language'), which enables AI-generated interaction with Trivy's data, but the evidence never shows Trivy itself producing AI-generated insights, remediation suggestions, or summaries beyond raw scan results being queryable. missing for 10: evidence of AI-generated remediation/insight suggestions produced by Trivy itself, first-party docs detailing what kind of AI reasoning the MCP server enables beyond Q&A, independent hands-on validation of the natural-language feature.", "evidenceIds": [ "trivy-docs-7", "trivy-docs-11", "trivy-docs-13", "trivy-probe-4" ] }, { "productId": "trivy", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Trivy is a CLI scanner that community evidence shows is easy to wire into CI/CD pipelines (trivy-comm-14) and it ships an MCP server (trivy-docs-13, trivy-probe-4) that lets agentic tools invoke scans via natural language, which supports background/automated invocation by an AI agent. However there is no documented native scheduler, daemon mode, or first-party 'autonomous background job' feature — automation depends entirely on external CI/cron infrastructure. Missing for 10: evidence of built-in scheduling/daemon mode, first-party autonomous trigger/orchestration docs, and independent confirmation of unattended long-running automation.", "evidenceIds": [ "trivy-comm-14", "trivy-docs-13", "trivy-probe-4", "trivy-docs-7" ] }, { "productId": "trivy", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Trivy is a CLI/scanner and provides an MCP server (trivy-mcp) so external AI assistants (e.g., in VS Code) can call Trivy's scanning capabilities — this is the reverse relationship of 'delegating tasks to a built-in AI assistant inside the product.' There is no evidence of an assistant embedded within Trivy itself that a user can delegate tasks to.", "evidenceIds": [ "trivy-docs-7", "trivy-docs-13", "trivy-probe-4" ] }, { "productId": "trivy", "storyId": "agentic-headless", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Trivy is a CLI-first tool distributed via brew/docker/binary with commands like `trivy image`, `trivy k8s --report summary`, designed for scripted/automated scanning, and community evidence confirms it's widely integrated into CI/CD pipelines ('super easy to integrate in every ci/cd pipeline', 'very easy to write automation around'). Missing for 10: no first-party CI-specific docs (e.g., GitHub Actions integration guide) cited in the pack, though community corroboration is strong.", "evidenceIds": [ "trivy-gh-1", "trivy-gh-2", "trivy-comm-11", "trivy-comm-12", "trivy-comm-14", "trivy-probe-5" ] }, { "productId": "trivy", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Trivy is a security/vulnerability scanner, not an AI agent or orchestrator that consumes external tools; the evidence only shows Trivy shipping its own MCP *server* (trivy-mcp) so that AI agents/IDEs can call Trivy's scanning capabilities, which is the opposite direction from this story (plugging MCP servers into Trivy so it can use their tools). This client-side 'consume other MCP servers' axis is a category error for a CLI scanning tool.", "evidenceIds": [ "trivy-docs-13", "trivy-probe-4" ] }, { "productId": "trivy", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Trivy ships an official trivy-mcp plugin that starts a Model Context Protocol server integrating Trivy's scanning with VS Code and other MCP-enabled tools, enabling natural-language security queries — directly matching the story. Missing for 10: independent/hands-on third-party corroboration of the MCP server's reliability beyond vendor docs.", "evidenceIds": [ "trivy-docs-13", "trivy-docs-7", "trivy-docs-11", "trivy-probe-4" ] }, { "productId": "trivy", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Trivy ships an official MCP server (trivy-mcp) explicitly advertised for 'Natural Language Scanning: Ask questions about security issues in natural language,' integrating with VS Code and other MCP-enabled tools, which enables natural-language operation via AI assistants. However, this is a separate plugin, not native to the core CLI, and evidence lacks independent hands-on validation of the natural-language experience quality. Missing for 10: independent/hands-on confirmation of natural-language query accuracy, broader detail on supported commands/scope beyond the one-line docs claim.", "evidenceIds": [ "trivy-docs-7", "trivy-docs-11", "trivy-docs-13", "trivy-probe-4" ] }, { "productId": "trivy", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Trivy is fundamentally shipped as an official CLI (trivy image, trivy fs, trivy k8s, trivy aws) distributed via brew/docker/github release, and community reports confirm it is easy to script and integrate into CI/CD pipelines. Missing for 10: dedicated independent benchmarking of CLI ergonomics for AI-agent use cases and any first-party AI-specific CLI flags (the AI-native/natural-language angle is handled by the separate MCP server, not the core CLI itself).", "evidenceIds": [ "trivy-gh-1", "trivy-gh-2", "trivy-probe-5", "trivy-docs-17", "trivy-comm-11", "trivy-comm-14" ] }, { "productId": "trivy", "storyId": "agentic-public-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Trivy exposes a documented CLI and an official MCP server (trivy-mcp) that explicitly targets AI-agent/natural-language driving of scans, which fits the agenticness/AI-native story. However, probes confirm no OpenAPI/REST API or machine-readable API spec (openapi.json, swagger.json all 404) and no llms.txt, so there's no general-purpose documented public API beyond the CLI/MCP surface. Missing for 10: a formal OpenAPI/REST API spec, broader API documentation beyond MCP, and independent confirmation of MCP server reliability/adoption.", "evidenceIds": [ "trivy-docs-7", "trivy-docs-11", "trivy-docs-13", "trivy-probe-3", "trivy-probe-4", "trivy-probe-1", "trivy-gh-2" ] }, { "productId": "trivy", "storyId": "agentic-scoped-keys", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Trivy is a security scanner CLI/tool, not an identity/credential-issuing platform; scoped API credential issuance for agents is not a capability this product category would offer, making this the wrong axis rather than a missing feature.", "evidenceIds": [] }, { "productId": "trivy", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Trivy's evidence pack documents a CLI, container/filesystem scanning, SBOM generation, and an MCP server plugin, but no official language SDKs (Go/Python/JS client libraries) for developers to build against are mentioned anywhere; probes for llms.txt/openapi also return 404. missing for 10: any documented official SDK/client library, language-specific package for programmatic integration, API reference for building custom tooling.", "evidenceIds": [ "trivy-probe-3", "trivy-probe-1", "trivy-docs-13", "trivy-gh-2" ] }, { "productId": "trivy", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Trivy is a CLI security scanner with an MCP server for AI querying, but there is no evidence of webhook subscription/event notification capability anywhere in the docs, GitHub repo, or probes. missing for 10: any webhook API/endpoint, event subscription mechanism, or documentation of push-based notifications.", "evidenceIds": [ "trivy-probe-3", "trivy-gh-1", "trivy-docs-13" ] }, { "productId": "trivy", "storyId": "ai-assisted-triage", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Trivy's official MCP server (trivy-mcp) enables natural-language querying/explanation of scan findings via AI-enabled tools like VS Code, which supports the 'explaining findings' part of the story, but there is no documented feature for AI-driven deduplication of findings or automated false-positive flagging. Missing for 10: explicit deduplication logic, false-positive triage/suppression via AI, and independent hands-on evidence that MCP-based triage actually reduces noise in practice.", "evidenceIds": [ "trivy-docs-7", "trivy-docs-11", "trivy-docs-13", "trivy-probe-4" ] }, { "productId": "trivy", "storyId": "ai-remediation-pr", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Trivy is a scanner that detects vulnerabilities, misconfigurations, secrets, and generates SBOMs; the evidence shows an MCP server for natural-language querying of scan results, but nowhere does it show Trivy or its MCP integration drafting remediation pull requests for review/merge. No evidence of automated PR generation capability.", "evidenceIds": [ "trivy-docs-7", "trivy-docs-13" ] }, { "productId": "trivy", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Trivy is a CLI security scanner with no public API reference; probes confirm no OpenAPI/Swagger spec exists (trivy-probe-3) and no llms.txt (trivy-probe-1). There is no evidence of any interactive API reference with runnable examples.", "evidenceIds": [ "trivy-probe-3", "trivy-probe-1" ] }, { "productId": "trivy", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Trivy is a CLI/scanner tool, not an API-driven service, but the story asks specifically for a downloadable machine-readable API spec; probes explicitly confirm no OpenAPI/swagger spec exists at any expected location and no llms.txt either. No evidence of any published API spec anywhere in the evidence pack.", "evidenceIds": [ "trivy-probe-3", "trivy-probe-1" ] }, { "productId": "trivy", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack shows Trivy scans real filesystems, images, repos, and IaC configs, and even documents an MCP plugin and CLI, but nothing describes a dedicated sandbox/test environment or non-production test mode that isolates it from production data. No mention of dry-run, mock targets, or staging environment support exists anywhere in the pack.", "evidenceIds": [] }, { "productId": "trivy", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of versioned APIs or a documented deprecation policy; probes show no OpenAPI spec, and docs evidence never mentions API versioning or deprecation guarantees.", "evidenceIds": [ "trivy-probe-3" ] }, { "productId": "trivy", "storyId": "autofix-findings", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Trivy is purely a scanner that detects vulnerabilities, misconfigurations, secrets, and licenses and reports findings; there is no evidence of any suggested-fix or auto-remediation capability in the docs, CLI, or MCP integration, which only supports natural-language querying of scan results.", "evidenceIds": [ "trivy-docs-1", "trivy-docs-4", "trivy-docs-3", "trivy-docs-13" ] }, { "productId": "trivy", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Trivy's CLI supports scanning multiple targets (images, filesystems, repos) and cluster-wide scans (e.g., trivy k8s cluster), and community evidence confirms it's easy to script and integrate into CI/CD pipelines for batch automation. However, there's no explicit documentation of a dedicated bulk/batch API, parallel multi-target job orchestration, or native support for processing large lists of items in one invocation beyond shell-level scripting. missing for 10: documented bulk/batch processing API, evidence of parallelized multi-target scanning at scale, first-party guidance on bulk operations beyond single-target CLI invocations.", "evidenceIds": [ "trivy-gh-1", "trivy-comm-11", "trivy-comm-14", "trivy-docs-2" ] }, { "productId": "trivy", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Trivy is a scanner run on-demand or in CI pipelines; the evidence shows no rule-based/event-triggered automation engine (e.g., defining rules that fire actions on events) within Trivy itself—only scan integrations into CI/CD and an MCP server for querying results conversationally. No evidence of a rules engine or event-driven action system.", "evidenceIds": [] }, { "productId": "trivy", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Trivy is a CLI security/vulnerability scanner invoked on-demand (in CI or manually); scheduling recurring jobs/workflows is a job orchestration concern belonging to CI/CD systems or cron, not a capability Trivy itself would ship. This is a category error for a scanner tool, not a missing feature.", "evidenceIds": [] }, { "productId": "trivy", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Trivy is a security scanner, not an automation/workflow platform; there is no concept of 'automations' to version, review, or roll back within its product category.", "evidenceIds": [] }, { "productId": "trivy", "storyId": "baseline-suppression", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no mention of Trivy's .trivyignore file, ignore-policies, baseline/suppression features, or VEX-based suppression that would let a security engineer mark existing findings as accepted/false-positive so only new issues alert. All docs excerpts focus on scanning capabilities (vulnerabilities, secrets, misconfig, SBOM) with none addressing triage/ignore workflows. Missing for 10: documentation of .trivyignore/ignore-file support, VEX suppression, baseline diffing, or any triage-noise reduction feature.", "evidenceIds": [] }, { "productId": "trivy", "storyId": "ci-pipeline-gate", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence confirms Trivy is widely used in CI/CD pipelines and is 'super easy to integrate' (trivy-comm-14, trivy-comm-11), and its scanning targets (image, filesystem, git) support PR-triggered scans, but the pack lacks any documentation of exit-code flags, fail-on-new-findings logic, or CI-specific configuration (e.g., GitHub Actions, exit-code thresholds). Missing for 10: explicit docs on --exit-code / --exit-on-eol flags, examples of failing CI builds on new vs. baseline findings, and first-party GitHub Actions/pipeline integration guides.", "evidenceIds": [ "trivy-comm-14", "trivy-comm-11", "trivy-docs-1", "trivy-docs-3" ] }, { "productId": "trivy", "storyId": "container-image-scan", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Trivy's core, well-documented capability is scanning container images for OS-package vulnerabilities (trivy-docs-1, trivy-docs-17) and application-layer/lock-file vulnerabilities (trivy-docs-2), backed by community corroboration of strong real-world CI/CD scanning use (trivy-comm-11, trivy-comm-12, trivy-comm-14). This directly matches the devops-lead's pre-deployment scanning need. missing for 10: independent benchmark/comparison of detection accuracy beyond community sentiment.", "evidenceIds": [ "trivy-docs-1", "trivy-docs-17", "trivy-docs-2", "trivy-comm-11", "trivy-comm-12", "trivy-comm-14" ] }, { "productId": "trivy", "storyId": "cross-file-dataflow", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Trivy is a vulnerability/misconfiguration/secret/SBOM scanner focused on known-CVE and dependency detection; the evidence pack shows no taint-analysis or cross-file/cross-function data-flow SAST capability (e.g. no mention of source-code taint tracking like Semgrep/CodeQL). This is an applicable axis for a security scanning tool, but there is no evidence Trivy performs taint-style SAST analysis.", "evidenceIds": [] }, { "productId": "trivy", "storyId": "custom-sast-rules", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Trivy's evidence covers vulnerability, misconfiguration, secret, license, and SBOM scanning, but there is no mention of a custom rule-writing syntax (e.g., Rego/OPA policies) or a mechanism for sharing custom rules across repositories in this evidence pack.", "evidenceIds": [] }, { "productId": "trivy", "storyId": "custom-secret-rules", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence confirms Trivy has built-in secret scanning for passwords, API keys, and tokens (trivy-docs-3, trivy-docs-10), but nothing in the pack describes an ability for users to define custom regex/rules for organization-specific token formats. Missing for 10: any documentation of custom secret rule configuration, regex pattern definition, or allow/deny-list rule authoring.", "evidenceIds": [ "trivy-docs-3", "trivy-docs-10" ] }, { "productId": "trivy", "storyId": "dependency-vulnerability-scan", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Trivy explicitly documents scanning lockfiles like Gemfile.lock and package-lock.json for known vulnerabilities, with strong first-party docs and positive independent community feedback on its CI/CD scanning usage. missing for 10: broader independent benchmark of lockfile-specific detection accuracy across more ecosystems.", "evidenceIds": [ "trivy-docs-2", "trivy-comm-11", "trivy-comm-14" ] }, { "productId": "trivy", "storyId": "detect-secrets-in-repo", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Trivy explicitly supports secret scanning across filesystem, container image, and git repository targets, enabled by default (trivy-docs-3, trivy-docs-10). However, the evidence never confirms that this scan traverses the full git commit history (e.g., detecting secrets committed and later removed) rather than just the current working tree/HEAD state. Missing for 10: explicit documentation or demonstration of full git-history secret scanning, independent verification of history-depth coverage.", "evidenceIds": [ "trivy-docs-3", "trivy-docs-10", "trivy-docs-9" ] }, { "productId": "trivy", "storyId": "fix-upgrade-guidance", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Trivy is purely a scanner/detector—evidence shows it detects vulnerabilities, misconfigurations, secrets, and generates SBOMs, but nothing indicates it suggests upgrade paths, fix versions, or opens automated remediation pull requests. No auto-fix PR or dependency-upgrade feature is documented anywhere in the evidence pack.", "evidenceIds": [ "trivy-docs-1", "trivy-docs-2", "trivy-docs-6" ] }, { "productId": "trivy", "storyId": "guardrails-for-agent-code", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Trivy ships an official MCP server (trivy-mcp) that integrates with VS Code and other MCP-enabled tools, letting coding agents invoke Trivy's scanning and query results in natural language — a direct fit for wiring into an AI coding agent. However, evidence is limited to first-party docs with no independent hands-on account of this specific workflow catching AI-generated code issues pre-merge. Missing for 10: independent/hands-on validation of the MCP integration in real agent workflows, and specifics on scanning AI-generated code specifically before it lands (e.g., pre-commit/pre-merge gating via the agent).", "evidenceIds": [ "trivy-docs-7", "trivy-docs-11", "trivy-docs-13", "trivy-probe-4" ] }, { "productId": "trivy", "storyId": "iac-misconfig-scan", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Trivy's docs explicitly state built-in misconfiguration checks for Docker, Kubernetes, Terraform, CloudFormation and more, plus k8s cluster scanning via `trivy k8s`, and community reviews corroborate strong CI/CD IaC scanning usage. Missing for 10: independent hands-on validation specifically of Terraform/K8s misconfig detection accuracy (only Docker/vuln scanning has community corroboration).", "evidenceIds": [ "trivy-docs-4", "trivy-gh-1", "trivy-comm-11", "trivy-comm-14" ] }, { "productId": "trivy", "storyId": "language-coverage", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Trivy documents scanning of container images, filesystems, and git repos for vulnerabilities via lock files (e.g., Gemfile.lock, package-lock.json), plus secrets, misconfigurations, licenses, and SBOM generation—covering many languages/ecosystems and package managers broadly. However, the evidence pack lacks an explicit enumerated list of 'all major languages/ecosystems' supported, and one community report notes a runtime panic on a specific scan type (AWS scanning), suggesting coverage gaps or bugs in some scan modes. missing for 10: an authoritative full list of supported languages/package managers, independent verification of breadth across ecosystems, and confirmation that edge-case scans (e.g., cloud scanning) are reliable.", "evidenceIds": [ "trivy-docs-1", "trivy-docs-2", "trivy-docs-3", "trivy-docs-4", "trivy-docs-5", "trivy-docs-6", "trivy-comm-15", "trivy-comm-9" ] }, { "productId": "trivy", "storyId": "license-compliance-scan", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Trivy documents dedicated license scanning that detects license files in container images and gives an opinionated risk categorization (trivy-docs-5), which covers the 'detect licenses' half of the story. However, the evidence pack contains no documentation of configurable allow/deny license lists or exit-code/policy enforcement gating a build on disallowed licenses—only the general misconfiguration/IaC policy engine is described (trivy-docs-4), not license-specific policy enforcement.\nmissing for 10: explicit license allow/deny-list configuration, CI/CD gating or exit-code enforcement tied to license violations, independent confirmation of license policy enforcement in practice.", "evidenceIds": [ "trivy-docs-5", "trivy-docs-4" ] }, { "productId": "trivy", "storyId": "machine-readable-output", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Trivy generates SBOM formats (CycloneDX/SPDX) and integrates well with CI/CD, but nowhere documents the --format flag or explicit JSON/SARIF vulnerability-scan output that this story asks about. missing for 10: explicit documentation of JSON/SARIF output format flags for vulnerability/misconfiguration scan results, any example of SARIF file generation, and confirmation of downstream tool consumption (e.g., GitHub Code Scanning integration).", "evidenceIds": [ "trivy-comm-9", "trivy-comm-14" ] }, { "productId": "trivy", "storyId": "openness-api-parity", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Trivy is a CLI-based security scanner with no evidenced graphical UI or REST API surface (openapi/API probes return 404); the API-vs-UI parity question is a category mismatch for a CLI-native tool, not a capability gap.", "evidenceIds": [ "trivy-probe-3", "trivy-gh-2", "trivy-probe-1" ] }, { "productId": "trivy", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Trivy generates SBOMs and scan reports in open, interoperable standards (CycloneDX, SPDX) rather than proprietary formats, and being open-source/self-hosted there is inherently no vendor lock-in to escape from. However, there's no explicit documentation framing this as 'exporting all your data to leave' — it's a scanner tool, not a data-storing platform, so the story is only partially applicable and evidenced. Missing for 10: explicit statement about exporting full scan history/config, independent corroboration that these formats are freely portable across tools, and clarity on what 'all data' means for a stateless CLI scanner.", "evidenceIds": [ "trivy-docs-6", "trivy-docs-12", "trivy-docs-18", "trivy-docs-15" ] }, { "productId": "trivy", "storyId": "openness-open-license", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Trivy's source is hosted publicly on GitHub (aquasecurity/trivy) and its own site describes it as an 'open source vulnerability scanning' tool, implying an open license, but no evidence pack item explicitly cites a license file (e.g., Apache-2.0) or license text. Missing for 10: explicit license identifier/citation, confirmation of license terms (permissions, redistribution), and independent corroboration of license compliance.", "evidenceIds": [ "trivy-gh-1", "trivy-gh-2", "trivy-docs-14", "trivy-probe-5" ] }, { "productId": "trivy", "storyId": "openness-self-host", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Trivy is an open-source CLI/binary distributed via package managers and Docker images, meaning the core scanning engine runs entirely on the user's own infrastructure with no mandatory SaaS dependency, satisfying the self-hosting expectation for an AI-native user who wants full control over the tool. Missing for 10: explicit documentation of a self-hosted 'Trivy Server' mode/architecture for shared caching, and independent hands-on confirmation of air-gapped/offline self-hosted deployments.", "evidenceIds": [ "trivy-gh-2", "trivy-probe-5", "trivy-docs-14" ] }, { "productId": "trivy", "storyId": "pr-comments-scm", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions PR/MR comments, GitHub/GitLab checks, or CI annotation integration; the pack only covers scanning targets, SBOM, MCP server, and unrelated security incidents. This is a plausible capability for a scanner (via CI actions), so the axis applies, but absence of evidence means 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "trivy", "storyId": "pre-commit-hook", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence shows Trivy scans secrets and filesystems and is easy to integrate into CI/CD generally, but no documentation or mention of a pre-commit hook framework integration (e.g., .pre-commit-hooks.yaml) is provided in the evidence pack.", "evidenceIds": [] }, { "productId": "trivy", "storyId": "privacy-data-residency", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Trivy is a self-hosted/CLI open-source scanner that runs locally or in CI; it does not store user data in a vendor cloud, so region/data-residency selection is not an applicable axis.", "evidenceIds": [] }, { "productId": "trivy", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Trivy is a security/vulnerability scanning tool; it has no data collection or AI-training relationship with user data, making an 'opt out of AI training' privacy control a category error for this product type.", "evidenceIds": [] }, { "productId": "trivy", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence describes any data retention, storage duration, or deletion controls for Trivy's scan results, cache, vulnerability DB, or MCP interactions; the docs focus solely on scanning capabilities and CLI/MCP setup, not privacy/retention policy.", "evidenceIds": [] }, { "productId": "trivy", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions telemetry, usage tracking, or an opt-out mechanism for Trivy or its MCP server; the axis is plausible for a CLI/MCP tool but no documentation confirms or denies it.", "evidenceIds": [] }, { "productId": "trivy", "storyId": "reachability-prioritization", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Trivy's evidence pack shows vulnerability detection, SBOM generation, secret scanning, and misconfiguration checks, but nothing about reachability analysis or code-path tracing to determine if vulnerable dependency code is actually invoked/exposed. No mention of call-graph analysis, static reachability, or exploitability-based prioritization.", "evidenceIds": [] }, { "productId": "trivy", "storyId": "sast-code-scanning", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Trivy's documented capabilities are vulnerability scanning (known CVEs in dependencies/images), secret detection, IaC misconfiguration checks, license and SBOM generation — none of the evidence shows a SAST engine that analyzes source code for vulnerability classes like injection or XSS. This is a fair axis for a security scanning product, but no evidence supports it, so it is 'none' rather than 'na'.", "evidenceIds": [ "trivy-docs-1", "trivy-docs-2", "trivy-docs-3", "trivy-docs-4", "trivy-docs-5", "trivy-docs-6" ] }, { "productId": "trivy", "storyId": "sbom-generation", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Trivy's docs explicitly document SBOM generation in both CycloneDX and SPDX formats for container images and filesystems/local projects, backed by a dedicated supply-chain SBOM guide and CLI usage. Missing for 10: independent/hands-on community corroboration specifically of SBOM output quality (community evidence focuses on vulnerability scanning and unrelated security incidents, not SBOM generation).", "evidenceIds": [ "trivy-docs-6", "trivy-docs-12", "trivy-docs-15", "trivy-docs-18" ] }, { "productId": "trivy", "storyId": "scan-beyond-git", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Trivy's secret scanner explicitly supports container images, filesystem, and git repositories (trivy-docs-3, trivy-docs-10), covering the 'Docker images' part of the story, but there is no evidence of secret scanning for S3 buckets or CI logs specifically. A 'trivy aws' subcommand exists (trivy-comm-15) but it's shown crashing and is not documented as a secret-detection target. missing for 10: documented S3 bucket secret scanning, documented CI log secret scanning, and hands-on confirmation these non-git sources work reliably.", "evidenceIds": [ "trivy-docs-3", "trivy-docs-10", "trivy-comm-15" ] }, { "productId": "trivy", "storyId": "severity-filtering", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no mention of severity or confidence filtering flags (e.g. --severity, --exit-code) or any gating mechanism to suppress low-value findings; all cited docs focus on scan targets (image, filesystem, secrets, misconfig, SBOM) rather than triage/filtering controls. Missing for 10: any documentation of severity/confidence thresholds, exit-code gating, or ignore-file mechanisms to reduce noise.", "evidenceIds": [] }, { "productId": "trivy", "storyId": "verify-live-secrets", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Trivy's docs describe detecting secrets (passwords, API keys, tokens) via pattern-based scanning, but there is no evidence of any feature that verifies detected secrets against the issuing service (e.g., calling an API to check if a credential is still active/live). No mention of live-verification capability anywhere in the evidence pack.", "evidenceIds": [ "trivy-docs-3", "trivy-docs-10" ] }, { "productId": "trufflehog", "storyId": "agent-runs-scan-headless", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "TruffleHog's CLI clearly supports headless, scriptable scanning (e.g. `trufflehog git ... --results=verified,unknown --fail`, `--include-detectors`, `--verify-detectors`) and Docker/CI usage patterns are well documented, and Enterprise routing to Jira/Slack/webhooks is mentioned. However, there is no evidence of a structured/JSON machine-readable output flag, nor any documented mechanism for the CLI itself to file issues (e.g. GitHub issue creation) based on findings. Missing for 10: explicit machine-readable output format documentation, and evidence of automated issue-filing integration from CLI results.", "evidenceIds": [ "trufflehog-docs-13", "trufflehog-docs-8", "trufflehog-docs-21", "trufflehog-docs-23", "trufflehog-docs-11", "trufflehog-gh-2" ] }, { "productId": "trufflehog", "storyId": "agentic-agent-docs", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "llms.txt itself returns a 404, but the docs site serves markdown versions of pages (e.g. docs.md, pre-commit-hooks.md, customizing-detection.md) that an agent could fetch directly, showing some agent-oriented doc structure without a formal llms.txt manifest. missing for 10: a working llms.txt file, any explicit agent/LLM-consumption guidance, and confirmation these .md endpoints are documented as an intentional agent-facing interface.", "evidenceIds": [ "trufflehog-probe-1", "trufflehog-probe-2", "trufflehog-docs-5", "trufflehog-docs-6" ] }, { "productId": "trufflehog", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "TruffleHog is a secrets-scanning/detection tool; the evidence pack describes verification, classification, and analyzer features but no AI-generated insights, summaries, or suggestions surfaced to users—no LLM-based feature is mentioned anywhere.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "TruffleHog supports background automation in the sense of continuous monitoring of sources (Git, Jira, Slack, etc.), hosted/self-hosted scanning, and automated alert routing (Slack, Jira, webhooks) that can run unattended, which fits the 'runs autonomously in the background' part of the story. However, this is generic security-scanning automation, not an AI-agent-specific or LLM-orchestrated automation framework, and there's no evidence of agent-triggered workflows, scheduling APIs, or AI-native orchestration hooks beyond CI/pre-commit hooks and continuous monitoring. Missing for 10: explicit AI-agent/LLM-triggered automation setup, scheduling/orchestration API for autonomous runs, and independent confirmation that background monitoring runs reliably unattended.", "evidenceIds": [ "trufflehog-gh-8", "trufflehog-gh-9", "trufflehog-docs-17", "trufflehog-docs-33", "trufflehog-docs-23", "trufflehog-docs-28", "trufflehog-docs-11" ] }, { "productId": "trufflehog", "storyId": "agentic-builtin-assistant", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "TruffleHog is a secrets-scanning CLI/platform, not an AI assistant product; there is no evidence of any built-in AI assistant to delegate tasks to. This axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "TruffleHog ships as a CLI and Docker image that can be invoked non-interactively (e.g. `trufflehog github --only-verified --repo ...`, `docker run ... trufflesecurity/trufflehog:latest github --org=...`) and supports automation-friendly flags like `--fail` and `--results=verified,unknown` for pipeline gating, plus pre-commit/pre-receive hook integration for git automation. missing for 10: explicit first-party CI/CD platform docs (e.g. GitHub Actions, GitLab CI templates) and independent hands-on confirmation of headless CI usage beyond docs.", "evidenceIds": [ "trufflehog-docs-1", "trufflehog-docs-13", "trufflehog-gh-2", "trufflehog-gh-6", "trufflehog-docs-5", "trufflehog-docs-19", "trufflehog-gh-1" ] }, { "productId": "trufflehog", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "TruffleHog is a secrets-scanning security tool, not an AI agent or assistant that consumes MCP tools; nothing in the evidence relates to MCP integration or agentic tool use, so this axis is a category error for this product.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "agentic-mcp-server", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "TruffleHog is a secrets-scanning CLI/platform, not an agentic coding assistant or MCP-serving product; the evidence pack shows CLI usage, scanning integrations, and enterprise dashboard features but no MCP server or agent-connectivity capability, and this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "agentic-nl-commands", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "TruffleHog is a CLI/platform secrets-scanning tool with flag-based commands (--only-verified, --include-detectors, etc.), not a conversational or agentic interface; natural-language command operation is a category mismatch for this type of product, not a missing feature.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "TruffleHog ships a well-documented official CLI (installable via brew/docker) with a rich set of subcommands and flags (git, github, scan, analyze, --include-detectors, --verify-detectors, --fail, --results) suitable for scripted/agentic invocation, and is explicitly used in automation contexts like pre-commit hooks and CI pipelines. Missing for 10: explicit documentation of machine-readable (JSON) output formats or exit-code semantics tailored for agent/automation consumption, and independent corroboration beyond vendor docs/GitHub.", "evidenceIds": [ "trufflehog-gh-1", "trufflehog-gh-2", "trufflehog-gh-6", "trufflehog-docs-13", "trufflehog-docs-14", "trufflehog-docs-7", "trufflehog-docs-21", "trufflehog-probe-4" ] }, { "productId": "trufflehog", "storyId": "agentic-public-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of a documented public API (REST/GraphQL/SDK) for programmatically driving TruffleHog; probes for llms.txt and openapi/swagger specs both 404, and only CLI flags/config and a webhook for custom-detector verification are documented, not a driving API.", "evidenceIds": [ "trufflehog-probe-1", "trufflehog-probe-3", "trufflehog-docs-27" ] }, { "productId": "trufflehog", "storyId": "agentic-scoped-keys", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "TruffleHog is a secrets-scanning/detection tool, not an identity/credential-issuance system; it discovers and analyzes existing secrets' permissions but does not issue scoped or least-privilege credentials for agents. This axis is a category error for this type of product.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack documents TruffleHog's CLI, Docker image, detectors, and API-like analyzer features, but contains no mention of an official SDK (Python, JS, Go client library, etc.) for developers to build against; probes for OpenAPI specs also returned 404s, indicating no such offering.", "evidenceIds": [ "trufflehog-probe-3", "trufflehog-gh-1", "trufflehog-docs-1" ] }, { "productId": "trufflehog", "storyId": "agentic-webhooks", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs mention routing findings to webhooks and setting up alerts across chosen platforms, indicating an outbound webhook notification capability exists, but there is no detailed documentation of webhook subscription/configuration, payload schema, or an API/OpenAPI spec (probe shows no OpenAPI found) to confirm depth of support. missing for 10: documented webhook configuration/subscription API, payload schema examples, independent/hands-on confirmation.", "evidenceIds": [ "trufflehog-docs-11", "trufflehog-docs-17", "trufflehog-docs-33", "trufflehog-probe-3" ] }, { "productId": "trufflehog", "storyId": "ai-assisted-triage", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "TruffleHog's evidence shows credential verification, classification, and permission analysis (analyzer) to reduce false positives, but none of this involves AI/LLM-based explanation, deduplication, or triage of findings — it's rule-based verification, not AI reasoning. No mention of AI, LLM, or natural-language explanation features appears anywhere in the pack.", "evidenceIds": [ "trufflehog-docs-3", "trufflehog-docs-9", "trufflehog-gh-4", "trufflehog-docs-29" ] }, { "productId": "trufflehog", "storyId": "ai-remediation-pr", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "TruffleHog is a secrets-scanning/detection tool that finds and verifies exposed credentials; it has no evidence of generating remediation code changes or AI-drafted pull requests. This story about AI-drafted remediation PRs is a category error for a secrets scanner (which surfaces findings/alerts rather than authoring code fixes), so the axis does not apply.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "TruffleHog is a CLI/scanning tool with docs and CLI examples, but there is no evidence of an interactive API reference (e.g., Swagger/OpenAPI explorer) with runnable examples; the openapi probe explicitly returned 404s for all candidate API spec paths and no interactive docs are mentioned.", "evidenceIds": [ "trufflehog-probe-3", "trufflehog-probe-1" ] }, { "productId": "trufflehog", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe for OpenAPI/swagger endpoints returned 404 for all candidate paths, and no evidence anywhere in the pack points to a downloadable machine-readable API spec.", "evidenceIds": [ "trufflehog-probe-3" ] }, { "productId": "trufflehog", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "TruffleHog's evidence only shows a demo scan against a public 'test_keys' GitHub repo used to illustrate CLI usage, not a dedicated sandbox/test environment for AI-native agents to validate behavior without touching production systems or secrets. There is no mention of an isolated sandbox mode, mock API, or non-production testing environment tailored for agentic use.", "evidenceIds": [ "trufflehog-docs-1", "trufflehog-gh-6" ] }, { "productId": "trufflehog", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of versioned APIs or a documented deprecation policy; probe results show no OpenAPI spec found (404s across all candidate paths), and no docs mention API versioning or deprecation practices.", "evidenceIds": [ "trufflehog-probe-3" ] }, { "productId": "trufflehog", "storyId": "autofix-findings", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "TruffleHog is a secrets-detection/scanning tool that finds, verifies, and reports on exposed secrets, and notifies teams to rotate/remediate them manually; there is no evidence of any auto-fix, patch-suggestion, or code-remediation capability—only alerting, tracking remediation status, and pre-commit/pre-receive blocking.", "evidenceIds": [ "trufflehog-docs-17", "trufflehog-docs-33", "trufflehog-docs-5", "trufflehog-docs-30" ] }, { "productId": "trufflehog", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "TruffleHog can scan across many sources at once (whole orgs, multiple repos, Docker images, Slack, Jira, Confluence, filesystems, etc.) and supports batch verification/classification of hundreds of secret types, which is a form of bulk operation. However, this is bulk *scanning*, not general-purpose bulk operations across arbitrary 'items' in an AI-native/automation sense (e.g., no evidence of bulk remediation, bulk API-driven item processing, or agent-orchestrated batch workflows beyond scanning). missing for 10: evidence of bulk actions beyond scanning (e.g., bulk remediation/rotation across many secrets programmatically), AI-agent-oriented batch orchestration, and independent confirmation of bulk-scale performance at scale.", "evidenceIds": [ "trufflehog-docs-12", "trufflehog-docs-18", "trufflehog-gh-8", "trufflehog-gh-9", "trufflehog-docs-2", "trufflehog-docs-3" ] }, { "productId": "trufflehog", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "TruffleHog supports event-driven automation via pre-commit/pre-receive git hooks that block commits/pushes containing secrets, custom detectors with webhook-based verification, and routing of findings to Slack/Jira/email/webhooks with continuous monitoring and alerting when remediation occurs. This is real rule-triggered action but it's scoped narrowly to secret-detection events rather than a general-purpose rule engine where arbitrary conditions map to arbitrary actions. Missing for 10: a documented general rules/policy engine with configurable conditions beyond hooks and webhooks, and independent hands-on confirmation that alerting/webhook routing works end-to-end in practice.", "evidenceIds": [ "trufflehog-docs-5", "trufflehog-docs-13", "trufflehog-docs-19", "trufflehog-docs-30", "trufflehog-docs-11", "trufflehog-docs-17", "trufflehog-docs-27", "trufflehog-docs-6" ] }, { "productId": "trufflehog", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "TruffleHog's docs mention 'continuously monitoring' sources and 'continuously tracks the status of all key types' (trufflehog-docs-17, trufflehog-gh-8/9), but there is no evidence of an actual scheduler, cron integration, or workflow/job orchestration feature that lets a user configure recurring scans or automation pipelines. The CLI and docs focus on one-off scans, CI/pre-commit hook triggers, and hosted dashboards, not recurring job scheduling.", "evidenceIds": [ "trufflehog-docs-17", "trufflehog-gh-8", "trufflehog-gh-9", "trufflehog-docs-23" ] }, { "productId": "trufflehog", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "TruffleHog is a secrets-scanning security tool, not an automation-builder platform; there is no concept of 'automations' to version, review, or roll back within its product scope. This story is a category error for this product type.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "baseline-suppression", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "TruffleHog supports scanning only new commits via `--since-commit` (used in pre-commit hooks) and reduces false positives via `--only-verified`/verification flags, which can approximate 'ignore existing/false positives, alert only new'. However there is no documented allowlist, ignore-file, or finding-suppression mechanism for marking specific findings as accepted/false-positive across full scans. missing for 10: explicit baseline/ignore-list feature, per-finding suppression or allowlisting, independent confirmation that verified-only filtering meaningfully reduces false positives in practice.", "evidenceIds": [ "trufflehog-docs-13", "trufflehog-docs-8", "trufflehog-docs-21", "trufflehog-docs-1" ] }, { "productId": "trufflehog", "storyId": "ci-pipeline-gate", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "TruffleHog docs show the `--fail` flag (trufflehog-docs-13) which exits non-zero on findings, and Docker/CLI invocations suitable for CI pipelines (trufflehog-gh-2, trufflehog-gh-6), plus verified-only filtering to reduce noise. However, the evidence is framed around pre-commit/pre-receive hooks rather than an explicit CI-on-PR integration (e.g., GitHub Actions example, diffing against base branch for 'new findings only'), and there's no first-party CI pipeline template or independent confirmation of CI usage. missing for 10: explicit CI/PR pipeline example (e.g. GitHub Actions workflow), documentation of scanning only new commits in a PR diff context, and independent/hands-on confirmation of CI fail-on-new-finding behavior.", "evidenceIds": [ "trufflehog-docs-13", "trufflehog-gh-2", "trufflehog-gh-6", "trufflehog-docs-5", "trufflehog-docs-30" ] }, { "productId": "trufflehog", "storyId": "container-image-scan", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "TruffleHog is a secrets-detection tool; while it can scan Docker images for exposed credentials, this story asks about OS-package/CVE and application vulnerability scanning, which is a fundamentally different product category (vulnerability scanners like Trivy/Grype) that TruffleHog does not address.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "cross-file-dataflow", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "TruffleHog is a secrets-detection tool that scans for hardcoded credentials/keys across sources; it does not perform SAST-style taint analysis tracking data flow across files and function boundaries. This story targets a different product category (static application security testing with taint tracking), so it is a category error for TruffleHog.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "custom-sast-rules", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "TruffleHog supports 'custom regex detectors' defined in a config file with optional webhook verification, which is a form of custom rule authoring in a simple (regex/YAML) syntax, and these config files can in principle be reused across scans/repos via `--config=config.yaml`. However, this is scoped narrowly to secret-detection regex patterns rather than general static-analysis rules, and there is no documented feature for centrally sharing/distributing these custom detector configs across repositories (e.g. a rule registry or team-wide rule sync). Missing for 10: broader SAST rule language beyond regex, explicit cross-repo rule-sharing/distribution mechanism, and independent evidence of teams using this for rule sharing.", "evidenceIds": [ "trufflehog-docs-6", "trufflehog-docs-27", "trufflehog-docs-14", "trufflehog-docs-7" ] }, { "productId": "trufflehog", "storyId": "custom-secret-rules", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "TruffleHog explicitly supports custom regex detectors for defining organization-specific secret formats, with optional webhook-based verification (trufflehog-docs-6, trufflehog-docs-27), plus CLI flags to include/exclude detectors and control verification (trufflehog-docs-7, trufflehog-docs-14, trufflehog-docs-21). This is documented first-party capability directly matching the story. Missing for 10: independent/community corroboration specifically of the custom-detector feature working in practice (community evidence in the pack discusses licensing/comparisons, not custom rules).", "evidenceIds": [ "trufflehog-docs-6", "trufflehog-docs-27", "trufflehog-docs-7", "trufflehog-docs-14", "trufflehog-docs-21" ] }, { "productId": "trufflehog", "storyId": "dependency-vulnerability-scan", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "TruffleHog is a secrets-scanning tool focused on detecting credentials/keys in code, git history, chat, and other sources; it does not analyze dependency manifests/lockfiles for known CVEs, which is the domain of SCA tools (e.g., Snyk, Dependabot). This is a category mismatch, not a missing feature.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "detect-secrets-in-repo", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "TruffleHog's core documented functionality is scanning git repos (and full history via commands like `trufflehog github --repo ...`) for hardcoded secrets, with 800+ detectors and live verification, corroborated by both official docs and GitHub README. Community commentary criticizes maturity/hype but does not contradict the core git-history scanning capability. Missing for 10: independent hands-on benchmark of full-history scan accuracy/performance beyond vendor docs.", "evidenceIds": [ "trufflehog-docs-1", "trufflehog-gh-6", "trufflehog-docs-25", "trufflehog-docs-29", "trufflehog-gh-3", "trufflehog-docs-13" ] }, { "productId": "trufflehog", "storyId": "fix-upgrade-guidance", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "TruffleHog is a secrets-scanning tool, not a software composition analysis (SCA) or dependency-vulnerability tool; it has no concept of dependency versions, vulnerability databases, or upgrade/fix-PR automation. This story asks about dependency vulnerability remediation, which is a wrong axis for a secrets detection product.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "guardrails-for-agent-code", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "TruffleHog ships well-documented pre-commit and pre-receive git hooks (trufflehog-docs-5, -13, -19, -20, -26, -30, -31) that scan code for secrets before it is committed, which can technically catch AI-generated code paths through standard git hooks or CI. However there is no evidence of any specific integration point for coding agents (e.g., an MCP server, IDE/agent plugin, or agent-specific pre-generation hook) — it's a generic pre-commit mechanism, not an AI-agent-aware wiring. Missing for 10: explicit coding-agent/AI-tool integration (MCP server, agent plugin, or IDE hook), evidence of scanning AI-generated code specifically, and independent confirmation the pre-commit flow works well with agent-driven commit workflows.", "evidenceIds": [ "trufflehog-docs-5", "trufflehog-docs-13", "trufflehog-docs-19", "trufflehog-docs-20", "trufflehog-docs-26", "trufflehog-docs-30", "trufflehog-docs-31" ] }, { "productId": "trufflehog", "storyId": "iac-misconfig-scan", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "TruffleHog is a secrets-scanning tool focused on detecting credentials/keys across repos, chats, and other sources; it does not perform IaC misconfiguration analysis (e.g., Terraform/Kubernetes policy checks) which is the domain of tools like Checkov or tfsec. This is a category mismatch, not a missing feature.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "language-coverage", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "TruffleHog scans raw content (git repos, filesystems, Docker images, chats, wikis, logs, object stores) rather than parsing by programming language, so it inherently ignores language/ecosystem boundaries for secret detection, and docs confirm broad source-type coverage beyond code repos. However there is no explicit documentation enumerating support across major languages or package-ecosystem-specific scanning (e.g., dependency manifests, language-specific SBOM/package files), so coverage-accuracy across ecosystems is implied rather than demonstrated. Missing for 10: explicit language/ecosystem coverage claims, package-manifest-aware scanning, and independent verification that all major ecosystems are handled equally well.", "evidenceIds": [ "trufflehog-docs-12", "trufflehog-docs-18", "trufflehog-gh-8", "trufflehog-gh-9" ] }, { "productId": "trufflehog", "storyId": "license-compliance-scan", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "TruffleHog is a secrets-detection tool focused on scanning for credentials/keys, not a dependency/license-compliance scanner (e.g., SBOM license analysis or SCA tooling); license policy enforcement is a wrong-axis question for this product category.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "machine-readable-output", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack describes scanning capabilities, verification, and CLI flags extensively, but no citation mentions JSON, SARIF, or any specific machine-readable output format for downstream tooling integration. Absence of evidence for this applicable capability yields 'none'.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "openness-api-parity", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While TruffleHog ships a CLI and an Enterprise UI/platform, the evidence pack shows no documented API (OpenAPI/swagger endpoints all 404) that would let a user replicate UI actions programmatically; only CLI flags and webhook-based verification for custom detectors are documented, not a general platform API.", "evidenceIds": [ "trufflehog-probe-3", "trufflehog-docs-24", "trufflehog-docs-10" ] }, { "productId": "trufflehog", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence covers scanning, detection, verification, and hosted/self-hosted deployment options, but there is no documentation of any data export feature, open export format, or migration/exit path for users of the hosted platform or CLI findings store.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "openness-open-license", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "TruffleHog's source is hosted publicly on GitHub (trufflehog-gh-1..8) and community discussion confirms it is licensed under AGPL (v3, shifted from GPL) — an OSI-recognized open license, satisfying the ability to read the source under an open license. Missing for 10: no direct citation of a LICENSE file or official docs statement confirming license terms, and no independent corroboration beyond the HN comment.", "evidenceIds": [ "trufflehog-comm-1", "trufflehog-gh-1", "trufflehog-probe-4" ] }, { "productId": "trufflehog", "storyId": "openness-self-host", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "TruffleHog is open source (CLI installable via brew/Docker/GitHub) and docs explicitly offer a self-hosted deployment option distinct from the hosted/managed offering, with getting-started guidance for running your own scanner. Missing for 10: detailed self-hosting infrastructure/deployment docs (e.g., Kubernetes/Helm charts, scaling guidance) and independent hands-on confirmation of self-hosting the full platform (vs just the OSS CLI).", "evidenceIds": [ "trufflehog-docs-23", "trufflehog-docs-28", "trufflehog-docs-10", "trufflehog-gh-1", "trufflehog-gh-2" ] }, { "productId": "trufflehog", "storyId": "pr-comments-scm", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers pre-commit hooks, pre-receive hooks, CLI scanning, and routing findings to Slack/Jira/email/webhooks, but there is no mention of PR comments or status checks in GitHub or GitLab CI workflows.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "pre-commit-hook", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "TruffleHog has dedicated first-party documentation for pre-commit hooks, including a specific example invocation (`trufflehog git file://. --since-commit HEAD --results=verified,unknown --fail`), guidance on repo-wide setup via `core.hooksPath`, Husky integration, a pre-receive hook alternative for server-side enforcement, and even the bypass flag (`--no-verify`) — directly matching the story of preventing secrets from entering git history at commit time. Missing for 10: independent/community hands-on confirmation that the pre-commit workflow works smoothly in practice (community evidence is mostly about unrelated aspects like licensing and other scanners).", "evidenceIds": [ "trufflehog-docs-5", "trufflehog-docs-13", "trufflehog-docs-19", "trufflehog-docs-20", "trufflehog-docs-26", "trufflehog-docs-30", "trufflehog-docs-31", "trufflehog-docs-16" ] }, { "productId": "trufflehog", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "TruffleHog offers a self-hosted deployment option (\"Choose hosted (managed by Truffle Security) or self-hosted (running in your environment)\"), which lets an org keep all scan data within its own infrastructure/region, giving indirect control over data residency. However, there is no explicit region-selection feature or documented multi-region hosting for the managed/enterprise offering. Missing for 10: explicit region/residency selection controls for the hosted service, and any documentation addressing data-locality guarantees.", "evidenceIds": [ "trufflehog-docs-28", "trufflehog-docs-23" ] }, { "productId": "trufflehog", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "TruffleHog is a secrets-scanning security tool, not an AI model or data-processing service that trains on user data; the axis of preventing data use for AI training is a category error for this product type.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "TruffleHog is a secrets-scanning tool (self-hosted or hosted scanner); the evidence pack shows scanning, detection, verification, and alerting capabilities but contains no mention of data retention policies, deletion controls, or user-facing settings for managing stored scan data/findings lifecycle. Since TruffleHog does offer a hosted enterprise option (docs-28), retention/deletion controls are a fair question but no evidence confirms them.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions telemetry, usage tracking, or any opt-out mechanism/flag for TruffleHog; the docs cover secret scanning features but are silent on data collection practices.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "reachability-prioritization", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "TruffleHog is a secrets-scanning tool focused on detecting and verifying credentials in code, chats, and infrastructure; it has no dependency/SCA vulnerability scanning or reachability analysis capability. Reachability-based prioritization of dependency vulnerabilities is a different product category and not something TruffleHog addresses.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "sast-code-scanning", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "TruffleHog is a secrets-scanning tool, not a SAST tool for vulnerability classes like injection or XSS; the entire evidence pack is about secret detection, verification, and monitoring, with no mention of code vulnerability analysis. This story targets a different product category (SAST) that TruffleHog does not address.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "sbom-generation", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "TruffleHog is a secrets-scanning tool focused on detecting and verifying credentials across repos, chats, images, etc.; SBOM generation (SPDX/CycloneDX) is a distinct supply-chain artifact-inventory capability unrelated to its product category, and no evidence anywhere in the pack mentions SBOM output.", "evidenceIds": [] }, { "productId": "trufflehog", "storyId": "scan-beyond-git", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "TruffleHog's docs claim broad non-git scanning coverage including Docker images, object stores, filesystems, and logs (trufflehog-docs-12, trufflehog-docs-18), but there is no first-party evidence naming S3 or CI logs specifically, and independent community evidence directly contradicts the Docker-image claim: a user reported they 'couldn't find anything comparable to Trufflehog for Docker images' and built their own tool instead (trufflehog-comm-6). Missing for 10: explicit named support/examples for S3 buckets and CI log scanning, and independent corroboration that Docker image scanning actually works as documented.", "evidenceIds": [ "trufflehog-docs-12", "trufflehog-docs-18", "trufflehog-comm-6" ] }, { "productId": "trufflehog", "storyId": "severity-filtering", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "TruffleHog supports filtering/gating via verification status rather than explicit severity/confidence scores — flags like --only-verified, --results=verified,unknown --fail, --include-detectors, and --verify-detectors/--no-verify-detectors let developers restrict findings and control build-blocking behavior (e.g., in pre-commit hooks). This addresses the triage-noise goal but doesn't map to a formal severity/confidence scale as the story implies. Missing for 10: explicit severity levels or confidence scoring, independent/hands-on confirmation of noise reduction in CI gating.", "evidenceIds": [ "trufflehog-docs-1", "trufflehog-docs-13", "trufflehog-docs-21", "trufflehog-docs-7", "trufflehog-docs-8" ] }, { "productId": "trufflehog", "storyId": "verify-live-secrets", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "TruffleHog's core credential verification feature logs into the issuing service to confirm if a secret is live, with CLI flags (--only-verified, --verify-detectors) and docs/README consistently describing this capability across hundreds of detectors. missing for 10: independent hands-on benchmark confirming verification accuracy/false-positive rates at scale.", "evidenceIds": [ "trufflehog-docs-1", "trufflehog-docs-3", "trufflehog-gh-3", "trufflehog-docs-21", "trufflehog-docs-8", "trufflehog-docs-29" ] } ]