[ { "productId": "anthropic-skills", "storyId": "active-maintenance", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence pack covers docs, specification, and usage patterns for Skills, but contains no information about release cadence, issue triage, or acceptance of community contributions to the anthropics/skills repository. No changelog, release notes, contributor stats, or issue-response evidence is present.", "evidenceIds": [] }, { "productId": "anthropic-skills", "storyId": "agent-authors-skill", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Anthropic ships a dedicated 'skill-creator' meta-skill for creating new skills and iteratively improving/benchmarking them, plus an official template SKILL.md and the open agentskills.io specification detailing required frontmatter, description rules, and folder structure (scripts/references/assets) — together these let an agent author and package a skill end-to-end per spec. Missing for 10: independent hands-on account of an agent successfully using skill-creator to author a new skill from scratch (community evidence only discusses skill triggering/invocation issues, not authoring/packaging).", "evidenceIds": [ "anthropic-skills-docs-13", "anthropic-skills-docs-14", "anthropic-skills-docs-30", "anthropic-skills-docs-29", "anthropic-skills-docs-17", "anthropic-skills-docs-26", "anthropic-skills-docs-44", "anthropic-skills-docs-51", "anthropic-skills-docs-52", "anthropic-skills-docs-41" ] }, { "productId": "anthropic-skills", "storyId": "agent-installs-skill", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs show scriptable install commands (`/plugin marketplace add`, `/plugin install ...`) and a Skills API for uploading/managing skills programmatically, which could in principle be run non-interactively by automation. However, none of the evidence explicitly documents a promptless, fully headless end-to-end install flow (e.g., a CLI flag or API call an agent invokes autonomously without any human-run slash command or confirmation step). Missing for 10: explicit non-interactive/headless install documentation, evidence of an agent autonomously choosing and installing a skill without human-issued commands, and confirmation that no interactive confirmation/prompt is required during install.", "evidenceIds": [ "anthropic-skills-docs-9", "anthropic-skills-docs-22", "anthropic-skills-docs-24", "anthropic-skills-docs-39", "anthropic-skills-gh-1", "anthropic-skills-gh-4", "anthropic-skills-gh-6" ] }, { "productId": "anthropic-skills", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Anthropic's docs site serves a working llms.txt (HTTP 200) plus .md variants of docs pages (e.g. skills.md), explicitly designed for agent/LLM consumption, and Claude Code skills follow the agentskills.io open spec. missing for 10: no independent/community confirmation that agents were actually pointed at llms.txt and successfully used it end-to-end.", "evidenceIds": [ "anthropic-skills-probe-1", "anthropic-skills-probe-2", "anthropic-skills-docs-47" ] }, { "productId": "anthropic-skills", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Skills can package data-handling capabilities (PDF, xlsx, docx extraction/manipulation) and are pitched for 'analyzing data using your organization's specific workflows,' giving Claude a path to generate insights from user data, but this is a general extensibility framework rather than a built-in insights/dashboard feature, and community reports show skills are frequently not invoked or unreliable in practice. missing for 10: a dedicated insights/analytics feature, proactive suggestion UI, and evidence that skills reliably surface unsolicited insights rather than requiring explicit triggering.", "evidenceIds": [ "anthropic-skills-docs-15", "anthropic-skills-docs-43", "anthropic-skills-gh-7", "anthropic-skills-gh-8", "anthropic-skills-comm-7", "anthropic-skills-comm-9" ] }, { "productId": "anthropic-skills", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Skills are packaged instructions/capabilities that Claude loads and uses during a session (invoked automatically or via /skill-name), but the evidence pack contains no mention of scheduling, triggers, or background/autonomous execution outside an active user session. Plugins and marketplaces cover distribution, not autonomous background automation.", "evidenceIds": [ "anthropic-skills-docs-1", "anthropic-skills-docs-2", "anthropic-skills-docs-25" ] }, { "productId": "anthropic-skills", "storyId": "agentic-builtin-assistant", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Anthropic's docs claim Claude will autonomously discover and load relevant Skills to perform delegated work ('Claude uses skills when relevant... transforms general-purpose agents into specialized agents'), which matches the story of delegating tasks to a built-in assistant. However, hands-on community reports directly contradict reliable automatic delegation: a Vercel eval found the skill was never invoked in 56% of cases despite being available, and multiple users report invocation is 'hit or miss' even with proper YAML triggers, often requiring explicit manual pointers. Missing for 10: independent benchmarks showing consistent autonomous task delegation, and resolution of the documented invocation-reliability gap.", "evidenceIds": [ "anthropic-skills-docs-2", "anthropic-skills-docs-43", "anthropic-skills-comm-7", "anthropic-skills-comm-8", "anthropic-skills-comm-9" ] }, { "productId": "anthropic-skills", "storyId": "agentic-headless", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Skills can be invoked programmatically via the Messages API (`container` parameter, Skills API for upload/management), which is inherently headless and scriptable, implying CI/automation use is possible. However, there is no explicit documentation or example of running Skills in a CI pipeline, headless mode, or automated build system — missing for 10: explicit CI/headless workflow examples, CLI flags for non-interactive automation, and independent evidence of production CI usage.", "evidenceIds": [ "anthropic-skills-docs-10", "anthropic-skills-docs-23", "anthropic-skills-docs-24", "anthropic-skills-docs-37", "anthropic-skills-docs-49" ] }, { "productId": "anthropic-skills", "storyId": "agentic-mcp-client", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs show that Claude Code plugins — the packaging mechanism used alongside Skills — can bundle MCP servers together with skills, agents, and hooks (anthropic-skills-docs-6, -7, -21, -35), implying an ai-native user could add an MCP server via the plugin/marketplace system. However, Skills themselves are a separate mechanism (plain SKILL.md instructions), and community commentary explicitly notes skills and MCP are distinct, sometimes competing approaches with skills lacking MCP's tool-calling functionality (anthropic-skills-comm-3, -10). Missing for 10: concrete first-party guide/example of installing an MCP server via a skill or plugin, hands-on confirmation that MCP tools become usable once added this way, and clarity on whether Skills (as opposed to Claude Code plugins broadly) directly expose MCP tool use.", "evidenceIds": [ "anthropic-skills-docs-6", "anthropic-skills-docs-7", "anthropic-skills-docs-21", "anthropic-skills-docs-35", "anthropic-skills-comm-3", "anthropic-skills-comm-10" ] }, { "productId": "anthropic-skills", "storyId": "agentic-mcp-server", "verdict": "na", "quality": 0, "confidence": "medium", "rationale": "Anthropic Skills is a packaging/format for capabilities (SKILL.md files, plugins, marketplaces) consumed by Claude/Claude Code, not a service or agent that itself exposes an official MCP server endpoint for other agents to connect to. Evidence mentions plugins can *bundle* MCP servers as one of several extension types, but this is about extending Claude Code, not Skills serving as an MCP server itself — the axis is a category mismatch for this product type.", "evidenceIds": [] }, { "productId": "anthropic-skills", "storyId": "agentic-nl-commands", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Anthropic's docs explicitly promise natural-language operation: Claude 'uses skills when relevant' by matching the task to a skill's description, in addition to explicit `/skill-name` invocation (anthropic-skills-docs-2, -3, -25, -34). However, hands-on community reports concretely contradict this: a Vercel eval found skills were never invoked in 56% of cases despite being available, and multiple practitioners report skills are 'hit or miss' or require explicitly telling Claude to use them even when asked (anthropic-skills-comm-7, -8, -9). Missing for 10: reliable first-party benchmark of natural-language trigger accuracy, and resolution of the discovery/triggering inconsistency reported by users.", "evidenceIds": [ "anthropic-skills-docs-2", "anthropic-skills-docs-3", "anthropic-skills-docs-25", "anthropic-skills-docs-34", "anthropic-skills-comm-7", "anthropic-skills-comm-8", "anthropic-skills-comm-9" ] }, { "productId": "anthropic-skills", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Skills are used and managed through Claude Code, which is described as a terminal-based agentic CLI tool, via slash commands like `/skill-name`, `/plugin marketplace add`, and `--plugin-dir` flags for local testing; the `skill-creator` skill also supports building/testing skills. However, there is no evidence of a dedicated standalone 'skills' CLI binary or command set (e.g., `skills create`, `skills validate`) separate from Claude Code's general slash-command interface, and API-based skill management (Skills API) is not CLI-based at all. Missing for 10: a purpose-built skills CLI tool, independent hands-on confirmation of CLI-based skill workflows, and CLI support outside the Claude Code product.", "evidenceIds": [ "anthropic-skills-docs-2", "anthropic-skills-docs-9", "anthropic-skills-docs-41", "anthropic-skills-docs-13", "anthropic-skills-probe-1" ] }, { "productId": "anthropic-skills", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Skills can be driven via the documented Messages API `container` parameter with `skill_id`, uploaded/managed through a dedicated Skills API, and invoked with up to 20 skills per request, giving AI-native users a clear programmatic path distinct from the chat UI. missing for 10: independent/hands-on corroboration of the API workflow itself (community evidence only covers Claude Code skill-triggering reliability, not the Messages/Skills API), and no evidence of API rate limits, SDKs, or error handling specifics.", "evidenceIds": [ "anthropic-skills-docs-10", "anthropic-skills-docs-23", "anthropic-skills-docs-24", "anthropic-skills-docs-37", "anthropic-skills-docs-39", "anthropic-skills-docs-49" ] }, { "productId": "anthropic-skills", "storyId": "agentic-scoped-keys", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Anthropic Skills is about packaging instructions/resources for Claude to load dynamically, not about credential/permission scoping or API key issuance for agents; no evidence pack content addresses scoped credential issuance.", "evidenceIds": [] }, { "productId": "anthropic-skills", "storyId": "agentic-sdks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs describe a dedicated Skills API and Messages API `container` parameter for programmatically attaching Skills (with `skill_id`, versioning, up to 20 per request), plus upload/management endpoints, which constitutes an official API/SDK surface to build against. Missing for 10: explicit language-specific SDK code samples (Python/TypeScript) and independent hands-on developer reports confirming building production integrations against this API.", "evidenceIds": [ "anthropic-skills-docs-10", "anthropic-skills-docs-23", "anthropic-skills-docs-24", "anthropic-skills-docs-37", "anthropic-skills-docs-39", "anthropic-skills-docs-49" ] }, { "productId": "anthropic-skills", "storyId": "agentic-webhooks", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Anthropic Skills is a mechanism for packaging instructions/scripts that agents load into context, not a service with an event system; webhook subscription is a wrong axis for this product type and no evidence suggests otherwise.", "evidenceIds": [] }, { "productId": "anthropic-skills", "storyId": "api-interactive-docs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Anthropic Skills is a mechanism for packaging agent capabilities (SKILL.md files, plugins), not an API-reference product with an interactive documentation explorer; 'runnable examples in an API reference' is a category mismatch for this product type.", "evidenceIds": [] }, { "productId": "anthropic-skills", "storyId": "api-machine-spec", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The evidence shows an open, machine-readable specification document (agentskills.io/specification.md) defining the SKILL.md format and frontmatter fields, plus references to a 'Skills API' for programmatic upload/management — these are the closest analogs to a downloadable machine-readable spec, but neither is an OpenAPI document nor explicitly offered as a downloadable API spec for the Skills/Messages API itself. Missing for 10: an actual OpenAPI/JSON-schema file for the Skills API endpoints, explicit download link/format, and independent confirmation that AI-native tooling consumes it.", "evidenceIds": [ "anthropic-skills-docs-17", "anthropic-skills-docs-26", "anthropic-skills-docs-51", "anthropic-skills-docs-24", "anthropic-skills-docs-49" ] }, { "productId": "anthropic-skills", "storyId": "api-sandbox", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Skills invoked via the Messages API run inside Anthropic's 'code execution environment' (a sandboxed container), and plugin docs mention testing skills locally with the `--plugin-dir` flag before sharing/distribution, which implies some separation from a live/production setup. However, there is no explicit documentation of a dedicated sandbox/staging environment for testing skills against non-production data, no discussion of data isolation guarantees, and no hands-on validation of this specific safety property. Missing for 10: explicit sandbox/staging environment documentation, data-isolation guarantees, and independent confirmation that local/test skill runs cannot touch production data.", "evidenceIds": [ "anthropic-skills-docs-10", "anthropic-skills-docs-41", "anthropic-skills-docs-23" ] }, { "productId": "anthropic-skills", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers Skills' structure, plugins, marketplaces, and API usage, but there is no mention of API versioning schemes or a documented deprecation policy for Skills/Claude API. Missing for 10: any documentation of API version numbers, backward-compatibility guarantees, or deprecation timelines/policy.", "evidenceIds": [] }, { "productId": "anthropic-skills", "storyId": "author-from-template", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "There is a documented open specification (agentskills.io) detailing required frontmatter (name, description, length constraints), file structure (SKILL.md plus optional scripts/references/assets folders), and even a template file (template/SKILL.md) with placeholder text, plus a dedicated skill-creator skill for authoring/improving skills without needing to reverse-engineer existing ones. Community evidence (comm-1) shows some users still had to ask others for example SKILL.md files, suggesting the template isn't universally discovered/used. missing for 10: independent hands-on confirmation that a developer successfully authored a skill purely from the template without consulting other examples, and more visibility/discoverability of the template in mainline docs.", "evidenceIds": [ "anthropic-skills-docs-26", "anthropic-skills-docs-29", "anthropic-skills-docs-44", "anthropic-skills-docs-51", "anthropic-skills-docs-52", "anthropic-skills-docs-13", "anthropic-skills-docs-30", "anthropic-skills-comm-1" ] }, { "productId": "anthropic-skills", "storyId": "authoring-meta-skill", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Anthropic ships a dedicated \"skill-creator\" meta-skill that explicitly guides creation, iteration, and improvement of skills, including benchmarking performance and optimizing description triggers for accuracy, and there's a template SKILL.md and open spec to follow. This directly matches the story of a meta-skill guiding authoring/improving/packaging skills, backed by first-party GitHub and docs evidence. Missing for 10: independent hands-on validation of the skill-creator workflow itself (community evidence only discusses skills generally, not this meta-skill specifically) and no evidence of a dedicated 'packaging for distribution' step within skill-creator beyond plugin/marketplace mechanisms.", "evidenceIds": [ "anthropic-skills-docs-13", "anthropic-skills-docs-14", "anthropic-skills-docs-30", "anthropic-skills-docs-29", "anthropic-skills-docs-17", "anthropic-skills-docs-51" ] }, { "productId": "anthropic-skills", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Skills can bundle scripts that operate on multiple files (e.g., the PDF skill 'merges multiple PDFs' and fills forms across documents), suggesting some batch/bulk-processing capability, and skills may include arbitrary scripts/executable code for such tasks. However, there is no explicit documentation or example demonstrating bulk operations across many items (e.g., batch-processing hundreds of records/files) as a core Skills feature, and community feedback focuses on skill-triggering reliability rather than bulk-processing performance. Missing for 10: explicit bulk/batch-processing examples or docs, evidence of scale (many items processed reliably), and independent confirmation that bulk workflows work as intended.", "evidenceIds": [ "anthropic-skills-docs-15", "anthropic-skills-docs-16", "anthropic-skills-docs-52", "anthropic-skills-comm-10" ] }, { "productId": "anthropic-skills", "storyId": "automation-rules-engine", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Anthropic's docs claim skills auto-trigger ('Claude uses skills when relevant... loads it automatically') based on description matching, which is the closest analog to event-driven rule triggering in this product, but this is relevance-based context loading, not true event/webhook/schedule triggers. Hands-on community reports directly contradict reliability of this claimed automation: Vercel's eval found skills were never invoked in 56% of cases despite being applicable, and multiple users report 'hit or miss' triggering even with proper YAML triggers, sometimes requiring explicit manual invocation. Missing for 10: genuine event-based triggers (webhooks, schedules, file-watchers), reliable automatic invocation without manual prompting, and independent confirmation that trigger accuracy is dependable in production.", "evidenceIds": [ "anthropic-skills-docs-2", "anthropic-skills-docs-25", "anthropic-skills-comm-7", "anthropic-skills-comm-8", "anthropic-skills-comm-9" ] }, { "productId": "anthropic-skills", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack describes Skills as on-demand or auto-triggered instruction modules invoked by Claude during a task (via /skill-name, semantic triggering, or API container calls), but nothing describes a scheduler, cron-like trigger, or persistent recurring job mechanism. Automation-depth for scheduling is a fair ask for an agentic tool, but no evidence shows Skills or Claude Code support recurring/scheduled execution.", "evidenceIds": [ "anthropic-skills-docs-2", "anthropic-skills-docs-25", "anthropic-skills-docs-5" ] }, { "productId": "anthropic-skills", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Docs mention plugin marketplaces provide 'version tracking' and 'versioned releases', and the API lets you specify an optional `version` for skill_id, implying some versioning support. However there is no documented review/approval workflow or explicit rollback mechanism — skills are just files/folders, so any versioning or rollback would rely on external git tooling not described as a first-class feature. missing for 10: explicit rollback command/feature, in-product review or approval workflow for skill changes, changelog/diff tooling, independent confirmation of version tracking in practice.", "evidenceIds": [ "anthropic-skills-docs-8", "anthropic-skills-docs-36", "anthropic-skills-docs-10", "anthropic-skills-docs-7" ] }, { "productId": "anthropic-skills", "storyId": "browse-searchable-catalog", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Anthropic provides a discoverable catalog via the official `anthropics/skills` GitHub repo (browsable list of example skills) and a formal 'plugin marketplace' concept described as providing 'centralized discovery, version tracking, automatic updates' that developers can add and install from (`/plugin marketplace add`, `/plugin install`). However, there's no evidence of a dedicated searchable registry UI, ratings/leaderboard, or cross-marketplace search — discovery is limited to browsing a GitHub repo or manually adding marketplace sources one at a time. Missing for 10: a searchable/filterable registry UI, popularity or leaderboard signals, and evidence of a unified cross-marketplace search experience.", "evidenceIds": [ "anthropic-skills-docs-8", "anthropic-skills-docs-9", "anthropic-skills-docs-36", "anthropic-skills-gh-1", "anthropic-skills-gh-2", "anthropic-skills-gh-4" ] }, { "productId": "anthropic-skills", "storyId": "clean-uninstall", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows how to create, add, invoke, and disable skills (e.g., disableBundledSkills, /plugin install), but there is no documentation of a command or workflow to list all installed skills or to cleanly uninstall/remove a skill and its plugin files, hooks, or references without leftovers.", "evidenceIds": [ "anthropic-skills-docs-4", "anthropic-skills-docs-19", "anthropic-skills-docs-9", "anthropic-skills-docs-33" ] }, { "productId": "anthropic-skills", "storyId": "documented-update-path", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs explicitly state plugin marketplaces provide 'centralized discovery, version tracking, automatic updates' for plugins that can bundle skills, and marketplace/plugin install commands are documented (e.g. /plugin marketplace add, /plugin install). However, this update path is scoped to the plugin-marketplace distribution mechanism rather than a general update command for individually-created or hand-copied skills (e.g. skills dropped into .claude/skills/), and there's no explicit 'update' CLI verb or independent confirmation of update behavior in practice. missing for 10: an explicit 'update' command example/output, confirmation this works for non-marketplace skills, independent/hands-on evidence of auto-update actually firing.", "evidenceIds": [ "anthropic-skills-docs-8", "anthropic-skills-docs-36", "anthropic-skills-docs-9", "anthropic-skills-gh-1", "anthropic-skills-docs-28" ] }, { "productId": "anthropic-skills", "storyId": "five-minute-quickstart", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs and GitHub show a genuinely lightweight path — a skill is 'just a folder with a SKILL.md file containing YAML frontmatter and instructions' (anthropic-skills-gh-3), installable via a single 'plugin marketplace add' + 'plugin install' command (anthropic-skills-gh-1, anthropic-skills-gh-4), which plausibly fits a five-minute window. However there's no dedicated timed 'quickstart' tutorial artifact, and community feedback shows friction getting Claude to actually invoke/use a newly installed skill reliably (anthropic-skills-comm-8, anthropic-skills-comm-9), meaning 'installed and working' isn't fully guaranteed in five minutes. Missing for 10: an explicit timed quickstart doc/tutorial, and independent hands-on confirmation of sub-5-minute install-to-working success.", "evidenceIds": [ "anthropic-skills-gh-3", "anthropic-skills-gh-1", "anthropic-skills-gh-4", "anthropic-skills-docs-1", "anthropic-skills-comm-8", "anthropic-skills-comm-9" ] }, { "productId": "anthropic-skills", "storyId": "inspect-before-install", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm skills are transparent, plain-text directories (SKILL.md plus optional scripts/references/assets folders) that a lead can browse in a repo/marketplace and read after installation, and plugins/marketplaces are just git repos or local paths a lead could inspect. However, there is no documented feature for listing or previewing a skill's full file contents specifically before installation (e.g., a dry-run/manifest-diff command), so the 'before install' half of the story is only implicit via manual repo browsing. missing for 10: a dedicated pre-install content-listing/manifest command, any audit/review tooling, and independent confirmation that installs can't run additional undisclosed files.", "evidenceIds": [ "anthropic-skills-docs-52", "anthropic-skills-docs-16", "anthropic-skills-docs-26", "anthropic-skills-docs-36", "anthropic-skills-gh-1", "anthropic-skills-docs-41" ] }, { "productId": "anthropic-skills", "storyId": "multi-harness-support", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Docs state Claude Code skills follow the open 'Agent Skills' standard which 'works across multiple AI tools' (agentskills.io spec), implying cross-agent portability, and the spec itself defines a tool-agnostic SKILL.md format. However, there is no concrete evidence of per-harness install instructions or documented support for Codex, Cursor, or other named agents — all install/marketplace instructions (plugin marketplace, /plugin install, container skill_id) are Claude-specific. Missing for 10: explicit Codex/Cursor installation docs, per-harness setup instructions, and independent confirmation that the same skill collection actually runs unmodified in non-Anthropic tools.", "evidenceIds": [ "anthropic-skills-docs-47", "anthropic-skills-docs-26", "anthropic-skills-docs-51", "anthropic-skills-docs-17" ] }, { "productId": "anthropic-skills", "storyId": "one-command-install", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Anthropic documents a clear CLI install path — `/plugin marketplace add anthropics/skills` followed by `/plugin install document-skills@anthropic-agent-skills` — which registers and installs a skill collection inside Claude Code, and docs imply skills become part of Claude's toolkit thereafter. However, this is a two-step sequence rather than a single one-liner, and there is no independent/hands-on confirmation that the installed skill persists and is reliably active in the very next session (community reports focus on activation/triggering reliability, not install itself). Missing for 10: a true single-command one-liner, and first-party or community confirmation of session-persistence after install.", "evidenceIds": [ "anthropic-skills-gh-1", "anthropic-skills-gh-4", "anthropic-skills-gh-6", "anthropic-skills-docs-9", "anthropic-skills-docs-22", "anthropic-skills-docs-8" ] }, { "productId": "anthropic-skills", "storyId": "open-spec-compliance", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Anthropic explicitly states Claude Code skills follow the open Agent Skills standard (agentskills.io) that works across multiple AI tools, and the same SKILL.md folder format (frontmatter + markdown, optional scripts/references/assets dirs) is documented both on the vendor docs and the independent agentskills.io spec site. Missing for 10: no third-party/independent tool (outside Anthropic) is shown actually consuming the same folder, and community commentary questions skill invocation reliability rather than spec portability.", "evidenceIds": [ "anthropic-skills-docs-47", "anthropic-skills-docs-26", "anthropic-skills-docs-51", "anthropic-skills-docs-52", "anthropic-skills-docs-17", "anthropic-skills-docs-44" ] }, { "productId": "anthropic-skills", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm Skills can be invoked and managed programmatically via the Messages API (container parameter, skill_id/type/version, up to 20 skills per request) and via a dedicated Skills API for upload/management, giving real API-level parity for core skill usage. However, other capabilities visible in the Claude Code/Claude.ai UI — plugin marketplaces, automatic relevance-based skill loading, /plugin and /skill-name invocation, bundled-skill toggling — are documented only as CLI/UI features with no evidence of an equivalent API path. Missing for 10: API equivalents for plugin marketplace distribution, automatic skill discovery/loading parity, and confirmation that all UI-configurable settings (e.g., disableBundledSkills) are reachable via API.", "evidenceIds": [ "anthropic-skills-docs-10", "anthropic-skills-docs-23", "anthropic-skills-docs-24", "anthropic-skills-docs-37", "anthropic-skills-docs-39", "anthropic-skills-docs-12", "anthropic-skills-docs-8", "anthropic-skills-docs-9" ] }, { "productId": "anthropic-skills", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "anthropic-skills", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While the anthropics/skills GitHub repo makes example skill files (SKILL.md, templates) publicly readable, none of the evidence cites an open-source license for Skills, Claude Code, or the underlying product; core Claude Code/Skills functionality itself is closed, proprietary tooling with no license grant shown.", "evidenceIds": [] }, { "productId": "anthropic-skills", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Anthropic Skills is a packaging/format layer (SKILL.md files, plugins, marketplaces) that runs on top of Claude Code or the hosted Claude API — it has no standalone server/model component of its own to self-host. Self-hosting is a category mismatch for a skills/plugin framework built atop a proprietary hosted LLM service, not an applicable axis for this product type.", "evidenceIds": [] }, { "productId": "anthropic-skills", "storyId": "per-skill-documentation", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Anthropic's spec strongly documents that every SKILL.md must include a description of what the skill does and when to use it (frontmatter is designed to be both machine- and human-readable for predicting activation), and Claude Code/Claude API docs describe controllable invocation triggers. However, multiple hands-on community reports (Vercel eval: skill never invoked in 56% of cases despite being documented and available; users reporting 'hit or miss' triggering even with well-written descriptions) concretely contradict the promise that documented descriptions reliably predict when the agent will actually activate a skill. missing for 10: consistent, benchmarked activation reliability matching documented triggers, and independent evidence resolving the invocation unpredictability reported by users.", "evidenceIds": [ "anthropic-skills-docs-17", "anthropic-skills-docs-44", "anthropic-skills-docs-32", "anthropic-skills-docs-3", "anthropic-skills-comm-7", "anthropic-skills-comm-8", "anthropic-skills-comm-9" ] }, { "productId": "anthropic-skills", "storyId": "plain-files-portability", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs confirm skills are just plain folders with a SKILL.md file (YAML frontmatter + markdown) plus optional scripts/references/assets, explicitly following the open Agent Skills standard (agentskills.io) that 'works across multiple AI tools,' and GitHub examples show these as ordinary files/folders anyone can read, copy, or fork. This directly supports the portability claim of non-proprietary, cross-harness markdown format. missing for 10: no independent hands-on report of someone actually carrying a skill folder to a different (non-Anthropic) harness and confirming it works unmodified.", "evidenceIds": [ "anthropic-skills-docs-26", "anthropic-skills-docs-51", "anthropic-skills-docs-52", "anthropic-skills-docs-47", "anthropic-skills-gh-3", "anthropic-skills-docs-17" ] }, { "productId": "anthropic-skills", "storyId": "privacy-data-residency", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Anthropic Skills is a feature for packaging instructions/scripts for Claude agents; data residency/region storage is an infrastructure/compliance concern of the underlying platform (Claude API/Claude.ai), not something Skills as a capability could expose or configure.", "evidenceIds": [] }, { "productId": "anthropic-skills", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Anthropic Skills is a feature/framework for packaging agent capabilities, not a data-privacy or training-opt-out control; the evidence pack contains no data-training-consent settings and this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "anthropic-skills", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers how Skills are created, invoked, packaged, and distributed, but contains no documentation about data retention policies, deletion controls, or privacy settings for skill data/usage. No mention of retention windows, user-initiated deletion, or data handling controls exists in this pack.", "evidenceIds": [] }, { "productId": "anthropic-skills", "storyId": "privacy-telemetry-optout", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Anthropic Skills is a feature/packaging format for extending Claude's capabilities, not a telemetry-collecting service with its own privacy/tracking controls to opt out of — this axis belongs to platform-level privacy settings (e.g., Claude.ai/Claude Code), not the Skills feature itself.", "evidenceIds": [] }, { "productId": "anthropic-skills", "storyId": "project-vs-global-install", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs show skills live in project-level directories like `.claude/skills/deploy/SKILL.md` and can be packaged into plugins shareable 'across projects and teams,' implying some notion of local vs shared scope, but there is no explicit documentation of a user-global (e.g. home-directory) skill install path or a direct project-vs-user scope toggle. missing for 10: explicit documentation of a user-global skill directory/location, explicit contrast between project-committed vs user-global install scope, and confirmation that both scopes are simultaneously supported and selectable by the developer.", "evidenceIds": [ "anthropic-skills-docs-33", "anthropic-skills-docs-21", "anthropic-skills-docs-35", "anthropic-skills-docs-36" ] }, { "productId": "anthropic-skills", "storyId": "release-notes", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows plugin marketplaces support 'version tracking' and skills/plugins have version fields in manifests, but there is no evidence of actual release notes or a changelog documenting what changed between skill versions before update.", "evidenceIds": [] }, { "productId": "anthropic-skills", "storyId": "reliable-auto-triggering", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Anthropic's docs explicitly describe automatic activation via engineered description fields (e.g. docs-17, docs-25, docs-32, docs-44) and even ship a skill-creator tool to optimize descriptions for 'triggering accuracy' (docs-14). However, hands-on community reports directly contradict reliable auto-triggering: Vercel's eval found the skill was never invoked in 56% of cases with no improvement over baseline, and multiple users report skills are 'hit or miss' or 'incredibly hard' to get invoked even with proper YAML descriptions (comm-7, comm-8, comm-9). missing for 10: independent benchmark showing consistent correct auto-activation, and resolution of the documented reliability gap.", "evidenceIds": [ "anthropic-skills-docs-17", "anthropic-skills-docs-25", "anthropic-skills-docs-14", "anthropic-skills-docs-32", "anthropic-skills-comm-7", "anthropic-skills-comm-8", "anthropic-skills-comm-9" ] }, { "productId": "anthropic-skills", "storyId": "security-posture-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows extensive functional docs (SKILL.md structure, plugin marketplaces, execution environment, disableBundledSkills toggle) but no dedicated security-posture documentation addressing what skills can execute (sandboxing, permissions), a trust model for vetting third-party/marketplace skills, or telemetry/data-collection disclosures tied to skills usage.", "evidenceIds": [] }, { "productId": "anthropic-skills", "storyId": "selective-install", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show marketplace-based installs where a user can add a marketplace and then install just one named plugin (e.g. `/plugin install document-skills@anthropic-agent-skills` vs `example-skills@anthropic-agent-skills`, or `quality-review-plugin@my-plugins`) rather than every plugin in the marketplace, and `disableBundledSkills` can turn off built-in skills en masse. This demonstrates selecting a specific plugin/skill bundle out of a larger marketplace, but there's no documented mechanism to cherry-pick an individual skill from inside a single plugin bundle. Missing for 10: explicit per-skill (sub-bundle) install granularity, and independent confirmation that partial installs work as described.", "evidenceIds": [ "anthropic-skills-gh-4", "anthropic-skills-docs-9", "anthropic-skills-docs-22", "anthropic-skills-docs-4", "anthropic-skills-docs-19" ] }, { "productId": "anthropic-skills", "storyId": "skill-testing-harness", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The skill-creator skill offers ad-hoc 'benchmark skill performance' tooling for authors (anthropic-skills-docs-14, -30), but there is no evidence of a maintained test suite, CI pipeline, or regression eval framework for the official skills collection itself. The only concrete eval-style evidence (Vercel's finding that skills were never invoked in 56% of cases) is a third-party community critique, not Anthropic's own maintained regression testing.", "evidenceIds": [ "anthropic-skills-docs-14", "anthropic-skills-docs-30", "anthropic-skills-comm-7" ] }, { "productId": "anthropic-skills", "storyId": "team-distribution", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Anthropic Skills supports distribution via plugin marketplaces (git repos or local paths), shared GitHub repos (e.g., anthropics/skills registered as a marketplace), and files committed directly to a project (.claude/skills/SKILL.md), covering all three distribution channels named in the story. Community feedback confirms real-world use of shared skill repos and marketplace installs, though adoption/triggering reliability is debated. missing for 10: independent case study of an engineering team formally rolling out a standardized skill set org-wide, and more detail on permissions/governance controls for enforcing a 'standard' team-wide set.", "evidenceIds": [ "anthropic-skills-docs-8", "anthropic-skills-docs-9", "anthropic-skills-docs-36", "anthropic-skills-gh-1", "anthropic-skills-gh-4", "anthropic-skills-docs-33", "anthropic-skills-docs-7" ] }, { "productId": "anthropic-skills", "storyId": "version-control-pinning", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "The Skills API explicitly supports pinning a specific version via the `container` parameter's optional `version` field when invoking a skill, and plugin manifests include a version field with marketplaces offering 'version tracking' (docs-10, docs-23, docs-27, docs-36). However, for Claude Code's locally-installed/bundled skills there is no lockfile or team-wide pinning mechanism described — marketplaces are instead touted for 'automatic updates', which is the opposite of controlled rollout, and no evidence shows a way to pin or freeze skill versions across a team's Claude Code installs. missing for 10: lockfile or pinned-version mechanism for Claude Code skill/plugin installs, evidence that automatic marketplace updates can be disabled or gated per-team, and any hands-on confirmation that version pinning works as documented.", "evidenceIds": [ "anthropic-skills-docs-10", "anthropic-skills-docs-23", "anthropic-skills-docs-27", "anthropic-skills-docs-36" ] }, { "productId": "codex-plugins", "storyId": "active-maintenance", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack documents plugin/skill features and how-tos but contains no information about release cadence, issue triage, or accepted community contributions for the openai/plugins repository — no changelog, commit history, issue tracker activity, or PR merge data is cited. Community comments present are about skills vs MCP design tradeoffs, not maintenance activity.", "evidenceIds": [] }, { "productId": "codex-plugins", "storyId": "agent-authors-skill", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs describe a full authoring pipeline: SKILL.md spec with required fields, a plugin-creator/skill-creator meta-skill that scaffolds .codex-plugin/plugin.json and MCP wiring, a recorder that drafts a skill from a demonstration, skill-installer for distribution, and a submission portal for publishing, matching author-and-package-end-to-end via the project's own spec, template, or meta-skill. Community comments corroborate real users creating and customizing skills with the skill-creator skill. Missing for 10: independent hands-on verification of the full creator-to-published-package flow, and more detail on packaging/versioning nuances beyond the manifest.", "evidenceIds": [ "codex-plugins-docs-10", "codex-plugins-docs-13", "codex-plugins-docs-15", "codex-plugins-docs-16", "codex-plugins-docs-18", "codex-plugins-docs-24", "codex-plugins-docs-30", "codex-plugins-docs-31", "codex-plugins-comm-1" ] }, { "productId": "codex-plugins", "storyId": "agent-installs-skill", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs show a `$skill-installer` skill and a `codex plugin marketplace add` CLI command that let a skill/plugin be installed by name without manual UI browsing (codex-plugins-docs-8, docs-11, docs-14, docs-21, docs-34), suggesting an agent could invoke these programmatically. However, none of the evidence confirms a fully non-interactive, promptless, end-to-end headless flow (e.g., a scripted CLI flag bypassing any session/config prompts, or documentation stating the agent can trigger this without human involvement) — install flows described (docs-25, docs-26) still reference starting a 'new session' via CLI/chat interaction rather than a scripted automation path. Missing for 10: explicit headless/CI-style install command or flag, confirmation the agent can self-invoke skill-installer without any human step, and evidence no interactive confirmation/prompt is required during install.", "evidenceIds": [ "codex-plugins-docs-8", "codex-plugins-docs-11", "codex-plugins-docs-14", "codex-plugins-docs-21", "codex-plugins-docs-25", "codex-plugins-docs-34" ] }, { "productId": "codex-plugins", "storyId": "agentic-agent-docs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Codex documentation itself is published as llms.txt/agent-oriented markdown (developers.openai.com/llms.txt returns 200 and indexes markdown docs), showing the product's own docs are agent-consumable, but there is no evidence that a Codex plugin/agent can be pointed at an arbitrary external llms.txt or agent-oriented docs site to ingest and act on it as a user-directed capability. missing for 10: explicit feature/skill letting a user supply/point the agent at a third-party llms.txt or docs URL for ingestion, and any hands-on confirmation of this workflow beyond the docs' own self-hosting.", "evidenceIds": [ "codex-plugins-probe-1", "codex-plugins-probe-2", "codex-plugins-docs-17" ] }, { "productId": "codex-plugins", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Codex Plugins connects to data sources (Gmail, Drive, Slack, Notion, Figma) and can summarize, draft, and act on that data via plugins/skills, giving AI-generated insights from connected data, but this is framed as tool-use/automation rather than a dedicated analytics/insights feature over the user's own stored data. missing for 10: no evidence of proactive dashboards, analytics summaries, or unprompted 'insights' generation from user data (e.g., trend detection, anomaly surfacing) beyond on-demand chat/skill invocation, and no independent hands-on validation of insight quality.", "evidenceIds": [ "codex-plugins-docs-2", "codex-plugins-docs-3", "codex-plugins-docs-4", "codex-plugins-gh-4", "codex-plugins-docs-1", "codex-plugins-docs-33" ] }, { "productId": "codex-plugins", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes installable plugins, skills, and MCP tool wiring, but nothing shows Codex Plugins can be configured to run autonomously in the background without a user-initiated session — sessions must be started manually after installation (docs-25) and skill triggers are described in terms of manual invocation (/skills, $mentions) rather than scheduled/autonomous execution. Missing for 10: any scheduler/cron/trigger mechanism, background/always-on execution mode, or evidence of unattended autonomous runs.", "evidenceIds": [ "codex-plugins-docs-25", "codex-plugins-docs-7", "codex-plugins-docs-20" ] }, { "productId": "codex-plugins", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Codex ships a built-in AI assistant (Codex itself) that can be extended via installed plugins to delegate tasks such as summarizing Slack channels, searching Gmail, working across Drive/Docs, or messaging via iMessage, all invoked directly from Codex CLI/chat sessions after installation. Docs also show skills and MCP-backed plugin tools becoming available to Codex for task delegation without leaving the product.\n\nmissing for 10: independent/hands-on evidence of reliable end-to-end task delegation (community comments focus on skills' pros/cons in general, not on Codex-plugin delegation specifically), and no benchmark of assistant task success rates.", "evidenceIds": [ "codex-plugins-docs-2", "codex-plugins-docs-3", "codex-plugins-docs-4", "codex-plugins-docs-5", "codex-plugins-docs-26", "codex-plugins-docs-25" ] }, { "productId": "codex-plugins", "storyId": "agentic-headless", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses running Codex Plugins headlessly or invoking it from CI/automation pipelines; all docs describe interactive install/browse flows (/plugins, /skills, plugin-creator) via CLI or ChatGPT UI. Missing for 10: any mention of a non-interactive/headless invocation mode, CI examples, or automation-friendly APIs for plugins/skills.", "evidenceIds": [ "codex-plugins-docs-6", "codex-plugins-docs-20", "codex-plugins-docs-25" ] }, { "productId": "codex-plugins", "storyId": "agentic-mcp-client", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Codex Plugins explicitly documents that plugins 'use the Model Context Protocol (MCP) to expose server-backed capabilities to ChatGPT and Codex,' with a plugin browser (/plugins), marketplace add command, and plugin-creator skill that 'adds MCP server wiring' — showing first-party, well-documented support for plugging in MCP servers and using their tools. missing for 10: independent hands-on verification of third-party MCP servers being installed and invoked successfully in Codex, and more detail on runtime tool-call behavior/limits.", "evidenceIds": [ "codex-plugins-docs-33", "codex-plugins-docs-15", "codex-plugins-docs-25", "codex-plugins-docs-26", "codex-plugins-docs-11", "codex-plugins-docs-10" ] }, { "productId": "codex-plugins", "storyId": "agentic-mcp-server", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Codex Plugins is the ecosystem that lets the Codex coding agent connect to (client-side) MCP-backed plugins like Slack, Notion, Figma, etc. Per the agent-role convention, this client-side MCP consumption doesn't make the 'serve an agent via official MCP server' axis applicable — there is no evidence Codex itself runs as an MCP server (e.g., an 'mcp serve' mode) for other agents to connect to.", "evidenceIds": [ "codex-plugins-docs-33", "codex-plugins-docs-15", "codex-plugins-docs-26" ] }, { "productId": "codex-plugins", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs show plugins/skills are used conversationally in ChatGPT/Codex chat (e.g., 'summarize channels or draft replies', skills auto-triggering based on natural-language description matching) alongside slash commands (/plugins, /skills) for management, indicating natural-language operation is the primary interaction mode. missing for 10: independent/hands-on confirmation that natural-language triggering reliably works, and clarification on how much slash/$ syntax is required vs pure natural language.", "evidenceIds": [ "codex-plugins-docs-4", "codex-plugins-docs-7", "codex-plugins-docs-20", "codex-plugins-docs-26", "codex-plugins-comm-5" ] }, { "productId": "codex-plugins", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Codex Plugins documentation repeatedly references the official Codex CLI, including specific commands (/plugins, /skills, $skill-installer, codex plugin marketplace add) that let an AI-native user browse, install, and manage plugins/skills directly from the CLI, confirming an official CLI workflow exists and is documented as first-class. Missing for 10: independent hands-on confirmation of CLI usage beyond vendor docs, and a full command reference showing breadth of CLI capabilities.", "evidenceIds": [ "codex-plugins-docs-6", "codex-plugins-docs-11", "codex-plugins-docs-20", "codex-plugins-docs-25", "codex-plugins-docs-8", "codex-plugins-probe-2" ] }, { "productId": "codex-plugins", "storyId": "agentic-public-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Codex Plugins uses MCP to let plugins expose capabilities *to* Codex/ChatGPT, and offers CLI commands (`codex plugin marketplace add`, `/plugins`) for local plugin management, but there is no documented public REST/GraphQL API for programmatically driving the Codex Plugins product itself — the direct OpenAPI probe returned 404 on all candidate spec paths, confirming no discoverable public API surface.", "evidenceIds": [ "codex-plugins-probe-3", "codex-plugins-docs-33", "codex-plugins-docs-11" ] }, { "productId": "codex-plugins", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers plugin installation, skills, and OAuth-style 'Sign in with ChatGPT' (which only shares name/email/photo) but nothing about issuing scoped or least-privilege API credentials/keys that a user can configure for an agent's access level.", "evidenceIds": [ "codex-plugins-docs-19", "codex-plugins-docs-23" ] }, { "productId": "codex-plugins", "storyId": "agentic-sdks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Codex Plugins has real developer-facing build documentation (plugin.json manifest schema, MCP-based server wiring, plugin-creator scaffolding tool, submission portal, marketplace CLI) that lets developers build against an official plugin/skill framework, but this is a manifest/CLI-based plugin system rather than a formal client SDK, and the probe found no OpenAPI/API spec artifact. missing for 10: an actual downloadable/importable SDK library (e.g. Python/JS package), versioned API reference, and independent hands-on confirmation that third-party devs successfully build with it.", "evidenceIds": [ "codex-plugins-docs-10", "codex-plugins-docs-15", "codex-plugins-docs-30", "codex-plugins-docs-33", "codex-plugins-docs-31", "codex-plugins-docs-11", "codex-plugins-probe-3" ] }, { "productId": "codex-plugins", "storyId": "agentic-webhooks", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Codex Plugins is a plugin/skill system for extending an AI coding agent with tools and skills, not an event-driven webhook subscription platform; the evidence pack covers plugin installation, skill authoring, and marketplace mechanics but nothing about webhook event subscriptions, which is a different product category axis.", "evidenceIds": [] }, { "productId": "codex-plugins", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an interactive API reference with runnable examples; OpenAPI probes returned 404 and docs are static markdown pages, not an interactive explorable API console.", "evidenceIds": [ "codex-plugins-probe-3" ] }, { "productId": "codex-plugins", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A direct probe for OpenAPI/swagger spec files at all standard paths returned 404, and no other evidence shows a downloadable machine-readable API spec for Codex Plugins; the plugin.json manifest and MCP protocol references describe plugin authoring, not a consumable API spec.", "evidenceIds": [ "codex-plugins-probe-3" ] }, { "productId": "codex-plugins", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of a sandbox/staging environment for testing plugins or skills against non-production data; docs only mention 'local marketplace entry for testing' which refers to plugin discovery/distribution, not data isolation from production systems.", "evidenceIds": [] }, { "productId": "codex-plugins", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of API versioning scheme or a documented deprecation policy for Codex Plugins; the probe for an OpenAPI spec even returned 404s across all candidate paths, and docs only cover plugin/skill creation and usage, not lifecycle/versioning guarantees.", "evidenceIds": [ "codex-plugins-probe-3" ] }, { "productId": "codex-plugins", "storyId": "author-from-template", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly define the SKILL.md format (name + description frontmatter, directory with optional scripts/references) and describe a guided `@plugin-creator`/skill-creator flow that asks what the skill does, when it triggers, and whether to include scripts — enabling authoring from a documented template rather than reverse-engineering. Community evidence corroborates developers building their own skills this way. Missing for 10: a full worked SKILL.md example/template file shown verbatim and independent hands-on verification of the authoring flow.", "evidenceIds": [ "codex-plugins-docs-13", "codex-plugins-docs-7", "codex-plugins-docs-10", "codex-plugins-docs-17", "codex-plugins-comm-1" ] }, { "productId": "codex-plugins", "storyId": "authoring-meta-skill", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Codex ships explicit meta-skills for this: the `@plugin-creator` skill scaffolds `.codex-plugin/plugin.json`, wires MCP servers, and generates a marketplace entry (docs-10/15/18/24), a skill-creation flow that interviews the user on triggers/scripts (docs-7), and a recorder that drafts a reusable skill from a demonstrated workflow (docs-16), covering write/improve/package end-to-end. Community commentary corroborates the general pattern of using a 'skill creator skill' to build proprietary skills (comm-1). Missing for 10: independent hands-on account specifically of using @plugin-creator/skill-installer rather than general skills discussion, and no evidence of an 'improve existing skill' iteration workflow beyond initial creation.", "evidenceIds": [ "codex-plugins-docs-10", "codex-plugins-docs-15", "codex-plugins-docs-18", "codex-plugins-docs-24", "codex-plugins-docs-7", "codex-plugins-docs-16", "codex-plugins-comm-1" ] }, { "productId": "codex-plugins", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers plugin/skill installation, marketplace browsing, and per-item workflows (Slack summarization, Drive docs, Notion, etc.), but nothing describes performing an operation across many items/records/files at once (e.g., batch processing, multi-item automation loops). No batch or bulk-operation capability is documented or implied.", "evidenceIds": [] }, { "productId": "codex-plugins", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Skill creation asks the author to define 'when it should trigger' (docs-7), implying some contextual auto-invocation logic, but all other evidence shows skills/plugins being invoked manually via `/skills`, `$` mention, or `/plugins` browser rather than firing automatically on external events like schedules, webhooks, or file changes. Missing for 10: explicit event-trigger/automation-rule mechanism (e.g., cron, webhook, file-watch), documentation of autonomous invocation without user prompt, and independent confirmation that trigger conditions actually fire without manual mention.", "evidenceIds": [ "codex-plugins-docs-7", "codex-plugins-docs-20", "codex-plugins-docs-6" ] }, { "productId": "codex-plugins", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers plugin installation, MCP wiring, and skill creation/curation, but there is no mention of scheduling, cron-like triggers, or recurring/automated workflow execution anywhere in the docs or community items.", "evidenceIds": [] }, { "productId": "codex-plugins", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence describes creating, installing, and browsing skills/plugins (SKILL.md files, `.agents/skills` directories, marketplace add/track commands) but contains no mention of version history, diffing/review workflows, or rollback/undo mechanisms for skills or plugins. Being file-based artifacts implies they could be manually versioned with external tools like git, but no such capability is documented as a product feature.", "evidenceIds": [ "codex-plugins-docs-13", "codex-plugins-docs-17", "codex-plugins-docs-11", "codex-plugins-docs-18" ] }, { "productId": "codex-plugins", "storyId": "browse-searchable-catalog", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs describe a `/plugins` command that opens a 'plugin browser' and a 'Plugins tab' to browse and install plugins in ChatGPT/Codex, backed by a marketplace.json listing plugin entries with categories and a `codex plugin marketplace add` CLI command; the openai/plugins GitHub repo also serves as a browsable catalog of available plugins (Figma, Notion, Expo, etc.). Missing for 10: independent/hands-on confirmation of the browsing UI's search/filter experience, and no evidence of ratings, leaderboards, or third-party marketplace aggregation beyond OpenAI's own listing.", "evidenceIds": [ "codex-plugins-docs-6", "codex-plugins-docs-25", "codex-plugins-docs-26", "codex-plugins-docs-11", "codex-plugins-docs-35", "codex-plugins-gh-1", "codex-plugins-gh-4", "codex-plugins-gh-9", "codex-plugins-gh-8" ] }, { "productId": "codex-plugins", "storyId": "clean-uninstall", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs show how to list/browse installed plugins and skills (the /plugins browser, Skills sidebar, /skills command) and how to install them via skill-installer or plugin marketplace add, but there is no documented uninstall/removal command or explanation of how installed skill files, manifests, or config.toml entries are cleaned up. Missing for 10: documented uninstall/remove command, confirmation that removal deletes .agents/skills files and marketplace/config entries without leaving orphaned instructions, and any hands-on evidence of clean removal.", "evidenceIds": [ "codex-plugins-docs-6", "codex-plugins-docs-26", "codex-plugins-docs-29", "codex-plugins-docs-20", "codex-plugins-docs-11", "codex-plugins-docs-8" ] }, { "productId": "codex-plugins", "storyId": "documented-update-path", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers plugin installation (`/plugins`, `codex plugin marketplace add`) and creation/submission workflows, but nowhere documents an explicit update/upgrade command or auto-update behavior for already-installed plugins to receive fixes without reinstalling.", "evidenceIds": [ "codex-plugins-docs-6", "codex-plugins-docs-11", "codex-plugins-docs-25", "codex-plugins-docs-26" ] }, { "productId": "codex-plugins", "storyId": "five-minute-quickstart", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs describe simple single-command paths to install a skill (`$skill-installer `, `/skills`, `/plugins`) and to scaffold one via `@plugin-creator`, suggesting a fast setup, but no source gives an explicit time-to-first-working-skill benchmark or a hands-on account confirming a sub-five-minute experience. Missing for 10: an explicit quickstart timing claim, a step-by-step first-run walkthrough, and independent/community confirmation that installation actually completes quickly.", "evidenceIds": [ "codex-plugins-docs-8", "codex-plugins-docs-14", "codex-plugins-docs-20", "codex-plugins-docs-24", "codex-plugins-docs-25" ] }, { "productId": "codex-plugins", "storyId": "inspect-before-install", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Skills are documented as plain directories (SKILL.md plus optional scripts/references) and plugins ship with an inspectable manifest (`.codex-plugin/plugin.json`), and curated skills/plugins live in public GitHub repos that can be browsed before installing via `$skill-installer` or the plugin marketplace. However, there is no documented in-product command or UI that explicitly lists a skill/plugin's file contents prior to installation, nor any stated post-install audit/read workflow for engineering leads. Missing for 10: a dedicated 'preview/list contents' command before install, and confirmation that all installed files (scripts, resources) are surfaced for review after install rather than just SKILL.md.", "evidenceIds": [ "codex-plugins-docs-13", "codex-plugins-docs-17", "codex-plugins-docs-18", "codex-plugins-docs-8", "codex-plugins-docs-34", "codex-plugins-gh-9" ] }, { "productId": "codex-plugins", "storyId": "multi-harness-support", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All evidence describes installing plugins/skills within OpenAI's own ecosystem (Codex CLI, ChatGPT desktop/Chat/Work) — there is no evidence of a single collection being installed into Claude Code, Cursor, or other third-party coding agents. In fact docs-32 implies the opposite: migrating an existing Claude Code plugin to OpenAI requires reviewing what needs to change, suggesting formats are not simply portable across harnesses.", "evidenceIds": [ "codex-plugins-docs-6", "codex-plugins-docs-25", "codex-plugins-docs-26", "codex-plugins-docs-32" ] }, { "productId": "codex-plugins", "storyId": "one-command-install", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs explicitly describe a one-command install flow — `/plugins` browser or `$skill-installer ` — and state that after installing a plugin you should 'start a new session before using its bundled skills or tools,' directly matching the story's install-then-next-session pattern. Additional CLI support (`codex plugin marketplace add`) reinforces multiple documented one-liner paths. Missing for 10: independent/hands-on confirmation that installed skills actually activate reliably in the next session (only vendor docs, no community verification of this specific flow).", "evidenceIds": [ "codex-plugins-docs-25", "codex-plugins-docs-8", "codex-plugins-docs-14", "codex-plugins-docs-21", "codex-plugins-docs-11", "codex-plugins-docs-6" ] }, { "productId": "codex-plugins", "storyId": "open-spec-compliance", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs describe skills as a directory with a SKILL.md containing required `name`/`description` fields plus optional scripts/references (docs-13), which matches the general shape of the open Agent Skills format, and community commentary links Codex Skills to the same lineage as Claude Skills (comm-6). However, no evidence explicitly states conformance to a named open 'Agent Skills specification' or demonstrates a skill folder authored elsewhere working unmodified in Codex. Missing for 10: explicit statement of spec compliance/versioning, and a concrete cross-vendor portability example or test.", "evidenceIds": [ "codex-plugins-docs-13", "codex-plugins-docs-28", "codex-plugins-comm-6" ] }, { "productId": "codex-plugins", "storyId": "openness-api-parity", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Codex Plugins documentation only describes UI (ChatGPT Plugins tab, desktop Skills sidebar) and CLI (`/plugins`, `$skill-installer`, `codex plugin marketplace add`) workflows for installing and managing plugins/skills; there is no mention of a public API for plugin/skill management, and a direct probe for an OpenAPI/swagger spec returned 404 on all candidate paths, indicating no documented API parity.", "evidenceIds": [ "codex-plugins-docs-6", "codex-plugins-docs-11", "codex-plugins-docs-26", "codex-plugins-docs-29", "codex-plugins-probe-3" ] }, { "productId": "codex-plugins", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of any data export feature, open-format export, or account/data portability mechanism for Codex Plugins; the documentation focuses on plugin/skill installation and creation, not user data export or exit.", "evidenceIds": [] }, { "productId": "codex-plugins", "storyId": "openness-open-license", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "There is a public GitHub repo (openai/plugins) exposing plugin source code and structure, offering some read access, but none of the evidence cites an explicit open-source license (e.g., MIT/Apache) governing the Codex Plugins product or its source. missing for 10: explicit license declaration, confirmation that core Codex Plugins platform code (not just example plugins) is open-sourced, independent corroboration of license terms.", "evidenceIds": [ "codex-plugins-gh-1", "codex-plugins-gh-9", "codex-plugins-docs-35" ] }, { "productId": "codex-plugins", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "codex-plugins", "storyId": "per-skill-documentation", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs describe the SKILL.md format (must include name and description) and state that the skill-creation flow explicitly asks 'what the skill does, when it should trigger, and whether it should stay instruction-only,' directly matching the story's ask for documented behavior and activation conditions. However, there's no evidence of a consistent, enforced convention across all published skills/plugins for describing trigger conditions, and community commentary flags real unpredictability in skill activation ('you just selectively append more text to prompt and pray'), reflecting some doubt about whether documented behavior always matches actual activation. Missing for 10: independent/hands-on verification that documented triggers reliably predict actual activation, and evidence that all shipped skills (not just the creator flow) enforce clear 'when it activates' documentation.", "evidenceIds": [ "codex-plugins-docs-7", "codex-plugins-docs-13", "codex-plugins-docs-17", "codex-plugins-comm-2" ] }, { "productId": "codex-plugins", "storyId": "plain-files-portability", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm skills are plain directories with a SKILL.md file plus optional scripts/references (not a proprietary binary), scanned from `.agents/skills` folders that teams can check into a repo alongside code, which supports the 'read/copy' portability claim. However, there is no explicit evidence describing moving a skill folder to a different harness (e.g., Claude Code) working out-of-the-box, and the plugin.json/MCP wiring layer around plugins is more proprietary/manifest-based than pure markdown. Missing for 10: explicit cross-harness carry demonstration, and clarity that plugin-level packaging (vs skill-level) is also portable.", "evidenceIds": [ "codex-plugins-docs-13", "codex-plugins-docs-17", "codex-plugins-docs-9", "codex-plugins-docs-28", "codex-plugins-docs-32" ] }, { "productId": "codex-plugins", "storyId": "privacy-data-residency", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Codex Plugins is an extensibility/marketplace layer for adding skills and tools to Codex/ChatGPT; data residency/region storage controls are an enterprise/platform-level data governance concern, not something a plugin ecosystem itself configures. No evidence in the pack even mentions region selection or data residency.", "evidenceIds": [] }, { "productId": "codex-plugins", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers plugin/skill installation, marketplace, and OAuth sign-in scope, but contains no mention of data-training opt-out or AI-training data controls for Codex Plugins.", "evidenceIds": [] }, { "productId": "codex-plugins", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence only describes what limited profile data is shared during plugin sign-in (name, email, picture) and that this doesn't grant automatic data access, but there is no mention of controls for retaining, exporting, or deleting plugin-related data or conversation history. Missing for 10: any documented retention policy, data deletion mechanism, or user-facing controls to manage stored data for plugins/skills.", "evidenceIds": [ "codex-plugins-docs-19", "codex-plugins-docs-23" ] }, { "productId": "codex-plugins", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers plugin/skill installation, marketplace, and sign-in data sharing scope, but contains no mention of telemetry or usage-tracking opt-out controls for Codex Plugins.", "evidenceIds": [] }, { "productId": "codex-plugins", "storyId": "project-vs-global-install", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Evidence shows skills can be committed to a repo (`.agents/skills` scanned from cwd up to repo root, teams can check in module-specific skills) and installed to a local Codex setup via `$skill-installer`, implying both project-local and user-global-like scopes exist, but there's no explicit documentation contrasting a 'project-local vs user-global' install flag or config for plugins/skills. missing for 10: explicit docs on a user-global (all-projects) install location/flag distinct from project-local, and confirmation that plugin (not just skill) installs support both scopes.", "evidenceIds": [ "codex-plugins-docs-17", "codex-plugins-docs-9", "codex-plugins-docs-8", "codex-plugins-docs-14" ] }, { "productId": "codex-plugins", "storyId": "release-notes", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions release notes, version changelogs, or any versioning information for plugins or skills before updating; the docs cover installation, creation, and marketplace submission but nothing about update transparency.", "evidenceIds": [] }, { "productId": "codex-plugins", "storyId": "reliable-auto-triggering", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs describe the mechanics needed for context-triggered skills: SKILL.md requires a `description` field, the skill-creator explicitly asks 'when it should trigger,' and Codex scans `.agents/skills` up the directory tree for applicable skills, implying automatic, description-driven activation. However, evidence is mostly first-party docs plus manual invocation commands (`/skills`, `$mention`), and community commentary voices skepticism about reliability ('you just append more text to prompt and pray') without a concrete hands-on failure, so this stays partial rather than full/disputed. Missing for 10: independent hands-on verification that description-matching reliably picks the correct skill at the correct moment, and quantitative/real-world evidence of trigger accuracy.", "evidenceIds": [ "codex-plugins-docs-7", "codex-plugins-docs-13", "codex-plugins-docs-17", "codex-plugins-docs-20", "codex-plugins-comm-5", "codex-plugins-comm-2" ] }, { "productId": "codex-plugins", "storyId": "security-posture-docs", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Docs disclose some security-relevant details (sign-in only shares name/email/photo, submission/review portal for publishing plugins, MCP-based server capability exposure) but there is no consolidated security posture document covering what skills can execute, an explicit trust model for third-party skills, or telemetry/data collection practices beyond OAuth sign-in scope. Missing for 10: dedicated security/trust-model documentation, explicit execution/sandboxing model for skills, and telemetry/data-collection disclosure beyond sign-in scopes.", "evidenceIds": [ "codex-plugins-docs-19", "codex-plugins-docs-23", "codex-plugins-docs-31", "codex-plugins-docs-33", "codex-plugins-docs-32" ] }, { "productId": "codex-plugins", "storyId": "selective-install", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show `$skill-installer ` to install individual curated skills (e.g. `$skill-installer linear`, `$skill-installer gh-address-comments`) rather than a whole bundle, and plugins are installed individually via `/plugins` browser. However, plugins themselves appear to be installed as whole packages (a plugin can bundle multiple skills/tools) with no documented way to cherry-pick a subset of skills from within a single installed plugin. missing for 10: evidence of selecting/excluding individual skills within one plugin bundle at install time, independent hands-on confirmation of partial-bundle installs.", "evidenceIds": [ "codex-plugins-docs-8", "codex-plugins-docs-14", "codex-plugins-docs-21", "codex-plugins-docs-34", "codex-plugins-docs-25", "codex-plugins-docs-26" ] }, { "productId": "codex-plugins", "storyId": "skill-testing-harness", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents skill/plugin creation, installation, and marketplace workflows but contains no mention of automated tests, evals, or regression suites for skills; a community comment even contrasts skills unfavorably with MCP by noting skills are 'append more text to prompt and pray' rather than testable. No vendor documentation describes any verification harness for skill quality.", "evidenceIds": [ "codex-plugins-comm-2", "codex-plugins-docs-13", "codex-plugins-docs-17" ] }, { "productId": "codex-plugins", "storyId": "team-distribution", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs describe all three distribution channels: checking skills into a repo/working folder (`.agents/skills` scanning, docs-9/docs-17), a marketplace mechanism (`codex plugin marketplace add`, marketplace.json, docs-11/docs-35), and a public submission portal for org-wide plugins (docs-31). Community commentary corroborates skills being used for 'cross-team standardization' (comm-3). Missing for 10: independent case study of a team actually rolling out a shared marketplace/repo at scale, and more detail on permissioning/governance for team-wide distribution.", "evidenceIds": [ "codex-plugins-docs-9", "codex-plugins-docs-17", "codex-plugins-docs-11", "codex-plugins-docs-18", "codex-plugins-docs-31", "codex-plugins-docs-35", "codex-plugins-comm-3" ] }, { "productId": "codex-plugins", "storyId": "version-control-pinning", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers installing plugins/skills, marketplace sources, and manifest scaffolding, but nothing describes version pinning, a lockfile, or any mechanism to prevent automatic/silent skill updates for a team.", "evidenceIds": [ "codex-plugins-docs-11", "codex-plugins-docs-25", "codex-plugins-docs-26" ] }, { "productId": "mattpocock-skills", "storyId": "active-maintenance", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "There's some indirect signal of active development — an ADR documenting a recent architectural decision (shipping as a Claude Code plugin) and acceptance into Claude Code's official marketplace, plus an update mechanism (`npx skills update`) implying ongoing releases — but no direct evidence of a release cadence, an issue tracker for the project itself being triaged, or accepted external community contributions/PRs. Community commentary (comm-1/2/3) shows engagement and critique of content quality but says nothing about maintenance cadence or contribution acceptance. missing for 10: changelog/release history, evidence of external PRs being merged, evidence of issues on the repo itself being triaged.", "evidenceIds": [ "mattpocock-skills-docs-10", "mattpocock-skills-docs-41", "mattpocock-skills-docs-4", "mattpocock-skills-comm-1", "mattpocock-skills-comm-3" ] }, { "productId": "mattpocock-skills", "storyId": "agent-authors-skill", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes numerous existing skills (tdd, grilling, triage, setup-matt-pocock-skills) and packaging/distribution mechanics (npx skills, Claude plugin marketplace), but nothing documents a meta-skill, spec, or template that an agent would follow to author and package a brand-new SKILL.md from scratch within this project's own conventions.", "evidenceIds": [] }, { "productId": "mattpocock-skills", "storyId": "agent-installs-skill", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "The docs describe simple CLI install commands (`npx skills`, `claude plugins install mattpocock-skills`, `npx skills update`) that could in principle be scripted, but the accompanying setup skill explicitly interviews the user ('Ask you which issue tracker you want to use') and other examples show a human typing a slash command, not a fully headless agent-run flow. missing for 10: explicit non-interactive/CI flag or documented flow for an agent to run install end-to-end without any prompts, and confirmation that the interactive setup step can be skipped or automated.", "evidenceIds": [ "mattpocock-skills-docs-10", "mattpocock-skills-docs-43", "mattpocock-skills-docs-3", "mattpocock-skills-docs-9", "mattpocock-skills-docs-31" ] }, { "productId": "mattpocock-skills", "storyId": "agentic-agent-docs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The product's entire mechanism is agent-oriented markdown docs (SKILL.md, CONTEXT.md files) explicitly written for agents to read and act on, and it documents raw.githubusercontent URLs an agent could be pointed at directly. However, there is no evidence of a dedicated llms.txt for this product itself — the probe shows only GitHub's own generic llms.txt (unrelated to this project) and a 404 for skills.md, so the specific llms.txt convention is not supported, only the broader 'agent-readable docs' pattern.\nmissing for 10: a product-specific llms.txt file, and confirmation that agents can be pointed at a single canonical docs entry point rather than individual SKILL.md files.", "evidenceIds": [ "mattpocock-skills-docs-5", "mattpocock-skills-docs-6", "mattpocock-skills-docs-9", "mattpocock-skills-gh-7", "mattpocock-skills-probe-1", "mattpocock-skills-probe-2" ] }, { "productId": "mattpocock-skills", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "The skill bundle includes several skills that produce AI-generated insights/suggestions from a user's own project data — a visual refactor-worthiness report (docs-18), bug diagnosis from a repro (docs-19), diff review against standards (docs-14), and issue triage/sorting (docs-30, docs-34) — but these are discrete slash-command skills rather than a unified 'insights' surface, and there's no evidence of a dashboard or proactive analytics view. missing for 10: a consolidated insights UI/report aggregating findings, independent hands-on evidence of the insight-generating skills actually producing useful output, and evidence these insights update automatically rather than being invoked per-skill.", "evidenceIds": [ "mattpocock-skills-docs-18", "mattpocock-skills-docs-19", "mattpocock-skills-docs-14", "mattpocock-skills-docs-30" ] }, { "productId": "mattpocock-skills", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The skills are built around interactive, human-in-the-loop workflows (grilling/interviewing the user, confirming test seams, triage state machines) rather than unattended background automation; the project's own philosophy explicitly rejects processes that 'take away your control' in favor of user-confirmed steps. No evidence of scheduling, background triggers, or autonomous execution without human interaction is present in the pack.", "evidenceIds": [ "mattpocock-skills-gh-1", "mattpocock-skills-docs-6", "mattpocock-skills-docs-8", "mattpocock-skills-gh-8" ] }, { "productId": "mattpocock-skills", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "mattpocock-skills", "storyId": "agentic-headless", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes skill files consumed by interactive coding agents (Claude Code, Cursor, etc.) that rely on human interviewing/grilling and manual slash-command invocation, with no mention of a CLI flag, non-interactive mode, or CI/automation pipeline usage. Nothing in the docs, GitHub, or community evidence discusses running the skills headlessly or in a CI pipeline.", "evidenceIds": [] }, { "productId": "mattpocock-skills", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "This product is a library of skill/prompt files installed into external coding agents (Claude Code, Cursor, Codex, Copilot); it is not itself an agent or platform that consumes or hosts MCP servers, so plugging MCP servers into it is a category error — that capability belongs to the host agents, not to this skills package.", "evidenceIds": [ "mattpocock-skills-docs-42", "mattpocock-skills-docs-10" ] }, { "productId": "mattpocock-skills", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows distribution via npm CLI, Claude Code plugin marketplace, and file-copy installation, but nowhere does it mention an MCP server for agents to connect to. Since this is a skills/plugin package (not itself an agent), the axis is a fair question, but there is no evidence of an official MCP server offering.", "evidenceIds": [ "mattpocock-skills-docs-10", "mattpocock-skills-docs-41", "mattpocock-skills-docs-42", "mattpocock-skills-docs-4" ] }, { "productId": "mattpocock-skills", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "The skills are designed to be invoked and operated conversationally: users type slash commands (e.g., /grill-with-docs) and the agent then interviews/grills the user in natural language to reach shared understanding before acting (docs-43, gh-6, docs-6/26/33). This natural-language interaction model is central and repeated across multiple skill docs (grilling, triage, TDD flows). Missing for 10: independent hands-on confirmation that the natural-language command flow works smoothly in practice (community evidence only critiques prose quality, not the interaction mechanism), and no demonstration of free-form (non-slash) natural language command parsing beyond the interview pattern.", "evidenceIds": [ "mattpocock-skills-docs-43", "mattpocock-skills-gh-6", "mattpocock-skills-docs-6", "mattpocock-skills-docs-33", "mattpocock-skills-gh-1", "mattpocock-skills-docs-11" ] }, { "productId": "mattpocock-skills", "storyId": "agentic-official-cli", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "The product ships an official CLI (`npx skills`, with `npx skills update` to pull latest changes) and is also installable via the Claude Code plugin CLI route (`claude plugins install mattpocock-skills`), matching the AI-native CLI story. Missing for 10: independent hands-on verification of the CLI's full command set and behavior beyond first-party docs.", "evidenceIds": [ "mattpocock-skills-docs-4", "mattpocock-skills-docs-31", "mattpocock-skills-docs-10", "mattpocock-skills-docs-1", "mattpocock-skills-docs-2" ] }, { "productId": "mattpocock-skills", "storyId": "agentic-public-api", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "This product is a static collection of skill/markdown files distributed via CLI installers (npx skills, claude plugins) and consumed inside a host agent's context — it is not a service or platform that exposes its own public API for programmatic control. The 'driven through a documented public API' axis is a category error for a skills-file bundle rather than an applicable-but-unmet capability.", "evidenceIds": [] }, { "productId": "mattpocock-skills", "storyId": "agentic-scoped-keys", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "This product is a collection of AI agent 'skills'/prompt workflows for coding tasks, not an identity/access-management or credentialing system; issuing scoped API credentials is entirely outside its category.", "evidenceIds": [] }, { "productId": "mattpocock-skills", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "mattpocock-skills", "storyId": "agentic-webhooks", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "This product is a skills/prompt library for AI coding agents, not an event-driven platform or service with a webhook subscription mechanism; nothing in the evidence pertains to webhooks or event subscriptions, and the concept doesn't fit this product's category.", "evidenceIds": [] }, { "productId": "mattpocock-skills", "storyId": "api-interactive-docs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Skills for Real Engineers is a set of agent skill files/prompts, not an API product with an interactive reference — this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "mattpocock-skills", "storyId": "api-machine-spec", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "This product is a collection of AI agent 'skill' files/instructions for coding workflows, not an API or service with an interface to document via OpenAPI. There is no evidence of an API surface that would warrant a machine-readable spec, making this axis a category error for this product type.", "evidenceIds": [] }, { "productId": "mattpocock-skills", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "mattpocock-skills", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes an update mechanism (`npx skills update`, 'nothing updates behind your back') but there is no documentation of semantic versioning, an API surface, or any deprecation policy for skills as they evolve or are removed. Missing for 10: any explicit version numbering scheme, changelog, or documented deprecation/backwards-compatibility policy for the skill files or plugin.", "evidenceIds": [ "mattpocock-skills-docs-4", "mattpocock-skills-docs-24", "mattpocock-skills-docs-31", "mattpocock-skills-docs-41" ] }, { "productId": "mattpocock-skills", "storyId": "author-from-template", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes an existing bundle of pre-built skills (tdd, grilling, triage, setup) that you can install, subscribe to, or copy as editable files, but there is no documented template, generator, or guide specifically for authoring a brand-new SKILL.md with frontmatter — the closest thing to a 'template' would be reverse-engineering the shipped example skills, which the story explicitly excludes. Community commentary even critiques the prose quality of existing skill files rather than pointing to any authoring template.", "evidenceIds": [ "mattpocock-skills-docs-2", "mattpocock-skills-docs-24", "mattpocock-skills-docs-9", "mattpocock-skills-comm-3" ] }, { "productId": "mattpocock-skills", "storyId": "authoring-meta-skill", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes many individual skills (TDD, triage, grilling, spec-writing, setup-per-repo config) but none of them describe a meta-skill that guides writing, improving, or packaging new skills for the collection itself — 'setup-matt-pocock-skills' only scaffolds per-repo config for existing skills, not skill authoring.", "evidenceIds": [] }, { "productId": "mattpocock-skills", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The skill set is built around structured, one-at-a-time workflows (grilling, triage state machine, TDD, spec-to-ticket) rather than any documented bulk/batch processing across many items simultaneously; no evidence describes running a skill across multiple issues, files, or specs in one operation. Missing for 10: any documented bulk/batch command or workflow, evidence of parallel/multi-item processing, and independent confirmation of such usage.", "evidenceIds": [ "mattpocock-skills-docs-7", "mattpocock-skills-docs-27", "mattpocock-skills-docs-9" ] }, { "productId": "mattpocock-skills", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The product ships a library of manually-invoked skill files triggered by slash commands or explicit user direction ('Install the ones you want, then type a slash command'; 'Use them every time you want to make a change'), not an event-driven rule/automation engine. No evidence describes defining rules that fire automatically on repo events, webhooks, or triggers without user invocation.", "evidenceIds": [ "mattpocock-skills-docs-43", "mattpocock-skills-gh-1", "mattpocock-skills-docs-9" ] }, { "productId": "mattpocock-skills", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "This product is a library of AI agent skills/prompts for coding workflows (TDD, triage, grilling, etc.), not a scheduler or automation platform that runs recurring jobs/workflows on a schedule. No evidence of cron-like scheduling or persistent job orchestration, and this capability is outside the product's category.", "evidenceIds": [] }, { "productId": "mattpocock-skills", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "The product offers two install modes—a locked, read-only bundle that only updates via an explicit `npx skills update` pull, or an editable copy where skills become 'ordinary files you own' in your repo—which gives some control over when changes land and implies normal git-based versioning/rollback, but there is no explicit changelog, diff view, or rollback command for the skills themselves. Missing for 10: dedicated version history or diff/review UI for skill changes, an explicit rollback mechanism, and any evidence of reviewing skill updates before applying them (beyond opting into `update`).", "evidenceIds": [ "mattpocock-skills-docs-1", "mattpocock-skills-docs-2", "mattpocock-skills-docs-4", "mattpocock-skills-docs-24", "mattpocock-skills-docs-31", "mattpocock-skills-docs-41" ] }, { "productId": "mattpocock-skills", "storyId": "browse-searchable-catalog", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "The aihero.dev/skills page functions as a lightweight catalog listing all 25 skills with one-line descriptions (docs-11 through docs-23, docs-30/35/37, docs-42/43), and the project is also listed in Claude Code's official plugin marketplace (docs-10), letting a developer browse before installing. However there's no evidence of search, filtering, ratings, or a leaderboard-style comparison across skills/authors. Missing for 10: searchable/filterable registry UI, ratings or usage leaderboard, independent confirmation of the marketplace listing's browsability.", "evidenceIds": [ "mattpocock-skills-docs-42", "mattpocock-skills-docs-43", "mattpocock-skills-docs-10", "mattpocock-skills-docs-23", "mattpocock-skills-docs-11", "mattpocock-skills-docs-30" ] }, { "productId": "mattpocock-skills", "storyId": "clean-uninstall", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers installation modes (editable copy vs. read-only subscribed bundle) and updating via `npx skills update`, but there is no mention of any command or mechanism to list installed skills or cleanly uninstall/remove them without leftover files.", "evidenceIds": [] }, { "productId": "mattpocock-skills", "storyId": "documented-update-path", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly document an update path: `npx skills update` pulls latest changes on demand (nothing auto-updates behind your back) for the copy-into-repo mode, and a separate managed/marketplace mode (Claude Code plugin, `claude plugins install mattpocock-skills`) that updates as the author ships. Both paths are documented first-party. missing for 10: independent/hands-on confirmation that `npx skills update` or marketplace auto-update actually works in practice, and no changelog/version-diff evidence showing successful update history.", "evidenceIds": [ "mattpocock-skills-docs-4", "mattpocock-skills-docs-31", "mattpocock-skills-docs-24", "mattpocock-skills-docs-10", "mattpocock-skills-docs-41", "mattpocock-skills-docs-1" ] }, { "productId": "mattpocock-skills", "storyId": "five-minute-quickstart", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show a simple two-step install path (\"Install the ones you want, then type a slash command\" and `claude plugins install mattpocock-skills`), plus npx-based add/update commands, suggesting a fast setup, but there is no explicit quickstart walkthrough or timed benchmark confirming a five-minute install-to-working-skill experience, and no independent hands-on report timing the process. missing for 10: an actual quickstart guide/tutorial with step timings, independent/hands-on confirmation of install speed, and evidence that a first skill run succeeds quickly without extra config.", "evidenceIds": [ "mattpocock-skills-docs-43", "mattpocock-skills-docs-10", "mattpocock-skills-docs-4", "mattpocock-skills-docs-24" ] }, { "productId": "mattpocock-skills", "storyId": "inspect-before-install", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "The skills ship as plain, open-source Markdown SKILL.md files (visible directly via raw GitHub links quoted in evidence) and are described as 'ordinary files you own and can edit' with no silent updates, which supports post-install readability and transparency. However, there is no documented pre-install 'list contents' or dry-run command shown in the evidence — inspection relies on browsing the public GitHub repo rather than a built-in review step. Missing for 10: an explicit CLI/list command to preview a skill's files before installing, and independent confirmation that all installed files match what's shown pre-install.", "evidenceIds": [ "mattpocock-skills-docs-2", "mattpocock-skills-docs-24", "mattpocock-skills-docs-31", "mattpocock-skills-docs-5", "mattpocock-skills-docs-9", "mattpocock-skills-docs-41" ] }, { "productId": "mattpocock-skills", "storyId": "multi-harness-support", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs explicitly claim the collection 'works with any agent' — listing Claude Code, Cursor, Codex, Copilot — and show at least one harness-specific install path (Claude Code plugin marketplace vs. generic `npx skills` copy/subscribe modes), supporting cross-agent installability. However, evidence lacks concrete per-harness instructions for Cursor, Codex, or Copilot individually (only Claude Code's plugin route is documented in detail), and no hands-on confirmation that the same skill set actually functions identically across harnesses. Missing for 10: explicit install/config steps for Cursor, Codex, and Copilot, and independent verification that per-harness behavior matches claims.", "evidenceIds": [ "mattpocock-skills-docs-42", "mattpocock-skills-docs-10", "mattpocock-skills-docs-1", "mattpocock-skills-docs-2", "mattpocock-skills-docs-24" ] }, { "productId": "mattpocock-skills", "storyId": "one-command-install", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "First-party docs give a genuine one-liner (`claude plugins install mattpocock-skills`) now that the pack is in Claude Code's official marketplace, plus an `npx skills update` command for the managed-bundle mode, both documented as first-party. However there's no independent/hands-on confirmation that the skill set is actually active in the next session, and the exact single-command install syntax for the other supported agents (Cursor, Codex, Copilot) beyond Claude Code isn't shown — only 'install the ones you want, then type a slash command' is vague. missing for 10: independent hands-on confirmation of post-install activation, explicit one-liner install commands for non-Claude agents.", "evidenceIds": [ "mattpocock-skills-docs-10", "mattpocock-skills-docs-4", "mattpocock-skills-docs-31", "mattpocock-skills-docs-42", "mattpocock-skills-docs-43" ] }, { "productId": "mattpocock-skills", "storyId": "open-spec-compliance", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Evidence shows skill folders are plain, editable files (SKILL.md) that work across multiple agents (Claude Code, Cursor, Codex, Copilot) and were accepted into Claude Code's official plugin marketplace, implying broad cross-tool portability consistent with an open skill format. However, none of the evidence explicitly names or cites conformance to the 'Agent Skills specification' itself, so spec-adherence is inferred rather than documented. Missing for 10: explicit reference to the Agent Skills spec, independent confirmation that the folder validates against that spec outside vendor claims.", "evidenceIds": [ "mattpocock-skills-docs-24", "mattpocock-skills-docs-42", "mattpocock-skills-docs-10" ] }, { "productId": "mattpocock-skills", "storyId": "openness-api-parity", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "mattpocock-skills", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The product ships skills as plain, MIT-licensed markdown files copied directly into the user's repo (mattpocock-skills-docs-2, docs-24, docs-42), meaning the artifacts themselves are already open, human-readable, and fully owned/editable with no proprietary lock-in or vendor updates without consent (docs-4, docs-31). However, there is no explicit 'export my data' feature or documentation addressing exporting configuration state (issue tracker settings, triage labels, ADRs) generated while using the skills, nor any statement about a formal data-portability/leave process. missing for 10: explicit data-export tooling/documentation, evidence about exporting generated artifacts (ADRs, triage state, configs) beyond the skill files themselves, and any independent confirmation of portability.", "evidenceIds": [ "mattpocock-skills-docs-2", "mattpocock-skills-docs-24", "mattpocock-skills-docs-42", "mattpocock-skills-docs-4", "mattpocock-skills-docs-31" ] }, { "productId": "mattpocock-skills", "storyId": "openness-open-license", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "The product's source lives in a public GitHub repo (github.com/mattpocock/skills) and is explicitly described as MIT-licensed with 25 skills, confirming both open-source hosting and license terms; docs also emphasize files are 'ordinary files you own and can edit,' reinforcing readability/openness of the source. Missing for 10: no explicit LICENSE file citation or independent third-party confirmation of the license text.", "evidenceIds": [ "mattpocock-skills-docs-42", "mattpocock-skills-docs-24", "mattpocock-skills-docs-2" ] }, { "productId": "mattpocock-skills", "storyId": "openness-self-host", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "The product is an MIT-licensed, MIT-open GitHub repo of skill files that are copied directly into the user's own repo as editable, ordinary files ('you own and can edit... nothing updates behind your back'), which is effectively full self-hosting since there is no server component to host beyond the files themselves. This is corroborated by both the docs and the ADR describing the plugin/fork model. missing for 10: no independent/hands-on confirmation of a full self-hosted install working end-to-end outside vendor docs, and no explicit statement addressing infrastructure/hosting concerns (e.g., private registries, offline use).", "evidenceIds": [ "mattpocock-skills-docs-2", "mattpocock-skills-docs-24", "mattpocock-skills-docs-31", "mattpocock-skills-docs-41" ] }, { "productId": "mattpocock-skills", "storyId": "per-skill-documentation", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Each skill ships a SKILL.md/one-line description stating its purpose and trigger (e.g., TDD skill's red-green rules, triage skill's state machine, grilling skill's interview process, and the aihero.dev list of 25 skills each with a one-line 'what it does' summary), and activation is explicit and user-controlled via install + slash command rather than silent background changes ('Nothing updates behind your back', 'Install the ones you want, then type a slash command'). A community thread does critique the prose quality/clarity of some SKILL.md files as having 'little utility,' which tempers confidence but doesn't concretely contradict that each skill documents what/when it activates. missing for 10: independent verification that every one of the 25 skills' docs clearly states activation triggers (not just a sample), and resolution of the community critique about jargon-heavy or low-utility prose in some skill docs.", "evidenceIds": [ "mattpocock-skills-docs-9", "mattpocock-skills-docs-5", "mattpocock-skills-docs-25", "mattpocock-skills-docs-7", "mattpocock-skills-docs-27", "mattpocock-skills-docs-11", "mattpocock-skills-docs-42", "mattpocock-skills-docs-43", "mattpocock-skills-docs-4", "mattpocock-skills-comm-3" ] }, { "productId": "mattpocock-skills", "storyId": "plain-files-portability", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly state skills are written as ordinary markdown files into the repo that you own and can edit, not a proprietary format, and work across multiple harnesses (Claude Code, Cursor, Codex, Copilot). This directly matches the story's plain-file, portable-across-harness claim. Missing for 10: independent hands-on confirmation that files are literally copy-pasteable markdown (only vendor docs/ADR cited) and no explicit demonstration of moving skills to a different harness in practice.", "evidenceIds": [ "mattpocock-skills-docs-2", "mattpocock-skills-docs-24", "mattpocock-skills-docs-42", "mattpocock-skills-docs-41" ] }, { "productId": "mattpocock-skills", "storyId": "privacy-data-residency", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "This product is a set of skill files/prompts for coding agents, not a data-hosting or storage service; data residency/region selection is not an applicable axis for this kind of tool.", "evidenceIds": [] }, { "productId": "mattpocock-skills", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "This product is a collection of AI agent 'skills'/prompt files for coding workflows, not a data-processing or model-training service; it has no data-handling relationship with end users' data being used for AI training, so an AI-training opt-out story is a category error for this kind of product.", "evidenceIds": [] }, { "productId": "mattpocock-skills", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "mattpocock-skills", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers installation modes, skill content, and community commentary but contains no mention of telemetry, analytics, or usage tracking of any kind, let alone an opt-out mechanism. Since this is a tool a buyer could reasonably ask about data collection, absence of any documentation on the topic means the axis applies but is unaddressed.", "evidenceIds": [] }, { "productId": "mattpocock-skills", "storyId": "project-vs-global-install", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "The docs describe two install modes—copying editable skill files into a repo (project-local, committed) versus subscribing to a managed, read-only, auto-updating bundle via the Claude Code plugin marketplace—but never explicitly frame the second mode as 'user-global across all projects' vs project-local; scope (per-project vs per-user) is never directly addressed. Missing for 10: explicit documentation of a user-global/home-directory install option and confirmation that the plugin/subscribe mode applies across all projects rather than just being non-editable.", "evidenceIds": [ "mattpocock-skills-docs-1", "mattpocock-skills-docs-2", "mattpocock-skills-docs-24", "mattpocock-skills-docs-41", "mattpocock-skills-docs-10" ] }, { "productId": "mattpocock-skills", "storyId": "release-notes", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence describes an update mechanism (`npx skills update`, subscribing to a read-only bundle) but nothing about release notes, a changelog, or version history that would let a developer see what changed before updating.", "evidenceIds": [] }, { "productId": "mattpocock-skills", "storyId": "reliable-auto-triggering", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes manual invocation — 'Install the ones you want, then type a slash command' (docs-43) and 'Use them every time you want to make a change' (gh-1/gh-4) — rather than automatic, context-triggered activation via engineered descriptions. There is a 'find out which skill to use' meta-skill (docs-23) but it's itself a skill you must invoke, not evidence of automatic description-based dispatch. No documentation or independent evidence shows the agent autonomously selecting/activating skills from task context.", "evidenceIds": [ "mattpocock-skills-docs-43", "mattpocock-skills-docs-23", "mattpocock-skills-gh-1", "mattpocock-skills-gh-4" ] }, { "productId": "mattpocock-skills", "storyId": "security-posture-docs", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "The docs give some trust-relevant transparency—skills are shipped as plain, ownable files that 'nothing updates behind your back' and can be pulled explicitly via `npx skills update`, plus acceptance into Claude Code's official marketplace as a vetted distribution channel—but there is no explicit security-posture document covering what skills can execute (tool/permission scope), a formal trust model for arbitrary third-party skills, or any statement on telemetry/data collection. missing for 10: explicit execution/permission model for skills, documented telemetry or data-collection policy, formal third-party skill vetting/trust framework beyond marketplace acceptance.", "evidenceIds": [ "mattpocock-skills-docs-4", "mattpocock-skills-docs-24", "mattpocock-skills-docs-31", "mattpocock-skills-docs-10", "mattpocock-skills-docs-41" ] }, { "productId": "mattpocock-skills", "storyId": "selective-install", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "The docs explicitly state \"Install the ones you want, then type a slash command\" (mattpocock-skills-docs-43), indicating selective installation is possible, and the file-copy mode writes only the skills you choose as editable files (mattpocock-skills-docs-24). However, the marketplace/plugin route is described as installing 'the whole set as a managed, read-only bundle' (mattpocock-skills-docs-1, docs-41), so whole-bundle install remains the primary documented path and there's no detailed CLI flag or command example showing per-skill selection. Missing for 10: a concrete CLI command/flag demonstrating selecting individual skills, and independent/hands-on confirmation that partial installs work as described.", "evidenceIds": [ "mattpocock-skills-docs-43", "mattpocock-skills-docs-24", "mattpocock-skills-docs-1", "mattpocock-skills-docs-41" ] }, { "productId": "mattpocock-skills", "storyId": "skill-testing-harness", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes TDD/testing as a discipline the skills teach users to apply to their own code, but nothing shows the maintainer running automated tests, evals, or CI against the skill prompts themselves to catch regressions. A community critique even suggests the SKILL.md prose lacks rigor and 'should be checked... by another LLM,' implying no such verification pipeline exists.", "evidenceIds": [ "mattpocock-skills-docs-25", "mattpocock-skills-docs-38", "mattpocock-skills-comm-3" ] }, { "productId": "mattpocock-skills", "storyId": "team-distribution", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "The product ships via Claude Code's official plugin marketplace (docs-10), as an npx-installed, updatable managed bundle (docs-1, docs-4/31, docs-41), or as editable files committed directly into the project repo (docs-2, docs-24), and is agent-agnostic/MIT-licensed so a lead can standardize it across a whole team's tools (docs-42). This covers all three named distribution paths (marketplace, subscribe/shared-source, and in-repo files). Missing for 10: no explicit 'shared git repo' distribution mode distinct from marketplace/npx, and no independent/community confirmation that team-wide rollout works smoothly in practice beyond first-party docs.", "evidenceIds": [ "mattpocock-skills-docs-10", "mattpocock-skills-docs-1", "mattpocock-skills-docs-2", "mattpocock-skills-docs-24", "mattpocock-skills-docs-4", "mattpocock-skills-docs-42" ] }, { "productId": "mattpocock-skills", "storyId": "version-control-pinning", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs explicitly state skills don't auto-update and changes only land when the user runs `npx skills update`, and offer a 'copy' mode where files become editable local copies you fully own — both give an engineering lead control over rollout timing. However, there is no mention of an actual lockfile, pinned semantic versions, or per-team version pinning mechanism, so the control is manual/all-or-nothing rather than granular version pinning. Missing for 10: explicit lockfile/version-pin mechanism, ability to pin to a specific historical version rather than just delaying `update`, independent confirmation of update-control behavior in practice.", "evidenceIds": [ "mattpocock-skills-docs-4", "mattpocock-skills-docs-24", "mattpocock-skills-docs-31", "mattpocock-skills-docs-1", "mattpocock-skills-docs-2", "mattpocock-skills-docs-41" ] }, { "productId": "skills-cli", "storyId": "active-maintenance", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains extensive CLI/feature documentation but no release history, changelog, issue triage activity, or accepted PR/contribution data indicating active maintenance; community comments even question missing package-management basics (updates, version-pinning, uninstalls), but this is skepticism, not concrete evidence of stale maintenance, so it doesn't meet the bar for 'disputed' either.", "evidenceIds": [ "skills-cli-comm-4" ] }, { "productId": "skills-cli", "storyId": "agent-authors-skill", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "The CLI ships a `skills init [name]` command that scaffolds a new SKILL.md template, the agentskills.io spec defines the file format and fields (e.g. allowed-tools), a `skills-ref validate` tool checks conformance, and a dedicated 'skill-creator' meta-skill is referenced for guided authoring — together covering the author-to-package workflow end to end. Missing for 10: a full worked example showing a new skill going from init through validate to packaged/published output, and independent (non-vendor) confirmation that agents can follow this spec unaided.", "evidenceIds": [ "skills-cli-gh-15", "skills-cli-gh-28", "skills-cli-gh-43", "skills-cli-docs-10", "skills-cli-docs-4", "skills-cli-docs-14", "skills-cli-gh-20" ] }, { "productId": "skills-cli", "storyId": "agent-installs-skill", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "The CLI documents an explicit non-interactive/CI-CD-friendly install path with a full example (`npx skills add ... --skill frontend-design -g -a claude-code -y`) that specifies target skill, agent, scope, and auto-confirms without prompts, enabling a coding agent to run it headlessly end-to-end. Missing for 10: independent/hands-on confirmation that this exact non-interactive flow works flawlessly in practice (only first-party docs cited).", "evidenceIds": [ "skills-cli-gh-8", "skills-cli-gh-22", "skills-cli-gh-31", "skills-cli-gh-5", "skills-cli-gh-7" ] }, { "productId": "skills-cli", "storyId": "agentic-agent-docs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The product ships an AGENTS.md file (raw.githubusercontent.com/vercel-labs/skills/HEAD/AGENTS.md) containing structured, agent-readable command references (skills-cli-docs-2,5,7,8,9,10,11), which is a recognized agent-oriented docs format a user could point an agent at. However there's no evidence of an llms.txt for skills.sh itself, and a probe for an alternate docs.md path returned 404, suggesting the agent-doc surface is limited to AGENTS.md rather than a broader llms.txt/docs strategy. Missing for 10: a dedicated llms.txt file for skills.sh/agentskills.io, confirmation the AGENTS.md is discoverable/linked from primary docs, and independent evidence an agent was successfully pointed at it.", "evidenceIds": [ "skills-cli-docs-2", "skills-cli-docs-8", "skills-cli-docs-9", "skills-cli-docs-10", "skills-cli-docs-11", "skills-cli-probe-2" ] }, { "productId": "skills-cli", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "skills-cli", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "skills-cli", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "skills-cli", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "skills.sh's CLI explicitly documents non-interactive, CI/CD-friendly installation flags (--skill, -a, -y, -g) and headless usage via npx skills add/use with flags that avoid interactive prompts, suitable for automation pipelines. missing for 10: no independent CI pipeline case study or official GitHub Actions integration example, and no REST/API-only automation path (a community request notes absence of a REST API).", "evidenceIds": [ "skills-cli-gh-8", "skills-cli-gh-22", "skills-cli-gh-31", "skills-cli-gh-9", "skills-cli-comm-5" ] }, { "productId": "skills-cli", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "skills-cli", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "skills.sh is a CLI package manager that installs skill files/prompts into agent directories (via add/use/symlink mechanisms), not an MCP server; no evidence anywhere in the pack mentions an MCP server or MCP protocol integration, only file-based/CLI installation across supported agents.", "evidenceIds": [ "skills-cli-gh-1", "skills-cli-gh-5", "skills-cli-gh-29", "skills-cli-docs-1" ] }, { "productId": "skills-cli", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "skills.sh is a structured CLI (npm-style syntax like `npx skills add`, `--skill`, `--agent` flags) with no evidence of a natural-language command interface for operating the tool itself; all documented usage requires exact flag syntax.", "evidenceIds": [ "skills-cli-gh-1", "skills-cli-gh-4", "skills-cli-gh-5", "skills-cli-gh-20", "skills-cli-gh-21" ] }, { "productId": "skills-cli", "storyId": "agentic-official-cli", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "skills.sh ships a fully documented official CLI (`npx skills`) with rich subcommands—add, use, list, search, update, remove, init, experimental_install/sync—supporting multiple agents (Claude Code, Codex, Cursor, etc.), symlink-based installs, CI/CD-friendly non-interactive mode, and private/public repo auth, all corroborated by first-party GitHub docs and independent HN discussion of real usage. Missing for 10: deeper independent hands-on verification of update/version-pinning reliability (one community comment questioned documentation of these, though the CLI docs do cover them) and no REST API alternative.", "evidenceIds": [ "skills-cli-gh-1", "skills-cli-gh-5", "skills-cli-gh-8", "skills-cli-gh-10", "skills-cli-gh-13", "skills-cli-gh-14", "skills-cli-gh-29", "skills-cli-docs-8", "skills-cli-docs-9", "skills-cli-comm-2", "skills-cli-comm-4" ] }, { "productId": "skills-cli", "storyId": "agentic-public-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "skills.sh exposes a well-documented CLI (`npx skills add/use/list/update/remove/init`, non-interactive/CI-friendly flags) that lets automated/agentic callers drive it programmatically, and AGENTS.md/specification.md document the interface in detail. However, this is a CLI, not a public API/SDK, and a community member explicitly requests 'Please make a rest API!' ([skills-cli-comm-5]), indicating no such API exists yet. Missing for 10: a documented REST/programmatic API (not just CLI), SDKs or webhooks, and independent confirmation of API-level access beyond the CLI wrapper.", "evidenceIds": [ "skills-cli-gh-8", "skills-cli-gh-31", "skills-cli-docs-8", "skills-cli-docs-9", "skills-cli-docs-10", "skills-cli-docs-11", "skills-cli-comm-5" ] }, { "productId": "skills-cli", "storyId": "agentic-scoped-keys", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "skills.sh is a CLI/package manager for distributing and installing agent 'skills' (prompt/instruction bundles) across coding agents; it has no relation to issuing or scoping API credentials/access tokens for agents. This axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "skills-cli", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only a CLI (`skills` npx tool) and a SKILL.md specification/validator, not an official SDK (e.g., a JS/Python library) for programmatic integration; no SDK is mentioned anywhere in the pack.", "evidenceIds": [] }, { "productId": "skills-cli", "storyId": "agentic-webhooks", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "skills.sh is a CLI for installing/managing agent skills, not an event-driven service with a webhook subscription model; there is no evidence of any event system to subscribe to, and the notion of webhooks is a category error for this product type.", "evidenceIds": [] }, { "productId": "skills-cli", "storyId": "api-interactive-docs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "skills.sh/vercel-labs skills is a CLI package manager for agent skills, not an API product; there is no evidence of an interactive API reference with runnable examples, and the concept of an 'API reference' is a category mismatch for this CLI tool's purpose.", "evidenceIds": [] }, { "productId": "skills-cli", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows a CLI tool and a Skill-format specification (SKILL.md schema, agentskills.io/specification.md), but no OpenAPI or machine-readable API spec for skills.sh itself; a community comment explicitly requests 'Please make a rest API!' implying none currently exists.", "evidenceIds": [ "skills-cli-docs-4", "skills-cli-comm-5" ] }, { "productId": "skills-cli", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "skills-cli", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack has no documentation of versioned APIs or a deprecation policy for skills.sh's CLI/registry; the closest matter is `skills update` for updating skill content, which is unrelated to API/version stability guarantees. Community feedback explicitly notes the absence of version-pinning/package-management documentation, reinforcing that no such policy exists.", "evidenceIds": [ "skills-cli-comm-4", "skills-cli-comm-5" ] }, { "productId": "skills-cli", "storyId": "author-from-template", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "The CLI ships a dedicated `skills init [name]` command that explicitly creates a new SKILL.md template, letting a developer scaffold a skill without copying an existing one, and the companion specification documents required frontmatter fields (name/description used for discovery) that such a template would include. Missing for 10: no example of the actual generated SKILL.md content/frontmatter shown in evidence, and no independent/community confirmation that `skills init` output is complete or bug-free.", "evidenceIds": [ "skills-cli-gh-15", "skills-cli-gh-28", "skills-cli-gh-43", "skills-cli-docs-10", "skills-cli-docs-3", "skills-cli-docs-14" ] }, { "productId": "skills-cli", "storyId": "authoring-meta-skill", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The CLI ships `skills init` to scaffold a new SKILL.md template and references a `skill-creator` skill by name as an installable skill, plus a separate `skills-ref validate` tool for checking a skill's structure — together these cover pieces of authoring/packaging support. However there's no detailed documentation of a guided 'writing/improving' workflow or what the skill-creator meta-skill actually instructs the agent to do. missing for 10: full documentation of the skill-creator/meta-skill's guidance content, an explicit 'improve an existing skill' workflow, and independent hands-on corroboration that authoring guidance works as intended.", "evidenceIds": [ "skills-cli-gh-15", "skills-cli-gh-28", "skills-cli-gh-43", "skills-cli-gh-20", "skills-cli-docs-4" ] }, { "productId": "skills-cli", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "skills.sh CLI supports bulk-style operations: installing multiple skills with repeated --skill flags, using '*' to install all skills in a repo, targeting multiple agents at once (-a claude-code -a opencode), searching/finding across an entire org's repositories, and listing/updating/removing installed skills in one command. However these are batch operations on skills/repos, not a generalized bulk-operation framework across arbitrary 'many items', and there's no evidence of bulk operations at scale (e.g., hundreds of skills, parallelism, progress reporting) or independent hands-on confirmation of true bulk behavior beyond CLI flag docs. missing for 10: evidence of large-scale/parallel bulk execution, independent hands-on validation of multi-item operations beyond documented flags, and any bulk-editing/config across many installed skills simultaneously.", "evidenceIds": [ "skills-cli-gh-20", "skills-cli-gh-21", "skills-cli-gh-4", "skills-cli-gh-12", "skills-cli-gh-25", "skills-cli-gh-42", "skills-cli-gh-27", "skills-cli-gh-10" ] }, { "productId": "skills-cli", "storyId": "automation-rules-engine", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "skills.sh is a CLI/package manager for installing and managing agent skills (SKILL.md files), not an automation/rules engine; nothing in the evidence describes event triggers or conditional rule-based actions, which is outside this product's category.", "evidenceIds": [] }, { "productId": "skills-cli", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "skills.sh is a CLI/registry for installing and managing agent skills (SKILL.md files), not a workflow/job scheduler or automation runner; scheduling recurring jobs is outside its product category (wrong axis).", "evidenceIds": [] }, { "productId": "skills-cli", "storyId": "automation-versioned-workflows", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "skills.sh documents an update command and an experimental skills-lock.json restore mechanism (skills experimental_install) that could serve as version/rollback primitives, and skills are stored in project dirs that can be committed to git for review. However, a hands-on community comment explicitly contradicts this, stating that none of these 'npm for Skills' tools document basic package-management features like updates, version-pinning, or uninstalls, undercutting the vendor's implied versioning story. There is no evidence of a review/diff workflow before applying changes, and rollback is only an 'experimental' feature. Missing for 10: documented version-pinning/semver support, a review/diff UI before install, and non-experimental rollback with independent confirmation it works.", "evidenceIds": [ "skills-cli-docs-2", "skills-cli-docs-5", "skills-cli-docs-7", "skills-cli-gh-13", "skills-cli-gh-26", "skills-cli-gh-24", "skills-cli-comm-4" ] }, { "productId": "skills-cli", "storyId": "browse-searchable-catalog", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "skills.sh hosts a public Skills Leaderboard for browsing available skills (skills-cli-docs-6), and the CLI supports interactive/keyword search and listing skills across repos/orgs before installing (skills-cli-gh-6, -11, -12, -25, -33). Community discussion confirms the leaderboard is live and ranks by download count, though some question ranking transparency/fairness. Missing for 10: no independent evidence of catalog breadth/quality beyond Vercel's own skills, and community skepticism about ranking bias (skills-cli-comm-1, -2) slightly undercuts trust in the leaderboard's neutrality.", "evidenceIds": [ "skills-cli-docs-6", "skills-cli-gh-6", "skills-cli-gh-11", "skills-cli-gh-12", "skills-cli-gh-25", "skills-cli-gh-33", "skills-cli-comm-1", "skills-cli-comm-2", "skills-cli-comm-3" ] }, { "productId": "skills-cli", "storyId": "clean-uninstall", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "The CLI ships explicit `skills list`/`ls` (list installed skills, akin to npm ls) and `skills remove` commands, and its symlink-based install architecture ('single source of truth, easy updates') implies removal is straightforward rather than leaving scattered copies. However, no evidence explicitly confirms that removal deletes all agent-specific copies/instructions without orphaned files, and a community commenter specifically questioned whether uninstall/version-pinning is well-documented at all, adding some doubt about thoroughness. missing for 10: explicit documentation or hands-on proof that `skills remove` cleans up all installed files/prompts with zero orphaned artifacts, and a rebuttal to the community claim that uninstall isn't clearly documented.", "evidenceIds": [ "skills-cli-gh-10", "skills-cli-docs-8", "skills-cli-gh-14", "skills-cli-gh-27", "skills-cli-gh-34", "skills-cli-gh-38", "skills-cli-gh-41", "skills-cli-comm-4" ] }, { "productId": "skills-cli", "storyId": "documented-update-path", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "The CLI documents an explicit `skills update [skills...]` command that updates installed skills to latest versions, either all at once or by name, and even a lock-file based `skills experimental_install` for restoring versions, giving developers a documented update path without full reinstall (gh-13, gh-26, gh-42, docs-9, docs-2). A HN commenter voices generic skepticism that 'npm for skills' tools lack update docs, but this is not a concrete hands-on failure of skills.sh's actual `update` command, so it reads as skepticism/confusion rather than a contradiction. Missing for 10: independent/hands-on confirmation that `skills update` actually fetches newer versions correctly, and no mention of automatic marketplace-triggered updates (only manual command).", "evidenceIds": [ "skills-cli-gh-13", "skills-cli-gh-26", "skills-cli-gh-42", "skills-cli-docs-9", "skills-cli-docs-2", "skills-cli-comm-4" ] }, { "productId": "skills-cli", "storyId": "five-minute-quickstart", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "The single command `npx skills add owner/repo` installs a working skill quickly and docs show a minimal quickstart flow (list, install, use with an agent), suggesting a fast path to a working skill. However, there's a community complaint that basic package-management docs (updates, version pinning, uninstall) are hard to find, and no independent timed hands-on report confirms the under-5-minutes claim. Missing for 10: an explicit timed quickstart walkthrough, independent confirmation of setup speed, and clearer documentation addressing the community's confusion about basic usage.", "evidenceIds": [ "skills-cli-gh-1", "skills-cli-gh-18", "skills-cli-gh-2", "skills-cli-comm-4" ] }, { "productId": "skills-cli", "storyId": "inspect-before-install", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "skills.sh supports listing skills before installing (`--list`, `skills find`, interactive search) and installs skills as visible files on disk (project or global directories, with symlinks to a canonical copy), so a lead can inspect files after install. However, there is no dedicated command or documented workflow for previewing full skill contents/scripts before installation or for auditing all bundled scripts after install — only skill names/descriptions are surfaced up front, and community discussion flags trust concerns about unreviewed instructions in skills. Missing for 10: a pre-install content/diff preview command, explicit documentation of a post-install file-audit workflow, and resolution of the community-raised trust concern about hidden instructions.", "evidenceIds": [ "skills-cli-gh-6", "skills-cli-gh-19", "skills-cli-gh-30", "skills-cli-gh-24", "skills-cli-gh-38", "skills-cli-docs-3", "skills-cli-comm-6" ] }, { "productId": "skills-cli", "storyId": "multi-harness-support", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "skills.sh CLI explicitly supports installing a skill collection into multiple named agents (e.g., `-a claude-code -a opencode`, `--agent claude-code cursor`) and lists broad support for Claude Code, Codex, Cursor, OpenCode plus 70+ more, with per-agent directory targeting (`.//skills/`) and symlink-based single-source-of-truth updates. Cross-agent reuse is also stated in docs ('Build a skill once and use it across any skills-compatible agent'). Missing for 10: independent hands-on verification that per-harness instructions actually differ/adapt correctly across agents beyond directory placement, and community commentary raises unanswered concerns about version-pinning/uninstall completeness.", "evidenceIds": [ "skills-cli-gh-5", "skills-cli-gh-21", "skills-cli-gh-27", "skills-cli-gh-29", "skills-cli-gh-45", "skills-cli-gh-40", "skills-cli-gh-38", "skills-cli-docs-1" ] }, { "productId": "skills-cli", "storyId": "one-command-install", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "The `npx skills add ` one-liner installs skills directly into the target agent's directory (project or global, via symlink or copy), making them available in the next session without extra steps, and this is documented extensively with concrete CLI examples and flags. missing for 10: independent hands-on confirmation that installed skills are actually picked up in a fresh agent session (only vendor docs/README evidence), and community comments raise concerns about missing version-pinning/uninstall documentation clarity.", "evidenceIds": [ "skills-cli-gh-1", "skills-cli-gh-18", "skills-cli-gh-24", "skills-cli-gh-37", "skills-cli-gh-38", "skills-cli-gh-29", "skills-cli-docs-1", "skills-cli-comm-4" ] }, { "productId": "skills-cli", "storyId": "open-spec-compliance", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "The CLI is built around the open Agent Skills specification (agentskills.io) which explicitly promises 'Build a skill once and use it across any skills-compatible agent,' includes a spec validator (skills-ref validate) and a SKILL.md template generator, and the tool itself supports 75+ different agents rather than locking skills to one vendor's runtime. No evidence contradicts spec portability itself (community complaints target leaderboard fairness and package-management UX, not spec compliance). Missing for 10: independent third-party confirmation that a skill authored via this tool works unmodified in a wholly separate, non-Vercel skills implementation.", "evidenceIds": [ "skills-cli-docs-1", "skills-cli-docs-4", "skills-cli-gh-45", "skills-cli-gh-15", "skills-cli-docs-14" ] }, { "productId": "skills-cli", "storyId": "openness-api-parity", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "The `skills` CLI (search, list, add, update, remove) lets an AI-native user perform most of what the skills.sh leaderboard/browsing UI shows, giving a non-UI, scriptable path to core functionality, but there is no documented REST/HTTP API — a Hacker News commenter explicitly asks 'Please make a rest API!', implying one does not exist and CLI-only access is required. missing for 10: a documented REST/programmatic API mirroring the website's leaderboard/ranking views, independent confirmation that all UI features (e.g., leaderboard filtering, skill descriptions) are reachable via CLI/API.", "evidenceIds": [ "skills-cli-gh-1", "skills-cli-gh-11", "skills-cli-gh-10", "skills-cli-gh-13", "skills-cli-gh-14", "skills-cli-comm-5", "skills-cli-docs-6" ] }, { "productId": "skills-cli", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Skills are stored locally as plain SKILL.md markdown files (not a proprietary DB), with symlinked canonical copies and a skills-lock.json manifest, so a user's installed skill data is inherently in an open, portable format and never trapped in a hosted service. However, there is no explicit 'export' command, no documented data-portability policy, and no first-party statement addressing leaving/migrating the skill library. missing for 10: explicit export/backup tooling, documented data-portability guarantee, independent confirmation that all state (not just skill files) is locally recoverable.", "evidenceIds": [ "skills-cli-gh-16", "skills-cli-gh-38", "skills-cli-docs-2", "skills-cli-docs-5", "skills-cli-gh-24", "skills-cli-gh-40" ] }, { "productId": "skills-cli", "storyId": "openness-open-license", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "The CLI's source is publicly hosted and readable on GitHub (github.com/vercel-labs/skills), with docs, code, and even AGENTS.md file linked directly, so an AI-native user can browse and read the source. However, no evidence pack item confirms an explicit open-source license (e.g., MIT/Apache) attached to the repo. Missing for 10: explicit license file/name, confirmation of license terms, any independent audit noting license type.", "evidenceIds": [ "skills-cli-probe-3", "skills-cli-gh-1", "skills-cli-docs-5" ] }, { "productId": "skills-cli", "storyId": "openness-self-host", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "The CLI (`npx skills`) is open-source on GitHub and runs entirely client-side against arbitrary git/HTTP sources with no vendor backend dependency, so a user can effectively run their own copies without a hosted service — but there is no explicit self-hosting guide, deployment doc, or Docker/server setup for the leaderboard site (skills.sh) itself. missing for 10: explicit self-host/deployment instructions for the skills.sh service, confirmation that the leaderboard or discovery backend can be run independently, and any first-party statement framing self-hosting as a supported use case.", "evidenceIds": [ "skills-cli-probe-3", "skills-cli-gh-1", "skills-cli-gh-9", "skills-cli-docs-6" ] }, { "productId": "skills-cli", "storyId": "per-skill-documentation", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "The SKILL.md spec and docs show skills carry a required name/description that agents load at startup specifically 'to know when it might be relevant,' plus optional fields like allowed-tools for scoping behavior, directly matching the story's ask for documented activation conditions. Missing for 10: independent hands-on confirmation that these descriptions reliably predict agent behavior, and no example showing a rich 'what it does' body beyond the discovery metadata.", "evidenceIds": [ "skills-cli-docs-3", "skills-cli-docs-12", "skills-cli-docs-14", "skills-cli-docs-13" ] }, { "productId": "skills-cli", "storyId": "plain-files-portability", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Skills are plain SKILL.md files/folders stored in canonical directories with symlinks to agent-specific paths, installable/removable/updatable via CLI, and explicitly designed for 'build once, use across any skills-compatible agent' with 75+ supported agents — this is markdown, not a proprietary binary. missing for 10: independent hands-on confirmation of actually moving a skill folder to a different harness and verifying it works unmodified, and clearer detail on how symlink/canonical-copy structure avoids lock-in on non-supported agents.", "evidenceIds": [ "skills-cli-gh-16", "skills-cli-gh-29", "skills-cli-gh-38", "skills-cli-docs-1", "skills-cli-gh-24", "skills-cli-gh-37", "skills-cli-docs-13" ] }, { "productId": "skills-cli", "storyId": "privacy-data-residency", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "skills.sh is a CLI/package-manager for AI agent skills, not a data storage/hosting service; there is no user data residency concept applicable since skills are pulled from GitHub/GitLab/URLs the user already controls, not stored by skills.sh itself.", "evidenceIds": [] }, { "productId": "skills-cli", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "skills.sh is a CLI/package manager for installing and managing AI agent skills, not a data-processing or AI training service; there is no data-training relationship with users to opt out of, making this privacy-posture axis a category error for this product type.", "evidenceIds": [] }, { "productId": "skills-cli", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Skills are installed as local files (project/global dirs or symlinks) and the CLI ships an explicit `remove`/`skills remove` command to delete installed skills per-agent, giving users direct control over what's stored and its removal. However, there's no documented policy on telemetry/usage data retention, no account-level data deletion, and a community comment even claims uninstall/versioning isn't well documented (though contradicted by the gh docs). Missing for 10: explicit data-retention/telemetry policy, account-data deletion guarantees, independent confirmation that remove fully purges data.", "evidenceIds": [ "skills-cli-gh-14", "skills-cli-gh-27", "skills-cli-gh-34", "skills-cli-gh-38", "skills-cli-comm-4" ] }, { "productId": "skills-cli", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions telemetry, usage tracking, analytics collection, or any opt-out mechanism for skills.sh/the skills CLI.", "evidenceIds": [] }, { "productId": "skills-cli", "storyId": "project-vs-global-install", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly define two install scopes: Project (default, `.//skills/`, committed with project) and Global (`-g` flag, `~//skills/`, available across all projects), with CLI examples using -g flag. This directly matches the story's project-local vs user-global choice. Missing for 10: no independent hands-on report verifying the -g flag behavior in practice beyond docs.", "evidenceIds": [ "skills-cli-gh-24", "skills-cli-gh-37", "skills-cli-gh-40", "skills-cli-gh-7", "skills-cli-gh-22" ] }, { "productId": "skills-cli", "storyId": "release-notes", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The CLI has an `update` command and lock-file restore mechanism, but there is no evidence anywhere in the pack of release notes, a changelog, or per-skill version history a developer could review before running an update; a community comment explicitly notes these 'npm for Skills' tools don't document versioning/update details at all.", "evidenceIds": [ "skills-cli-gh-13", "skills-cli-gh-26", "skills-cli-docs-5", "skills-cli-comm-4" ] }, { "productId": "skills-cli", "storyId": "reliable-auto-triggering", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "The Agent Skills spec/docs describe the underlying discovery mechanism—agents load only name+description at startup 'just enough to know when it might be relevant'—which is the technical basis for auto-triggering, and skills.sh's CLI is the distribution layer for these SKILL.md files. However, there's no first-party guidance on engineering descriptions for reliable activation, and no independent/hands-on evidence confirming skills actually trigger correctly at the right moment (community comments focus on ranking/leaderboard fairness, not trigger accuracy). Missing for 10: documentation on description-engineering best practices, hands-on validation that the right skill activates at the right time, and independent corroboration beyond vendor docs.", "evidenceIds": [ "skills-cli-docs-3", "skills-cli-docs-12", "skills-cli-docs-13" ] }, { "productId": "skills-cli", "storyId": "security-posture-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows only a single experimental 'allowed-tools' field for pre-approving tools a skill may use, but no explicit documentation of a security model, sandboxing/execution boundaries, trust model for third-party skill authors, or telemetry/data-collection disclosure. Community commentary even raises unaddressed trust concerns about running other people's skill instructions, with no vendor response documenting mitigations.", "evidenceIds": [ "skills-cli-docs-14", "skills-cli-comm-6", "skills-cli-comm-4" ] }, { "productId": "skills-cli", "storyId": "selective-install", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "The CLI explicitly supports installing named skills via `--skill` flags (e.g., `--skill frontend-design --skill skill-creator`) or `'*'` for all, plus `--list` to preview skills before installing, directly enabling selective installation instead of a whole bundle. missing for 10: independent/hands-on third-party confirmation beyond the vendor's own docs/README (community commentary is about ranking/versioning, not selective install itself).", "evidenceIds": [ "skills-cli-gh-4", "skills-cli-gh-20", "skills-cli-gh-19", "skills-cli-gh-6", "skills-cli-gh-30" ] }, { "productId": "skills-cli", "storyId": "skill-testing-harness", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "skills.sh/vercel-labs skills is a CLI for installing, updating, and managing skills packages, but nothing in the evidence pack mentions tests, evals, CI checks, or regression verification for the skills themselves; community comments even criticize lack of basic package-management documentation, and there is no mention of quality assurance for skill content.", "evidenceIds": [ "skills-cli-comm-4" ] }, { "productId": "skills-cli", "storyId": "team-distribution", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "skills.sh's CLI supports installing skills from GitHub repos (public/private), a marketplace-like registry with search/leaderboard, and project-level installs that are committed to the repo and shared with the team (`.//skills/` default, symlink mode for single source of truth), plus CI/CD-friendly non-interactive install and update/remove commands for team-wide standardization. Community feedback confirms real-world adoption but raises concerns about package-management maturity (versioning/uninstall docs). missing for 10: no case study of an actual team-wide rollout process, no built-in access-control/marketplace curation features beyond org-wide search, and community skepticism about update/versioning robustness.", "evidenceIds": [ "skills-cli-gh-24", "skills-cli-gh-40", "skills-cli-gh-38", "skills-cli-gh-9", "skills-cli-gh-8", "skills-cli-gh-12", "skills-cli-docs-6", "skills-cli-comm-4" ] }, { "productId": "skills-cli", "storyId": "version-control-pinning", "verdict": "disputed", "quality": 3, "confidence": "medium", "rationale": "Docs mention an experimental `skills-lock.json` and `experimental_install`/`experimental_sync` commands that hint at a lockfile mechanism, but these are explicitly labeled experimental and there's no first-party documentation of pinning specific skill versions or preventing silent updates — `skills update` appears to always pull latest. A community reviewer directly disputes this, asking why none of these 'npm for skills' tools document version-pinning or controlled updates at all. missing for 10: documented pin/lock command with team workflow guidance, evidence the lockfile actually prevents silent updates, independent confirmation the experimental restore feature works as a version-control mechanism.", "evidenceIds": [ "skills-cli-docs-2", "skills-cli-docs-5", "skills-cli-docs-7", "skills-cli-comm-4" ] }, { "productId": "superpowers", "storyId": "active-maintenance", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "RELEASE-NOTES.md shows frequent, detailed updates (new harness support, workflow fixes, workspace scoping, TDD doc rewrites) and HN community threads confirm users are tracking recent changes, indicating active maintenance and recent releases. However, there is no evidence of issue triage practices or accepted community pull requests/contributions. Missing for 10: evidence of issue triage process, evidence of merged community PRs/contributions, independent contributor activity beyond the single maintainer.", "evidenceIds": [ "superpowers-docs-1", "superpowers-docs-41", "superpowers-docs-52", "superpowers-docs-54", "superpowers-comm-3" ] }, { "productId": "superpowers", "storyId": "agent-authors-skill", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Superpowers includes a dedicated writing-skills SKILL.md meta-skill describing a full TDD-style workflow for authoring skills (write test cases, watch fail, write skill, watch pass, refactor) and a harness-porting spec/template for extending skills across agents, backed by an eval harness that verifies skill compliance. Community evidence corroborates the skill-authoring and self-review workflow exists and is used in practice, though with some workflow friction noted. Missing for 10: independent hands-on account of an agent actually authoring a brand-new skill end-to-end via this meta-skill, and packaging/publishing steps are only documented, not demonstrated by a third party.", "evidenceIds": [ "superpowers-docs-15", "superpowers-docs-16", "superpowers-docs-13", "superpowers-docs-59", "superpowers-docs-38", "superpowers-comm-1" ] }, { "productId": "superpowers", "storyId": "agent-installs-skill", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All installation evidence describes human-run commands (`/plugin marketplace add`, `devin plugins install`, git clone steps) that differ per harness and are documented as manual steps a user performs, not a single headless, promptless path the agent runs itself end-to-end. Community commentary even notes the install mechanism is 'curl|bash but with added LLM agents,' underscoring it's not a clean self-serve agent install. No evidence shows an agent autonomously invoking these install commands without human initiation.", "evidenceIds": [ "superpowers-gh-5", "superpowers-docs-36", "superpowers-docs-7", "superpowers-comm-4" ] }, { "productId": "superpowers", "storyId": "agentic-agent-docs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Superpowers ships extensive agent-oriented documentation (SKILL.md files as harness-agnostic 'source of truth', support for AGENTS.md/CLAUDE.md/GEMINI.md conventions, and a cross-runtime `~/.agents/skills/` alias), which is the spirit of pointing an agent at agent-native docs. However, a direct probe for an actual llms.txt on the project itself returned 404, and the only llms.txt success in evidence is GitHub's own generic file, not one authored by Superpowers. Missing for 10: an official llms.txt endpoint for the Superpowers project/docs, and confirmation that agents are explicitly pointed at it as an entry format.", "evidenceIds": [ "superpowers-docs-13", "superpowers-docs-31", "superpowers-docs-32", "superpowers-probe-2", "superpowers-probe-1" ] }, { "productId": "superpowers", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "superpowers", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Superpowers enables an agent to work autonomously for extended periods once dispatched—\"work autonomously for a couple hours at a time without deviating from the plan\" via subagent-driven-development with review loops (gh-6, gh-10, docs-11)—which is real unsupervised agentic execution. But this is autonomy within an active coding session triggered by a user prompt, not a background/scheduled automation that runs independent of an open session (e.g., cron-like triggers, headless unattended jobs). Missing for 10: evidence of scheduling/triggering mechanisms, headless/background execution outside an active session, or automation independent of user-initiated sessions.", "evidenceIds": [ "superpowers-gh-6", "superpowers-gh-10", "superpowers-docs-11", "superpowers-docs-6" ] }, { "productId": "superpowers", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Superpowers' core workflow explicitly delegates engineering tasks to subagents that implement, review, and iterate autonomously for hours (superpowers-gh-3, superpowers-gh-6, superpowers-gh-10, superpowers-docs-11, superpowers-docs-60), which matches the 'delegate to a built-in AI assistant' story. However, community reports note the self-review subagent step was recently folded back into the main agent (superpowers-comm-3) and mixed real-world effectiveness (superpowers-comm-2), and the 'assistant' is not a standalone built-in AI but relies on an external host agent (Claude Code, Devin, etc.) to run. Missing for 10: independent verification that delegated subagent work is reliably higher quality, and evidence of a fully self-contained assistant not dependent on a third-party host CLI.", "evidenceIds": [ "superpowers-gh-3", "superpowers-gh-6", "superpowers-gh-10", "superpowers-docs-11", "superpowers-docs-60", "superpowers-comm-2", "superpowers-comm-3" ] }, { "productId": "superpowers", "storyId": "agentic-headless", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Superpowers is described as a Claude-Code-style plugin/skills system that repeatedly pauses for human approval (brainstorming step asks the user what they're trying to do, 'every path still stops for your approval before implementation', worktree conflicts ask rather than force) — this is an interactive workflow, and no evidence pack item mentions a headless mode, CI flag, non-interactive invocation, or automation pipeline usage.", "evidenceIds": [ "superpowers-docs-1", "superpowers-docs-56", "superpowers-docs-9", "superpowers-docs-4" ] }, { "productId": "superpowers", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Superpowers is a skills/plugin framework layered on top of existing coding agents (Claude Code, Devin, Codex, etc.), not itself an agent host that consumes external MCP servers as its own tool-use mechanism. The evidence describes it porting 'skills' to different agent harnesses, not plugging MCP servers into Superpowers itself, so this axis is a category mismatch for the product's role.", "evidenceIds": [ "superpowers-docs-13", "superpowers-docs-59", "superpowers-gh-5" ] }, { "productId": "superpowers", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Superpowers is a skills/plugin framework installed into various agent harnesses (Claude Code, Devin, Codex, Gemini CLI, etc.), not an agent itself, so an official MCP server is a plausible axis for this kind of product—but no evidence anywhere in the pack mentions Superpowers exposing an MCP server for agents to connect to; installation is via harness-specific plugin mechanisms, not MCP.", "evidenceIds": [] }, { "productId": "superpowers", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Superpowers operates entirely via natural-language interaction and slash-commands (/brainstorm, /execute-plan) layered on top of chat-based agents, with users directing tasks conversationally and the system interpreting intent, clarifying goals, and driving implementation without requiring code-level commands. Community evidence corroborates real-world natural-language-driven workflows (brainstorm/spec/TDD loop), though some users find the resulting plans/specs not always intuitive to read. missing for 10: independent third-party benchmarking of NL command robustness across edge cases, and no evidence of non-command free-form conversational control beyond the documented skill/slash-command triggers.", "evidenceIds": [ "superpowers-gh-1", "superpowers-docs-9", "superpowers-docs-18", "superpowers-docs-20", "superpowers-comm-1", "superpowers-comm-6" ] }, { "productId": "superpowers", "storyId": "agentic-official-cli", "verdict": "na", "quality": 0, "confidence": "medium", "rationale": "Superpowers is a skills/plugin package that installs into and runs through other agents' native CLIs (Claude Code, Codex, Gemini CLI, Devin CLI, etc.) rather than shipping its own standalone CLI product; slash commands like /brainstorm and /execute-plan are Claude Code plugin commands, not an independent 'Superpowers CLI'. Asking whether this plugin framework ships its own official CLI is a category mismatch given its explicit design as a cross-harness plugin/skills layer.", "evidenceIds": [ "superpowers-gh-5", "superpowers-docs-13", "superpowers-docs-18", "superpowers-docs-59", "superpowers-docs-48" ] }, { "productId": "superpowers", "storyId": "agentic-public-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Superpowers ships as skills/plugins consumed via harness-specific install commands and Claude Code slash commands (/brainstorm, /execute-plan) rather than a documented public API (REST, SDK, etc.) that an external AI agent could call to drive the product programmatically; no such API is described anywhere in the evidence.", "evidenceIds": [] }, { "productId": "superpowers", "storyId": "agentic-scoped-keys", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Superpowers is a Claude Code skills/plugin framework for agent workflows (planning, TDD, subagent orchestration); it has no concept of issuing or managing API credentials/IAM scoping for agents. This is a wrong-axis question for this product category.", "evidenceIds": [] }, { "productId": "superpowers", "storyId": "agentic-sdks", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Superpowers is a skills library rather than an API/service product, but its 'porting-to-a-new-harness' docs, harness-agnostic skills format, and tool-mapping/bootstrap injector amount to a quasi-SDK for extending the system to new agent runtimes; there's also a testing/eval harness for building compliant skills. Missing for 10: no formally branded 'SDK', no language-specific client libraries, no versioned API reference, and no independent developer accounts of building against it as an SDK.", "evidenceIds": [ "superpowers-docs-13", "superpowers-docs-28", "superpowers-docs-37", "superpowers-docs-48", "superpowers-docs-14" ] }, { "productId": "superpowers", "storyId": "agentic-webhooks", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Superpowers is a Claude Code skills/agent framework for coding workflows, not a service exposing webhook subscriptions for external event notifications; the evidence pack contains nothing about webhooks and the axis is a category mismatch for this kind of product.", "evidenceIds": [] }, { "productId": "superpowers", "storyId": "api-interactive-docs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Superpowers is an agent skills/plugin framework for coding workflows, not an API product with a reference or runnable examples explorer; this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "superpowers", "storyId": "api-machine-spec", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Superpowers is a Claude Code skills/plugin framework, not an API service; there is no evidence of any API surface that would warrant an OpenAPI spec, making this axis a category error for this product type.", "evidenceIds": [] }, { "productId": "superpowers", "storyId": "api-sandbox", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Superpowers isolates agent work via git worktrees so parallel tasks 'don't clobber each other' and blocks destructive actions like force-removing trees with uncommitted work, which functions as a code-level sandbox, but there is no evidence of a dedicated staging/test-data environment or safeguards against touching production databases/services. Missing for 10: explicit production-data isolation, staging/test environment support, and evidence of data-level (not just git-worktree) sandboxing.", "evidenceIds": [ "superpowers-docs-10", "superpowers-docs-45", "superpowers-docs-4", "superpowers-docs-41", "superpowers-docs-17" ] }, { "productId": "superpowers", "storyId": "api-versioning-policy", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Superpowers is a skills/plugin framework for coding agents, not a hosted API product; the evidence pack shows no versioned API surface, so a deprecation-policy story is a category error rather than an unmet capability.", "evidenceIds": [] }, { "productId": "superpowers", "storyId": "author-from-template", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "The product ships a dedicated 'writing-skills' SKILL.md that documents a TDD-like process for authoring skills (write test cases, watch fail, write skill doc, watch pass, refactor) and there's a 'skill authoring' capability mentioned in the plugin docs, implying some documented authoring path exists. However, no evidence explicitly shows a SKILL.md template with name/description frontmatter fields, nor confirms a developer can author without referencing/reverse-engineering existing skills — missing for 10: an explicit template/schema example with frontmatter fields, and confirmation the process doesn't require studying existing skill files.", "evidenceIds": [ "superpowers-docs-15", "superpowers-docs-50", "superpowers-docs-31" ] }, { "productId": "superpowers", "storyId": "authoring-meta-skill", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "The repo ships a dedicated `writing-skills` SKILL.md meta-skill that walks through writing test cases (pressure scenarios), watching them fail, authoring the skill doc, verifying compliance, and refactoring to close loopholes — directly matching the write/improve/package-a-skill workflow — and docs confirm 'Claude learns... skill authoring through Superpowers' plus cross-runtime packaging notes (harness-agnostic skills, `~/.agents/skills/` alias). Missing for 10: independent/community hands-on validation specifically of the skill-authoring meta-skill (community evidence only discusses the brainstorm/plan/TDD workflow, not skill-creation itself), and no concrete example of an end-to-end 'packaged' new skill produced via this meta-skill.", "evidenceIds": [ "superpowers-docs-15", "superpowers-docs-31", "superpowers-docs-50", "superpowers-docs-13", "superpowers-docs-20" ] }, { "productId": "superpowers", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Release notes describe a specific batching capability where 'small same-shape tasks batch into one dispatch, cutting subagent cost sharply on micro-task plans,' with batch reviews verifying every file in the brief made it into the diff — this is a real bulk-operation mechanism for micro-tasks in a plan. However, this is narrow (limited to same-shape subagent task batching within SDD plans) rather than general-purpose bulk operations across arbitrary items, and comes only from first-party release notes with no independent or hands-on corroboration. Missing for 10: evidence of bulk operations beyond same-shape micro-tasks (e.g., bulk file edits, bulk API calls across many independent items), and community/hands-on validation of the batching feature's reliability.", "evidenceIds": [ "superpowers-docs-3", "superpowers-docs-23", "superpowers-docs-57" ] }, { "productId": "superpowers", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Superpowers skills auto-trigger on contextual events (session start, failed fix attempts, git conflicts, worktree issues) and users can author new skills that encode conditional trigger/action rules, but this is a semantic skill-matching system rather than a general-purpose 'if event X then action Y' rule engine that AI-native users can freely define for arbitrary events. missing for 10: an explicit user-facing rule/trigger definition interface (vs. embedding logic in skill docs), support for arbitrary custom events beyond built-in dev workflow triggers, and independent confirmation that user-authored triggers reliably fire.", "evidenceIds": [ "superpowers-docs-7", "superpowers-docs-19", "superpowers-docs-4", "superpowers-docs-2", "superpowers-gh-11", "superpowers-docs-16", "superpowers-docs-15" ] }, { "productId": "superpowers", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Superpowers is a skills/agent-workflow framework for coding tasks (brainstorming, plan, TDD, subagent review), not a scheduling or automation-orchestration tool; there is no concept of recurring jobs or cron-like workflows in the evidence, and this is a category mismatch rather than a missing feature.", "evidenceIds": [] }, { "productId": "superpowers", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Superpowers uses git worktrees, PR/merge workflows, and review-fix loops with a code-reviewer agent that checks implementations against plans, and states 'git history is the durable record' after workspace cleanup — giving some version/review/rollback capability for code changes it makes. However, this is about versioning the code it produces, not about versioning, reviewing, or rolling back the automations/skills/workflows themselves, and no dedicated rollback or automation-versioning mechanism is documented. Missing for 10: explicit versioning/rollback of the skills/automations themselves (not just git history of generated code), a rollback command or mechanism, and independent evidence that review/rollback works reliably in practice.", "evidenceIds": [ "superpowers-docs-47", "superpowers-docs-12", "superpowers-docs-60", "superpowers-docs-6", "superpowers-docs-4" ] }, { "productId": "superpowers", "storyId": "browse-searchable-catalog", "verdict": "partial", "quality": 6, "confidence": "low", "rationale": "The project ships an 'obra/superpowers-marketplace' repo listing 20+ skills plus a 'Skills-search tool for discovery' and a `/plugin marketplace add` command, giving developers a way to see/search available skills before installing individual ones. However, there's no evidence of a rich browsable UI, ratings, leaderboard, or independent confirmation that search works well pre-install — it's mostly first-party doc mentions. Missing for 10: independent/hands-on confirmation of the search tool, a UI or leaderboard-style catalog, and community commentary on discoverability.", "evidenceIds": [ "superpowers-docs-20", "superpowers-docs-40", "superpowers-docs-51" ] }, { "productId": "superpowers", "storyId": "clean-uninstall", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers installation across many harnesses (plugin marketplace, git clone, devin/hermes/grok CLIs) but there is no mention of a command or mechanism to list installed skills or cleanly uninstall/remove them without leftover files or instructions. missing for 10: a list-installed-skills command, an uninstall/remove workflow, and confirmation that removal leaves no orphaned files or lingering context injections.", "evidenceIds": [ "superpowers-gh-5", "superpowers-docs-48", "superpowers-docs-59" ] }, { "productId": "superpowers", "storyId": "documented-update-path", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "There is a documented update path — for the Antigravity harness, the docs explicitly state 'Reinstall with the same command to update,' and the marketplace install command (`/plugin marketplace add`/`/plugin install`) plus an actively maintained RELEASE-NOTES.md show a real versioning/update cadence. However, this is not a one-click marketplace auto-update: install/update differs by harness (separate reinstall per harness), and there's no first-party 'update' subcommand shown for most integrations. Missing for 10: an explicit auto-update mechanism or dedicated update command (vs. reinstall) that works uniformly across harnesses, and independent confirmation that reinstalling actually preserves user config/state.", "evidenceIds": [ "superpowers-gh-8", "superpowers-gh-5", "superpowers-docs-51", "superpowers-docs-36", "superpowers-docs-55" ] }, { "productId": "superpowers", "storyId": "five-minute-quickstart", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show simple one-line install commands across multiple harnesses (e.g. `/plugin marketplace add obra/superpowers-marketplace` + `/plugin install superpowers@superpowers-marketplace`, `devin plugins install obra/superpowers`, git-clone for Hermes) and skills auto-trigger at session start without extra developer action, suggesting a fast path to a working skill. However, no first-party quickstart doc explicitly times the process, and a community comment calls the codex/opencode install mechanism 'interesting... curl|bash but with added LLM agents,' hinting at friction not captured in the marketing copy. missing for 10: an explicit timed quickstart walkthrough, and independent hands-on confirmation of sub-5-minute setup across harnesses.", "evidenceIds": [ "superpowers-docs-36", "superpowers-docs-51", "superpowers-docs-7", "superpowers-docs-8", "superpowers-gh-11", "superpowers-comm-4" ] }, { "productId": "superpowers", "storyId": "inspect-before-install", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Skills are shipped as plain-text SKILL.md files in the public GitHub repo ('skills/' is the source of truth, shared verbatim) and a skills-search tool exists for discovery, so a lead can in principle browse/clone the repo and read every file before and after install. However, there is no dedicated 'list contents before install' review command, and community commentary notes the install mechanism for some harnesses is 'like curl|bash but with added LLM agents,' which cuts against an explicit, structured audit workflow. Missing for 10: a built-in pre-install manifest/listing command, documented post-install file-audit tooling, and independent confirmation that installers don't execute anything before a human can inspect files.", "evidenceIds": [ "superpowers-docs-13", "superpowers-docs-20", "superpowers-docs-40", "superpowers-comm-4", "superpowers-docs-48" ] }, { "productId": "superpowers", "storyId": "multi-harness-support", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Superpowers documents skills as harness-agnostic source of truth (docs-13) with explicit per-harness install/porting instructions for Claude Code, Codex, Devin CLI, Hermes Agent, Grok Build CLI, Cursor-class tools, Gemini CLI, Copilot CLI, and Antigravity (docs-7,8,31,54,gh-8), plus a porting guide (docs-37,59) and a testing harness running real tmux sessions across Claude Code/Codex/Gemini CLI (docs-14). It explicitly notes installation differs by harness and must be done separately for each (gh-5), matching the story's 'per-harness instructions' framing. missing for 10: no independent hands-on confirmation that identical behavior/parity is achieved across all these harnesses (only vendor docs and release notes), and community evidence focuses mainly on Claude Code with only a passing, slightly negative mention of the Codex/OpenCode install mechanism (comm-4).", "evidenceIds": [ "superpowers-docs-13", "superpowers-docs-7", "superpowers-docs-8", "superpowers-docs-31", "superpowers-docs-54", "superpowers-gh-8", "superpowers-gh-5", "superpowers-docs-37", "superpowers-docs-59", "superpowers-docs-14", "superpowers-comm-4" ] }, { "productId": "superpowers", "storyId": "one-command-install", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Superpowers documents a clean one-liner install for Claude Code ('/plugin marketplace add obra/superpowers-marketplace' + '/plugin install superpowers@superpowers-marketplace') and equivalent one-liners for other harnesses (e.g. 'devin plugins install obra/superpowers'), with explicit claims that skills auto-trigger at session start via a SessionStart hook so the collection is active on the very next session without further setup. This is corroborated across multiple harness release notes (Devin, Hermes, Antigravity) and a first-party blog walkthrough. Missing for 10: independent hands-on confirmation that the one-liner installs cleanly on a fresh machine (community notes call the install mechanism for some harnesses 'interesting... like curl|bash but with added LLM agents' and note it must be repeated per-harness), and no non-vendor benchmark of session-start activation timing.", "evidenceIds": [ "superpowers-docs-36", "superpowers-docs-51", "superpowers-docs-7", "superpowers-docs-8", "superpowers-gh-8", "superpowers-gh-11", "superpowers-gh-5", "superpowers-comm-4" ] }, { "productId": "superpowers", "storyId": "open-spec-compliance", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs show skills are harness-agnostic — 'Everything in skills/ is the source of truth, shared verbatim by every harness' (docs-13), porting only adds a bootstrap/tool-mapping layer and 'never reaches into skills/*/SKILL.md to swap tool names' (docs-37), and the same skill folder is confirmed working across Claude Code, Devin, Hermes, Codex, Gemini CLI, Copilot CLI and Grok Build CLI via a cross-runtime `~/.agents/skills/` alias (docs-31, docs-52-54). This directly supports the claim that the skill folder is valid beyond one vendor's tooling. Missing for 10: an explicit citation naming the 'Agent Skills' open specification itself, and independent (non-vendor) confirmation that the folder format is spec-conformant rather than just multi-harness-compatible.", "evidenceIds": [ "superpowers-docs-13", "superpowers-docs-31", "superpowers-docs-37", "superpowers-docs-48", "superpowers-docs-52", "superpowers-docs-53", "superpowers-docs-54", "superpowers-gh-5" ] }, { "productId": "superpowers", "storyId": "openness-api-parity", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "superpowers", "storyId": "openness-full-export", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Superpowers is a skills/plugin framework for coding agents, not a data-storage or SaaS product that holds user data to export; there's no concept of exportable user data or lock-in to leave from. This axis is a category error for this kind of product.", "evidenceIds": [] }, { "productId": "superpowers", "storyId": "openness-open-license", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "The product is hosted as a public GitHub repo with source files (skills/*.md, docs/, RELEASE-NOTES.md) directly viewable and clonable, implying open readability, but no evidence pack item cites an explicit OSS license (e.g., MIT/Apache) governing the code. missing for 10: explicit license file/declaration, independent confirmation of license terms, any mention of licensing terms at all.", "evidenceIds": [ "superpowers-gh-5", "superpowers-docs-13", "superpowers-docs-59", "superpowers-probe-1" ] }, { "productId": "superpowers", "storyId": "openness-self-host", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Superpowers is fully open-source (GitHub repo) and its skills are installed and run entirely locally within whatever agent harness you already run (Claude Code, Devin, Hermes, Codex, etc.) via git clone or plugin install, meaning there's no cloud service to depend on — evidence of harness-agnostic 'skills/' as source of truth and multi-harness install instructions supports this. However, there's no explicit documentation framing this as 'self-hosting' or describing any server/infra component to host, since it's not a hosted service at all. Missing for 10: explicit self-hosting/deployment guide, any discussion of infra/server components, independent confirmation of running fully offline/air-gapped.", "evidenceIds": [ "superpowers-docs-13", "superpowers-gh-5", "superpowers-docs-48", "superpowers-docs-31" ] }, { "productId": "superpowers", "storyId": "per-skill-documentation", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Each skill ships as a SKILL.md with explicit activation rules (e.g. 'Invoke relevant or requested skills BEFORE any response or action', 'NO PRODUCTION CODE WITHOUT A FAILING TEST FIRST'), a documented precedence order (user instructions > skills > defaults), and an eval harness that tests whether agents actually comply with each skill's documented behavior, which together let a developer predict activation and behavior. However, community reports note skill behavior has changed between updates in ways not obviously reflected to users in the moment (e.g. self-review subagents silently removed), and one experienced user found the plan/implementation docs didn't match what they expected in practice, showing documentation-to-behavior fidelity isn't perfect. Missing for 10: an independent audit confirming every one of the 20+ skills documents activation triggers consistently, and stronger evidence that doc updates keep pace with behavior changes.", "evidenceIds": [ "superpowers-docs-16", "superpowers-docs-17", "superpowers-docs-32", "superpowers-docs-15", "superpowers-docs-30", "superpowers-comm-3", "superpowers-comm-6" ] }, { "productId": "superpowers", "storyId": "plain-files-portability", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly confirm skills are harness-agnostic markdown source-of-truth files (skills/*/SKILL.md) shared verbatim across harnesses, with per-harness install adapters that never modify the skill files themselves, and community/vendor evidence of usage across Claude Code, Codex, Gemini CLI, Devin, Hermes, Grok Build, etc. missing for 10: independent hands-on confirmation of copying skill folders manually into a new harness without the official installer, and explicit statement that no binary/compiled artifacts are involved.", "evidenceIds": [ "superpowers-docs-13", "superpowers-docs-31", "superpowers-docs-37", "superpowers-docs-48", "superpowers-docs-59", "superpowers-gh-5" ] }, { "productId": "superpowers", "storyId": "privacy-data-residency", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Superpowers is a Claude Code skills/plugin framework for agentic dev workflows, not a data storage or hosting service; data residency/region selection is not an applicable axis for this product type.", "evidenceIds": [] }, { "productId": "superpowers", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Superpowers is a Claude Code skills/plugin framework for coding workflows (brainstorming, TDD, subagent orchestration); it has no data-collection or AI-training pipeline of its own, so an AI-training opt-out privacy control is a category error for this product type.", "evidenceIds": [] }, { "productId": "superpowers", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "superpowers", "storyId": "privacy-telemetry-optout", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Superpowers is a skills/plugin framework layered on coding agent harnesses, not a telemetry-collecting SaaS product; there is no evidence it collects usage/telemetry data at all, so an opt-out control is not a meaningful axis for this kind of open-source skill library.", "evidenceIds": [] }, { "productId": "superpowers", "storyId": "project-vs-global-install", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence describes installation via harness-specific plugin managers (Claude Code marketplace, Devin CLI, Hermes) and a global `~/.agents/skills/` alias, but nowhere mentions an explicit choice between project-local (repo-committed) and user-global install scopes as a deliberate feature.", "evidenceIds": [ "superpowers-docs-31", "superpowers-gh-5", "superpowers-docs-36" ] }, { "productId": "superpowers", "storyId": "release-notes", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "There is a maintained RELEASE-NOTES.md with per-release entries describing specific changes (worktree removal safety, workspace scoping, batching, TDD doc rewrites) plus new harness support additions like Devin/Hermes/Grok CLI, giving developers real changelog visibility before updating. missing for 10: no independent/community corroboration that release notes are consulted or trusted pre-update, and no explicit versioning scheme (semver/version tags) tying notes to installable releases.", "evidenceIds": [ "superpowers-docs-1", "superpowers-docs-2", "superpowers-docs-3", "superpowers-docs-4", "superpowers-docs-5", "superpowers-docs-6", "superpowers-docs-7", "superpowers-docs-8", "superpowers-docs-44", "superpowers-docs-54" ] }, { "productId": "superpowers", "storyId": "reliable-auto-triggering", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Multiple first-party docs describe automatic, harness-agnostic skill triggering ('skills trigger automatically... you don't need to do anything special', 'Invoke relevant or requested skills BEFORE any response or action', session-start bootstrap injection across Claude Code, Devin CLI, Hermes, Antigravity) and a testing/eval harness that specifically judges skill-compliance triggering. Community discussion confirms the core brainstorm→plan→TDD workflow functions but doesn't contest trigger timing specifically. Missing for 10: independent hands-on verification that skill descriptions reliably select the *correct* skill at the *right* moment across diverse tasks (community comments focus on workflow quality/spec-plan friction rather than mis-triggering), and confirmation this holds uniformly across all listed harnesses rather than mainly Claude Code.", "evidenceIds": [ "superpowers-gh-11", "superpowers-docs-16", "superpowers-docs-7", "superpowers-docs-8", "superpowers-docs-28", "superpowers-gh-8", "superpowers-docs-30", "superpowers-comm-1", "superpowers-comm-3" ] }, { "productId": "superpowers", "storyId": "security-posture-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item documents a security posture statement, execution/permission model for skills, a trust model for vetting third-party skills (e.g., marketplace skills), or any telemetry/data-collection disclosure. Items like 'never edit the user's files' or worktree safety are incidental engineering safeguards, not a documented security posture for engineering-lead review.", "evidenceIds": [] }, { "productId": "superpowers", "storyId": "selective-install", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All installation evidence describes installing the entire Superpowers plugin/skill bundle as one unit (e.g., '/plugin install superpowers@superpowers-marketplace', 'devin plugins install obra/superpowers', 'install from a git clone'), with no mechanism shown for selecting individual skills to install rather than the whole collection.", "evidenceIds": [ "superpowers-docs-36", "superpowers-docs-7", "superpowers-docs-8", "superpowers-gh-5", "superpowers-docs-51" ] }, { "productId": "superpowers", "storyId": "skill-testing-harness", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "The repo's testing docs describe a two-tier verification system: `tests/` for non-LLM code (bash/node/python integration tests) and `evals/` — a Python harness driving real tmux sessions across multiple agent CLIs with an LLM actor and verifier judging skill compliance — plus a documented TDD-style process for writing skills themselves (write pressure-scenario tests, watch them fail, write the skill, watch them pass, refactor to close loopholes). This is concrete, specific first-party evidence that skills aren't shipped on vibes but are eval-gated. Missing for 10: independent/community confirmation that the eval suite actually catches regressions in practice, and quantitative pass-rate or CI-run evidence beyond the docs themselves.", "evidenceIds": [ "superpowers-docs-14", "superpowers-docs-15", "superpowers-docs-29", "superpowers-docs-30", "superpowers-docs-38", "superpowers-docs-49" ] }, { "productId": "superpowers", "storyId": "team-distribution", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Superpowers ships as skills files (harness-agnostic, source of truth in `skills/`) that can be committed to a project, cloned from a git repo, or installed via the official `obra/superpowers-marketplace` plugin marketplace with a single `/plugin marketplace add` command, supporting many agent harnesses (Claude Code, Devin, Hermes, Codex, Gemini CLI, etc.), which is exactly the distribution story an engineering lead needs. Community evidence (HN) confirms real-world install/adoption across a team, though it also notes the install flow feels 'like curl|bash' — a minor rough edge. Missing for 10: no explicit team-permission/ACL or org-wide governance controls, and no first-party guidance on version-pinning a shared skill set across a team.", "evidenceIds": [ "superpowers-docs-36", "superpowers-docs-51", "superpowers-docs-13", "superpowers-docs-31", "superpowers-gh-5", "superpowers-comm-4", "superpowers-docs-7", "superpowers-docs-8" ] }, { "productId": "superpowers", "storyId": "version-control-pinning", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes install/update mechanisms (git clone, plugin marketplace, harness-specific installers, manual reinstall to update) but nowhere mentions version pinning, a lockfile, or any mechanism letting a lead control/freeze which skill version reaches the team versus picking up the latest on reinstall or auto-update.", "evidenceIds": [] } ]