[
  {
    "id": "crawl4ai-docs-1",
    "tier": "claimed-docs",
    "url": "https://docs.crawl4ai.com",
    "excerpt": "async with AsyncWebCrawler() as crawler: result = await crawler.arun(url=\"https://crawl4ai.com\") print(result.markdown)",
    "fetchedAt": "2026-08-28T01:16:28.120Z"
  },
  {
    "id": "crawl4ai-docs-2",
    "tier": "claimed-docs",
    "url": "https://docs.crawl4ai.com",
    "excerpt": "Structured Extraction: Parse repeated patterns with CSS, XPath, or LLM-based extraction.",
    "fetchedAt": "2026-08-28T01:16:28.120Z"
  },
  {
    "id": "crawl4ai-gh-1",
    "tier": "github",
    "url": "https://github.com/unclecode/crawl4ai",
    "excerpt": "LLM-Driven Extraction: Supports all LLMs (open-source and proprietary) for structured data extraction.",
    "fetchedAt": "2026-08-28T01:16:28.120Z"
  },
  {
    "id": "crawl4ai-gh-2",
    "tier": "github",
    "url": "https://github.com/unclecode/crawl4ai",
    "excerpt": "crwl https://docs.crawl4ai.com --deep-crawl bfs --max-pages 10",
    "fetchedAt": "2026-08-28T01:16:28.120Z"
  },
  {
    "id": "crawl4ai-docs-3",
    "tier": "claimed-docs",
    "url": "https://docs.crawl4ai.com",
    "excerpt": "Crawl4AI now features intelligent adaptive crawling that knows when to stop! Using advanced information foraging algorithms, it determines when sufficient information has been gathered to answer your query.",
    "fetchedAt": "2026-08-28T01:16:28.120Z"
  },
  {
    "id": "crawl4ai-gh-3",
    "tier": "github",
    "url": "https://github.com/unclecode/crawl4ai",
    "excerpt": "Browser Profiler: Create and manage persistent profiles with saved authentication states, cookies, and settings.",
    "fetchedAt": "2026-08-28T01:16:28.120Z"
  },
  {
    "id": "crawl4ai-gh-4",
    "tier": "github",
    "url": "https://github.com/unclecode/crawl4ai",
    "excerpt": "Automatic retry with proxy chain and fallback fetch function",
    "fetchedAt": "2026-08-28T01:16:28.120Z"
  },
  {
    "id": "crawl4ai-gh-5",
    "tier": "github",
    "url": "https://github.com/unclecode/crawl4ai",
    "excerpt": "Dockerized Setup: Optimized Docker image with FastAPI server for easy deployment.",
    "fetchedAt": "2026-08-28T01:16:28.120Z"
  },
  {
    "id": "crawl4ai-gh-6",
    "tier": "github",
    "url": "https://github.com/unclecode/crawl4ai",
    "excerpt": "Real-time Monitoring Dashboard with live system metrics and browser pool visibility",
    "fetchedAt": "2026-08-28T01:16:28.120Z"
  },
  {
    "id": "crawl4ai-docs-4",
    "tier": "claimed-docs",
    "url": "https://docs.crawl4ai.com",
    "excerpt": "Open Source: No forced API keys, no paywalls—everyone can access their data.",
    "fetchedAt": "2026-08-28T01:16:28.120Z"
  },
  {
    "id": "crawl4ai-gh-7",
    "tier": "github",
    "url": "https://github.com/unclecode/crawl4ai",
    "excerpt": "LLMTableExtraction: Revolutionary table extraction with intelligent chunking for massive tables",
    "fetchedAt": "2026-08-28T01:16:28.120Z"
  },
  {
    "id": "crawl4ai-gh-8",
    "tier": "github",
    "url": "https://github.com/unclecode/crawl4ai",
    "excerpt": "resume_state parameter to continue from a saved checkpoint",
    "fetchedAt": "2026-08-28T01:16:28.120Z"
  },
  {
    "id": "crawl4ai-gh-9",
    "tier": "github",
    "url": "https://github.com/unclecode/crawl4ai",
    "excerpt": "Undetected Browser Support: Bypass sophisticated bot detection systems",
    "fetchedAt": "2026-08-28T01:16:28.120Z"
  },
  {
    "id": "crawl4ai-gh-10",
    "tier": "github",
    "url": "https://github.com/unclecode/crawl4ai",
    "excerpt": "Multi-URL Configuration: Different strategies for different URL patterns in one batch",
    "fetchedAt": "2026-08-28T01:16:28.120Z"
  },
  {
    "id": "crawl4ai-comm-1",
    "tier": "community",
    "url": "https://news.ycombinator.com/item?id=43827993",
    "excerpt": "sweet, i am testing this out",
    "fetchedAt": "2026-08-28T01:20:29.112Z"
  },
  {
    "id": "crawl4ai-comm-2",
    "tier": "community",
    "url": "https://news.ycombinator.com/item?id=47481953",
    "excerpt": "Promising foundation if you're willing to own the policy layer + quality gates.",
    "fetchedAt": "2026-08-28T01:20:29.112Z"
  },
  {
    "id": "crawl4ai-comm-3",
    "tier": "community",
    "url": "https://news.ycombinator.com/item?id=47481953",
    "excerpt": "Worth calling out the boring production bits: robots/ToS, rate limiting, bot mitigation, login/session handling, and not accidentally hoovering up PII.",
    "fetchedAt": "2026-08-28T01:20:29.112Z"
  },
  {
    "id": "crawl4ai-comm-4",
    "tier": "community",
    "url": "https://news.ycombinator.com/item?id=46646798",
    "excerpt": "Crawl4AI is an amazing open-source library that solves many LLM-scraping headaches.",
    "fetchedAt": "2026-08-28T01:20:29.112Z"
  },
  {
    "id": "crawl4ai-comm-5",
    "tier": "community",
    "url": "https://news.ycombinator.com/item?id=46646798",
    "excerpt": "New developers often struggle with production configurations—specifically how to use Crawl4AI with MCP servers for Cursor, or how to bridge it with automation tools like n8n.",
    "fetchedAt": "2026-08-28T01:20:29.112Z"
  },
  {
    "id": "crawl4ai-comm-6",
    "tier": "community",
    "url": "https://news.ycombinator.com/item?id=46646798",
    "excerpt": "Built crawl4ai.dev as a community-driven documentation hub with one-click Docker setups for n8n/FastAPI and production-ready MCP server guides for Cursor & Claude, including cost/performance benchmarks vs proprietary tools like Firecrawl.",
    "fetchedAt": "2026-08-28T01:20:29.112Z"
  },
  {
    "id": "crawl4ai-probe-1",
    "tier": "probe",
    "url": "https://docs.crawl4ai.com/llms.txt",
    "excerpt": "PROBE llms.txt: HTTP 404 at https://docs.crawl4ai.com/llms.txt",
    "fetchedAt": "2026-08-28T21:08:58.791Z"
  },
  {
    "id": "crawl4ai-probe-2",
    "tier": "probe",
    "url": "https://docs.crawl4ai.com/openapi.json",
    "excerpt": "PROBE openapi: all candidate paths 404 (https://docs.crawl4ai.com/openapi.json, https://docs.crawl4ai.com/swagger.json, https://docs.crawl4ai.com/api/openapi.json, https://docs.crawl4ai.com/.well-known/openapi.json)",
    "fetchedAt": "2026-08-28T21:08:58.791Z"
  },
  {
    "id": "crawl4ai-probe-3",
    "tier": "probe",
    "url": "https://docs.crawl4ai.com/core/self-hosting/#mcp-model-context-protocol-support",
    "excerpt": "official MCP server documented at https://docs.crawl4ai.com/core/self-hosting/#mcp-model-context-protocol-support",
    "fetchedAt": "2026-08-28T21:08:58.791Z"
  },
  {
    "id": "crawl4ai-probe-4",
    "tier": "probe",
    "url": "https://docs.crawl4ai.com/core/cli/",
    "excerpt": "official CLI documented at https://docs.crawl4ai.com/core/cli/",
    "fetchedAt": "2026-08-28T21:08:58.791Z"
  }
]