Async URL Seeder (crawl4ai/async_url_seeder.py)

High‑performance asynchronous URL discovery using Common Crawl, robots.txt, and sitemap streaming with disk caching and global rate limiting.

# Source: https://github.com/mperkins0155/crawl4ai-cobaltai | License: Apache-2.0
# Provided via justclaw.it.com; keep this notice with the code.
"""
async_url_seeder.py
Fast async URL discovery for Crawl4AI

Features
--------
* Common-Crawl streaming via httpx.AsyncClient (HTTP/2, keep-alive)
* robots.txt → sitemap chain (.gz + nested indexes) via async httpx
* Per-domain CDX result cache on disk (~/.crawl4ai/<index>_<domain>_<hash>.jsonl)
* Optional HEAD-only liveness check
* Optional partial <head> download + meta parsing
* Global hits-per-second rate-limit via asyncio.Semaphore
* Concurrency in the thousands — fine on a single event-loop
"""

from __future__ import annotations
import aiofiles
import asyncio
import gzip
import hashlib
import io
import json
import os
import pathlib
impor

... (truncated -- full source via MCP)