Crawl4AI Core Utilities (crawl4ai/utils.py)

Provides helper classes and functions for version management, robots parsing, HTML sanitization, chunking, concurrency, and system resource queries used across the library.

# Source: https://github.com/mperkins0155/crawl4ai-cobaltai | License: Apache-2.0
# Provided via justclaw.it.com; keep this notice with the code.
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from bs4 import BeautifulSoup, Comment, element, Tag, NavigableString
import json
import html
import lxml
import re
import os
import subprocess
import platform
from .prompts import PROMPT_EXTRACT_BLOCKS
from array import array
from .html2text import html2text, CustomHTML2Text
# from .config import *
from .config import MIN_WORD_THRESHOLD, IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD, IMAGE_SCORE_THRESHOLD, DEFAULT_PROVIDER, PROVIDER_MODELS
import httpx
from socket import gaierror
from pathlib import Path
from typing import Dict, Any, List, Optional, Callable, Generator, Tuple, It

... (truncated -- full source via MCP)