Async Crawler Configuration (crawl4ai/async_configs.py)

crawl4ai-cobaltai · scraping, Scraping / Crawling, python, config, async, crawling, proxy

Typed configuration objects for browser, proxy, geolocation, deep crawling, and LLM settings with serialization helpers for async crawling.

# Source: https://github.com/mperkins0155/crawl4ai-cobaltai | License: Apache-2.0
# Provided via justclaw.it.com; keep this notice with the code.
import copy
import functools
import importlib
import os
import warnings
import requests
from .config import (
    DEFAULT_PROVIDER,
    DEFAULT_PROVIDER_API_KEY,
    MIN_WORD_THRESHOLD,
    IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD,
    PROVIDER_MODELS,
    PROVIDER_MODELS_PREFIXES,
    SCREENSHOT_HEIGHT_TRESHOLD,
    PAGE_TIMEOUT,
    IMAGE_SCORE_THRESHOLD,
    SOCIAL_MEDIA_DOMAINS,
)

from .user_agent_generator import UAGen, ValidUAGenerator  # , OnlineUAGenerator
from .extraction_strategy import ExtractionStrategy, LLMExtractionStrategy
from .chunking_strategy import ChunkingStrategy, RegexChunking

from .markdown_generation_strategy import Markdow

... (truncated -- full source via MCP)

See the full source, get the GitHub permalink, and search 40K more like it.

Get a free API key