Typed configuration objects for browser, proxy, geolocation, deep crawling, and LLM settings with serialization helpers for async crawling.
# Source: https://github.com/mperkins0155/crawl4ai-cobaltai | License: Apache-2.0
# Provided via justclaw.it.com; keep this notice with the code.
import copy
import functools
import importlib
import os
import warnings
import requests
from .config import (
DEFAULT_PROVIDER,
DEFAULT_PROVIDER_API_KEY,
MIN_WORD_THRESHOLD,
IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD,
PROVIDER_MODELS,
PROVIDER_MODELS_PREFIXES,
SCREENSHOT_HEIGHT_TRESHOLD,
PAGE_TIMEOUT,
IMAGE_SCORE_THRESHOLD,
SOCIAL_MEDIA_DOMAINS,
)
from .user_agent_generator import UAGen, ValidUAGenerator # , OnlineUAGenerator
from .extraction_strategy import ExtractionStrategy, LLMExtractionStrategy
from .chunking_strategy import ChunkingStrategy, RegexChunking
from .markdown_generation_strategy import Markdow
... (truncated -- full source via MCP)
See the full source, get the GitHub permalink, and search 40K more like it.
Get a free API key