Async Crawler Strategy Base (crawl4ai/async_crawler_strategy.py)

crawl4ai-cobaltai · scraping, Scraping / Crawling, async, crawler, playwright, http, strategy

Abstract base and concrete async crawling strategies using Playwright or raw HTTP for retrieving web content.

# Source: https://github.com/mperkins0155/crawl4ai-cobaltai | License: Apache-2.0
# Provided via justclaw.it.com; keep this notice with the code.
from __future__ import annotations

import asyncio
import base64
import time
from abc import ABC, abstractmethod
from typing import Callable, Dict, Any, List, Union
from typing import Optional, AsyncGenerator, Final
import os
from playwright.async_api import Page, Error
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
from io import BytesIO
from PIL import Image, ImageDraw, ImageFont
import hashlib
import random
import uuid
from .js_snippet import load_js_script
from .models import AsyncCrawlResponse
from .config import SCREENSHOT_HEIGHT_TRESHOLD
from .async_configs import BrowserConfig, CrawlerRunConfig, HTTPCrawlerConfig


... (truncated -- full source via MCP)

See the full source, get the GitHub permalink, and search 40K more like it.

Get a free API key