ExtractionStrategy (extraction_strategy.py)

Defines extraction strategies: LLM, cosine similarity, topic modeling, CSS selectors for crawling content.

# Source: https://github.com/nerdy-tech-com-gitub/crawl4ai | License: Apache-2.0
# Provided via justclaw.it.com; keep this notice with the code.
from abc import ABC, abstractmethod
from typing import Any, List, Dict, Optional, Union
from concurrent.futures import ThreadPoolExecutor, as_completed
import json, time
# from optimum.intel import IPEXModel
from .prompts import *
from .config import *
from .utils import *
from functools import partial
from .model_loader import *
import math
import numpy as np
from lxml import etree

class ExtractionStrategy(ABC):
    """
    Abstract base class for all extraction strategies.
    """
    
    def __init__(self, **kwargs):
        self.DEL = "<|DEL|>"
        self.name = self.__class__.__name__
        self.verbose = kwargs.get("verbose", False)

 

... (truncated -- full source via MCP)