mild_strip.py

Cleans HTML by removing comments, scripts, styles, and other non-content tags while trimming attribute values to 100 characters for web scraping processing.

# Source: https://github.com/nerdy-tech-com-gitub/dendrite-python-sdk | License: MIT | Copyright 2024 Dendrite Systems
# Provided via justclaw.it.com; keep this notice with the code.
from bs4 import BeautifulSoup, Doctype, Tag, Comment


def mild_strip(soup: Tag, keep_d_id: bool = True) -> BeautifulSoup:
    new_soup = BeautifulSoup(str(soup), "html.parser")
    _mild_strip(new_soup, keep_d_id)
    return new_soup


def mild_strip_in_place(soup: BeautifulSoup, keep_d_id: bool = True) -> None:
    _mild_strip(soup, keep_d_id)


def _mild_strip(soup: BeautifulSoup, keep_d_id: bool = True) -> None:
    for element in soup(text=lambda text: isinstance(text, Comment)):
        element.extract()

    # for text in soup.find_all(text=lambda text: isinstance(text, NavigableString)):
    #     if l

... (truncated -- full source via MCP)