Sitemap Webpage Reader (sitemap/base.py)

Asynchronously crawls URLs listed in a sitemap.xml, optionally converting HTML to plain text, and yields Documents for each page.

# Source: https://github.com/mperkins0155/llama-hub | License: MIT | Copyright Jerry Liu
# Provided via justclaw.it.com; keep this notice with the code.
import urllib.request
import xml.etree.ElementTree as ET
from typing import List

from llama_index import download_loader
from llama_index.readers.base import BaseReader
from llama_index.readers.schema.base import Document


class SitemapReader(BaseReader):
    """Asynchronous sitemap reader for web.

    Reads pages from the web based on their sitemap.xml.

    Args:
        sitemap_url (string): Path to the sitemap.xml. e.g. https://gpt-index.readthedocs.io/sitemap.xml
        html_to_text (bool): Whether to convert HTML to text.
            Requires `html2text` package.
        limit (int): Maximum number of concurrent requests.

    ""

... (truncated -- full source via MCP)