Remote Depth Crawler (remote_depth/base.py)

Fetches a remote page or file and recursively follows links up to a configurable depth, applying optional file extractors to each resource.

# Source: https://github.com/mperkins0155/llama-hub | License: MIT | Copyright Jerry Liu
# Provided via justclaw.it.com; keep this notice with the code.
"""Remote file reader.

A loader that fetches any remote page or file by URL and retrieves child pages with certain constraints. The class also parses the contents of each page and provides access to the parsed data.
"""
from typing import Any, Dict, List, Optional, Union

import requests
from llama_index import download_loader
from llama_index.readers.base import BaseReader
from llama_index.readers.schema.base import Document


class RemoteDepthReader(BaseReader):
    def __init__(
        self,
        *args: Any,
        file_extractor: Optional[Dict[str, Union[str, BaseReader]]] = None,
        depth: int = 1,
        domain_lock: bool

... (truncated -- full source via MCP)