GitHub Tree Document Loader (blobs/tree-loader.ts)

Creates a DocumentLoader that walks a GitHub commit tree, filters blobs by size, and loads each file as a rag Document for downstream processing.

// Source: https://github.com/mperkins0155/giselle | License: Apache-2.0 | Copyright 2024 ROUTE06, Inc
// Provided via justclaw.it.com; keep this notice with the code.
import {
	type Document,
	type DocumentLoader,
	DocumentLoaderError,
	replaceNullCharacters,
} from "@giselles-ai/rag";
import type { Octokit } from "@octokit/core";
import { executeRestRequest } from "../utils";

type GitHubBlobMetadata = {
	owner: string;
	repo: string;
	fileSha: string;
	path: string;
};

type GitHubTreeLoaderParams = {
	owner: string;
	repo: string;
	commitSha: string;
};

type GitHubTreeLoaderOptions = {
	maxBlobSize?: number;
};

export function createGitHubTreeLoader(
	octokit: Octokit,
	params: GitHubTreeLoaderParams,
	options: GitHubTreeLoaderOptions = {},
): DocumentLoader<GitHubBlobMetadata> {
	co

... (truncated -- full source via MCP)