Evaluator Benchmark Pack (evaluator_benchmarker/base.py)

Benchmarks a custom evaluator against labelled datasets, producing predictions and performance metrics.

# Source: https://github.com/mperkins0155/llama-hub | License: MIT | Copyright Jerry Liu
# Provided via justclaw.it.com; keep this notice with the code.
from typing import Union
from llama_index.llama_pack.base import BaseLlamaPack
from llama_index.llama_dataset.evaluator_evaluation import (
    LabelledEvaluatorDataset,
    LabelledPairwiseEvaluatorDataset,
    PairwiseEvaluatorPredictionDataset,
    EvaluatorPredictionDataset,
)
from llama_index.evaluation import BaseEvaluator
import warnings
import pandas as pd
import numpy as np


class EvaluatorBenchmarkerPack(BaseLlamaPack):
    """A pack for benchmarking/evaluating your own evaluator

    Args:
        evaluator (BaseEvaluator): The evaluator to evaluate/benchmark.
        eval_dataset (LabelledEvaluatorDataset | LabelledPairwiseEva

... (truncated -- full source via MCP)