Camel.benchmarks.base

​

BaseBenchmark

class BaseBenchmark(ABC):

• name (str): Name of the benchmark.
• data_dir (str): Path to the data directory.
• save_to (str): Path to save the results.
• processes (int): Number of processes to use for parallel processing. :(default: :obj:1)

​

init

def __init__(
    self,
    name: str,
    data_dir: str,
    save_to: str,
    processes: int = 1
):

• name (str): Name of the benchmark.
• data_dir (str): Path to the data directory.
• save_to (str): Path to save the results.
• processes (int): Number of processes to use for parallel processing. :(default: :obj:1)

​

download

def download(self):

​

load

def load(self, force_download: bool = False):

• force_download (bool): Whether to force download the data.

​

train

def train(self):

​

valid

def valid(self):

​

test

def test(self):

​

run

def run(
    self,
    agent: ChatAgent,
    on: Literal['train', 'valid', 'test'],
    randomize: bool = False,
    subset: Optional[int] = None,
    *args,
    **kwargs
):

• agent (ChatAgent): The chat agent.
• on (str): The data split to run the benchmark on.
• randomize (bool): Whether to randomize the data.
• subset (int): The subset of the data to run the benchmark on.

​

results

def results(self):