Camel.benchmarks.gaia

​

RetrieverProtocol

class RetrieverProtocol(Protocol):

​

retrieve

def retrieve(
    self,
    query: str,
    contents: List[str],
    **kwargs: Dict[str, Any]
):

• query (str): The query to retrieve the content for.
• contents (List[str]): The list of contents to search in. **kwargs (Dict[str, Any]): Additional keyword arguments.

​

reset

def reset(self, **kwargs):

​

DefaultGAIARetriever

class DefaultGAIARetriever(AutoRetriever):

​

retrieve

def retrieve(
    self,
    query: str,
    contents: List[str],
    **kwargs: Any
):

• query (str): The query to search for.
• contents (List[str]): The list of contents to search from. **kwargs (Any): The keyword arguments to pass to the retriever.

​

reset

def reset(self, **kwargs: Any):

​

GAIABenchmark

class GAIABenchmark(BaseBenchmark):

• data_dir (str): The directory to save the data.
• save_to (str): The file to save the results.
• retriever (Optional[RetrieverProtocol]): The retriever to use. (default: :obj:None)
• processes (int, optional): The number of processes to use. (default: :obj:1)

​

init

def __init__(
    self,
    data_dir: str,
    save_to: str,
    retriever: Optional[RetrieverProtocol] = None,
    processes: int = 1
):

• data_dir (str): The directory to save the data.
• save_to (str): The file to save the results.
• retriever (Optional[RetrieverProtocol], optional): The retriever to use. (default: :obj:None)
• processes (int, optional): The number of processes to use for parallel processing. (default: :obj:1)

​

download

def download(self):

​

load

def load(self, force_download = False):

• force_download (bool, optional): Whether to force download the data.

​

train

def train(self):

​

run

def run(
    self,
    agent: ChatAgent,
    on: Literal['train', 'valid', 'test'],
    level: Union[int, List[int], Literal['all']],
    randomize: bool = False,
    subset: Optional[int] = None
):

• agent (ChatAgent): The agent to run the benchmark.
• on (Literal["valid", "test"]): The set to run the benchmark.
• level (Union[int, List[int], Literal["all"]]): The level to run the benchmark.
• randomize (bool, optional): Whether to randomize the data. (default: :obj:False)
• subset (Optional[int], optional): The subset of data to run. (default: :obj:None)

​

_prepare_task

def _prepare_task(self, task: Dict[str, Any]):

​

_create_user_message

def _create_user_message(self, task: Dict[str, Any]):

​

_process_result

def _process_result(
    self,
    agent: ChatAgent,
    task: Dict[str, Any],
    result: Any,
    file_obj: Any
):

​

_handle_error

def _handle_error(
    self,
    task: Dict[str, Any],
    error: Exception,
    file_obj: Any
):

​

_generate_summary

def _generate_summary(self):

​

question_scorer

def question_scorer(self, model_answer: str, ground_truth: str):

• model_answer (str): The model answer.
• ground_truth (str): The ground truth answer.

​

normalize_number_str

def normalize_number_str(self, number_str: str):

​

split_string

def split_string(self, s: str, char_list: Optional[List[str]] = None):

• s (str): The string to split.
• char_list (Optional[List[str]], optional): T he list of characters to split on. (default: :obj:None)

​

normalize_str

def normalize_str(self, input_str, remove_punct = True):

• input_str: The input string to normalize.
• remove_punct: Whether to remove punctuation.

​

get_final_answer

def get_final_answer(self, content: str):

• content (str): The content to extract the final answer from.