Camel.benchmarks.apibank

​

process_messages

def process_messages(chat_history: List[Dict[str, Any]], prompt: str):

• chat_history (List[Dict[str, Any]): A list of dictionaries representing the chat history.
• prompt (str): A prompt to be set as the system message.

• ‘role’: The role of the message (‘system’, ‘user’, or ‘assistant’).
• ‘content’: The content of the message, including formatted API responses when applicable.

​

APIBankBenchmark

class APIBankBenchmark(BaseBenchmark):

• save_to (str): The file to save the results.
• processes (int, optional): The number of processes to use. (default: :obj:1)

​

init

def __init__(self, save_to: str, processes: int = 1):

• save_to (str): The file to save the results.
• processes (int, optional): The number of processes to use for parallel processing. (default: :obj:1)

​

download

def download(self):

​

load

def load(self, level: str, force_download: bool = False):

• level (str): Level to run benchmark on.
• force_download (bool, optional): Whether to force download the data.

​

run

def run(
    self,
    agent: ChatAgent,
    level: Literal['level-1', 'level-2'],
    api_test_enabled = True,
    randomize: bool = False,
    subset: Optional[int] = None
):

• agent (ChatAgent): The agent to run the benchmark.
• level (Literal['level-1', 'level-2']): The level to run the benchmark on.
• randomize (bool, optional): Whether to randomize the data.
• api_test_enabled (bool): Whether to test API calling (True) or response (False) (default: :obj:False)
• subset (Optional[int], optional): The subset of data to run. (default: :obj:None)

​

agent_call

def agent_call(messages: List[Dict], agent: ChatAgent):

​

calculate_rouge_l_score

def calculate_rouge_l_score(reference, hypothesis):

​

get_api_call

def get_api_call(model_output):

​

APIBankSample

class APIBankSample:

​

init

def __init__(
    self,
    chat_history,
    apis,
    ground_truth
):

​

repr

def __repr__(self):

​

from_chat_history

def from_chat_history(cls, chat_history):

​

Evaluator

class Evaluator:

​

init

def __init__(self, samples: List[APIBankSample]):

​

get_all_sample_ids

def get_all_sample_ids(self):

​

get_api_description

def get_api_description(self, api_name):

​

get_model_input

def get_model_input(self, sample_id: int):

​

evaluate

def evaluate(self, sample_id, model_output):