EvalTest

import { EvalTest } from "@mcpjam/sdk";

new EvalTest(options: EvalTestConfig)

const test = new EvalTest({
  name: "books the venue",
  expectedToolCalls: [{ toolName: "book_venue" }],
  matchOptions: { toolCallOrder: "strict" },
  predicates: [
    { type: "noToolErrors" },
    { type: "responseContains", needle: "confirmed" },
  ],
  test: async (executor) => {
    const r = await executor.run("Book the venue for Friday.");
    return r.hasToolCall("book_venue");
  },
});

type TestFunction = (executor: HostExecutor) => boolean | Promise<boolean>;

const test = new EvalTest({
  name: "addition-accuracy",
  test: async (agent) => {
    const result = await agent.run("Add 2 and 3");
    return result.hasToolCall("add");
  },
});

run(executor: HostExecutor, options: EvalTestRunOptions): Promise<EvalRunResult>

type ProgressCallback = (completed: number, total: number) => void;

await test.run(agent, {
  iterations: 30,
  concurrency: 5,
  retries: 2,
  timeoutMs: 30000,
  mcpjam: {
    suiteName: "Addition Eval",
    strict: false,
  },
  onProgress: (done, total) => {
    console.log(`${done}/${total}`);
  },
  onFailure: (report) => {
    console.error(report);
  },
});

accuracy(): number

console.log(`Accuracy: ${(test.accuracy() * 100).toFixed(1)}%`);
// "Accuracy: 96.7%"

precision(): number

recall(): number

truePositiveRate(): number

unexpectedToolCallRate(): number

falsePositiveRate(): number

averageTokenUse(): number

console.log(`Avg tokens: ${test.averageTokenUse()}`);

getResults(): EvalRunResult | null

getName(): string

getConfig(): EvalTestConfig

getAllIterations(): IterationResult[]

getFailedIterations(): IterationResult[]

const failures = test.getFailedIterations();
console.log(`${failures.length} failures`);

for (const fail of failures) {
  console.log(`  Error: ${fail.error}`);
}

getSuccessfulIterations(): IterationResult[]

getFailureReport(): string

await test.run(agent, { iterations: 30 });

if (test.accuracy() < 0.9) {
  console.error(test.getFailureReport());
}

test.getName(); // "addition-accuracy"

test: async (agent) => {
  const result = await agent.run("Add 5 and 3");
  return result.hasToolCall("add");
};

test: async (agent) => {
  const result = await agent.run("Add 10 and 20");
  const args = result.getToolArguments("add");
  return args?.a === 10 && args?.b === 20;
};

test: async (agent) => {
  const result = await agent.run("What is 5 + 5?");
  return result.getText().includes("10");
};

test: async (agent) => {
  const result = await agent.run("Calculate 5 * 3");
  return (
    result.hasToolCall("multiply") &&
    !result.hasError() &&
    result.getText().length > 0
  );
};

test: async (agent) => {
  const r1 = await agent.run("Create a project");
  const r2 = await agent.run("Add a task to it", { context: r1 });
  return r1.hasToolCall("createProject") && r2.hasToolCall("createTask");
};

import { matchToolCallWithArgs } from "@mcpjam/sdk";

test: async (agent) => {
  const result = await agent.run("Add 2 and 3");
  return matchToolCallWithArgs("add", { a: 2, b: 3 }, result.getToolCalls());
};

import { MCPClientManager, HostRunner, EvalTest } from "@mcpjam/sdk";

async function main() {
  const manager = new MCPClientManager({
    everything: {
      command: "npx",
      args: ["-y", "@modelcontextprotocol/server-everything"],
    },
  });
  await manager.connectToServer("everything");

  const agent = new HostRunner({
    tools: await manager.getTools(),
    model: "anthropic/claude-sonnet-5",
    apiKey: process.env.ANTHROPIC_API_KEY,
    temperature: 0.1,
  });

  const test = new EvalTest({
    name: "addition",
    test: async (agent) => {
      const r = await agent.run("Add 2 and 3");
      return r.hasToolCall("add");
    },
  });

  console.log("Running evaluation...");

  const result = await test.run(agent, {
    iterations: 30,
    concurrency: 5,
    mcpjam: { suiteName: "Addition Eval" },
    onProgress: (done, total) => {
      process.stdout.write(`\r${done}/${total}`);
    },
    onFailure: (report) => {
      console.error(report);
    },
  });

  console.log("\n\nResults:");
  console.log(`  Accuracy: ${(test.accuracy() * 100).toFixed(1)}%`);
  console.log(`  Precision: ${(test.precision() * 100).toFixed(1)}%`);
  console.log(`  Recall: ${(test.recall() * 100).toFixed(1)}%`);
  console.log(`  Avg tokens: ${test.averageTokenUse()}`);
  console.log(`  Iterations: ${result.iterations}`);
  console.log(`  Successes: ${result.successes}`);

  await manager.disconnectServer("everything");
}