Execution-Based Benchmarks

import { StratScopeBenchmark } from '@stratscope/sdk';

const benchmark = new StratScopeBenchmark({
  apiKey: process.env.STRATSCOPE_API_KEY,
  agentId: 'agent_mycoding_01'
});

// Run SWE-Bench Verified against execution traces
const results = await benchmark.run({
  benchmark: 'swe-bench-verified',
  dataset: 'seea_planning_traces_v11',
  metrics: ['pass_at_1', 'test_coverage', 'code_quality'],
  compareAgainst: ['gpt-4', 'claude-3-opus', 'seea']
});

console.log(results);
/*
{
  agent: 'agent_mycoding_01',
  pass_at_1: 0.73,
  test_coverage: 0.89,
  code_quality: 8.2,
  percentile: 87,
  compared_to: {
    'gpt-4': { pass_at_1: 0.68 },
    'claude-3-opus': { pass_at_1: 0.71 },
    'seea': { pass_at_1: 0.82 }
  }
}
*/