#!/usr/bin/env node
/**
 * Time-to-interactive (TTI) benchmark for a StateSet Sandbox API.
 *
 * TTI = from the moment the client sends the create request to the moment it
 * holds the first command's output (the ComputeSDK methodology: create(),
 * then run one command, timed end to end from the client). Each run:
 *
 *   t0 -> POST /sandbox/create -> 201          (create_ms)
 *      -> POST /sandbox/:id/execute "echo"     (exec_ms, output verified)
 *   tti_ms = output received - t0
 *   then POST /sandbox/:id/stop (not timed into TTI)
 *
 * Both responses carry a Server-Timing header (controller >= v2.7.x): the
 * server's own phases. The report puts every run's client time next to the
 * server's phases, and the difference (network, TLS, ingress, routing, auth
 * middleware) as `client_overhead`, so every millisecond is attributed.
 *
 * Usage:
 *   STATESET_API_KEY=sk_... node scripts/bench-tti.mjs \
 *     --url https://api.example.test --runs 50 [--delay-ms 3000] \
 *     [--cpus 1 --memory 1Gi] [--warmup 1] [--out results.json] [--format md|json]
 *
 * Prints a markdown report (default, ready for docs/BENCHMARKS.md) or the JSON
 * document (--format json) to stdout; --out always writes the JSON document.
 * Never run it against an environment you do not own: every run creates and
 * bills a sandbox.
 */
import { writeFileSync } from 'node:fs';
import { pathToFileURL } from 'node:url';
import { performance } from 'node:perf_hooks';

export function parseArgs(argv, env = process.env) {
  const opts = {
    url: env.STATESET_API_URL || 'http://localhost:8080',
    apiKey: env.STATESET_API_KEY || '',
    apiPath: '/api/v1',
    runs: 50,
    warmup: 0,
    delayMs: 2000,
    cpus: undefined,
    memory: undefined,
    command: 'echo tti-ok',
    expect: 'tti-ok',
    timeoutMs: 120_000,
    out: undefined,
    format: 'md',
  };
  for (let i = 0; i < argv.length; i++) {
    const arg = argv[i];
    const next = () => {
      const value = argv[++i];
      if (value === undefined) throw new Error(`${arg} needs a value`);
      return value;
    };
    switch (arg) {
      case '--url': opts.url = next(); break;
      case '--api-key': opts.apiKey = next(); break;
      case '--api-path': opts.apiPath = next(); break;
      case '--runs': opts.runs = Number(next()); break;
      case '--warmup': opts.warmup = Number(next()); break;
      case '--delay-ms': opts.delayMs = Number(next()); break;
      case '--cpus': opts.cpus = next(); break;
      case '--memory': opts.memory = next(); break;
      case '--command': opts.command = next(); break;
      case '--expect': opts.expect = next(); break;
      case '--timeout-ms': opts.timeoutMs = Number(next()); break;
      case '--out': opts.out = next(); break;
      case '--format': opts.format = next(); break;
      case '-h': case '--help': opts.help = true; break;
      default: throw new Error(`Unknown argument: ${arg}`);
    }
  }
  if (!opts.help) {
    if (!opts.apiKey) throw new Error('An API key is required (STATESET_API_KEY or --api-key)');
    if (!Number.isSafeInteger(opts.runs) || opts.runs < 1) throw new Error('--runs must be a positive integer');
    if (!Number.isSafeInteger(opts.warmup) || opts.warmup < 0) throw new Error('--warmup must be >= 0');
    if (!Number.isSafeInteger(opts.delayMs) || opts.delayMs < 0 || opts.delayMs > 2_147_483_647) throw new Error('--delay-ms must be an integer in 0..2147483647');
    if (!Number.isSafeInteger(opts.timeoutMs) || opts.timeoutMs < 1 || opts.timeoutMs > 4_294_967_295) throw new Error('--timeout-ms must be an integer in 1..4294967295');
    if (opts.format !== 'md' && opts.format !== 'json') throw new Error('--format must be md or json');
    const target = new URL(opts.url);
    if (!['http:', 'https:'].includes(target.protocol) || target.username || target.password || target.search || target.hash) throw new Error('--url must be HTTP(S), without credentials, query or fragment');
  }
  opts.url = opts.url.replace(/\/+$/, '');
  return opts;
}

/** Nearest-rank percentile (p in 0..100) of a numeric array; null when empty. */
export function percentile(values, p) {
  if (values.length === 0) return null;
  const sorted = [...values].sort((a, b) => a - b);
  const rank = Math.ceil((p / 100) * sorted.length);
  return sorted[Math.min(sorted.length - 1, Math.max(0, rank - 1))];
}

export function summarize(values) {
  const finite = values.filter((v) => typeof v === 'number' && Number.isFinite(v));
  return {
    n: finite.length,
    p50: percentile(finite, 50),
    p90: percentile(finite, 90),
    p95: percentile(finite, 95),
    p99: percentile(finite, 99),
    min: finite.length ? Math.min(...finite) : null,
    max: finite.length ? Math.max(...finite) : null,
  };
}

/** Parse a Server-Timing header into { name: durationMs }; entries without dur are skipped. */
export function parseServerTiming(header) {
  const out = {};
  if (!header) return out;
  for (const entry of String(header).split(',')) {
    const [rawName, ...params] = entry.trim().split(';');
    const name = rawName?.trim();
    if (!name) continue;
    for (const param of params) {
      const [key, value] = param.trim().split('=');
      if (key?.trim() === 'dur' && value !== undefined && Number.isFinite(Number(value))) {
        out[name] = Number(value);
      }
    }
  }
  return out;
}

/** A create served from the warm pool reports a warm_pool_claim phase. */
export function isWarm(sandbox) {
  const phases = sandbox?.startup_metrics?.phases;
  return Array.isArray(phases) && phases.some((phase) => phase?.name === 'warm_pool_claim');
}

async function request(opts, method, path, body) {
  const res = await fetch(`${opts.url}${opts.apiPath}${path}`, {
    method,
    headers: {
      'Content-Type': 'application/json',
      Authorization: `ApiKey ${opts.apiKey}`,
    },
    body: body === undefined ? undefined : JSON.stringify(body),
    signal: AbortSignal.timeout(opts.timeoutMs),
  });
  const text = await res.text();
  let json;
  try { json = text ? JSON.parse(text) : undefined; } catch { json = undefined; }
  return { status: res.status, json, text, serverTiming: parseServerTiming(res.headers.get('server-timing')) };
}

/** Client time minus the server's own total: network, TLS, ingress, middleware before the handler. */
function overhead(clientMs, serverTiming) {
  return typeof serverTiming?.total === 'number' ? Math.max(0, Math.round(clientMs - serverTiming.total)) : null;
}

export async function runOnce(opts) {
  const createBody = { timeout_seconds: 300 };
  if (opts.cpus) createBody.cpus = opts.cpus;
  if (opts.memory) createBody.memory = opts.memory;

  const t0 = performance.now();
  let created;
  try {
    created = await request(opts, 'POST', '/sandbox/create', createBody);
  } catch (error) {
    return { status: 'error', failure_stage: 'create', error: String(error), cleanup_error: 'Create outcome unknown; cleanup unconfirmed (300s TTL requested)', create_ms: Math.round(performance.now() - t0) };
  }
  const tCreated = performance.now();
  if (created.status === 202) {
    return { status: 'error', failure_stage: 'create', error: 'queued (202): concurrency limit reached', cleanup_error: 'Queued creation requires reconciliation', create_ms: Math.round(tCreated - t0) };
  }
  if (created.status !== 201 && created.status !== 200) {
    return { status: 'error', failure_stage: 'create', error: `create ${created.status}: ${created.text.slice(0, 200)}`, create_ms: Math.round(tCreated - t0) };
  }
  const sandbox = created.json ?? {};
  const sandboxId = sandbox.sandbox_id ?? sandbox.id;
  if (typeof sandboxId !== 'string' || !/^[a-zA-Z0-9-]+$/.test(sandboxId)) {
    return { status: 'error', failure_stage: 'create', error: 'create response has no valid sandbox ID', cleanup_error: 'Created sandbox cannot be identified; cleanup unconfirmed (300s TTL requested)', create_ms: Math.round(tCreated - t0) };
  }
  const run = {
    status: 'ok',
    sandbox_id: sandboxId,
    warm_pool: Array.isArray(sandbox.startup_metrics?.phases) && sandbox.startup_metrics.phases.length
      ? isWarm(sandbox) : null,
    create_ms: Math.round(tCreated - t0),
    server_create_ms: sandbox.startup_metrics?.total_ms ?? null,
    server_phases: sandbox.startup_metrics?.phases ?? [],
    create_timing: created.serverTiming,
    create_overhead_ms: overhead(tCreated - t0, created.serverTiming),
  };
  try {
    const executed = await request(opts, 'POST', `/sandbox/${sandboxId}/execute`, {
      command: ['sh', '-c', opts.command],
    });
    const tOutput = performance.now();
    run.exec_ms = Math.round(tOutput - tCreated);
    run.tti_ms = Math.round(tOutput - t0);
    run.exec_timing = executed.serverTiming;
    run.exec_overhead_ms = overhead(tOutput - tCreated, executed.serverTiming);
    const stdout = executed.json?.stdout;
    if (executed.status !== 200 || executed.json?.exit_code !== 0 || typeof stdout !== 'string'
        || stdout.replace(/\r\n/g, '\n').trimEnd() !== opts.expect.replace(/\r\n/g, '\n').trimEnd()
        || executed.json?.output_truncated === true) {
      run.status = 'error';
      run.failure_stage = 'execute';
      run.error = `execute ${executed.status}: ${executed.text.slice(0, 200)}`;
    }
  } catch (error) {
    run.status = 'error';
    run.failure_stage = 'execute';
    run.error = String(error);
    run.exec_ms = Math.round(performance.now() - tCreated);
  } finally {
    const tStop = performance.now();
    const stopped = await request(opts, 'POST', `/sandbox/${sandboxId}/stop`).catch((error) => ({ status: 0, text: String(error) }));
    run.cleanup_ms = Math.round(performance.now() - tStop);
    if (stopped.status >= 300 || stopped.status === 0) {
      run.cleanup_error = `stop ${stopped.status}`;
      if (run.status === 'ok') {
        run.status = 'error';
        run.failure_stage = 'cleanup';
        run.error = run.cleanup_error;
      }
    }
  }
  return run;
}

/** Per-phase summaries of a set of runs' Server-Timing maps, in first-seen order. */
function phaseSummaries(runs, key) {
  const values = new Map();
  for (const run of runs) {
    for (const [name, ms] of Object.entries(run[key] ?? {})) {
      if (!values.has(name)) values.set(name, []);
      values.get(name).push(ms);
    }
  }
  return Object.fromEntries([...values].map(([name, list]) => [name, summarize(list)]));
}

export function buildReport(opts, runs, startedAt, warmups = []) {
  const ok = runs.filter((run) => run.status === 'ok');
  const warm = ok.filter((run) => run.warm_pool === true);
  const cold = ok.filter((run) => run.warm_pool === false);
  const unknown = ok.filter((run) => run.warm_pool !== true && run.warm_pool !== false);
  const phaseMedians = {};
  for (const run of warm) {
    for (const phase of run.server_phases ?? []) {
      (phaseMedians[phase.name] ??= []).push(phase.durationMs);
    }
  }
  const attribution = (set) => ({
    create: phaseSummaries(set, 'create_timing'),
    create_client_overhead: summarize(set.map((r) => r.create_overhead_ms)),
    exec: phaseSummaries(set, 'exec_timing'),
    exec_client_overhead: summarize(set.map((r) => r.exec_overhead_ms)),
  });
  return {
    tool: 'bench-tti',
    methodology: 'client-observed: create request sent -> first command output received; sequential runs',
    url: opts.url,
    command: opts.command,
    configuration: {
      cpus: opts.cpus ?? null, memory: opts.memory ?? null,
      expected_stdout: opts.expect, timeout_ms: opts.timeoutMs, delay_ms: opts.delayMs,
      api_path: opts.apiPath, node: process.version, platform: process.platform, arch: process.arch,
    },
    started_at: startedAt,
    runs_requested: opts.runs,
    warmup_runs: warmups.length,
    warmup_runs_requested: opts.warmup,
    success: `${ok.length}/${runs.length}`,
    failures: runs.length - ok.length,
    cleanup_failures: [...warmups, ...runs].filter(run => run.cleanup_error).length,
    cleanup_ms: summarize(runs.map(run => run.cleanup_ms)),
    warmups,
    warmup_failures: warmups.filter(run => run.status !== 'ok').length,
    aborted_for_cleanup_failure: [...warmups, ...runs].some(run => run.cleanup_error),
    warm_pool_hits: warm.length,
    warm_pool_unknown: unknown.length,
    tti_ms: { all: summarize(ok.map((r) => r.tti_ms)), warm: summarize(warm.map((r) => r.tti_ms)), cold: summarize(cold.map((r) => r.tti_ms)), unknown: summarize(unknown.map((r) => r.tti_ms)) },
    create_ms: { warm: summarize(warm.map((r) => r.create_ms)), cold: summarize(cold.map((r) => r.create_ms)) },
    exec_ms: { warm: summarize(warm.map((r) => r.exec_ms)), cold: summarize(cold.map((r) => r.exec_ms)) },
    server_warm_phase_p50_ms: Object.fromEntries(
      Object.entries(phaseMedians).map(([name, values]) => [name, percentile(values, 50)])
    ),
    server_timing: { warm: attribution(warm), cold: attribution(cold) },
    runs: runs.map(({ server_phases: _phases, ...rest }) => rest),
  };
}

const fmt = (v) => (v === null || v === undefined ? '-' : String(Math.round(v * 10) / 10));
const row = (label, s) => `| ${label} | ${s.n} | ${fmt(s.p50)} | ${fmt(s.p90)} | ${fmt(s.p99)} | ${fmt(s.min)} | ${fmt(s.max)} |`;
const HEADER = ['| | n | p50 | p90 | p99 | min | max |', '|---|---|---|---|---|---|---|'];

/** The report as markdown tables (ms), ready to paste into docs/BENCHMARKS.md. */
export function renderMarkdown(report) {
  const lines = [
    `### TTI: create -> first \`${report.command}\` output (${report.started_at.slice(0, 10)})`,
    '',
    `Target \`${report.url}\`, ${report.success} runs succeeded, ${report.warm_pool_hits} served warm, ${report.warm_pool_unknown} with unknown pool origin, sequential, client-observed (ms).`,
    `Failures: ${report.failures}; cleanup failures: ${report.cleanup_failures}; warmup failures: ${report.warmup_failures}. Failed runs are retained in JSON and excluded from latency percentiles.`,
    '',
    ...HEADER,
    row('**TTI, warm**', report.tti_ms.warm),
    row('TTI, cold', report.tti_ms.cold),
    row('TTI, pool origin unknown', report.tti_ms.unknown),
    row('TTI, all', report.tti_ms.all),
    row('create (client), warm', report.create_ms.warm),
    row('first exec (client), warm', report.exec_ms.warm),
  ];
  for (const kind of ['warm', 'cold']) {
    const a = report.server_timing[kind];
    const names = [...Object.keys(a.create), ...Object.keys(a.exec)];
    if (names.length === 0) continue;
    lines.push('', `#### Where the time goes (${kind} runs, Server-Timing)`, '', ...HEADER);
    for (const [name, s] of Object.entries(a.create)) lines.push(row(`create: ${name}`, s));
    lines.push(row('create: client overhead (network, ingress, middleware)', a.create_client_overhead));
    for (const [name, s] of Object.entries(a.exec)) lines.push(row(`exec: ${name}`, s));
    lines.push(row('exec: client overhead (network, ingress, middleware)', a.exec_client_overhead));
  }
  return `${lines.join('\n')}\n`;
}

export async function main(argv = process.argv.slice(2)) {
  const opts = parseArgs(argv);
  if (opts.help) {
    process.stdout.write('Usage: STATESET_API_KEY=... node scripts/bench-tti.mjs --url <api> [--runs N] [--warmup N] [--delay-ms MS] [--cpus C --memory M] [--out file.json] [--format md|json]\n');
    return null;
  }
  const startedAt = new Date().toISOString();
  const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
  const warmups = [];
  for (let i = 0; i < opts.warmup; i++) {
    warmups.push(await runOnce(opts).catch(error => ({ status: 'error', error: String(error) })));
    if (warmups.at(-1).cleanup_error) break;
    await sleep(opts.delayMs);
  }
  const runs = [];
  for (let i = 0; i < opts.runs && !warmups.some(run => run.cleanup_error); i++) {
    try {
      runs.push(await runOnce(opts));
    } catch (error) {
      runs.push({ status: 'error', error: error instanceof Error ? error.message : String(error) });
    }
    process.stderr.write(`run ${i + 1}/${opts.runs}: ${JSON.stringify({ status: runs[i].status, warm: runs[i].warm_pool, tti_ms: runs[i].tti_ms })}\n`);
    if (runs[i].cleanup_error) break;
    // Lets the warm pool replenish, so warm and cold runs are not an artefact of pace.
    if (i + 1 < opts.runs) await sleep(opts.delayMs);
  }
  const report = buildReport(opts, runs, startedAt, warmups);
  const json = JSON.stringify(report, null, 2);
  if (opts.out) writeFileSync(opts.out, `${json}\n`);
  process.stdout.write(opts.format === 'json' ? `${json}\n` : renderMarkdown(report));
  return report;
}

if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
  main().then(report => {
    if (report && (report.failures || report.warmup_failures)) process.exitCode = 1;
  }).catch((error) => {
    process.stderr.write(`bench-tti: ${error instanceof Error ? error.message : String(error)}\n`);
    process.exit(1);
  });
}
