253 lines
10 KiB
TypeScript
253 lines
10 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
import { createSession, runTurn } from "./agent";
|
|
import { KeyedQueue, Limiter, MAX_PARALLEL_TOOLS } from "./tool-runner";
|
|
import type { AgentUI, LLMProvider, StreamEvent, Tool } from "./types";
|
|
|
|
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
|
|
|
function providerCalling(calls: { name: string; input: Record<string, unknown> }[]): LLMProvider {
|
|
let n = 0;
|
|
return {
|
|
id: "fake",
|
|
model: "fake",
|
|
capabilities: { tools: true },
|
|
async *chat(): AsyncIterable<StreamEvent> {
|
|
n++;
|
|
if (n === 1) {
|
|
for (const [i, c] of calls.entries()) yield { type: "tool_use", id: `t${i}`, name: c.name, input: c.input };
|
|
yield { type: "done", stopReason: "tool_use" };
|
|
} else {
|
|
yield { type: "text_delta", text: "ok" };
|
|
yield { type: "done", stopReason: "end_turn" };
|
|
}
|
|
},
|
|
};
|
|
}
|
|
|
|
function setup(tools: Tool[], calls: { name: string; input: Record<string, unknown> }[], ui: Partial<AgentUI> = {}) {
|
|
const events: string[] = [];
|
|
const fullUI: AgentUI = {
|
|
onThinking() {},
|
|
onText() {},
|
|
onTextEnd() {},
|
|
onToolCall: (n, s) => void events.push(`call:${n}:${s}`),
|
|
onToolResult: (n, o, e) => void events.push(`result:${n}:${o}:${e}`),
|
|
confirm: async () => true,
|
|
...ui,
|
|
};
|
|
const session = createSession({ provider: providerCalling(calls), init: { systemPrompt: "s", tools }, ui: fullUI, cwd: process.cwd() });
|
|
return { session, events };
|
|
}
|
|
|
|
const slow = (name: string, ms: number, extra: Partial<Tool> = {}): Tool => ({
|
|
spec: { name, description: name, inputSchema: { type: "object" } },
|
|
permission: "auto",
|
|
describe: (i) => String(i.v ?? ""),
|
|
execute: async (i) => {
|
|
await sleep(ms);
|
|
return `${name}:${String(i.v ?? "")}`;
|
|
},
|
|
...extra,
|
|
});
|
|
|
|
const resultsOf = (s: ReturnType<typeof setup>["session"]) =>
|
|
(s.messages[2]?.content ?? []).map((b) => (b.type === "tool_result" ? b.content : ""));
|
|
|
|
describe("exécution parallèle des outils", () => {
|
|
it("lance les outils en parallèle dès 2 appels et garde l'ordre demandé", async () => {
|
|
const { session } = setup([slow("A", 150), slow("B", 50), slow("C", 100)], [
|
|
{ name: "A", input: { v: 1 } },
|
|
{ name: "B", input: { v: 2 } },
|
|
{ name: "C", input: { v: 3 } },
|
|
]);
|
|
const t0 = Date.now();
|
|
await runTurn(session, "go");
|
|
expect(Date.now() - t0).toBeLessThan(290);
|
|
expect(resultsOf(session)).toEqual(["A:1", "B:2", "C:3"]);
|
|
});
|
|
|
|
it("un seul appel reste au premier plan : annonce, résultat, dans l'ordre", async () => {
|
|
const running: unknown[] = [];
|
|
const { session, events } = setup([slow("A", 5)], [{ name: "A", input: { v: 1 } }], { onToolsRunning: (s) => void running.push(s) });
|
|
await runTurn(session, "go");
|
|
expect(events).toEqual(["call:A:1", "result:A:A:1:false"]);
|
|
expect(running).toEqual([]);
|
|
});
|
|
|
|
it("affiche appels et résultats dans l'ordre demandé et signale l'activité", async () => {
|
|
const running: ({ names: string[]; done: number } | null)[] = [];
|
|
const { session, events } = setup([slow("A", 60), slow("B", 10)], [
|
|
{ name: "A", input: { v: 1 } },
|
|
{ name: "B", input: { v: 2 } },
|
|
], { onToolsRunning: (s) => void running.push(s && { names: [...s.names], done: s.done }) });
|
|
await runTurn(session, "go");
|
|
expect(events).toEqual(["call:A:1", "result:A:A:1:false", "call:B:2", "result:B:B:2:false"]);
|
|
expect(running[0]).toEqual({ names: ["A", "B"], done: 0 });
|
|
expect(running.at(-1)).toBeNull();
|
|
expect(running.some((r) => r?.done === 2)).toBe(true);
|
|
});
|
|
|
|
it("une erreur d'un outil n'annule pas les autres", async () => {
|
|
const bad: Tool = { ...slow("Bad", 5), execute: async () => { throw new Error("boum"); } };
|
|
const { session } = setup([bad, slow("A", 30)], [
|
|
{ name: "Bad", input: {} },
|
|
{ name: "A", input: { v: 1 } },
|
|
]);
|
|
await runTurn(session, "go");
|
|
expect(resultsOf(session)).toEqual(["boum", "A:1"]);
|
|
expect(session.messages[2]?.content[0]).toMatchObject({ isError: true });
|
|
});
|
|
|
|
it("outil inconnu : erreur pour le LLM, les autres s'exécutent", async () => {
|
|
const { session } = setup([slow("A", 5)], [
|
|
{ name: "Nope", input: {} },
|
|
{ name: "A", input: { v: 1 } },
|
|
]);
|
|
await runTurn(session, "go");
|
|
expect(resultsOf(session)).toEqual(["Outil inconnu : Nope", "A:1"]);
|
|
});
|
|
|
|
it("confirmations posées une par une ; un refus ne bloque pas les autres", async () => {
|
|
const asked: string[] = [];
|
|
let concurrent = 0;
|
|
let maxConcurrent = 0;
|
|
const ui = {
|
|
confirm: async (m: string) => {
|
|
asked.push(m);
|
|
concurrent++;
|
|
maxConcurrent = Math.max(maxConcurrent, concurrent);
|
|
await sleep(10);
|
|
concurrent--;
|
|
return !m.includes("B");
|
|
},
|
|
};
|
|
const ask = (n: string): Tool => slow(n, 20, { permission: "ask" });
|
|
const { session, events } = setup([ask("A"), ask("B"), ask("C")], [
|
|
{ name: "A", input: { v: 1 } },
|
|
{ name: "B", input: { v: 2 } },
|
|
{ name: "C", input: { v: 3 } },
|
|
], ui);
|
|
await runTurn(session, "go");
|
|
expect(asked).toHaveLength(3);
|
|
expect(maxConcurrent).toBe(1);
|
|
expect(resultsOf(session)).toEqual(["A:1", "L'utilisateur a refusé cette action.", "C:3"]);
|
|
expect(events).toContain("result:B:Refusé par l'utilisateur:true");
|
|
});
|
|
|
|
it("règle deny : refusé sans exécution", async () => {
|
|
let ran = false;
|
|
const a = slow("A", 5, { execute: async () => { ran = true; return "x"; } });
|
|
const { session } = setup([a, slow("B", 5)], [
|
|
{ name: "A", input: {} },
|
|
{ name: "B", input: { v: 1 } },
|
|
]);
|
|
session.permissions = { check: (name: string) => ({ decision: name === "A" ? "deny" : "allow" }) } as never;
|
|
await runTurn(session, "go");
|
|
expect(ran).toBe(false);
|
|
expect(resultsOf(session)[1]).toBe("B:1");
|
|
expect(resultsOf(session)[0]).toContain("refusée");
|
|
});
|
|
|
|
it("même lockKey : exécution l'une après l'autre dans l'ordre ; clés différentes : en parallèle", async () => {
|
|
const log: string[] = [];
|
|
const mk = (name: string): Tool =>
|
|
slow(name, 0, {
|
|
lockKey: (i) => String(i.k),
|
|
execute: async (i) => {
|
|
log.push(`start${i.v}`);
|
|
await sleep(40);
|
|
log.push(`end${i.v}`);
|
|
return String(i.v);
|
|
},
|
|
});
|
|
const { session } = setup([mk("E")], [
|
|
{ name: "E", input: { k: "f1", v: 1 } },
|
|
{ name: "E", input: { k: "f1", v: 2 } },
|
|
{ name: "E", input: { k: "f2", v: 3 } },
|
|
]);
|
|
await runTurn(session, "go");
|
|
expect(log.indexOf("end1")).toBeLessThan(log.indexOf("start2"));
|
|
expect(log.indexOf("start3")).toBeLessThan(log.indexOf("end1"));
|
|
expect(resultsOf(session)).toEqual(["1", "2", "3"]);
|
|
});
|
|
|
|
it("jamais plus de MAX_PARALLEL_TOOLS outils en même temps", async () => {
|
|
let active = 0;
|
|
let peak = 0;
|
|
const t: Tool = slow("T", 0, {
|
|
execute: async () => {
|
|
peak = Math.max(peak, ++active);
|
|
await sleep(20);
|
|
active--;
|
|
return "ok";
|
|
},
|
|
});
|
|
const { session } = setup([t], Array.from({ length: 20 }, () => ({ name: "T", input: {} })));
|
|
await runTurn(session, "go");
|
|
expect(peak).toBe(MAX_PARALLEL_TOOLS);
|
|
expect(resultsOf(session)).toHaveLength(20);
|
|
});
|
|
});
|
|
|
|
describe("Limiter / KeyedQueue", () => {
|
|
it("Limiter limite la concurrence", async () => {
|
|
const l = new Limiter(2);
|
|
let a = 0;
|
|
let peak = 0;
|
|
await Promise.all(Array.from({ length: 6 }, () => l.run(async () => { peak = Math.max(peak, ++a); await sleep(5); a--; })));
|
|
expect(peak).toBe(2);
|
|
});
|
|
it("KeyedQueue continue après une erreur", async () => {
|
|
const q = new KeyedQueue();
|
|
const first = q.run("k", async () => { throw new Error("x"); });
|
|
const second = q.run("k", async () => "ok");
|
|
await expect(first).rejects.toThrow("x");
|
|
await expect(second).resolves.toBe("ok");
|
|
});
|
|
});
|
|
|
|
describe("réponse « stop » à une demande d'autorisation", () => {
|
|
it("un appel : rien n'est exécuté, le tour s'arrête sans rollback, la saisie suivante complète le contexte", async () => {
|
|
let ran = false;
|
|
const a = slow("A", 5, { permission: "ask", execute: async () => { ran = true; return "x"; } });
|
|
const { session, events } = setup([a], [{ name: "A", input: { v: 1 } }], { ask: async () => "stop" });
|
|
expect(await runTurn(session, "go")).toBe("stopped");
|
|
expect(ran).toBe(false);
|
|
// Contexte conservé : demande, appel d'outil, résultat « stop » (le LLM n'est pas rappelé)
|
|
expect(session.messages.map((m) => m.role)).toEqual(["user", "assistant", "user"]);
|
|
expect(resultsOf(session)[0]).toMatch(/stop/);
|
|
expect(events).toContain("result:A:Arrêt demandé (stop) : action non exécutée:true");
|
|
// Nouvelle saisie : ajoutée au message des résultats (alternance user/assistant respectée), conversation reprise
|
|
expect(await runTurn(session, "voici une info oubliée")).toBe("done");
|
|
expect(session.messages.map((m) => m.role)).toEqual(["user", "assistant", "user", "assistant"]);
|
|
const merged = session.messages[2]!.content;
|
|
expect(merged.map((b) => b.type)).toEqual(["tool_result", "text"]);
|
|
});
|
|
|
|
it("plusieurs appels : les outils déjà autorisés finissent, le « stop » et les suivants ne sont pas lancés", async () => {
|
|
const asked: string[] = [];
|
|
const ask = (n: string): Tool => slow(n, 20, { permission: "ask" });
|
|
const { session } = setup([ask("A"), ask("B"), ask("C")], [
|
|
{ name: "A", input: { v: 1 } },
|
|
{ name: "B", input: { v: 2 } },
|
|
{ name: "C", input: { v: 3 } },
|
|
], {
|
|
ask: async (m: string) => {
|
|
asked.push(m);
|
|
return m.includes("B") ? "stop" : "yes";
|
|
},
|
|
});
|
|
expect(await runTurn(session, "go")).toBe("stopped");
|
|
expect(asked).toHaveLength(2); // C n'est même pas demandé
|
|
const r = resultsOf(session);
|
|
expect(r[0]).toBe("A:1");
|
|
expect(r[1]).toMatch(/« stop »/);
|
|
expect(r[2]).toMatch(/non exécutée/);
|
|
});
|
|
|
|
it("sans méthode ask, la confirmation booléenne sert (oui / non uniquement)", async () => {
|
|
const { session } = setup([slow("A", 5, { permission: "ask" })], [{ name: "A", input: { v: 1 } }], { confirm: async () => false });
|
|
expect(await runTurn(session, "go")).toBe("done");
|
|
expect(resultsOf(session)).toEqual(["L'utilisateur a refusé cette action."]);
|
|
});
|
|
});
|