9f3badd79d
- Add HOW_To_Agents.md, generated_page.md, llm_avis.md to .gitignore - Add SPECS_TECH.md technical specifications - Add test files for bash and write tools - Update various core, agent, provider and UI components
269 lines
9.9 KiB
TypeScript
269 lines
9.9 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
import { createSession, runTurn, sessionContextTokens } from "./agent";
|
|
import { countTextTokens } from "./context";
|
|
import type { AgentUI, LLMProvider, PermissionPolicy, StreamEvent, Tool } from "./types";
|
|
|
|
const silentUI: AgentUI = {
|
|
onThinking() {},
|
|
onText() {},
|
|
onTextEnd() {},
|
|
onToolCall() {},
|
|
onToolResult() {},
|
|
confirm: async () => true,
|
|
};
|
|
|
|
/** Provider factice : 1er appel → demande d'outil, 2e appel → réponse texte. */
|
|
function fakeProvider(toolName = "echo"): LLMProvider {
|
|
let call = 0;
|
|
return {
|
|
id: "fake",
|
|
model: "fake",
|
|
capabilities: { tools: true },
|
|
async *chat(): AsyncIterable<StreamEvent> {
|
|
call++;
|
|
if (call === 1) {
|
|
yield { type: "tool_use", id: "t1", name: toolName, input: { value: "salut" } };
|
|
yield { type: "done", stopReason: "tool_use" };
|
|
} else {
|
|
yield { type: "text_delta", text: "terminé" };
|
|
yield { type: "done", stopReason: "end_turn" };
|
|
}
|
|
},
|
|
};
|
|
}
|
|
|
|
describe("runTurn", () => {
|
|
it("exécute l'outil demandé puis renvoie la réponse finale", async () => {
|
|
const echo: Tool = {
|
|
spec: { name: "echo", description: "echo", inputSchema: { type: "object" } },
|
|
permission: "auto",
|
|
execute: async (input) => String(input.value),
|
|
};
|
|
const session = createSession({
|
|
provider: fakeProvider(),
|
|
init: { systemPrompt: "test", tools: [echo] },
|
|
ui: silentUI,
|
|
cwd: process.cwd(),
|
|
});
|
|
|
|
await runTurn(session, "bonjour");
|
|
|
|
// user, assistant(tool_use), user(tool_result), assistant(texte)
|
|
expect(session.messages).toHaveLength(4);
|
|
expect(session.messages[2]?.content[0]).toMatchObject({
|
|
type: "tool_result",
|
|
content: "salut",
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("politique de permissions", () => {
|
|
const makeTool = (ran: string[]): Tool => ({
|
|
spec: { name: "danger", description: "d", inputSchema: { type: "object" } },
|
|
permission: "ask",
|
|
execute: async () => {
|
|
ran.push("x");
|
|
return "ok";
|
|
},
|
|
});
|
|
const policy = (decision: "allow" | "ask" | "deny"): PermissionPolicy => ({
|
|
check: () => ({ decision, reason: "règle de test" }),
|
|
});
|
|
|
|
async function run(decision: "allow" | "ask" | "deny", answer: boolean) {
|
|
const ran: string[] = [];
|
|
const confirms: string[] = [];
|
|
const session = createSession({
|
|
provider: fakeProvider("danger"),
|
|
init: { systemPrompt: "t", tools: [makeTool(ran)], permissions: policy(decision) },
|
|
ui: { ...silentUI, confirm: async (m) => (confirms.push(m), answer) },
|
|
cwd: process.cwd(),
|
|
});
|
|
await runTurn(session, "go");
|
|
return { ran, confirms, session };
|
|
}
|
|
|
|
it("allow : exécute sans confirmation", async () => {
|
|
const { ran, confirms } = await run("allow", false);
|
|
expect(ran).toHaveLength(1);
|
|
expect(confirms).toHaveLength(0);
|
|
});
|
|
|
|
it("deny : refuse sans demander", async () => {
|
|
const { ran, confirms, session } = await run("deny", true);
|
|
expect(ran).toHaveLength(0);
|
|
expect(confirms).toHaveLength(0);
|
|
expect(session.messages[2]?.content[0]).toMatchObject({ type: "tool_result", isError: true });
|
|
});
|
|
|
|
it("ask : demande confirmation même pour un outil automatique", async () => {
|
|
const { ran, confirms } = await run("ask", true);
|
|
expect(confirms).toHaveLength(1);
|
|
expect(ran).toHaveLength(1);
|
|
});
|
|
});
|
|
|
|
describe("contexte en attente", () => {
|
|
it("est joint à la prochaine demande puis vidé", async () => {
|
|
const session = createSession({
|
|
provider: fakeProvider("echo"),
|
|
init: {
|
|
systemPrompt: "t",
|
|
tools: [{ spec: { name: "echo", description: "e", inputSchema: { type: "object" } }, permission: "auto", execute: async () => "ok" }],
|
|
},
|
|
ui: silentUI,
|
|
cwd: process.cwd(),
|
|
});
|
|
session.pendingContext.push("CONTEXTE-A", "CONTEXTE-B");
|
|
|
|
await runTurn(session, "ma demande");
|
|
|
|
const first = session.messages[0]?.content[0];
|
|
expect(first).toMatchObject({ type: "text", text: "CONTEXTE-A\n\nCONTEXTE-B\n\nma demande" });
|
|
expect(session.pendingContext).toEqual([]);
|
|
});
|
|
});
|
|
|
|
describe("taille du contexte", () => {
|
|
const echo: Tool = {
|
|
spec: { name: "echo", description: "echo", inputSchema: { type: "object" } },
|
|
permission: "auto",
|
|
execute: async () => "résultat de l'outil",
|
|
};
|
|
|
|
/** Provider qui répond par un texte, après un appel d'outil au 1er tour ; il renvoie des consommations absurdes (ignorées). */
|
|
function chatty(): LLMProvider {
|
|
let n = 0;
|
|
return {
|
|
id: "fake",
|
|
model: "fake",
|
|
capabilities: { tools: true },
|
|
async *chat(req): AsyncIterable<StreamEvent> {
|
|
n++;
|
|
const last = req.messages.at(-1);
|
|
const afterTool = last?.content.some((b) => b.type === "tool_result");
|
|
if (!afterTool && n % 2 === 1) {
|
|
yield { type: "tool_use", id: `t${n}`, name: "echo", input: { n } };
|
|
yield { type: "usage", inputTokens: 7, outputTokens: 1 };
|
|
yield { type: "done", stopReason: "tool_use" };
|
|
} else {
|
|
yield { type: "text_delta", text: `réponse numéro ${n} `.repeat(5) };
|
|
yield { type: "usage", inputTokens: 999_999 - n * 10_000, outputTokens: 3 };
|
|
yield { type: "done", stopReason: "end_turn" };
|
|
}
|
|
},
|
|
};
|
|
}
|
|
|
|
it("ne fait que croître quand la conversation s'enrichit (aucune variation due au serveur)", async () => {
|
|
const reported: number[] = [];
|
|
const session = createSession({
|
|
provider: chatty(),
|
|
init: { systemPrompt: "prompt système", tools: [echo] },
|
|
ui: { ...silentUI, onContextUsage: (n) => reported.push(n) },
|
|
cwd: process.cwd(),
|
|
});
|
|
const beforePrompts: number[] = [sessionContextTokens(session)];
|
|
for (const prompt of ["premier message", "deuxième message un peu plus long", "troisième", "quatrième"]) {
|
|
await runTurn(session, prompt);
|
|
beforePrompts.push(sessionContextTokens(session)); // valeur affichée avant la saisie suivante
|
|
}
|
|
for (let i = 1; i < beforePrompts.length; i++) expect(beforePrompts[i]!).toBeGreaterThan(beforePrompts[i - 1]!);
|
|
for (let i = 1; i < reported.length; i++) expect(reported[i]!).toBeGreaterThan(reported[i - 1]!);
|
|
expect(reported.at(-1)).toBe(beforePrompts.at(-1));
|
|
});
|
|
|
|
it("la valeur envoyée à l'UI après chaque réponse est le comptage local de la session", async () => {
|
|
const seen: number[] = [];
|
|
const session = createSession({
|
|
provider: chatty(),
|
|
init: { systemPrompt: "x".repeat(4000), tools: [echo] },
|
|
ui: { ...silentUI, onContextUsage: (n) => seen.push(n) },
|
|
cwd: process.cwd(),
|
|
});
|
|
await runTurn(session, "go");
|
|
expect(seen).toHaveLength(2); // une mise à jour par réponse du LLM (appel d'outil, puis réponse finale)
|
|
expect(seen[0]).toBeGreaterThan(0);
|
|
expect(seen[1]).toBeGreaterThan(seen[0]!);
|
|
expect(seen[1]).toBe(sessionContextTokens(session));
|
|
expect(seen[1]).toBeLessThan(100_000); // la consommation du serveur (≈ 990 000) est ignorée
|
|
});
|
|
|
|
it("le contexte en attente (commandes « ! ») est compté dans la taille de la session", () => {
|
|
const session = createSession({ provider: chatty(), init: { systemPrompt: "t", tools: [] }, ui: silentUI, cwd: "." });
|
|
const base = sessionContextTokens(session);
|
|
session.pendingContext.push("y ".repeat(2000));
|
|
expect(sessionContextTokens(session)).toBe(base + countTextTokens("y ".repeat(2000)));
|
|
session.pendingContext.length = 0;
|
|
expect(sessionContextTokens(session)).toBe(base);
|
|
});
|
|
});
|
|
|
|
describe("réponse coupée par la limite de tokens de sortie", () => {
|
|
it("avertit l'utilisateur quand le serveur signale max_tokens", async () => {
|
|
const results: string[] = [];
|
|
const provider: LLMProvider = {
|
|
id: "f",
|
|
model: "f",
|
|
capabilities: { tools: true },
|
|
async *chat(): AsyncIterable<StreamEvent> {
|
|
yield { type: "text_delta", text: "début de réponse" };
|
|
yield { type: "done", stopReason: "max_tokens" };
|
|
},
|
|
};
|
|
const session = createSession({
|
|
provider,
|
|
init: { systemPrompt: "t", tools: [] },
|
|
ui: { ...silentUI, onToolResult: (name, out, isError) => void results.push(`${name}|${isError}|${out}`) },
|
|
cwd: ".",
|
|
});
|
|
await runTurn(session, "écris un long fichier");
|
|
expect(results).toHaveLength(1);
|
|
expect(results[0]).toContain("agent|true|Réponse");
|
|
expect(results[0]).toContain("MaxOutputTokens");
|
|
});
|
|
});
|
|
|
|
describe("appel d'outil « réparé » par le serveur après une coupure", () => {
|
|
const writeTool: Tool = {
|
|
spec: { name: "Write", description: "w", inputSchema: { type: "object" } },
|
|
permission: "auto",
|
|
execute: async () => "ok",
|
|
};
|
|
function run(outputTokens: number) {
|
|
const results: string[] = [];
|
|
let n = 0;
|
|
const provider: LLMProvider = {
|
|
id: "f",
|
|
model: "f",
|
|
capabilities: { tools: true },
|
|
async *chat(): AsyncIterable<StreamEvent> {
|
|
n++;
|
|
if (n === 1) {
|
|
yield { type: "tool_use", id: "t", name: "Write", input: { path: "a.md" } };
|
|
yield { type: "usage", inputTokens: 47_000, outputTokens };
|
|
yield { type: "done", stopReason: "tool_use" };
|
|
} else {
|
|
yield { type: "text_delta", text: "fin" };
|
|
yield { type: "done", stopReason: "end_turn" };
|
|
}
|
|
},
|
|
};
|
|
const session = createSession({
|
|
provider,
|
|
init: { systemPrompt: "t", tools: [writeTool] },
|
|
ui: { ...silentUI, onToolResult: (name, out) => void results.push(`${name}|${out}`) },
|
|
cwd: ".",
|
|
});
|
|
return runTurn(session, "écris").then(() => results);
|
|
}
|
|
|
|
it("sortie de 4096 tokens pile avec un appel d'outil : avertissement", async () => {
|
|
const r = await run(4096);
|
|
expect(r.some((x) => x.startsWith("agent|Réponse probablement coupée"))).toBe(true);
|
|
});
|
|
it("sortie ordinaire : pas d'avertissement", async () => {
|
|
const r = await run(137);
|
|
expect(r.some((x) => x.startsWith("agent|"))).toBe(false);
|
|
});
|
|
});
|