fix: improve brain init debug output
This commit is contained in:
@@ -1,6 +1,6 @@
|
|||||||
import { randomUUID } from "node:crypto";
|
import { randomUUID } from "node:crypto";
|
||||||
import { config } from "@/config";
|
import { config } from "@/config";
|
||||||
import { IdentityDB, type Space } from "identitydb";
|
import { IdentityDB, type ExtractedFact, type Space } from "identitydb";
|
||||||
import { llm } from "@/openrouter";
|
import { llm } from "@/openrouter";
|
||||||
import { loadPrompt } from "@/openrouter/promptLoader";
|
import { loadPrompt } from "@/openrouter/promptLoader";
|
||||||
import {
|
import {
|
||||||
@@ -28,6 +28,19 @@ export interface DebugOptions {
|
|||||||
personality: string;
|
personality: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface BrainCreateResult {
|
||||||
|
brain: Brain;
|
||||||
|
description: string;
|
||||||
|
baseSystemPrompt: string;
|
||||||
|
/**
|
||||||
|
* Raw facts as returned by `factExtractor.extract(description)`. Populated
|
||||||
|
* only when `Brain.create` is called with `debug: true`; in production
|
||||||
|
* (the default), facts are persisted via `db.ingestStatements` which does
|
||||||
|
* not surface the raw extractor output to the caller.
|
||||||
|
*/
|
||||||
|
extractedFacts?: ExtractedFact[];
|
||||||
|
}
|
||||||
|
|
||||||
export class Brain {
|
export class Brain {
|
||||||
private availabilityCache: Map<string, AvailabilityWindows> = new Map();
|
private availabilityCache: Map<string, AvailabilityWindows> = new Map();
|
||||||
|
|
||||||
@@ -280,8 +293,8 @@ export class Brain {
|
|||||||
static async create(
|
static async create(
|
||||||
displayName: string,
|
displayName: string,
|
||||||
seed: string,
|
seed: string,
|
||||||
options: { dbPath?: string; braindbPath?: string } = {},
|
options: { dbPath?: string; braindbPath?: string; debug?: boolean } = {},
|
||||||
): Promise<Brain | null> {
|
): Promise<BrainCreateResult | null> {
|
||||||
const dbPath = options.dbPath ?? config.dbPath;
|
const dbPath = options.dbPath ?? config.dbPath;
|
||||||
const manager = options.braindbPath
|
const manager = options.braindbPath
|
||||||
? new BrainDBManager(options.braindbPath)
|
? new BrainDBManager(options.braindbPath)
|
||||||
@@ -321,10 +334,26 @@ export class Brain {
|
|||||||
description: displayName,
|
description: displayName,
|
||||||
});
|
});
|
||||||
|
|
||||||
await db.ingestStatement(description, {
|
let extractedFacts: ExtractedFact[] | undefined;
|
||||||
extractor: factExtractor,
|
if (options.debug) {
|
||||||
spaceName,
|
extractedFacts = await factExtractor.extract(description);
|
||||||
});
|
for (const fact of extractedFacts) {
|
||||||
|
await db.addFact({
|
||||||
|
spaceName,
|
||||||
|
statement: fact.statement ?? description,
|
||||||
|
summary: fact.summary,
|
||||||
|
source: fact.source,
|
||||||
|
confidence: fact.confidence,
|
||||||
|
topics: fact.topics,
|
||||||
|
metadata: fact.metadata,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
await db.ingestStatements(description, {
|
||||||
|
extractor: factExtractor,
|
||||||
|
spaceName,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
const brainbase: BrainItem = {
|
const brainbase: BrainItem = {
|
||||||
brainId,
|
brainId,
|
||||||
@@ -334,7 +363,8 @@ export class Brain {
|
|||||||
};
|
};
|
||||||
await manager.saveBrain(brainId, brainbase);
|
await manager.saveBrain(brainId, brainbase);
|
||||||
|
|
||||||
return new Brain(db, space, brainbase);
|
const brain = new Brain(db, space, brainbase);
|
||||||
|
return { brain, description, baseSystemPrompt, extractedFacts };
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const reason = error instanceof Error ? error.message : String(error);
|
const reason = error instanceof Error ? error.message : String(error);
|
||||||
logger.error(`Failed to create brain "${displayName}": ${reason}`);
|
logger.error(`Failed to create brain "${displayName}": ${reason}`);
|
||||||
|
|||||||
@@ -1,5 +1,8 @@
|
|||||||
|
import { randomUUID } from "node:crypto";
|
||||||
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
||||||
import { existsSync, readdirSync } from "fs";
|
import { existsSync, readdirSync } from "fs";
|
||||||
|
import { unlink, writeFile } from "fs/promises";
|
||||||
|
import type { ExtractedFact } from "identitydb";
|
||||||
import { tmpdir } from "os";
|
import { tmpdir } from "os";
|
||||||
|
|
||||||
interface RecordedCall {
|
interface RecordedCall {
|
||||||
@@ -16,18 +19,7 @@ const llmCalls: RecordedCall[] = [];
|
|||||||
const PERSONA_DESCRIPTION = "A 34yo night-shift nurse, hides exhaustion behind sarcasm.";
|
const PERSONA_DESCRIPTION = "A 34yo night-shift nurse, hides exhaustion behind sarcasm.";
|
||||||
const GENERATED_BASE_SYSTEM_PROMPT =
|
const GENERATED_BASE_SYSTEM_PROMPT =
|
||||||
"You are Maren. You text in lowercase. You use '...' when tired.";
|
"You are Maren. You text in lowercase. You use '...' when tired.";
|
||||||
const EXTRACTED_FACTS: Array<{
|
const EXTRACTED_FACTS: ExtractedFact[] = [
|
||||||
statement: string;
|
|
||||||
summary: string;
|
|
||||||
source: string;
|
|
||||||
confidence: number;
|
|
||||||
topics: Array<{
|
|
||||||
name: string;
|
|
||||||
category: string;
|
|
||||||
granularity: string;
|
|
||||||
role: string;
|
|
||||||
}>;
|
|
||||||
}> = [
|
|
||||||
{
|
{
|
||||||
statement: "Maren is 34 years old.",
|
statement: "Maren is 34 years old.",
|
||||||
summary: "Maren is 34 years old.",
|
summary: "Maren is 34 years old.",
|
||||||
@@ -96,6 +88,7 @@ mock.module("@/config", () => ({
|
|||||||
}));
|
}));
|
||||||
|
|
||||||
const { runDebugBrainInit } = await import("./brain");
|
const { runDebugBrainInit } = await import("./brain");
|
||||||
|
const { Brain: ProdBrain } = await import("@/brain");
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
llmCalls.length = 0;
|
llmCalls.length = 0;
|
||||||
@@ -115,7 +108,7 @@ afterEach(async () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe("runDebugBrainInit", () => {
|
describe("runDebugBrainInit", () => {
|
||||||
test("B1: returns ok result with brainId, spaceName, baseSystemPrompt, and uses the supplied seed", async () => {
|
test("B1: returns ok result with full description, baseSystemPrompt, extractedFacts, and uses the supplied seed", async () => {
|
||||||
const result = await runDebugBrainInit({
|
const result = await runDebugBrainInit({
|
||||||
displayName: "Maren",
|
displayName: "Maren",
|
||||||
seed: "Maren, 34, night-shift nurse, hides exhaustion behind sarcasm",
|
seed: "Maren, 34, night-shift nurse, hides exhaustion behind sarcasm",
|
||||||
@@ -131,6 +124,8 @@ describe("runDebugBrainInit", () => {
|
|||||||
);
|
);
|
||||||
expect(result.spaceName).toBe(`brain:${result.brainId}`);
|
expect(result.spaceName).toBe(`brain:${result.brainId}`);
|
||||||
|
|
||||||
|
expect(result.description).toBe(PERSONA_DESCRIPTION);
|
||||||
|
|
||||||
expect(result.baseSystemPrompt).toContain(GENERATED_BASE_SYSTEM_PROMPT);
|
expect(result.baseSystemPrompt).toContain(GENERATED_BASE_SYSTEM_PROMPT);
|
||||||
expect(result.baseSystemPrompt).toContain("You exist in a text chat.");
|
expect(result.baseSystemPrompt).toContain("You exist in a text chat.");
|
||||||
expect(result.baseSystemPrompt).toBe(
|
expect(result.baseSystemPrompt).toBe(
|
||||||
@@ -139,6 +134,8 @@ describe("runDebugBrainInit", () => {
|
|||||||
GENERATED_BASE_SYSTEM_PROMPT.length + 2,
|
GENERATED_BASE_SYSTEM_PROMPT.length + 2,
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
expect(result.extractedFacts).toEqual(EXTRACTED_FACTS);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("B2: invokes the LLM exactly 3 times — PERSONA_INIT, PERSONA_BASE_SYSTEM_PROMPT, fact-extractor", async () => {
|
test("B2: invokes the LLM exactly 3 times — PERSONA_INIT, PERSONA_BASE_SYSTEM_PROMPT, fact-extractor", async () => {
|
||||||
@@ -207,3 +204,49 @@ describe("runDebugBrainInit", () => {
|
|||||||
expect(result.ok).toBe(true);
|
expect(result.ok).toBe(true);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("Brain.create (production path — debug: false)", () => {
|
||||||
|
test("B6: with debug omitted (default), uses db.ingestStatements and does NOT return extractedFacts", async () => {
|
||||||
|
const braindbPath = `${tmpdir()}/brainbox-prod-brain-${randomUUID()}.json`;
|
||||||
|
await writeFile(braindbPath, "{}", { encoding: "utf-8" });
|
||||||
|
|
||||||
|
const result = await ProdBrain.create("ProdMaren", "a prod seed", {
|
||||||
|
dbPath: ":memory:",
|
||||||
|
braindbPath,
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
if (!result) throw new Error("expected result");
|
||||||
|
expect(result.brain).toBeDefined();
|
||||||
|
expect(result.description).toBe(PERSONA_DESCRIPTION);
|
||||||
|
expect(result.baseSystemPrompt).toContain(GENERATED_BASE_SYSTEM_PROMPT);
|
||||||
|
expect(result.extractedFacts).toBeUndefined();
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
await unlink(braindbPath);
|
||||||
|
} catch {}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("B7: with debug: false (explicit), same as default — uses db.ingestStatements, no extractedFacts", async () => {
|
||||||
|
const braindbPath = `${tmpdir()}/brainbox-prod-brain-${randomUUID()}.json`;
|
||||||
|
await writeFile(braindbPath, "{}", { encoding: "utf-8" });
|
||||||
|
|
||||||
|
const result = await ProdBrain.create("ProdMaren2", "seed", {
|
||||||
|
dbPath: ":memory:",
|
||||||
|
braindbPath,
|
||||||
|
debug: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
if (!result) throw new Error("expected result");
|
||||||
|
expect(result.extractedFacts).toBeUndefined();
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
await unlink(braindbPath);
|
||||||
|
} catch {}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import { tmpdir } from "os";
|
|||||||
import { join } from "path";
|
import { join } from "path";
|
||||||
import type { Command } from "commander";
|
import type { Command } from "commander";
|
||||||
import ora from "ora";
|
import ora from "ora";
|
||||||
|
import type { ExtractedFact } from "identitydb";
|
||||||
import { Brain } from "@/brain";
|
import { Brain } from "@/brain";
|
||||||
import { logger } from "@/utils/logger";
|
import { logger } from "@/utils/logger";
|
||||||
|
|
||||||
@@ -19,18 +20,26 @@ export type BrainInitResult =
|
|||||||
displayName: string;
|
displayName: string;
|
||||||
brainId: string;
|
brainId: string;
|
||||||
spaceName: string;
|
spaceName: string;
|
||||||
|
description: string;
|
||||||
baseSystemPrompt: string;
|
baseSystemPrompt: string;
|
||||||
|
extractedFacts: ExtractedFact[];
|
||||||
}
|
}
|
||||||
| { ok: false; error: string };
|
| { ok: false; error: string };
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Exercise the full `Brain.create` flow (PERSONA_INIT → PERSONA_BASE_SYSTEM_PROMPT
|
* Exercise the full `Brain.create` flow (PERSONA_INIT → PERSONA_BASE_SYSTEM_PROMPT
|
||||||
* LLM calls → SQLite DB upsert → fact extraction → braindb save) without
|
* LLM calls → SQLite DB upsert → fact extraction via `factExtractor.extract` →
|
||||||
* touching real on-disk state.
|
* braindb save) without touching real on-disk state.
|
||||||
*
|
*
|
||||||
* - SQLite DB uses `:memory:` (ephemeral, dies with the process).
|
* - SQLite DB uses `:memory:` (ephemeral, dies with the process).
|
||||||
* - The braindb JSON is written to a fresh temp file under `os.tmpdir()`
|
* - The braindb JSON is written to a fresh temp file under `os.tmpdir()`
|
||||||
* and unlinked after the run.
|
* and unlinked after the run.
|
||||||
|
*
|
||||||
|
* Prints the full text of:
|
||||||
|
* 1. the generated `description` (PERSONA_INIT output)
|
||||||
|
* 2. the concatenated `baseSystemPrompt` (generated + fixed)
|
||||||
|
* 3. the `extractedFacts` (obtained by directly calling
|
||||||
|
* `factExtractor.extract(description)`)
|
||||||
*/
|
*/
|
||||||
export async function runDebugBrainInit(
|
export async function runDebugBrainInit(
|
||||||
opts: BrainInitOptions,
|
opts: BrainInitOptions,
|
||||||
@@ -44,27 +53,53 @@ export async function runDebugBrainInit(
|
|||||||
`Initializing brain "${opts.displayName}" with LLM (debug, no real disk state)...`,
|
`Initializing brain "${opts.displayName}" with LLM (debug, no real disk state)...`,
|
||||||
).start();
|
).start();
|
||||||
try {
|
try {
|
||||||
const brain = await Brain.create(opts.displayName, opts.seed, {
|
const result = await Brain.create(opts.displayName, opts.seed, {
|
||||||
dbPath: ":memory:",
|
dbPath: ":memory:",
|
||||||
braindbPath,
|
braindbPath,
|
||||||
|
debug: true,
|
||||||
});
|
});
|
||||||
if (!brain) {
|
if (!result) {
|
||||||
spinner.fail("Brain initialization failed");
|
spinner.fail("Brain initialization failed");
|
||||||
return { ok: false, error: "Brain initialization failed" };
|
return { ok: false, error: "Brain initialization failed" };
|
||||||
}
|
}
|
||||||
|
const {
|
||||||
|
brain,
|
||||||
|
description,
|
||||||
|
baseSystemPrompt,
|
||||||
|
extractedFacts,
|
||||||
|
} = result;
|
||||||
|
const factCount = extractedFacts?.length ?? 0;
|
||||||
spinner.succeed(
|
spinner.succeed(
|
||||||
`Brain initialized (id=${brain.brainbase.brainId}, space=${brain.brainbase.spaceName})`,
|
`Brain initialized (id=${brain.brainbase.brainId}, space=${brain.brainbase.spaceName}, ${factCount} fact(s) extracted)`,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
printSection(`Description (PERSONA_INIT output)`);
|
||||||
|
console.log(description);
|
||||||
|
console.log();
|
||||||
|
|
||||||
|
printSection(`baseSystemPrompt (PERSONA_BASE_SYSTEM_PROMPT + FIXED)`);
|
||||||
|
console.log(baseSystemPrompt);
|
||||||
|
console.log();
|
||||||
|
|
||||||
printSection(
|
printSection(
|
||||||
`Brain — ${opts.displayName} (${brain.brainbase.brainId})`,
|
`Extracted facts (factExtractor.extract — ${factCount})`,
|
||||||
);
|
|
||||||
console.log(`spaceName: ${brain.brainbase.spaceName}`);
|
|
||||||
console.log(`displayName: ${brain.brainbase.displayName}`);
|
|
||||||
console.log(`baseSystemPrompt (first 240 chars):`);
|
|
||||||
console.log(
|
|
||||||
` ${brain.brainbase.baseSystemPrompt.slice(0, 240).replace(/\n/g, "\n ")}${brain.brainbase.baseSystemPrompt.length > 240 ? "..." : ""}`,
|
|
||||||
);
|
);
|
||||||
|
if (extractedFacts && extractedFacts.length > 0) {
|
||||||
|
extractedFacts.forEach((fact, i) => {
|
||||||
|
console.log(`\n[${i + 1}/${extractedFacts.length}]`);
|
||||||
|
console.log(` statement: ${fact.statement ?? ""}`);
|
||||||
|
console.log(` summary: ${fact.summary ?? ""}`);
|
||||||
|
console.log(` source: ${fact.source ?? ""}`);
|
||||||
|
console.log(` confidence: ${fact.confidence ?? ""}`);
|
||||||
|
console.log(` topics: ${JSON.stringify(fact.topics)}`);
|
||||||
|
if (fact.metadata) {
|
||||||
|
console.log(` metadata: ${JSON.stringify(fact.metadata)}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
console.log(" (no facts extracted)");
|
||||||
|
}
|
||||||
|
console.log();
|
||||||
|
|
||||||
logger.info("Debug run complete. Nothing was written to real disk.");
|
logger.info("Debug run complete. Nothing was written to real disk.");
|
||||||
|
|
||||||
@@ -74,14 +109,15 @@ export async function runDebugBrainInit(
|
|||||||
displayName: opts.displayName,
|
displayName: opts.displayName,
|
||||||
brainId: brain.brainbase.brainId,
|
brainId: brain.brainbase.brainId,
|
||||||
spaceName: brain.brainbase.spaceName,
|
spaceName: brain.brainbase.spaceName,
|
||||||
baseSystemPrompt: brain.brainbase.baseSystemPrompt,
|
description,
|
||||||
|
baseSystemPrompt,
|
||||||
|
extractedFacts: extractedFacts ?? [],
|
||||||
};
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const reason = error instanceof Error ? error.message : String(error);
|
const reason = error instanceof Error ? error.message : String(error);
|
||||||
spinner.fail("Brain initialization failed");
|
spinner.fail("Brain initialization failed");
|
||||||
return { ok: false, error: reason };
|
return { ok: false, error: reason };
|
||||||
} finally {
|
} finally {
|
||||||
// Clean up the temp braindb file regardless of success/failure.
|
|
||||||
try {
|
try {
|
||||||
await unlink(braindbPath);
|
await unlink(braindbPath);
|
||||||
} catch {}
|
} catch {}
|
||||||
|
|||||||
Reference in New Issue
Block a user