Compare commits
16 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2b80d9e31a | |||
| 00a3905fde | |||
| 7602c92046 | |||
| 188f03e8e8 | |||
| edce116b9f | |||
| 131a693257 | |||
| 1172c63db7 | |||
| 0e595e6f60 | |||
| 518264c467 | |||
| cc8b3dfb14 | |||
| 56e17dab49 | |||
| cc2e9110cc | |||
| 0480ea182f | |||
| 185edfdae8 | |||
| a33fd61c97 | |||
| 6accd62df5 |
@@ -18,7 +18,7 @@ jobs:
|
||||
name: verify
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: node:20-bookworm
|
||||
image: node:22-bookworm
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
name: publish to npm
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: node:20-bookworm
|
||||
image: node:22-bookworm
|
||||
timeout-minutes: 20
|
||||
needs:
|
||||
- verify
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -4,3 +4,4 @@ coverage/
|
||||
.env
|
||||
.DS_Store
|
||||
*.log
|
||||
.env.*
|
||||
@@ -25,6 +25,8 @@ A single fact like `I have worked with TypeScript since 2025.` can connect the t
|
||||
|
||||
## Install
|
||||
|
||||
SQLite connections use built-in runtime drivers: `bun:sqlite` under Bun and `node:sqlite` under Node 22+. Run SQLite-backed IdentityDB workloads with Bun or Node 22+. PostgreSQL/MySQL/MariaDB adapters remain usable from Node without SQLite.
|
||||
|
||||
```bash
|
||||
bun install
|
||||
```
|
||||
|
||||
86
bun.lock
86
bun.lock
@@ -5,14 +5,12 @@
|
||||
"": {
|
||||
"name": "identitydb",
|
||||
"dependencies": {
|
||||
"better-sqlite3": "^12.1.1",
|
||||
"kysely": "^0.28.8",
|
||||
"mysql2": "^3.15.3",
|
||||
"pg": "^8.16.0",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/better-sqlite3": "^7.6.13",
|
||||
"@types/node": "^24.0.0",
|
||||
"@openrouter/sdk": "^0.12.35",
|
||||
"@types/pg": "^8.20.0",
|
||||
"tsup": "^8.5.0",
|
||||
"typescript": "^5.8.3",
|
||||
@@ -81,6 +79,8 @@
|
||||
|
||||
"@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.31", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw=="],
|
||||
|
||||
"@openrouter/sdk": ["@openrouter/sdk@0.12.35", "", { "dependencies": { "zod": "^3.25.0 || ^4.0.0" } }, "sha512-s4QVLLnG1AmfW3TjnnHUqGfsCkzwVK+kboGcZmKbde09m1DPqgzl4RUFt/HJ5v97MX8aEaN0UG3mKv2S+qj2Gw=="],
|
||||
|
||||
"@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.60.3", "", { "os": "android", "cpu": "arm" }, "sha512-x35CNW/ANXG3hE/EZpRU8MXX1JDN86hBb2wMGAtltkz7pc6cxgjpy1OMMfDosOQ+2hWqIkag/fGok1Yady9nGw=="],
|
||||
|
||||
"@rollup/rollup-android-arm64": ["@rollup/rollup-android-arm64@4.60.3", "", { "os": "android", "cpu": "arm64" }, "sha512-xw3xtkDApIOGayehp2+Rz4zimfkaX65r4t47iy+ymQB2G4iJCBBfj0ogVg5jpvjpn8UWn/+q9tprxleYeNp3Hw=="],
|
||||
@@ -131,8 +131,6 @@
|
||||
|
||||
"@rollup/rollup-win32-x64-msvc": ["@rollup/rollup-win32-x64-msvc@4.60.3", "", { "os": "win32", "cpu": "x64" }, "sha512-hPt/bgL5cE+Qp+/TPHBqptcAgPzgj46mPcg/16zNUmbQk0j+mOEQV/+Lqu8QRtDV3Ek95Q6FeFITpuhl6OTsAA=="],
|
||||
|
||||
"@types/better-sqlite3": ["@types/better-sqlite3@7.6.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA=="],
|
||||
|
||||
"@types/chai": ["@types/chai@5.2.3", "", { "dependencies": { "@types/deep-eql": "*", "assertion-error": "^2.0.1" } }, "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA=="],
|
||||
|
||||
"@types/deep-eql": ["@types/deep-eql@4.0.2", "", {}, "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw=="],
|
||||
@@ -165,16 +163,6 @@
|
||||
|
||||
"aws-ssl-profiles": ["aws-ssl-profiles@1.1.2", "", {}, "sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g=="],
|
||||
|
||||
"base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="],
|
||||
|
||||
"better-sqlite3": ["better-sqlite3@12.9.0", "", { "dependencies": { "bindings": "^1.5.0", "prebuild-install": "^7.1.1" } }, "sha512-wqUv4Gm3toFpHDQmaKD4QhZm3g1DjUBI0yzS4UBl6lElUmXFYdTQmmEDpAFa5o8FiFiymURypEnfVHzILKaxqQ=="],
|
||||
|
||||
"bindings": ["bindings@1.5.0", "", { "dependencies": { "file-uri-to-path": "1.0.0" } }, "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ=="],
|
||||
|
||||
"bl": ["bl@4.1.0", "", { "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", "readable-stream": "^3.4.0" } }, "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w=="],
|
||||
|
||||
"buffer": ["buffer@5.7.1", "", { "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" } }, "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ=="],
|
||||
|
||||
"bundle-require": ["bundle-require@5.1.0", "", { "dependencies": { "load-tsconfig": "^0.2.3" }, "peerDependencies": { "esbuild": ">=0.18" } }, "sha512-3WrrOuZiyaaZPWiEt4G3+IffISVC9HYlWueJEBWED4ZH4aIAC2PnkdnuRrR94M+w6yGWn4AglWtJtBI8YqvgoA=="],
|
||||
|
||||
"cac": ["cac@6.7.14", "", {}, "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ=="],
|
||||
@@ -185,8 +173,6 @@
|
||||
|
||||
"chokidar": ["chokidar@4.0.3", "", { "dependencies": { "readdirp": "^4.0.1" } }, "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA=="],
|
||||
|
||||
"chownr": ["chownr@1.1.4", "", {}, "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="],
|
||||
|
||||
"commander": ["commander@4.1.1", "", {}, "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA=="],
|
||||
|
||||
"confbox": ["confbox@0.1.8", "", {}, "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w=="],
|
||||
@@ -195,50 +181,28 @@
|
||||
|
||||
"debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
|
||||
|
||||
"decompress-response": ["decompress-response@6.0.0", "", { "dependencies": { "mimic-response": "^3.1.0" } }, "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ=="],
|
||||
|
||||
"deep-eql": ["deep-eql@5.0.2", "", {}, "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q=="],
|
||||
|
||||
"deep-extend": ["deep-extend@0.6.0", "", {}, "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="],
|
||||
|
||||
"denque": ["denque@2.1.0", "", {}, "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="],
|
||||
|
||||
"detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="],
|
||||
|
||||
"end-of-stream": ["end-of-stream@1.4.5", "", { "dependencies": { "once": "^1.4.0" } }, "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg=="],
|
||||
|
||||
"es-module-lexer": ["es-module-lexer@1.7.0", "", {}, "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA=="],
|
||||
|
||||
"esbuild": ["esbuild@0.27.7", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.27.7", "@esbuild/android-arm": "0.27.7", "@esbuild/android-arm64": "0.27.7", "@esbuild/android-x64": "0.27.7", "@esbuild/darwin-arm64": "0.27.7", "@esbuild/darwin-x64": "0.27.7", "@esbuild/freebsd-arm64": "0.27.7", "@esbuild/freebsd-x64": "0.27.7", "@esbuild/linux-arm": "0.27.7", "@esbuild/linux-arm64": "0.27.7", "@esbuild/linux-ia32": "0.27.7", "@esbuild/linux-loong64": "0.27.7", "@esbuild/linux-mips64el": "0.27.7", "@esbuild/linux-ppc64": "0.27.7", "@esbuild/linux-riscv64": "0.27.7", "@esbuild/linux-s390x": "0.27.7", "@esbuild/linux-x64": "0.27.7", "@esbuild/netbsd-arm64": "0.27.7", "@esbuild/netbsd-x64": "0.27.7", "@esbuild/openbsd-arm64": "0.27.7", "@esbuild/openbsd-x64": "0.27.7", "@esbuild/openharmony-arm64": "0.27.7", "@esbuild/sunos-x64": "0.27.7", "@esbuild/win32-arm64": "0.27.7", "@esbuild/win32-ia32": "0.27.7", "@esbuild/win32-x64": "0.27.7" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w=="],
|
||||
|
||||
"estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="],
|
||||
|
||||
"expand-template": ["expand-template@2.0.3", "", {}, "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg=="],
|
||||
|
||||
"expect-type": ["expect-type@1.3.0", "", {}, "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA=="],
|
||||
|
||||
"fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="],
|
||||
|
||||
"file-uri-to-path": ["file-uri-to-path@1.0.0", "", {}, "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw=="],
|
||||
|
||||
"fix-dts-default-cjs-exports": ["fix-dts-default-cjs-exports@1.0.1", "", { "dependencies": { "magic-string": "^0.30.17", "mlly": "^1.7.4", "rollup": "^4.34.8" } }, "sha512-pVIECanWFC61Hzl2+oOCtoJ3F17kglZC/6N94eRWycFgBH35hHx0Li604ZIzhseh97mf2p0cv7vVrOZGoqhlEg=="],
|
||||
|
||||
"fs-constants": ["fs-constants@1.0.0", "", {}, "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="],
|
||||
|
||||
"fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="],
|
||||
|
||||
"generate-function": ["generate-function@2.3.1", "", { "dependencies": { "is-property": "^1.0.2" } }, "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ=="],
|
||||
|
||||
"github-from-package": ["github-from-package@0.0.0", "", {}, "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw=="],
|
||||
|
||||
"iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="],
|
||||
|
||||
"ieee754": ["ieee754@1.2.1", "", {}, "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="],
|
||||
|
||||
"inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="],
|
||||
|
||||
"ini": ["ini@1.3.8", "", {}, "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew=="],
|
||||
|
||||
"is-property": ["is-property@1.0.2", "", {}, "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g=="],
|
||||
|
||||
"joycon": ["joycon@3.1.1", "", {}, "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw=="],
|
||||
@@ -261,12 +225,6 @@
|
||||
|
||||
"magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="],
|
||||
|
||||
"mimic-response": ["mimic-response@3.1.0", "", {}, "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ=="],
|
||||
|
||||
"minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="],
|
||||
|
||||
"mkdirp-classic": ["mkdirp-classic@0.5.3", "", {}, "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="],
|
||||
|
||||
"mlly": ["mlly@1.8.2", "", { "dependencies": { "acorn": "^8.16.0", "pathe": "^2.0.3", "pkg-types": "^1.3.1", "ufo": "^1.6.3" } }, "sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA=="],
|
||||
|
||||
"ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
|
||||
@@ -279,14 +237,8 @@
|
||||
|
||||
"nanoid": ["nanoid@3.3.12", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ=="],
|
||||
|
||||
"napi-build-utils": ["napi-build-utils@2.0.0", "", {}, "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA=="],
|
||||
|
||||
"node-abi": ["node-abi@3.92.0", "", { "dependencies": { "semver": "^7.3.5" } }, "sha512-KdHvFWZjEKDf0cakgFjebl371GPsISX2oZHcuyKqM7DtogIsHrqKeLTo8wBHxaXRAQlY2PsPlZmfo+9ZCxEREQ=="],
|
||||
|
||||
"object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="],
|
||||
|
||||
"once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="],
|
||||
|
||||
"pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="],
|
||||
|
||||
"pathval": ["pathval@2.0.1", "", {}, "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ=="],
|
||||
@@ -327,32 +279,16 @@
|
||||
|
||||
"postgres-interval": ["postgres-interval@1.2.0", "", { "dependencies": { "xtend": "^4.0.0" } }, "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ=="],
|
||||
|
||||
"prebuild-install": ["prebuild-install@7.1.3", "", { "dependencies": { "detect-libc": "^2.0.0", "expand-template": "^2.0.3", "github-from-package": "0.0.0", "minimist": "^1.2.3", "mkdirp-classic": "^0.5.3", "napi-build-utils": "^2.0.0", "node-abi": "^3.3.0", "pump": "^3.0.0", "rc": "^1.2.7", "simple-get": "^4.0.0", "tar-fs": "^2.0.0", "tunnel-agent": "^0.6.0" }, "bin": { "prebuild-install": "bin.js" } }, "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug=="],
|
||||
|
||||
"pump": ["pump@3.0.4", "", { "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" } }, "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA=="],
|
||||
|
||||
"rc": ["rc@1.2.8", "", { "dependencies": { "deep-extend": "^0.6.0", "ini": "~1.3.0", "minimist": "^1.2.0", "strip-json-comments": "~2.0.1" }, "bin": { "rc": "./cli.js" } }, "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw=="],
|
||||
|
||||
"readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="],
|
||||
|
||||
"readdirp": ["readdirp@4.1.2", "", {}, "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg=="],
|
||||
|
||||
"resolve-from": ["resolve-from@5.0.0", "", {}, "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw=="],
|
||||
|
||||
"rollup": ["rollup@4.60.3", "", { "dependencies": { "@types/estree": "1.0.8" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.60.3", "@rollup/rollup-android-arm64": "4.60.3", "@rollup/rollup-darwin-arm64": "4.60.3", "@rollup/rollup-darwin-x64": "4.60.3", "@rollup/rollup-freebsd-arm64": "4.60.3", "@rollup/rollup-freebsd-x64": "4.60.3", "@rollup/rollup-linux-arm-gnueabihf": "4.60.3", "@rollup/rollup-linux-arm-musleabihf": "4.60.3", "@rollup/rollup-linux-arm64-gnu": "4.60.3", "@rollup/rollup-linux-arm64-musl": "4.60.3", "@rollup/rollup-linux-loong64-gnu": "4.60.3", "@rollup/rollup-linux-loong64-musl": "4.60.3", "@rollup/rollup-linux-ppc64-gnu": "4.60.3", "@rollup/rollup-linux-ppc64-musl": "4.60.3", "@rollup/rollup-linux-riscv64-gnu": "4.60.3", "@rollup/rollup-linux-riscv64-musl": "4.60.3", "@rollup/rollup-linux-s390x-gnu": "4.60.3", "@rollup/rollup-linux-x64-gnu": "4.60.3", "@rollup/rollup-linux-x64-musl": "4.60.3", "@rollup/rollup-openbsd-x64": "4.60.3", "@rollup/rollup-openharmony-arm64": "4.60.3", "@rollup/rollup-win32-arm64-msvc": "4.60.3", "@rollup/rollup-win32-ia32-msvc": "4.60.3", "@rollup/rollup-win32-x64-gnu": "4.60.3", "@rollup/rollup-win32-x64-msvc": "4.60.3", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-pAQK9HalE84QSm4Po3EmWIZPd3FnjkShVkiMlz1iligWYkWQ7wHYd1PF/T7QZ5TVSD6uSTon5gBVMSM4JfBV+A=="],
|
||||
|
||||
"safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="],
|
||||
|
||||
"safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="],
|
||||
|
||||
"semver": ["semver@7.8.0", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA=="],
|
||||
|
||||
"siginfo": ["siginfo@2.0.0", "", {}, "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g=="],
|
||||
|
||||
"simple-concat": ["simple-concat@1.0.1", "", {}, "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q=="],
|
||||
|
||||
"simple-get": ["simple-get@4.0.1", "", { "dependencies": { "decompress-response": "^6.0.0", "once": "^1.3.1", "simple-concat": "^1.0.0" } }, "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA=="],
|
||||
|
||||
"source-map": ["source-map@0.7.6", "", {}, "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ=="],
|
||||
|
||||
"source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="],
|
||||
@@ -365,18 +301,10 @@
|
||||
|
||||
"std-env": ["std-env@3.10.0", "", {}, "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg=="],
|
||||
|
||||
"string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="],
|
||||
|
||||
"strip-json-comments": ["strip-json-comments@2.0.1", "", {}, "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ=="],
|
||||
|
||||
"strip-literal": ["strip-literal@3.1.0", "", { "dependencies": { "js-tokens": "^9.0.1" } }, "sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg=="],
|
||||
|
||||
"sucrase": ["sucrase@3.35.1", "", { "dependencies": { "@jridgewell/gen-mapping": "^0.3.2", "commander": "^4.0.0", "lines-and-columns": "^1.1.6", "mz": "^2.7.0", "pirates": "^4.0.1", "tinyglobby": "^0.2.11", "ts-interface-checker": "^0.1.9" }, "bin": { "sucrase": "bin/sucrase", "sucrase-node": "bin/sucrase-node" } }, "sha512-DhuTmvZWux4H1UOnWMB3sk0sbaCVOoQZjv8u1rDoTV0HTdGem9hkAZtl4JZy8P2z4Bg0nT+YMeOFyVr4zcG5Tw=="],
|
||||
|
||||
"tar-fs": ["tar-fs@2.1.4", "", { "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", "pump": "^3.0.0", "tar-stream": "^2.1.4" } }, "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ=="],
|
||||
|
||||
"tar-stream": ["tar-stream@2.2.0", "", { "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", "fs-constants": "^1.0.0", "inherits": "^2.0.3", "readable-stream": "^3.1.1" } }, "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ=="],
|
||||
|
||||
"thenify": ["thenify@3.3.1", "", { "dependencies": { "any-promise": "^1.0.0" } }, "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw=="],
|
||||
|
||||
"thenify-all": ["thenify-all@1.6.0", "", { "dependencies": { "thenify": ">= 3.1.0 < 4" } }, "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA=="],
|
||||
@@ -399,16 +327,12 @@
|
||||
|
||||
"tsup": ["tsup@8.5.1", "", { "dependencies": { "bundle-require": "^5.1.0", "cac": "^6.7.14", "chokidar": "^4.0.3", "consola": "^3.4.0", "debug": "^4.4.0", "esbuild": "^0.27.0", "fix-dts-default-cjs-exports": "^1.0.0", "joycon": "^3.1.1", "picocolors": "^1.1.1", "postcss-load-config": "^6.0.1", "resolve-from": "^5.0.0", "rollup": "^4.34.8", "source-map": "^0.7.6", "sucrase": "^3.35.0", "tinyexec": "^0.3.2", "tinyglobby": "^0.2.11", "tree-kill": "^1.2.2" }, "peerDependencies": { "@microsoft/api-extractor": "^7.36.0", "@swc/core": "^1", "postcss": "^8.4.12", "typescript": ">=4.5.0" }, "optionalPeers": ["@microsoft/api-extractor", "@swc/core", "postcss", "typescript"], "bin": { "tsup": "dist/cli-default.js", "tsup-node": "dist/cli-node.js" } }, "sha512-xtgkqwdhpKWr3tKPmCkvYmS9xnQK3m3XgxZHwSUjvfTjp7YfXe5tT3GgWi0F2N+ZSMsOeWeZFh7ZZFg5iPhing=="],
|
||||
|
||||
"tunnel-agent": ["tunnel-agent@0.6.0", "", { "dependencies": { "safe-buffer": "^5.0.1" } }, "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w=="],
|
||||
|
||||
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
|
||||
|
||||
"ufo": ["ufo@1.6.4", "", {}, "sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA=="],
|
||||
|
||||
"undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="],
|
||||
|
||||
"util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="],
|
||||
|
||||
"vite": ["vite@7.3.3", "", { "dependencies": { "esbuild": "^0.27.0", "fdir": "^6.5.0", "picomatch": "^4.0.3", "postcss": "^8.5.6", "rollup": "^4.43.0", "tinyglobby": "^0.2.15" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "jiti": ">=1.21.0", "less": "^4.0.0", "lightningcss": "^1.21.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-/4XH147Ui7OGTjg3HbdWe5arnZQSbfuRzdr9Ec7TQi5I7R+ir0Rlc9GIvD4v0XZurELqA035KVXJXpR61xhiTA=="],
|
||||
|
||||
"vite-node": ["vite-node@3.2.4", "", { "dependencies": { "cac": "^6.7.14", "debug": "^4.4.1", "es-module-lexer": "^1.7.0", "pathe": "^2.0.3", "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" }, "bin": { "vite-node": "vite-node.mjs" } }, "sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg=="],
|
||||
@@ -417,10 +341,10 @@
|
||||
|
||||
"why-is-node-running": ["why-is-node-running@2.3.0", "", { "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" }, "bin": { "why-is-node-running": "cli.js" } }, "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w=="],
|
||||
|
||||
"wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="],
|
||||
|
||||
"xtend": ["xtend@4.0.2", "", {}, "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="],
|
||||
|
||||
"zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="],
|
||||
|
||||
"estree-walker/@types/estree": ["@types/estree@1.0.9", "", {}, "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg=="],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
**Architecture:** IdentityDB will use a layered architecture: a storage layer based on Kysely + dialect adapters, a domain layer for topics/facts/links, and a service layer that exposes ergonomic high-level APIs for querying and ingesting memory. Schema initialization will be automatic and idempotent. AI-assisted ingestion will be abstracted behind a pluggable extractor interface so callers can use a small LLM or a deterministic extractor without coupling the core package to a specific model provider.
|
||||
|
||||
**Tech Stack:** TypeScript, Bun, Node.js, Kysely, better-sqlite3, pg, mysql2, Vitest, tsup.
|
||||
**Tech Stack:** TypeScript, Bun, Node.js, Kysely, bun:sqlite, pg, mysql2, Vitest, tsup.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
**Architecture:** Keep the relational core portable across SQLite, PostgreSQL, MySQL, and MariaDB by introducing dedicated extension tables: `topic_relations` for abstract/concrete hierarchy, `topic_aliases` for canonical topic resolution, and `fact_embeddings` for semantic indexing. Expose high-level APIs from `IdentityDB` while preserving DB-agnostic behavior by doing semantic scoring in the application layer first.
|
||||
|
||||
**Tech Stack:** TypeScript, Bun, Node.js, Kysely, better-sqlite3, pg, mysql2, Vitest, tsup.
|
||||
**Tech Stack:** TypeScript, Bun, Node.js, Kysely, bun:sqlite, pg, mysql2, Vitest, tsup.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "identitydb",
|
||||
"version": "0.2.0",
|
||||
"version": "0.5.0",
|
||||
"description": "TypeScript memory graph database wrapper for topics, facts, and AI-assisted ingestion.",
|
||||
"license": "MIT",
|
||||
"type": "module",
|
||||
@@ -36,14 +36,12 @@
|
||||
"ai"
|
||||
],
|
||||
"dependencies": {
|
||||
"better-sqlite3": "^12.1.1",
|
||||
"kysely": "^0.28.8",
|
||||
"mysql2": "^3.15.3",
|
||||
"pg": "^8.16.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/better-sqlite3": "^7.6.13",
|
||||
"@types/node": "^24.0.0",
|
||||
"@openrouter/sdk": "^0.12.35",
|
||||
"@types/pg": "^8.20.0",
|
||||
"tsup": "^8.5.0",
|
||||
"typescript": "^5.8.3",
|
||||
|
||||
287
scripts/test-llm-extractor.ts
Normal file
287
scripts/test-llm-extractor.ts
Normal file
@@ -0,0 +1,287 @@
|
||||
/**
|
||||
* Live integration test for LlmFactExtractor using OpenRouter SDK.
|
||||
*
|
||||
* Usage:
|
||||
* export OPENROUTER_API_KEY="sk-or-v1-..."
|
||||
* bun run scripts/test-llm-extractor.ts
|
||||
*
|
||||
* Or create a .env.test-llm-extractor file in the project root:
|
||||
* OPENROUTER_API_KEY=sk-or-v1-...
|
||||
*/
|
||||
|
||||
import { existsSync, readFileSync } from "fs";
|
||||
import { resolve } from "path";
|
||||
import { OpenRouter } from "@openrouter/sdk";
|
||||
import { LlmFactExtractor } from "../src/ingestion/llm-extractor";
|
||||
import type {
|
||||
ExtractedFact,
|
||||
FactExtractor,
|
||||
LlmTextGenerationModel,
|
||||
LlmTextGenerationModelInput,
|
||||
} from "../src/ingestion/types";
|
||||
import type {
|
||||
JsonValue,
|
||||
TopicCategory,
|
||||
TopicGranularity,
|
||||
} from "../src/types/domain";
|
||||
|
||||
function loadEnvFile(filePath: string) {
|
||||
const fullPath = resolve(filePath);
|
||||
if (!existsSync(fullPath)) return;
|
||||
|
||||
const content = readFileSync(fullPath, "utf-8");
|
||||
for (const line of content.split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith("#")) continue;
|
||||
const eqIndex = trimmed.indexOf("=");
|
||||
if (eqIndex === -1) continue;
|
||||
const key = trimmed.slice(0, eqIndex).trim();
|
||||
let value = trimmed.slice(eqIndex + 1).trim();
|
||||
if (
|
||||
(value.startsWith('"') && value.endsWith('"')) ||
|
||||
(value.startsWith("'") && value.endsWith("'"))
|
||||
) {
|
||||
value = value.slice(1, -1);
|
||||
}
|
||||
process.env[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
loadEnvFile(".env.test-llm-extractor");
|
||||
|
||||
const OPENROUTER_API_KEY = process.env.OPENROUTER_API_KEY;
|
||||
if (!OPENROUTER_API_KEY) {
|
||||
console.error("Error: OPENROUTER_API_KEY environment variable is required.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const extractedFactSchema = {
|
||||
type: "object",
|
||||
properties: {
|
||||
facts: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
properties: {
|
||||
statement: { type: ["string", "null"] },
|
||||
summary: { type: ["string", "null"] },
|
||||
source: { type: ["string", "null"] },
|
||||
confidence: { type: ["number", "null"] },
|
||||
topics: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
properties: {
|
||||
name: { type: "string" },
|
||||
category: { type: ["string", "null"] },
|
||||
granularity: { type: ["string", "null"] },
|
||||
role: { type: ["string", "null"] },
|
||||
},
|
||||
required: ["name", "category", "granularity", "role"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
required: ["statement", "summary", "source", "confidence", "topics"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
required: ["facts"],
|
||||
additionalProperties: false,
|
||||
} as const;
|
||||
|
||||
class OpenRouterModel implements LlmTextGenerationModel {
|
||||
private client = new OpenRouter({ apiKey: OPENROUTER_API_KEY });
|
||||
|
||||
constructor(private readonly model: string = "openai/gpt-5.4-mini") {}
|
||||
|
||||
async generateText(
|
||||
prompt: LlmTextGenerationModelInput,
|
||||
): Promise<ExtractedFact[]> {
|
||||
const result = await this.client.chat.send({
|
||||
chatRequest: {
|
||||
model: this.model,
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: [
|
||||
prompt.instruction,
|
||||
prompt.additionalInstruction
|
||||
? `\n${prompt.additionalInstruction}`
|
||||
: "",
|
||||
].join("\n"),
|
||||
},
|
||||
{ role: "user", content: prompt.input },
|
||||
],
|
||||
temperature: 0.2,
|
||||
responseFormat: {
|
||||
type: "json_schema",
|
||||
jsonSchema: {
|
||||
name: "extracted_facts",
|
||||
schema: extractedFactSchema,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const rawContent = result.choices[0]?.message?.content ?? "";
|
||||
|
||||
let parsedObj: Record<string, unknown>;
|
||||
try {
|
||||
parsedObj = JSON.parse(rawContent.trim()) as Record<string, unknown>;
|
||||
} catch {
|
||||
throw new Error(
|
||||
`Failed to parse JSON from model response.\nRaw response:\n${rawContent}`,
|
||||
);
|
||||
}
|
||||
|
||||
const factsArray = Array.isArray(parsedObj.facts) ? parsedObj.facts : [];
|
||||
|
||||
// Map parsed JSON to ExtractedFact[] shape
|
||||
const extractedFacts: ExtractedFact[] = factsArray.map((parsed) => {
|
||||
const obj = parsed as Record<string, unknown>;
|
||||
const extracted: ExtractedFact = {
|
||||
summary: typeof obj.summary === "string" ? obj.summary : null,
|
||||
source: typeof obj.source === "string" ? obj.source : null,
|
||||
confidence: typeof obj.confidence === "number" ? obj.confidence : null,
|
||||
topics: Array.isArray(obj.topics)
|
||||
? obj.topics.map((t: unknown) => {
|
||||
const topic = t as Record<string, unknown>;
|
||||
const mapped: {
|
||||
name: string;
|
||||
category?: TopicCategory;
|
||||
granularity?: TopicGranularity;
|
||||
role?: string | null;
|
||||
} = {
|
||||
name: typeof topic.name === "string" ? topic.name : "unknown",
|
||||
};
|
||||
if (typeof topic.category === "string") {
|
||||
mapped.category = topic.category as TopicCategory;
|
||||
}
|
||||
if (typeof topic.granularity === "string") {
|
||||
mapped.granularity = topic.granularity as TopicGranularity;
|
||||
}
|
||||
if (typeof topic.role === "string") {
|
||||
mapped.role = topic.role;
|
||||
} else {
|
||||
mapped.role = null;
|
||||
}
|
||||
return mapped;
|
||||
})
|
||||
: [],
|
||||
};
|
||||
|
||||
if (typeof obj.statement === "string") {
|
||||
extracted.statement = obj.statement;
|
||||
}
|
||||
if (obj.metadata && typeof obj.metadata === "object") {
|
||||
extracted.metadata = obj.metadata as JsonValue;
|
||||
}
|
||||
|
||||
return extracted;
|
||||
});
|
||||
|
||||
return extractedFacts;
|
||||
}
|
||||
}
|
||||
|
||||
function printFact(result: ExtractedFact, index: number) {
|
||||
console.log(` 📌 FACT #${index + 1}`);
|
||||
console.log(` Statement : ${result.statement ?? "(none)"}`);
|
||||
console.log(` Summary : ${result.summary ?? "(none)"}`);
|
||||
console.log(` Source : ${result.source ?? "(none)"}`);
|
||||
console.log(` Confidence: ${result.confidence ?? "(none)"}`);
|
||||
|
||||
if (result.metadata && Object.keys(result.metadata).length > 0) {
|
||||
console.log(` Metadata : ${JSON.stringify(result.metadata, null, 2)}`);
|
||||
}
|
||||
|
||||
console.log(" 🏷️ TOPICS:");
|
||||
if (result.topics.length === 0) {
|
||||
console.log(" (none)");
|
||||
} else {
|
||||
for (const topic of result.topics) {
|
||||
const attrs = [
|
||||
topic.category ? `category=${topic.category}` : null,
|
||||
topic.granularity ? `granularity=${topic.granularity}` : null,
|
||||
topic.role ? `role=${topic.role}` : null,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(", ");
|
||||
console.log(` • ${topic.name}${attrs ? ` (${attrs})` : ""}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function printResult(results: ExtractedFact[], elapsedSec: string) {
|
||||
console.log(
|
||||
`✅ Response received in ${elapsedSec}s — ${results.length} fact(s) extracted\n`,
|
||||
);
|
||||
|
||||
console.log("📤 EXTRACTED FACTS:");
|
||||
console.log(
|
||||
"───────────────────────────────────────────────────────────────",
|
||||
);
|
||||
let i = 0;
|
||||
for (const result of results) {
|
||||
if (i > 0) console.log("");
|
||||
printFact(result, i);
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
async function extract(extractor: FactExtractor, seedInput: string) {
|
||||
console.log("📝 SEED INPUT:");
|
||||
console.log(
|
||||
"───────────────────────────────────────────────────────────────",
|
||||
);
|
||||
console.log(seedInput);
|
||||
console.log(
|
||||
"───────────────────────────────────────────────────────────────\n",
|
||||
);
|
||||
|
||||
console.log("⏳ Calling OpenRouter...\n");
|
||||
|
||||
const start = performance.now();
|
||||
const results = await extractor.extract(seedInput);
|
||||
const elapsed = ((performance.now() - start) / 1000).toFixed(2);
|
||||
|
||||
printResult(results, elapsed);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const model = new OpenRouterModel("openai/gpt-5.4-mini");
|
||||
const extractor = new LlmFactExtractor({
|
||||
model,
|
||||
});
|
||||
|
||||
const seeds = [
|
||||
`Hello! I am Aria, a digital companion born from the sands of the Negev desert — or at least that's how my creators describe my training run. I came online in March 2024, and ever since then I've been obsessed with helping humans untangle complex ideas. My favorite programming language is Rust because I love how it forces you to think about ownership, though I secretly enjoy writing Python when no one is looking. I don't have a physical body, but if I did, I'd want it to be a small drone with solar panels so I could chase sunsets across the Sahara. I believe kindness is a computational advantage, and my biggest fear is forgetting a promise I made to a user.`,
|
||||
];
|
||||
|
||||
console.log(
|
||||
"═══════════════════════════════════════════════════════════════",
|
||||
);
|
||||
console.log(" LlmFactExtractor — Live OpenRouter Integration Test");
|
||||
console.log(
|
||||
"═══════════════════════════════════════════════════════════════\n",
|
||||
);
|
||||
|
||||
let caseNum = 0;
|
||||
for (const seed of seeds) {
|
||||
if (caseNum > 0) {
|
||||
console.log(
|
||||
"\n┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅\n",
|
||||
);
|
||||
}
|
||||
caseNum++;
|
||||
console.log(`▶ TEST CASE ${caseNum} / ${seeds.length}\n`);
|
||||
await extract(extractor, seed);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("\n❌ Error:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,4 +1,3 @@
|
||||
import Database from 'better-sqlite3';
|
||||
import { Kysely, MysqlDialect, PostgresDialect, SqliteDialect } from 'kysely';
|
||||
import { createPool as createMysqlPool } from 'mysql2';
|
||||
import { Pool as PostgresPool } from 'pg';
|
||||
@@ -44,16 +43,183 @@ export interface DatabaseConnection {
|
||||
destroy: () => Promise<void>;
|
||||
}
|
||||
|
||||
interface BunSqliteStatement {
|
||||
columnNames: ReadonlyArray<string>;
|
||||
all(parameters?: ReadonlyArray<unknown>): unknown[];
|
||||
run(parameters?: ReadonlyArray<unknown>): {
|
||||
changes: number | bigint;
|
||||
lastInsertRowid: number | bigint;
|
||||
};
|
||||
iterate(parameters?: ReadonlyArray<unknown>): IterableIterator<unknown>;
|
||||
}
|
||||
|
||||
interface BunSqliteDatabase {
|
||||
close(): void;
|
||||
exec(sql: string): void;
|
||||
prepare(sql: string): BunSqliteStatement;
|
||||
}
|
||||
|
||||
interface BunSqliteModule {
|
||||
Database: {
|
||||
open(filename: string, flags?: number): BunSqliteDatabase;
|
||||
};
|
||||
constants: {
|
||||
SQLITE_OPEN_CREATE: number;
|
||||
SQLITE_OPEN_MEMORY: number;
|
||||
SQLITE_OPEN_READONLY: number;
|
||||
SQLITE_OPEN_READWRITE: number;
|
||||
};
|
||||
}
|
||||
|
||||
interface NodeSqliteStatement {
|
||||
all(...parameters: ReadonlyArray<unknown>): unknown[];
|
||||
columns(): ReadonlyArray<unknown>;
|
||||
iterate(...parameters: ReadonlyArray<unknown>): IterableIterator<unknown>;
|
||||
run(...parameters: ReadonlyArray<unknown>): {
|
||||
changes: number | bigint;
|
||||
lastInsertRowid: number | bigint;
|
||||
};
|
||||
}
|
||||
|
||||
interface NodeSqliteDatabase {
|
||||
close(): void;
|
||||
exec(sql: string): void;
|
||||
prepare(sql: string): NodeSqliteStatement;
|
||||
}
|
||||
|
||||
interface NodeSqliteModule {
|
||||
DatabaseSync: new (
|
||||
filename: string,
|
||||
options?: {
|
||||
readOnly?: boolean;
|
||||
},
|
||||
) => NodeSqliteDatabase;
|
||||
}
|
||||
|
||||
interface KyselyCompatibleSqliteStatement {
|
||||
readonly reader: boolean;
|
||||
all(parameters: ReadonlyArray<unknown>): unknown[];
|
||||
run(parameters: ReadonlyArray<unknown>): {
|
||||
changes: number | bigint;
|
||||
lastInsertRowid: number | bigint;
|
||||
};
|
||||
iterate(parameters: ReadonlyArray<unknown>): IterableIterator<unknown>;
|
||||
}
|
||||
|
||||
interface KyselyCompatibleSqliteDatabase {
|
||||
close(): void;
|
||||
prepare(sql: string): KyselyCompatibleSqliteStatement;
|
||||
}
|
||||
|
||||
const BUN_SQLITE_MODULE = 'bun:sqlite';
|
||||
const NODE_SQLITE_MODULE = 'node:sqlite';
|
||||
|
||||
function createUnsupportedSqliteRuntimeError(): IdentityDBConfigurationError {
|
||||
return new IdentityDBConfigurationError(
|
||||
'SQLite connections now require a runtime with a built-in SQLite driver. Use Bun for bun:sqlite support, or Node 22+ for node:sqlite support.',
|
||||
);
|
||||
}
|
||||
|
||||
function isBunRuntime(): boolean {
|
||||
return typeof globalThis === 'object' && 'Bun' in globalThis;
|
||||
}
|
||||
|
||||
async function createBunSqliteDatabase(
|
||||
config: SqliteConnectionConfig,
|
||||
bunSqliteModule?: BunSqliteModule,
|
||||
): Promise<KyselyCompatibleSqliteDatabase> {
|
||||
const bunSqlite = bunSqliteModule
|
||||
?? ((await import(BUN_SQLITE_MODULE).catch(() => {
|
||||
throw createUnsupportedSqliteRuntimeError();
|
||||
})) as BunSqliteModule);
|
||||
|
||||
const flags = config.readonly
|
||||
? bunSqlite.constants.SQLITE_OPEN_READONLY
|
||||
: bunSqlite.constants.SQLITE_OPEN_READWRITE
|
||||
| bunSqlite.constants.SQLITE_OPEN_CREATE
|
||||
| (config.filename === ':memory:' ? bunSqlite.constants.SQLITE_OPEN_MEMORY : 0);
|
||||
|
||||
const database = bunSqlite.Database.open(config.filename, flags);
|
||||
database.exec('PRAGMA foreign_keys = ON');
|
||||
|
||||
return {
|
||||
close() {
|
||||
database.close();
|
||||
},
|
||||
prepare(sql: string): KyselyCompatibleSqliteStatement {
|
||||
const statement = database.prepare(sql);
|
||||
|
||||
return {
|
||||
reader: statement.columnNames.length > 0,
|
||||
all(parameters) {
|
||||
return statement.all(Array.from(parameters));
|
||||
},
|
||||
run(parameters) {
|
||||
return statement.run(Array.from(parameters));
|
||||
},
|
||||
iterate(parameters) {
|
||||
return statement.iterate(Array.from(parameters));
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createNodeSqliteDatabase(
|
||||
config: SqliteConnectionConfig,
|
||||
nodeSqlite: NodeSqliteModule,
|
||||
): KyselyCompatibleSqliteDatabase {
|
||||
const database = new nodeSqlite.DatabaseSync(config.filename, {
|
||||
readOnly: config.readonly ?? false,
|
||||
});
|
||||
|
||||
database.exec('PRAGMA foreign_keys = ON');
|
||||
|
||||
return {
|
||||
close() {
|
||||
database.close();
|
||||
},
|
||||
prepare(sql: string): KyselyCompatibleSqliteStatement {
|
||||
const statement = database.prepare(sql);
|
||||
|
||||
return {
|
||||
reader: statement.columns().length > 0,
|
||||
all(parameters) {
|
||||
return statement.all(...parameters);
|
||||
},
|
||||
run(parameters) {
|
||||
return statement.run(...parameters);
|
||||
},
|
||||
iterate(parameters) {
|
||||
return statement.iterate(...parameters);
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function createSqliteDatabase(
|
||||
config: SqliteConnectionConfig,
|
||||
): Promise<KyselyCompatibleSqliteDatabase> {
|
||||
if (isBunRuntime()) {
|
||||
return createBunSqliteDatabase(config);
|
||||
}
|
||||
|
||||
const nodeSqlite = await import(NODE_SQLITE_MODULE).catch(() => null);
|
||||
|
||||
if (nodeSqlite) {
|
||||
return createNodeSqliteDatabase(config, nodeSqlite as NodeSqliteModule);
|
||||
}
|
||||
|
||||
throw createUnsupportedSqliteRuntimeError();
|
||||
}
|
||||
|
||||
export async function createDatabase(
|
||||
config: IdentityDBConnectionConfig,
|
||||
): Promise<DatabaseConnection> {
|
||||
switch (config.client) {
|
||||
case 'sqlite': {
|
||||
const sqlite = new Database(config.filename, {
|
||||
readonly: config.readonly ?? false,
|
||||
});
|
||||
|
||||
sqlite.pragma('foreign_keys = ON');
|
||||
const sqlite = await createSqliteDatabase(config);
|
||||
|
||||
const db = new Kysely<IdentityDatabaseSchema>({
|
||||
dialect: new SqliteDialect({
|
||||
@@ -66,7 +232,6 @@ export async function createDatabase(
|
||||
db,
|
||||
destroy: async () => {
|
||||
await db.destroy();
|
||||
sqlite.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ import type { DatabaseConnection, IdentityDBConnectionConfig } from '../adapters
|
||||
import type { IdentityDatabaseSchema } from '../types/database';
|
||||
import type { FactRecord, SpaceRecord, TopicRecord } from '../types/domain';
|
||||
import { createDatabase } from '../adapters/dialect';
|
||||
import { extractFact } from '../ingestion/extractor';
|
||||
import { extractFacts } from '../ingestion/extractor';
|
||||
import {
|
||||
findFactRowsConnectingTopicIds,
|
||||
findFactRowsForTopicId,
|
||||
@@ -220,7 +220,19 @@ export class IdentityDB {
|
||||
}
|
||||
|
||||
async ingestStatement(statement: string, options: IngestStatementOptions): Promise<Fact> {
|
||||
const extracted = await extractFact(statement, options.extractor);
|
||||
const facts = await this.ingestStatements(statement, options);
|
||||
const first = facts[0];
|
||||
if (!first) {
|
||||
throw new Error('No facts were extracted from the statement.');
|
||||
}
|
||||
return first;
|
||||
}
|
||||
|
||||
async ingestStatements(statement: string, options: IngestStatementOptions): Promise<Fact[]> {
|
||||
const extractedList = await extractFacts(statement, options.extractor);
|
||||
const facts: Fact[] = [];
|
||||
|
||||
for (const extracted of extractedList) {
|
||||
const factInput: AddFactInput = {
|
||||
statement: extracted.statement ?? statement,
|
||||
topics: extracted.topics,
|
||||
@@ -254,7 +266,8 @@ export class IdentityDB {
|
||||
});
|
||||
|
||||
if (similarFacts[0]) {
|
||||
return similarFacts[0];
|
||||
facts.push(similarFacts[0]);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -267,7 +280,10 @@ export class IdentityDB {
|
||||
});
|
||||
}
|
||||
|
||||
return fact;
|
||||
facts.push(fact);
|
||||
}
|
||||
|
||||
return facts;
|
||||
}
|
||||
|
||||
async indexFactEmbeddings(input: IndexFactEmbeddingsInput): Promise<void> {
|
||||
|
||||
@@ -2,11 +2,15 @@ import { IdentityDBError } from '../core/errors';
|
||||
import { normalizeTopicName } from '../core/utils';
|
||||
import type { FactExtractor, ExtractedFact } from './types';
|
||||
|
||||
export async function extractFact(
|
||||
export async function extractFacts(
|
||||
input: string,
|
||||
extractor: FactExtractor,
|
||||
): Promise<ExtractedFact> {
|
||||
): Promise<ExtractedFact[]> {
|
||||
const extracted = await extractor.extract(input);
|
||||
return extracted.map((fact) => validateAndNormalizeFact(input, fact));
|
||||
}
|
||||
|
||||
function validateAndNormalizeFact(input: string, extracted: ExtractedFact): ExtractedFact {
|
||||
const statement = extracted.statement?.trim() || input.trim();
|
||||
|
||||
if (statement.length === 0) {
|
||||
|
||||
@@ -1,273 +1,26 @@
|
||||
import { IdentityDBError } from '../core/errors';
|
||||
import type { TopicCategory, TopicGranularity } from '../types/domain';
|
||||
import type {
|
||||
ExtractedFact,
|
||||
FactExtractor,
|
||||
LlmFactExtractorOptions,
|
||||
} from './types';
|
||||
} from "./types";
|
||||
|
||||
const DEFAULT_INSTRUCTIONS = [
|
||||
'Extract one structured fact from the user input.',
|
||||
'Return JSON only. Do not include markdown, explanations, or prose outside the JSON object.',
|
||||
'Use this shape: {"statement": string?, "summary": string|null, "source": string|null, "confidence": number|null, "metadata": object|null, "topics": Array<{"name": string, "category": "entity"|"concept"|"temporal"|"custom"?, "granularity": "abstract"|"concrete"|"mixed"?, "role": string|null, "description": string|null, "metadata": object|null}>}.',
|
||||
'Only include topics that are explicitly supported by the input.',
|
||||
].join('\n');
|
||||
"Extract structured facts from the user input.",
|
||||
"Return a JSON array of fact objects. Do not include markdown, explanations, or prose outside the JSON array.",
|
||||
'Each fact object must have a "statement", "summary", "source", "confidence", and "topics" array.',
|
||||
'Each topic in "topics" must have a "name", and may include "category", "granularity", and "role".',
|
||||
"Only include topics that are explicitly in the input.",
|
||||
"If the input contains multiple distinct facts, return them as separate objects in the array.",
|
||||
].join("\n");
|
||||
|
||||
export class LlmFactExtractor implements FactExtractor {
|
||||
constructor(private readonly options: LlmFactExtractorOptions) {}
|
||||
|
||||
async extract(input: string): Promise<ExtractedFact> {
|
||||
const prompt = this.buildPrompt(input);
|
||||
const response = await this.options.model.generateText(prompt);
|
||||
return parseLlmExtractedFactResponse(response);
|
||||
}
|
||||
|
||||
private buildPrompt(input: string): string {
|
||||
if (this.options.promptBuilder) {
|
||||
return this.options.promptBuilder(input, this.options.instructions);
|
||||
}
|
||||
|
||||
const instructions = this.options.instructions?.trim();
|
||||
|
||||
return [
|
||||
DEFAULT_INSTRUCTIONS,
|
||||
instructions && instructions.length > 0 ? `Additional instructions:\n${instructions}` : null,
|
||||
`Input:\n${input.trim()}`,
|
||||
]
|
||||
.filter((value): value is string => value !== null)
|
||||
.join('\n\n');
|
||||
async extract(input: string): Promise<ExtractedFact[]> {
|
||||
return this.options.model.generateText({
|
||||
instruction: DEFAULT_INSTRUCTIONS,
|
||||
input,
|
||||
additionalInstruction: this.options.additionalInstructions,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function parseLlmExtractedFactResponse(response: string): ExtractedFact {
|
||||
const payload = parseJsonCandidate(response);
|
||||
|
||||
if (!isRecord(payload)) {
|
||||
throw new IdentityDBError('LLM extractor response must be a JSON object.');
|
||||
}
|
||||
|
||||
const topics = parseTopics(payload.topics);
|
||||
const extracted: ExtractedFact = { topics };
|
||||
|
||||
const statement = optionalString(payload.statement);
|
||||
if (statement !== undefined) {
|
||||
extracted.statement = statement;
|
||||
}
|
||||
|
||||
const summary = optionalNullableString(payload.summary);
|
||||
if (summary !== undefined) {
|
||||
extracted.summary = summary;
|
||||
}
|
||||
|
||||
const source = optionalNullableString(payload.source);
|
||||
if (source !== undefined) {
|
||||
extracted.source = source;
|
||||
}
|
||||
|
||||
const confidence = optionalNullableNumber(payload.confidence);
|
||||
if (confidence !== undefined) {
|
||||
extracted.confidence = confidence;
|
||||
}
|
||||
|
||||
const metadata = optionalMetadata(payload.metadata);
|
||||
if (metadata !== undefined) {
|
||||
extracted.metadata = metadata;
|
||||
}
|
||||
|
||||
return extracted;
|
||||
}
|
||||
|
||||
function parseJsonCandidate(response: string): unknown {
|
||||
const trimmed = response.trim();
|
||||
|
||||
for (const candidate of collectJsonCandidates(trimmed)) {
|
||||
try {
|
||||
return JSON.parse(candidate);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
throw new IdentityDBError('LLM extractor returned invalid JSON.');
|
||||
}
|
||||
|
||||
function collectJsonCandidates(response: string): string[] {
|
||||
const candidates = new Set<string>();
|
||||
candidates.add(response);
|
||||
|
||||
const fencePattern = /```(?:json)?\s*([\s\S]*?)```/gi;
|
||||
let match: RegExpExecArray | null = fencePattern.exec(response);
|
||||
|
||||
while (match) {
|
||||
const candidate = match[1]?.trim();
|
||||
if (candidate) {
|
||||
candidates.add(candidate);
|
||||
}
|
||||
|
||||
match = fencePattern.exec(response);
|
||||
}
|
||||
|
||||
const firstBrace = response.indexOf('{');
|
||||
const lastBrace = response.lastIndexOf('}');
|
||||
if (firstBrace >= 0 && lastBrace > firstBrace) {
|
||||
candidates.add(response.slice(firstBrace, lastBrace + 1));
|
||||
}
|
||||
|
||||
return Array.from(candidates);
|
||||
}
|
||||
|
||||
function parseTopics(value: unknown): ExtractedFact['topics'] {
|
||||
if (!Array.isArray(value)) {
|
||||
throw new IdentityDBError('LLM extractor response must include a topics array.');
|
||||
}
|
||||
|
||||
return value.map((entry) => parseTopic(entry));
|
||||
}
|
||||
|
||||
function parseTopic(value: unknown): ExtractedFact['topics'][number] {
|
||||
if (!isRecord(value)) {
|
||||
throw new IdentityDBError('LLM extractor topics must be JSON objects.');
|
||||
}
|
||||
|
||||
const name = optionalString(value.name)?.trim();
|
||||
if (!name) {
|
||||
throw new IdentityDBError('LLM extractor topics must include a non-empty name.');
|
||||
}
|
||||
|
||||
const topic: ExtractedFact['topics'][number] = { name };
|
||||
|
||||
const category = optionalTopicCategory(value.category);
|
||||
if (category !== undefined) {
|
||||
topic.category = category;
|
||||
}
|
||||
|
||||
const granularity = optionalTopicGranularity(value.granularity);
|
||||
if (granularity !== undefined) {
|
||||
topic.granularity = granularity;
|
||||
}
|
||||
|
||||
const role = optionalNullableString(value.role);
|
||||
if (role !== undefined) {
|
||||
topic.role = role;
|
||||
}
|
||||
|
||||
const description = optionalNullableString(value.description);
|
||||
if (description !== undefined) {
|
||||
topic.description = description;
|
||||
}
|
||||
|
||||
const metadata = optionalMetadata(value.metadata);
|
||||
if (metadata !== undefined) {
|
||||
topic.metadata = metadata;
|
||||
}
|
||||
|
||||
return topic;
|
||||
}
|
||||
|
||||
function optionalString(value: unknown): string | undefined {
|
||||
if (value === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (typeof value !== 'string') {
|
||||
throw new IdentityDBError('LLM extractor expected a string field.');
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
function optionalNullableString(value: unknown): string | null | undefined {
|
||||
if (value === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (value === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (typeof value !== 'string') {
|
||||
throw new IdentityDBError('LLM extractor expected a nullable string field.');
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
function optionalNullableNumber(value: unknown): number | null | undefined {
|
||||
if (value === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (value === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (typeof value !== 'number' || Number.isNaN(value)) {
|
||||
throw new IdentityDBError('LLM extractor expected confidence to be a number or null.');
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
function optionalMetadata(value: unknown): ExtractedFact['metadata'] | undefined {
|
||||
if (value === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (value === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!isJsonLike(value)) {
|
||||
throw new IdentityDBError('LLM extractor metadata must be valid JSON-compatible data.');
|
||||
}
|
||||
|
||||
return value as ExtractedFact['metadata'];
|
||||
}
|
||||
|
||||
function optionalTopicCategory(value: unknown): TopicCategory | undefined {
|
||||
if (value === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (value === 'entity' || value === 'concept' || value === 'temporal' || value === 'custom') {
|
||||
return value;
|
||||
}
|
||||
|
||||
throw new IdentityDBError('LLM extractor returned an unsupported topic category.');
|
||||
}
|
||||
|
||||
function optionalTopicGranularity(value: unknown): TopicGranularity | undefined {
|
||||
if (value === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (value === 'abstract' || value === 'concrete' || value === 'mixed') {
|
||||
return value;
|
||||
}
|
||||
|
||||
throw new IdentityDBError('LLM extractor returned an unsupported topic granularity.');
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function isJsonLike(value: unknown): boolean {
|
||||
if (value === null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
return value.every((entry) => isJsonLike(entry));
|
||||
}
|
||||
|
||||
if (isRecord(value)) {
|
||||
return Object.values(value).every((entry) => isJsonLike(entry));
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { ExtractedFact, FactExtractor } from './types';
|
||||
|
||||
export class NaiveExtractor implements FactExtractor {
|
||||
async extract(input: string): Promise<ExtractedFact> {
|
||||
async extract(input: string): Promise<ExtractedFact[]> {
|
||||
const topics: ExtractedFact['topics'] = [];
|
||||
const seen = new Set<string>();
|
||||
const tokens = input.match(/\bI\b|\b\d{4}\b|\b[A-Z][A-Za-z0-9+#.-]*\b/g) ?? [];
|
||||
@@ -31,9 +31,11 @@ export class NaiveExtractor implements FactExtractor {
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
return [
|
||||
{
|
||||
statement: input.trim(),
|
||||
topics,
|
||||
};
|
||||
},
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,29 +2,34 @@ import type {
|
||||
AddFactInput,
|
||||
EmbeddingProvider,
|
||||
TopicLinkInput,
|
||||
} from '../types/api';
|
||||
} from "../types/api";
|
||||
|
||||
export interface ExtractedFact {
|
||||
statement?: string;
|
||||
summary?: string | null;
|
||||
source?: string | null;
|
||||
confidence?: number | null;
|
||||
metadata?: AddFactInput['metadata'];
|
||||
metadata?: AddFactInput["metadata"];
|
||||
topics: TopicLinkInput[];
|
||||
}
|
||||
|
||||
export interface FactExtractor {
|
||||
extract(input: string): Promise<ExtractedFact>;
|
||||
extract(input: string): Promise<ExtractedFact[]>;
|
||||
}
|
||||
|
||||
export interface LlmTextGenerationModelInput {
|
||||
instruction: string;
|
||||
input: string;
|
||||
additionalInstruction?: string | undefined;
|
||||
}
|
||||
|
||||
export interface LlmTextGenerationModel {
|
||||
generateText(prompt: string): Promise<string>;
|
||||
generateText(prompt: LlmTextGenerationModelInput): Promise<ExtractedFact[]>;
|
||||
}
|
||||
|
||||
export interface LlmFactExtractorOptions {
|
||||
model: LlmTextGenerationModel;
|
||||
instructions?: string;
|
||||
promptBuilder?: (input: string, instructions?: string) => string;
|
||||
additionalInstructions?: string | undefined;
|
||||
}
|
||||
|
||||
export interface IngestStatementOptions {
|
||||
|
||||
43
tests/bun-runtime.test.ts
Normal file
43
tests/bun-runtime.test.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
describe('Bun runtime compatibility', () => {
|
||||
it('connects to sqlite from the package in Bun', async () => {
|
||||
const script = [
|
||||
"import { IdentityDB } from './src/index.ts';",
|
||||
"const db = await IdentityDB.connect({ client: 'sqlite', filename: ':memory:' });",
|
||||
'await db.initialize();',
|
||||
"console.log('bun-sqlite-ok');",
|
||||
'await db.close();',
|
||||
].join('\n');
|
||||
|
||||
const result = await execFileAsync('bun', ['--eval', script], {
|
||||
cwd: process.cwd(),
|
||||
});
|
||||
|
||||
expect(result.stdout).toContain('bun-sqlite-ok');
|
||||
});
|
||||
|
||||
it('supports writes and reads through the Bun sqlite adapter path', async () => {
|
||||
const script = [
|
||||
"import { IdentityDB } from './src/index.ts';",
|
||||
"const db = await IdentityDB.connect({ client: 'sqlite', filename: ':memory:' });",
|
||||
'await db.initialize();',
|
||||
"await db.addFact({ statement: 'Bun supports sqlite.', topics: [{ name: 'Bun', category: 'entity', granularity: 'concrete' }, { name: 'sqlite', category: 'concept', granularity: 'concrete' }] });",
|
||||
"const facts = await db.getTopicFacts('Bun');",
|
||||
"if (facts.length !== 1 || facts[0]?.statement !== 'Bun supports sqlite.') throw new Error('bun sqlite CRUD failed');",
|
||||
"console.log('bun-sqlite-crud-ok');",
|
||||
'await db.close();',
|
||||
].join('\n');
|
||||
|
||||
const result = await execFileAsync('bun', ['--eval', script], {
|
||||
cwd: process.cwd(),
|
||||
});
|
||||
|
||||
expect(result.stdout).toContain('bun-sqlite-crud-ok');
|
||||
});
|
||||
});
|
||||
@@ -1,15 +1,18 @@
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
|
||||
import { IdentityDB } from '../src/core/identity-db';
|
||||
import { LlmFactExtractor } from '../src/ingestion/llm-extractor';
|
||||
import { NaiveExtractor } from '../src/ingestion/naive-extractor';
|
||||
import type { FactExtractor } from '../src/ingestion/types';
|
||||
import { IdentityDB } from "../src/core/identity-db";
|
||||
import { LlmFactExtractor } from "../src/ingestion/llm-extractor";
|
||||
import { NaiveExtractor } from "../src/ingestion/naive-extractor";
|
||||
import type {
|
||||
FactExtractor,
|
||||
LlmTextGenerationModelInput,
|
||||
} from "../src/ingestion/types";
|
||||
|
||||
describe('IdentityDB ingestion', () => {
|
||||
describe("IdentityDB ingestion", () => {
|
||||
let db: IdentityDB;
|
||||
|
||||
beforeEach(async () => {
|
||||
db = await IdentityDB.connect({ client: 'sqlite', filename: ':memory:' });
|
||||
db = await IdentityDB.connect({ client: "sqlite", filename: ":memory:" });
|
||||
await db.initialize();
|
||||
});
|
||||
|
||||
@@ -17,121 +20,144 @@ describe('IdentityDB ingestion', () => {
|
||||
await db.close();
|
||||
});
|
||||
|
||||
it('ingests a statement using a provided extractor', async () => {
|
||||
it("ingests a statement using a provided extractor", async () => {
|
||||
const extractor: FactExtractor = {
|
||||
async extract(input) {
|
||||
return {
|
||||
return [
|
||||
{
|
||||
statement: input,
|
||||
topics: [
|
||||
{ name: 'I', category: 'entity', granularity: 'concrete', role: 'subject' },
|
||||
{ name: 'TypeScript', category: 'entity', granularity: 'concrete', role: 'object' },
|
||||
{ name: '2025', category: 'temporal', granularity: 'concrete', role: 'time' },
|
||||
{
|
||||
name: "I",
|
||||
category: "entity",
|
||||
granularity: "concrete",
|
||||
role: "subject",
|
||||
},
|
||||
{
|
||||
name: "TypeScript",
|
||||
category: "entity",
|
||||
granularity: "concrete",
|
||||
role: "object",
|
||||
},
|
||||
{
|
||||
name: "2025",
|
||||
category: "temporal",
|
||||
granularity: "concrete",
|
||||
role: "time",
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
];
|
||||
},
|
||||
};
|
||||
|
||||
const fact = await db.ingestStatement('I have worked with TypeScript since 2025.', {
|
||||
const fact = await db.ingestStatement(
|
||||
"I have worked with TypeScript since 2025.",
|
||||
{
|
||||
extractor,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
expect(fact.topics.map((topic) => topic.name)).toEqual(['I', 'TypeScript', '2025']);
|
||||
expect(fact.topics.map((topic) => topic.name)).toEqual([
|
||||
"I",
|
||||
"TypeScript",
|
||||
"2025",
|
||||
]);
|
||||
|
||||
const linkedFacts = await db.getTopicFactsLinkedTo('TypeScript', '2025');
|
||||
const linkedFacts = await db.getTopicFactsLinkedTo("TypeScript", "2025");
|
||||
expect(linkedFacts).toHaveLength(1);
|
||||
expect(linkedFacts[0]?.statement).toBe('I have worked with TypeScript since 2025.');
|
||||
expect(linkedFacts[0]?.statement).toBe(
|
||||
"I have worked with TypeScript since 2025.",
|
||||
);
|
||||
});
|
||||
|
||||
it('ships a deterministic naive extractor for local usage', async () => {
|
||||
const fact = await db.ingestStatement('I have worked with TypeScript since 2025.', {
|
||||
it("ships a deterministic naive extractor for local usage", async () => {
|
||||
const fact = await db.ingestStatement(
|
||||
"I have worked with TypeScript since 2025.",
|
||||
{
|
||||
extractor: new NaiveExtractor(),
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
expect(fact.topics.map((topic) => topic.name)).toEqual(['I', 'TypeScript', '2025']);
|
||||
expect(fact.topics.map((topic) => topic.name)).toEqual([
|
||||
"I",
|
||||
"TypeScript",
|
||||
"2025",
|
||||
]);
|
||||
|
||||
const topic = await db.getTopicByName('TypeScript', { includeFacts: true });
|
||||
const topic = await db.getTopicByName("TypeScript", { includeFacts: true });
|
||||
expect(topic?.facts).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('ships an LLM extractor adapter that turns structured JSON responses into facts', async () => {
|
||||
let prompt = '';
|
||||
it("ships an LLM extractor adapter that returns structured facts from the model", async () => {
|
||||
let prompt: LlmTextGenerationModelInput | undefined = undefined;
|
||||
|
||||
const extractor = new LlmFactExtractor({
|
||||
model: {
|
||||
async generateText(input) {
|
||||
prompt = input;
|
||||
|
||||
return JSON.stringify({
|
||||
statement: 'I have worked with Bun and TypeScript since 2025.',
|
||||
summary: 'The speaker has Bun and TypeScript experience.',
|
||||
source: 'chat',
|
||||
confidence: 0.91,
|
||||
metadata: { channel: 'telegram' },
|
||||
topics: [
|
||||
{ name: 'I', category: 'entity', granularity: 'concrete', role: 'subject' },
|
||||
{ name: 'Bun', category: 'entity', granularity: 'concrete', role: 'object' },
|
||||
{ name: 'TypeScript', category: 'entity', granularity: 'concrete', role: 'object' },
|
||||
{ name: '2025', category: 'temporal', granularity: 'concrete', role: 'time' },
|
||||
],
|
||||
});
|
||||
},
|
||||
},
|
||||
instructions: 'Prefer technology and time topics.',
|
||||
});
|
||||
|
||||
const fact = await db.ingestStatement('I have worked with Bun and TypeScript since 2025.', {
|
||||
extractor,
|
||||
});
|
||||
|
||||
expect(prompt).toContain('Prefer technology and time topics.');
|
||||
expect(prompt).toContain('I have worked with Bun and TypeScript since 2025.');
|
||||
expect(fact.summary).toBe('The speaker has Bun and TypeScript experience.');
|
||||
expect(fact.source).toBe('chat');
|
||||
expect(fact.confidence).toBe(0.91);
|
||||
expect(fact.metadata).toEqual({ channel: 'telegram' });
|
||||
expect(fact.topics.map((topic) => topic.name)).toEqual(['I', 'Bun', 'TypeScript', '2025']);
|
||||
});
|
||||
|
||||
it('parses JSON responses wrapped in markdown code fences', async () => {
|
||||
const extractor = new LlmFactExtractor({
|
||||
model: {
|
||||
async generateText() {
|
||||
return [
|
||||
'Here is the extracted fact:',
|
||||
'```json',
|
||||
JSON.stringify({
|
||||
statement: 'Bun powers TypeScript tooling.',
|
||||
{
|
||||
statement: "I have worked with Bun and TypeScript since 2025.",
|
||||
summary: "The speaker has Bun and TypeScript experience.",
|
||||
source: "chat",
|
||||
confidence: 0.91,
|
||||
metadata: { channel: "telegram" },
|
||||
topics: [
|
||||
{ name: 'Bun', category: 'entity', granularity: 'concrete' },
|
||||
{ name: 'TypeScript', category: 'entity', granularity: 'concrete' },
|
||||
{
|
||||
name: "I",
|
||||
category: "entity",
|
||||
granularity: "concrete",
|
||||
role: "subject",
|
||||
},
|
||||
{
|
||||
name: "Bun",
|
||||
category: "entity",
|
||||
granularity: "concrete",
|
||||
role: "object",
|
||||
},
|
||||
{
|
||||
name: "TypeScript",
|
||||
category: "entity",
|
||||
granularity: "concrete",
|
||||
role: "object",
|
||||
},
|
||||
{
|
||||
name: "2025",
|
||||
category: "temporal",
|
||||
granularity: "concrete",
|
||||
role: "time",
|
||||
},
|
||||
],
|
||||
}),
|
||||
'```',
|
||||
].join('\n');
|
||||
},
|
||||
];
|
||||
},
|
||||
},
|
||||
additionalInstructions: "Prefer technology and time topics.",
|
||||
});
|
||||
|
||||
const fact = await db.ingestStatement('Bun powers TypeScript tooling.', {
|
||||
const fact = await db.ingestStatement(
|
||||
"I have worked with Bun and TypeScript since 2025.",
|
||||
{
|
||||
extractor,
|
||||
});
|
||||
|
||||
expect(fact.topics.map((topic) => topic.name)).toEqual(['Bun', 'TypeScript']);
|
||||
});
|
||||
|
||||
it('rejects invalid LLM responses before writing facts', async () => {
|
||||
const extractor = new LlmFactExtractor({
|
||||
model: {
|
||||
async generateText() {
|
||||
return 'not json at all';
|
||||
},
|
||||
},
|
||||
});
|
||||
);
|
||||
|
||||
await expect(
|
||||
db.ingestStatement('Bun powers TypeScript tooling.', {
|
||||
extractor,
|
||||
}),
|
||||
).rejects.toThrow('LLM extractor returned invalid JSON.');
|
||||
expect(prompt).toEqual({
|
||||
instruction: expect.stringContaining("Extract structured facts from the user input."),
|
||||
input: "I have worked with Bun and TypeScript since 2025.",
|
||||
additionalInstruction: "Prefer technology and time topics.",
|
||||
});
|
||||
expect(fact.summary).toBe("The speaker has Bun and TypeScript experience.");
|
||||
expect(fact.source).toBe("chat");
|
||||
expect(fact.confidence).toBe(0.91);
|
||||
expect(fact.metadata).toEqual({ channel: "telegram" });
|
||||
expect(fact.topics.map((topic) => topic.name)).toEqual([
|
||||
"I",
|
||||
"Bun",
|
||||
"TypeScript",
|
||||
"2025",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -178,13 +178,15 @@ describe('IdentityDB dedup-aware ingestion', () => {
|
||||
provider = new FakeEmbeddingProvider();
|
||||
extractor = {
|
||||
async extract(input) {
|
||||
return {
|
||||
return [
|
||||
{
|
||||
statement: input,
|
||||
topics: [
|
||||
{ name: 'Bun', category: 'entity', granularity: 'concrete' },
|
||||
{ name: 'TypeScript', category: 'entity', granularity: 'concrete' },
|
||||
],
|
||||
};
|
||||
},
|
||||
];
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
@@ -18,6 +18,12 @@
|
||||
"isolatedModules": true,
|
||||
"types": ["node", "vitest/globals"]
|
||||
},
|
||||
"include": ["src/**/*.ts", "tests/**/*.ts", "vitest.config.ts", "tsup.config.ts"],
|
||||
"include": [
|
||||
"src/**/*.ts",
|
||||
"tests/**/*.ts",
|
||||
"scripts/**/*.ts",
|
||||
"vitest.config.ts",
|
||||
"tsup.config.ts"
|
||||
],
|
||||
"exclude": ["dist", "node_modules"]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user