feat: add pluggable fact extraction pipeline

src/core/identity-db.ts

@@ -9,6 +9,7 @@ import {
   type UpsertTopicInput,
   type AddFactInput,
 } from '../types/api';
+import type { IngestStatementOptions } from '../ingestion/types';
 import type { DatabaseConnection, IdentityDBConnectionConfig } from '../adapters/dialect';
 import type { IdentityDatabaseSchema } from '../types/database';
 import type { FactRecord, TopicRecord } from '../types/domain';
@@ -24,6 +25,7 @@ import {
   nowIsoString,
   serializeMetadata,
 } from './utils';
+import { extractFact } from '../ingestion/extractor';
 import {
   findFactRowsConnectingTopicIds,
   findFactRowsForTopicId,
@@ -85,7 +87,8 @@ export class IdentityDB {
 
       const topics: FactTopic[] = [];
 
-      for (const [index, topicInput] of input.topics.entries()) {
+      for (let index = 0; index < input.topics.length; index += 1) {
+        const topicInput = input.topics[index]!;
         const topic = await this.upsertTopicInExecutor(trx, topicInput);
 
         await trx
@@ -120,6 +123,35 @@ export class IdentityDB {
     });
   }
 
+  async ingestStatement(
+    statement: string,
+    options: IngestStatementOptions,
+  ): Promise<Fact> {
+    const extracted = await extractFact(statement, options.extractor);
+    const factInput: AddFactInput = {
+      statement: extracted.statement ?? statement,
+      topics: extracted.topics,
+    };
+
+    if (extracted.summary !== undefined) {
+      factInput.summary = extracted.summary;
+    }
+
+    if (extracted.source !== undefined) {
+      factInput.source = extracted.source;
+    }
+
+    if (extracted.confidence !== undefined) {
+      factInput.confidence = extracted.confidence;
+    }
+
+    if (extracted.metadata !== undefined) {
+      factInput.metadata = extracted.metadata;
+    }
+
+    return this.addFact(factInput);
+  }
+
   async getTopicFacts(name: string): Promise<Fact[]> {
     const topicRow = await this.getRequiredTopicRow(name);