diff --git a/.trellis/spec/backend/ai-sdk-integration.md b/.trellis/spec/backend/ai-sdk-integration.md index a869772..08977b4 100644 --- a/.trellis/spec/backend/ai-sdk-integration.md +++ b/.trellis/spec/backend/ai-sdk-integration.md @@ -350,7 +350,7 @@ GOOGLE_GENERATIVE_AI_API_KEY=... ANTHROPIC_API_KEY=sk-ant-... ``` -## Scenario: Elder AI Analysis MVP With LangChain And pgvector +## Scenario: Elder AI Analysis MVP With LangChain And PostgreSQL JSONB Embeddings ### 1. Scope / Trigger @@ -369,7 +369,7 @@ ANTHROPIC_API_KEY=sk-ant-... - `DELETE /api/ai/knowledge/[id]` - DB: - `ai_knowledge_entries` - - `ai_knowledge_chunks` with `vector(1536)` + - `ai_knowledge_chunks` with `jsonb` `embedding` arrays - `elder_ai_analyses` - Runtime env: - `AI_API_KEY` required for generation and embedding. @@ -394,6 +394,7 @@ ANTHROPIC_API_KEY=sk-ant-... - `errorCategory` / `errorReason`: sanitized only for failed rows - Failed rows must not store prompts, full resident context snapshots, API keys, or raw provider errors. - Knowledge retrieval may return enabled platform knowledge and enabled active-organization knowledge only. +- Knowledge retrieval first filters enabled platform / active-organization chunks in SQL, then computes cosine similarity in `modules/ai/server/knowledge.ts`; this keeps the MVP deployable on plain PostgreSQL without the `vector` extension. - Completed elder analysis `resultJson.citations` must be derived from citation IDs actually referenced by `keyFindings[].citationIds` and `recommendations[].citationIds`; do not append unused available citations. ### 4. Validation & Error Matrix @@ -420,7 +421,7 @@ ANTHROPIC_API_KEY=sk-ant-... ### 6. Tests Required - Permission seeding assertions for `ai:*` and `knowledge:*` default role grants. -- Knowledge retrieval tests for platform shared, own organization, other organization, and disabled entry filtering. +- Knowledge retrieval tests for platform shared, own organization, other organization, disabled entry filtering, and cosine-score ordering from JSONB embeddings. - Analysis tests for missing config, schema validation failure, failed-history sanitization, and data-scope redaction. - Analysis tests for unknown citation rejection and referenced-only citation persistence. - Route tests for auth/permission failures and standard response shape. diff --git a/.trellis/tasks/07-05-07-06-harden-ai-analysis-path/design.md b/.trellis/tasks/07-05-07-06-harden-ai-analysis-path/design.md index e1cfd8c..da9df73 100644 --- a/.trellis/tasks/07-05-07-06-harden-ai-analysis-path/design.md +++ b/.trellis/tasks/07-05-07-06-harden-ai-analysis-path/design.md @@ -63,8 +63,8 @@ Tests should mock database/provider boundaries and assert observable contracts, ## Compatibility -No database migration is required for this hardening pass. No new dependency is required. +Deployment hardening additionally changes the still-unapplied `0007` AI migration from pgvector to JSONB embeddings, so production PostgreSQL 18 Alpine can run the migration without a `vector` extension. Existing persisted `resultJson.citations` remain readable; new analyses will have stricter citation lists. ## Rollback diff --git a/.trellis/tasks/07-05-07-06-harden-ai-analysis-path/implement.md b/.trellis/tasks/07-05-07-06-harden-ai-analysis-path/implement.md index 7afe86e..9b6e9e4 100644 --- a/.trellis/tasks/07-05-07-06-harden-ai-analysis-path/implement.md +++ b/.trellis/tasks/07-05-07-06-harden-ai-analysis-path/implement.md @@ -9,7 +9,8 @@ 5. Add AI route tests for permission failures, invalid input, service failure propagation, success responses, and audit calls. 6. Improve knowledge UI and analysis dialog failure copy. 7. Clarify `ai:manage` permission description. -8. Run targeted AI tests, then project verification. +8. Align AI knowledge embedding storage to JSONB for production PostgreSQL without pgvector. +9. Run targeted AI tests, then project verification. ## Validation Commands diff --git a/.trellis/tasks/07-05-07-06-harden-ai-analysis-path/prd.md b/.trellis/tasks/07-05-07-06-harden-ai-analysis-path/prd.md index fe28211..bc5f01a 100644 --- a/.trellis/tasks/07-05-07-06-harden-ai-analysis-path/prd.md +++ b/.trellis/tasks/07-05-07-06-harden-ai-analysis-path/prd.md @@ -6,13 +6,13 @@ Harden the existing elder AI analysis and knowledge retrieval MVP so the critica ## Background -The current MVP already implements LangChain-backed elder AI analysis, pgvector-backed knowledge retrieval, permission-scoped resident context, persisted analysis history, and knowledge management UI. +The current MVP now implements LangChain-backed elder AI analysis, PostgreSQL JSONB-backed knowledge retrieval, permission-scoped resident context, persisted analysis history, and knowledge management UI. Current evidence from repository inspection: - AI service files live under `modules/ai/`. - API routes live under `app/api/ai/`. -- AI tables and vector extension are in `modules/core/server/schema.ts` and `drizzle/0007_purple_puff_adder.sql`. +- AI tables and JSONB embedding storage are in `modules/core/server/schema.ts` and `drizzle/0007_purple_puff_adder.sql`; the production target does not require PostgreSQL `vector` extension support. - Existing project verification passes: `pnpm lint`, `pnpm type-check`, and `pnpm test`. - No AI-specific test file currently covers schema validation, permission scoping, history redaction, knowledge organization isolation, disabled knowledge exclusion, or provider failure behavior. - `normalizeCitations` currently appends allowed citations that the model did not necessarily cite, weakening the evidence chain. diff --git a/drizzle/0007_purple_puff_adder.sql b/drizzle/0007_purple_puff_adder.sql index f586ec8..72d1f4e 100644 --- a/drizzle/0007_purple_puff_adder.sql +++ b/drizzle/0007_purple_puff_adder.sql @@ -1,4 +1,3 @@ -CREATE EXTENSION IF NOT EXISTS vector;--> statement-breakpoint CREATE TYPE "public"."ai_knowledge_scope" AS ENUM('platform', 'organization');--> statement-breakpoint CREATE TYPE "public"."ai_knowledge_status" AS ENUM('enabled', 'disabled');--> statement-breakpoint CREATE TYPE "public"."elder_ai_analysis_status" AS ENUM('completed', 'failed');--> statement-breakpoint @@ -9,7 +8,7 @@ CREATE TABLE "ai_knowledge_chunks" ( "scope" "ai_knowledge_scope" NOT NULL, "chunk_index" integer NOT NULL, "content" text NOT NULL, - "embedding" vector(1536) NOT NULL, + "embedding" jsonb DEFAULT '[]'::jsonb NOT NULL, "source_title" text NOT NULL, "source_category" text DEFAULT '' NOT NULL, "created_at" timestamp with time zone DEFAULT now() NOT NULL @@ -55,7 +54,6 @@ ALTER TABLE "elder_ai_analyses" ADD CONSTRAINT "elder_ai_analyses_elder_id_elder ALTER TABLE "elder_ai_analyses" ADD CONSTRAINT "elder_ai_analyses_actor_account_id_accounts_id_fk" FOREIGN KEY ("actor_account_id") REFERENCES "public"."accounts"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint CREATE INDEX "ai_knowledge_chunks_entry_lookup" ON "ai_knowledge_chunks" USING btree ("entry_id");--> statement-breakpoint CREATE INDEX "ai_knowledge_chunks_scope_lookup" ON "ai_knowledge_chunks" USING btree ("scope","organization_id");--> statement-breakpoint -CREATE INDEX "ai_knowledge_chunks_embedding_cosine_idx" ON "ai_knowledge_chunks" USING hnsw ("embedding" vector_cosine_ops);--> statement-breakpoint CREATE INDEX "ai_knowledge_entries_scope_lookup" ON "ai_knowledge_entries" USING btree ("scope","status");--> statement-breakpoint CREATE INDEX "ai_knowledge_entries_organization_lookup" ON "ai_knowledge_entries" USING btree ("organization_id","status");--> statement-breakpoint CREATE INDEX "elder_ai_analyses_elder_lookup" ON "elder_ai_analyses" USING btree ("organization_id","elder_id","created_at");--> statement-breakpoint diff --git a/drizzle/meta/0007_snapshot.json b/drizzle/meta/0007_snapshot.json index a5c4705..d5a6d3e 100644 --- a/drizzle/meta/0007_snapshot.json +++ b/drizzle/meta/0007_snapshot.json @@ -277,9 +277,10 @@ }, "embedding": { "name": "embedding", - "type": "vector(1536)", + "type": "jsonb", "primaryKey": false, - "notNull": true + "notNull": true, + "default": "'[]'::jsonb" }, "source_title": { "name": "source_title", diff --git a/modules/ai/server/knowledge.test.ts b/modules/ai/server/knowledge.test.ts index 1431d85..322eaa4 100644 --- a/modules/ai/server/knowledge.test.ts +++ b/modules/ai/server/knowledge.test.ts @@ -130,7 +130,7 @@ type RetrievedChunkRow = { title: string; category: string; content: string; - distance: number; + embedding: number[]; }; type KnowledgeDatabaseFake = { @@ -325,7 +325,7 @@ class SelectBuilder implements PromiseLike { title: chunk.sourceTitle, category: chunk.sourceCategory, content: chunk.content, - distance: 0.1 + rows.length * 0.1, + embedding: chunk.embedding, }); } } @@ -407,6 +407,31 @@ describe("AI knowledge service", () => { expect(embeddingMocks.embedDocuments).toHaveBeenCalledWith(["night fall risk"]); }); + it("orders retrieved chunks by cosine similarity after scope filtering", async () => { + const weakEntry = createEntry({ id: "entry-weak", title: "Weak guidance" }); + const strongEntry = createEntry({ id: "entry-strong", title: "Strong guidance" }); + useDatabase(createDatabaseFake({ + entries: [weakEntry, strongEntry], + chunks: [ + createChunk(weakEntry, { embedding: [0, 1, 0] }), + createChunk(strongEntry, { embedding: [1, 0, 0] }), + ], + })); + embeddingMocks.embedDocuments.mockResolvedValueOnce([[1, 0, 0]]); + + const result = await retrieveKnowledge("org-1", "night fall risk", { limit: 2 }); + + expect(result).toEqual({ + success: true, + data: { + results: [ + expect.objectContaining({ entryId: "entry-strong", score: 1 }), + expect.objectContaining({ entryId: "entry-weak", score: 0 }), + ], + }, + }); + }); + it("requires platform management permission before mutating platform knowledge", async () => { const existingPlatformEntry = createEntry({ id: "entry-platform", scope: "platform", organizationId: null }); useDatabase(createDatabaseFake({ entries: [existingPlatformEntry] })); diff --git a/modules/ai/server/knowledge.ts b/modules/ai/server/knowledge.ts index 07353a8..1a7be69 100644 --- a/modules/ai/server/knowledge.ts +++ b/modules/ai/server/knowledge.ts @@ -1,7 +1,7 @@ import "server-only"; import { OpenAIEmbeddings } from "@langchain/openai"; -import { and, desc, eq, isNull, or, sql } from "drizzle-orm"; +import { and, desc, eq, isNull, or } from "drizzle-orm"; import { getAiRuntimeConfig } from "@/modules/ai/server/config"; import type { @@ -316,6 +316,29 @@ export async function deleteKnowledgeEntry(context: AuthContext, id: string): Pr return { success: true, data: { entry: toKnowledgeEntry(row) } }; } +function calculateCosineSimilarity(left: readonly number[], right: readonly number[]): number { + if (left.length === 0 || right.length === 0 || left.length !== right.length) { + return 0; + } + + let dotProduct = 0; + let leftMagnitude = 0; + let rightMagnitude = 0; + for (let index = 0; index < left.length; index += 1) { + const leftValue = left[index] ?? 0; + const rightValue = right[index] ?? 0; + dotProduct += leftValue * rightValue; + leftMagnitude += leftValue * leftValue; + rightMagnitude += rightValue * rightValue; + } + + if (leftMagnitude === 0 || rightMagnitude === 0) { + return 0; + } + + return dotProduct / (Math.sqrt(leftMagnitude) * Math.sqrt(rightMagnitude)); +} + export async function retrieveKnowledge( organizationId: string, query: string, @@ -333,7 +356,6 @@ export async function retrieveKnowledge( } const database = getDatabase(); - const distance = sql`${aiKnowledgeChunks.embedding} <=> ${JSON.stringify(embedding)}::vector`; const rows = await database .select({ entryId: aiKnowledgeChunks.entryId, @@ -341,25 +363,26 @@ export async function retrieveKnowledge( title: aiKnowledgeChunks.sourceTitle, category: aiKnowledgeChunks.sourceCategory, content: aiKnowledgeChunks.content, - distance, + embedding: aiKnowledgeChunks.embedding, }) .from(aiKnowledgeChunks) .innerJoin(aiKnowledgeEntries, eq(aiKnowledgeEntries.id, aiKnowledgeChunks.entryId)) - .where(getEnabledChunkWhere(organizationId)) - .orderBy(distance) - .limit(options?.limit ?? DEFAULT_RETRIEVAL_LIMIT); + .where(getEnabledChunkWhere(organizationId)); return { success: true, data: { - results: rows.map((row) => ({ - entryId: row.entryId, - chunkId: row.chunkId, - title: row.title, - category: row.category, - content: row.content, - score: 1 - row.distance, - })), + results: rows + .map((row) => ({ + entryId: row.entryId, + chunkId: row.chunkId, + title: row.title, + category: row.category, + content: row.content, + score: calculateCosineSimilarity(row.embedding, embedding), + })) + .sort((left, right) => right.score - left.score) + .slice(0, options?.limit ?? DEFAULT_RETRIEVAL_LIMIT), }, }; } diff --git a/modules/core/server/schema.ts b/modules/core/server/schema.ts index e9225c6..5a3b69e 100644 --- a/modules/core/server/schema.ts +++ b/modules/core/server/schema.ts @@ -11,7 +11,6 @@ import { timestamp, uniqueIndex, uuid, - vector, } from "drizzle-orm/pg-core"; export const accountStatusEnum = pgEnum("account_status", ["active", "disabled", "pending"]); @@ -555,7 +554,7 @@ export const aiKnowledgeChunks = pgTable("ai_knowledge_chunks", { scope: aiKnowledgeScopeEnum("scope").notNull(), chunkIndex: integer("chunk_index").notNull(), content: text("content").notNull(), - embedding: vector("embedding", { dimensions: AI_KNOWLEDGE_EMBEDDING_DIMENSIONS }).notNull(), + embedding: jsonb("embedding").$type().notNull().default(sql`'[]'::jsonb`), sourceTitle: text("source_title").notNull(), sourceCategory: text("source_category").notNull().default(""), createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),