fix: harden AI analysis and seed defaults

modules/ai/server/knowledge.ts

@@ -1,9 +1,7 @@
 import "server-only";
 
-import { OpenAIEmbeddings } from "@langchain/openai";
 import { and, desc, eq, isNull, or } from "drizzle-orm";
 
-import { getAiRuntimeConfig } from "@/modules/ai/server/config";
 import type {
   AiKnowledgeScope,
   KnowledgeEntry,
@@ -41,22 +39,21 @@ function toKnowledgeEntry(row: KnowledgeRow): KnowledgeEntry {
   };
 }
 
-function createEmbeddings() {
-  const configResult = getAiRuntimeConfig();
-  if (!configResult.success) {
-    return configResult;
+function normalizeSearchText(value: string): string[] {
+  const tokens = new Set<string>();
+  const normalizedTokens = value.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [];
+  for (const token of normalizedTokens) {
+    if (token.length >= 2) {
+      tokens.add(token);
+    }
+    const cjkSequences = token.match(/[\u3400-\u9fff]+/gu) ?? [];
+    for (const sequence of cjkSequences) {
+      for (let index = 0; index < sequence.length - 1; index += 1) {
+        tokens.add(sequence.slice(index, index + 2));
+      }
+    }
   }
-  const configuration = configResult.config.baseUrl ? { baseURL: configResult.config.baseUrl } : undefined;
-  return {
-    success: true as const,
-    config: configResult.config,
-    embeddings: new OpenAIEmbeddings({
-      apiKey: configResult.config.apiKey,
-      model: configResult.config.embeddingModel,
-      dimensions: configResult.config.embeddingDimensions,
-      configuration,
-    }),
-  };
+  return Array.from(tokens);
 }
 
 function chunkKnowledgeText(input: KnowledgeEntryInput): string[] {
@@ -118,19 +115,8 @@ function getEnabledChunkWhere(organizationId: string) {
   );
 }
 
-async function buildKnowledgeChunks(input: KnowledgeEntryInput): Promise<ServiceResult<{ chunks: string[]; vectors: number[][] }>> {
-  const embeddingsResult = createEmbeddings();
-  if (!embeddingsResult.success) {
-    return { success: false, reason: embeddingsResult.reason, status: 500 };
-  }
-
-  const chunks = chunkKnowledgeText(input);
-  try {
-    const vectors = await embeddingsResult.embeddings.embedDocuments(chunks);
-    return { success: true, data: { chunks, vectors } };
-  } catch {
-    return { success: false, reason: "知识库向量生成失败", status: 500 };
-  }
+function buildKnowledgeChunks(input: KnowledgeEntryInput): string[] {
+  return chunkKnowledgeText(input);
 }
 
 export async function listKnowledgeEntries(context: AuthContext): Promise<KnowledgeEntry[]> {
@@ -152,11 +138,8 @@ export async function createKnowledgeEntry(
     return scope;
   }
 
+  const chunks = buildKnowledgeChunks(input);
   const database = getDatabase();
-  const build = await buildKnowledgeChunks(input);
-  if (!build.success) {
-    return build;
-  }
 
   const row = await database.transaction(async (transaction) => {
     const rows = await transaction
@@ -178,15 +161,15 @@ export async function createKnowledgeEntry(
     if (!entry) {
       return null;
     }
-    if (build.data.chunks.length > 0) {
+    if (chunks.length > 0) {
       await transaction.insert(aiKnowledgeChunks).values(
-        build.data.chunks.map((content, index) => ({
+        chunks.map((content, index) => ({
           entryId: entry.id,
           organizationId: entry.organizationId,
           scope: entry.scope,
           chunkIndex: index,
           content,
-          embedding: build.data.vectors[index] ?? [],
+          embedding: [],
           sourceTitle: entry.title,
           sourceCategory: entry.category,
         })),
@@ -208,10 +191,7 @@ async function updateEntryWithChunks(
   input: KnowledgeEntryInput,
   organizationId: string | null,
 ): Promise<KnowledgeRow | null> {
-  const build = await buildKnowledgeChunks(input);
-  if (!build.success) {
-    throw new Error(build.reason);
-  }
+  const chunks = buildKnowledgeChunks(input);
 
   const database = getDatabase();
   return database.transaction(async (transaction) => {
@@ -236,15 +216,15 @@ async function updateEntryWithChunks(
     }
 
     await transaction.delete(aiKnowledgeChunks).where(eq(aiKnowledgeChunks.entryId, row.id));
-    if (build.data.chunks.length > 0) {
+    if (chunks.length > 0) {
       await transaction.insert(aiKnowledgeChunks).values(
-        build.data.chunks.map((content, index) => ({
+        chunks.map((content, index) => ({
           entryId: row.id,
           organizationId: row.organizationId,
           scope: row.scope,
           chunkIndex: index,
           content,
-          embedding: build.data.vectors[index] ?? [],
+          embedding: [],
           sourceTitle: row.title,
           sourceCategory: row.category,
         })),
@@ -284,7 +264,7 @@ export async function updateKnowledgeEntry(
   try {
     row = await updateEntryWithChunks(id, context, input, scope.data.organizationId);
   } catch {
-    return { success: false, reason: "AI_API_KEY 未配置或知识库向量生成失败", status: 500 };
+    return { success: false, reason: "知识库更新失败", status: 500 };
   }
 
   if (!row) {
@@ -316,27 +296,22 @@ export async function deleteKnowledgeEntry(context: AuthContext, id: string): Pr
   return { success: true, data: { entry: toKnowledgeEntry(row) } };
 }
 
-function calculateCosineSimilarity(left: readonly number[], right: readonly number[]): number {
-  if (left.length === 0 || right.length === 0 || left.length !== right.length) {
+function scoreKnowledgeChunk(
+  chunk: Pick<KnowledgeRetrievalResult, "title" | "category" | "content">,
+  queryTokens: readonly string[],
+): number {
+  if (queryTokens.length === 0) {
     return 0;
   }
 
-  let dotProduct = 0;
-  let leftMagnitude = 0;
-  let rightMagnitude = 0;
-  for (let index = 0; index < left.length; index += 1) {
-    const leftValue = left[index] ?? 0;
-    const rightValue = right[index] ?? 0;
-    dotProduct += leftValue * rightValue;
-    leftMagnitude += leftValue * leftValue;
-    rightMagnitude += rightValue * rightValue;
+  const chunkTokens = new Set(normalizeSearchText(`${chunk.title}\n${chunk.category}\n${chunk.content}`));
+  let matchedTokens = 0;
+  for (const token of queryTokens) {
+    if (chunkTokens.has(token)) {
+      matchedTokens += 1;
+    }
   }
-
-  if (leftMagnitude === 0 || rightMagnitude === 0) {
-    return 0;
-  }
-
-  return dotProduct / (Math.sqrt(leftMagnitude) * Math.sqrt(rightMagnitude));
+  return matchedTokens / queryTokens.length;
 }
 
 export async function retrieveKnowledge(
@@ -344,16 +319,7 @@ export async function retrieveKnowledge(
   query: string,
   options?: { limit?: number },
 ): Promise<ServiceResult<{ results: KnowledgeRetrievalResult[] }>> {
-  const embeddingsResult = createEmbeddings();
-  if (!embeddingsResult.success) {
-    return { success: false, reason: embeddingsResult.reason, status: 500 };
-  }
-
-  const vectors = await embeddingsResult.embeddings.embedDocuments([query]).catch(() => []);
-  const embedding = vectors[0];
-  if (!embedding) {
-    return { success: false, reason: "知识库向量生成失败", status: 500 };
-  }
+  const queryTokens = normalizeSearchText(query);
 
   const database = getDatabase();
   const rows = await database
@@ -363,7 +329,6 @@ export async function retrieveKnowledge(
       title: aiKnowledgeChunks.sourceTitle,
       category: aiKnowledgeChunks.sourceCategory,
       content: aiKnowledgeChunks.content,
-      embedding: aiKnowledgeChunks.embedding,
     })
     .from(aiKnowledgeChunks)
     .innerJoin(aiKnowledgeEntries, eq(aiKnowledgeEntries.id, aiKnowledgeChunks.entryId))
@@ -379,8 +344,9 @@ export async function retrieveKnowledge(
           title: row.title,
           category: row.category,
           content: row.content,
-          score: calculateCosineSimilarity(row.embedding, embedding),
+          score: scoreKnowledgeChunk(row, queryTokens),
         }))
+        .filter((row) => row.score > 0)
         .sort((left, right) => right.score - left.score)
         .slice(0, options?.limit ?? DEFAULT_RETRIEVAL_LIMIT),
     },