feat: add AI knowledge analysis MVP
This commit is contained in:
364
modules/ai/server/knowledge.ts
Normal file
364
modules/ai/server/knowledge.ts
Normal file
@@ -0,0 +1,364 @@
|
||||
import "server-only";
|
||||
|
||||
import { OpenAIEmbeddings } from "@langchain/openai";
|
||||
import { and, desc, eq, isNull, or, sql } from "drizzle-orm";
|
||||
|
||||
import { getAiRuntimeConfig } from "@/modules/ai/server/config";
|
||||
import type {
|
||||
AiKnowledgeScope,
|
||||
KnowledgeEntry,
|
||||
KnowledgeEntryInput,
|
||||
KnowledgeRetrievalResult,
|
||||
} from "@/modules/ai/types";
|
||||
import { getDatabase } from "@/modules/core/server/db";
|
||||
import { aiKnowledgeChunks, aiKnowledgeEntries } from "@/modules/core/server/schema";
|
||||
import type { AuthContext } from "@/modules/core/types";
|
||||
|
||||
type KnowledgeRow = typeof aiKnowledgeEntries.$inferSelect;
|
||||
|
||||
type ServiceResult<T> = { success: true; data: T } | { success: false; reason: string; status: number };
|
||||
|
||||
const CHUNK_TARGET_SIZE = 900;
|
||||
const CHUNK_OVERLAP_SIZE = 160;
|
||||
const DEFAULT_RETRIEVAL_LIMIT = 6;
|
||||
|
||||
function toIsoString(value: Date): string {
|
||||
return value.toISOString();
|
||||
}
|
||||
|
||||
function toKnowledgeEntry(row: KnowledgeRow): KnowledgeEntry {
|
||||
return {
|
||||
id: row.id,
|
||||
scope: row.scope,
|
||||
organizationId: row.organizationId ?? undefined,
|
||||
title: row.title,
|
||||
category: row.category,
|
||||
tags: row.tags,
|
||||
body: row.body,
|
||||
status: row.status,
|
||||
createdAt: toIsoString(row.createdAt),
|
||||
updatedAt: toIsoString(row.updatedAt),
|
||||
};
|
||||
}
|
||||
|
||||
function createEmbeddings() {
|
||||
const configResult = getAiRuntimeConfig();
|
||||
if (!configResult.success) {
|
||||
return configResult;
|
||||
}
|
||||
const configuration = configResult.config.baseUrl ? { baseURL: configResult.config.baseUrl } : undefined;
|
||||
return {
|
||||
success: true as const,
|
||||
config: configResult.config,
|
||||
embeddings: new OpenAIEmbeddings({
|
||||
apiKey: configResult.config.apiKey,
|
||||
model: configResult.config.embeddingModel,
|
||||
dimensions: configResult.config.embeddingDimensions,
|
||||
configuration,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function chunkKnowledgeText(input: KnowledgeEntryInput): string[] {
|
||||
const prefix = [input.title, input.category, input.tags].filter(Boolean).join("\n");
|
||||
const text = `${prefix}\n\n${input.body}`.trim();
|
||||
if (text.length <= CHUNK_TARGET_SIZE) {
|
||||
return [text];
|
||||
}
|
||||
|
||||
const chunks: string[] = [];
|
||||
let start = 0;
|
||||
while (start < text.length) {
|
||||
const end = Math.min(start + CHUNK_TARGET_SIZE, text.length);
|
||||
chunks.push(text.slice(start, end).trim());
|
||||
if (end >= text.length) {
|
||||
break;
|
||||
}
|
||||
start = Math.max(end - CHUNK_OVERLAP_SIZE, start + 1);
|
||||
}
|
||||
return chunks.filter(Boolean);
|
||||
}
|
||||
|
||||
function ensureWritableScope(context: AuthContext, input: KnowledgeEntryInput): ServiceResult<{ organizationId: string | null }> {
|
||||
if (input.scope === "platform") {
|
||||
if (!context.permissions.includes("platform:manage")) {
|
||||
return { success: false, reason: "只有平台管理员可以维护平台知识", status: 403 };
|
||||
}
|
||||
return { success: true, data: { organizationId: null } };
|
||||
}
|
||||
|
||||
const organizationId = context.organization?.id;
|
||||
if (!organizationId) {
|
||||
return { success: false, reason: "请选择机构后维护知识库", status: 400 };
|
||||
}
|
||||
if (input.organizationId && input.organizationId !== organizationId) {
|
||||
return { success: false, reason: "不能维护其他机构的知识库", status: 403 };
|
||||
}
|
||||
return { success: true, data: { organizationId } };
|
||||
}
|
||||
|
||||
function getVisibleKnowledgeWhere(organizationId?: string) {
|
||||
const platformFilter = and(eq(aiKnowledgeEntries.scope, "platform"), isNull(aiKnowledgeEntries.organizationId));
|
||||
if (!organizationId) {
|
||||
return platformFilter;
|
||||
}
|
||||
return or(
|
||||
platformFilter,
|
||||
and(eq(aiKnowledgeEntries.scope, "organization"), eq(aiKnowledgeEntries.organizationId, organizationId)),
|
||||
);
|
||||
}
|
||||
|
||||
function getEnabledChunkWhere(organizationId: string) {
|
||||
return and(
|
||||
eq(aiKnowledgeEntries.status, "enabled"),
|
||||
or(
|
||||
and(eq(aiKnowledgeChunks.scope, "platform"), isNull(aiKnowledgeChunks.organizationId)),
|
||||
and(eq(aiKnowledgeChunks.scope, "organization"), eq(aiKnowledgeChunks.organizationId, organizationId)),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async function buildKnowledgeChunks(input: KnowledgeEntryInput): Promise<ServiceResult<{ chunks: string[]; vectors: number[][] }>> {
|
||||
const embeddingsResult = createEmbeddings();
|
||||
if (!embeddingsResult.success) {
|
||||
return { success: false, reason: embeddingsResult.reason, status: 500 };
|
||||
}
|
||||
|
||||
const chunks = chunkKnowledgeText(input);
|
||||
const vectors = await embeddingsResult.embeddings.embedDocuments(chunks);
|
||||
return { success: true, data: { chunks, vectors } };
|
||||
}
|
||||
|
||||
export async function listKnowledgeEntries(context: AuthContext): Promise<KnowledgeEntry[]> {
|
||||
const database = getDatabase();
|
||||
const rows = await database
|
||||
.select()
|
||||
.from(aiKnowledgeEntries)
|
||||
.where(getVisibleKnowledgeWhere(context.organization?.id))
|
||||
.orderBy(desc(aiKnowledgeEntries.updatedAt));
|
||||
return rows.map(toKnowledgeEntry);
|
||||
}
|
||||
|
||||
export async function createKnowledgeEntry(
|
||||
context: AuthContext,
|
||||
input: KnowledgeEntryInput,
|
||||
): Promise<ServiceResult<{ entry: KnowledgeEntry }>> {
|
||||
const scope = ensureWritableScope(context, input);
|
||||
if (!scope.success) {
|
||||
return scope;
|
||||
}
|
||||
|
||||
const database = getDatabase();
|
||||
const build = await buildKnowledgeChunks(input);
|
||||
if (!build.success) {
|
||||
return build;
|
||||
}
|
||||
|
||||
const row = await database.transaction(async (transaction) => {
|
||||
const rows = await transaction
|
||||
.insert(aiKnowledgeEntries)
|
||||
.values({
|
||||
scope: input.scope,
|
||||
organizationId: scope.data.organizationId,
|
||||
title: input.title,
|
||||
category: input.category,
|
||||
tags: input.tags,
|
||||
body: input.body,
|
||||
status: input.status,
|
||||
createdByAccountId: context.account.id,
|
||||
updatedByAccountId: context.account.id,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.returning();
|
||||
const entry = rows[0];
|
||||
if (!entry) {
|
||||
return null;
|
||||
}
|
||||
if (build.data.chunks.length > 0) {
|
||||
await transaction.insert(aiKnowledgeChunks).values(
|
||||
build.data.chunks.map((content, index) => ({
|
||||
entryId: entry.id,
|
||||
organizationId: entry.organizationId,
|
||||
scope: entry.scope,
|
||||
chunkIndex: index,
|
||||
content,
|
||||
embedding: build.data.vectors[index] ?? [],
|
||||
sourceTitle: entry.title,
|
||||
sourceCategory: entry.category,
|
||||
})),
|
||||
);
|
||||
}
|
||||
return entry;
|
||||
});
|
||||
|
||||
if (!row) {
|
||||
return { success: false, reason: "知识库创建失败", status: 500 };
|
||||
}
|
||||
|
||||
return { success: true, data: { entry: toKnowledgeEntry(row) } };
|
||||
}
|
||||
|
||||
async function updateEntryWithChunks(
|
||||
id: string,
|
||||
context: AuthContext,
|
||||
input: KnowledgeEntryInput,
|
||||
organizationId: string | null,
|
||||
): Promise<KnowledgeRow | null> {
|
||||
const build = await buildKnowledgeChunks(input);
|
||||
if (!build.success) {
|
||||
throw new Error(build.reason);
|
||||
}
|
||||
|
||||
const database = getDatabase();
|
||||
return database.transaction(async (transaction) => {
|
||||
const rows = await transaction
|
||||
.update(aiKnowledgeEntries)
|
||||
.set({
|
||||
scope: input.scope,
|
||||
organizationId,
|
||||
title: input.title,
|
||||
category: input.category,
|
||||
tags: input.tags,
|
||||
body: input.body,
|
||||
status: input.status,
|
||||
updatedByAccountId: context.account.id,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(aiKnowledgeEntries.id, id))
|
||||
.returning();
|
||||
const row = rows[0];
|
||||
if (!row) {
|
||||
return null;
|
||||
}
|
||||
|
||||
await transaction.delete(aiKnowledgeChunks).where(eq(aiKnowledgeChunks.entryId, row.id));
|
||||
if (build.data.chunks.length > 0) {
|
||||
await transaction.insert(aiKnowledgeChunks).values(
|
||||
build.data.chunks.map((content, index) => ({
|
||||
entryId: row.id,
|
||||
organizationId: row.organizationId,
|
||||
scope: row.scope,
|
||||
chunkIndex: index,
|
||||
content,
|
||||
embedding: build.data.vectors[index] ?? [],
|
||||
sourceTitle: row.title,
|
||||
sourceCategory: row.category,
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
return row;
|
||||
});
|
||||
}
|
||||
|
||||
export async function updateKnowledgeEntry(
|
||||
context: AuthContext,
|
||||
id: string,
|
||||
input: KnowledgeEntryInput,
|
||||
): Promise<ServiceResult<{ entry: KnowledgeEntry }>> {
|
||||
const scope = ensureWritableScope(context, input);
|
||||
if (!scope.success) {
|
||||
return scope;
|
||||
}
|
||||
|
||||
const database = getDatabase();
|
||||
const existingRows = await database
|
||||
.select()
|
||||
.from(aiKnowledgeEntries)
|
||||
.where(and(eq(aiKnowledgeEntries.id, id), getVisibleKnowledgeWhere(context.organization?.id)))
|
||||
.limit(1);
|
||||
const existing = existingRows[0];
|
||||
if (!existing) {
|
||||
return { success: false, reason: "知识库条目不存在", status: 404 };
|
||||
}
|
||||
|
||||
if (existing.scope === "platform" && !context.permissions.includes("platform:manage")) {
|
||||
return { success: false, reason: "只有平台管理员可以维护平台知识", status: 403 };
|
||||
}
|
||||
|
||||
let row: KnowledgeRow | null;
|
||||
try {
|
||||
row = await updateEntryWithChunks(id, context, input, scope.data.organizationId);
|
||||
} catch {
|
||||
return { success: false, reason: "AI_API_KEY 未配置或知识库向量生成失败", status: 500 };
|
||||
}
|
||||
|
||||
if (!row) {
|
||||
return { success: false, reason: "知识库更新失败", status: 500 };
|
||||
}
|
||||
return { success: true, data: { entry: toKnowledgeEntry(row) } };
|
||||
}
|
||||
|
||||
export async function deleteKnowledgeEntry(context: AuthContext, id: string): Promise<ServiceResult<{ entry: KnowledgeEntry }>> {
|
||||
const database = getDatabase();
|
||||
const existingRows = await database
|
||||
.select()
|
||||
.from(aiKnowledgeEntries)
|
||||
.where(and(eq(aiKnowledgeEntries.id, id), getVisibleKnowledgeWhere(context.organization?.id)))
|
||||
.limit(1);
|
||||
const existing = existingRows[0];
|
||||
if (!existing) {
|
||||
return { success: false, reason: "知识库条目不存在", status: 404 };
|
||||
}
|
||||
if (existing.scope === "platform" && !context.permissions.includes("platform:manage")) {
|
||||
return { success: false, reason: "只有平台管理员可以维护平台知识", status: 403 };
|
||||
}
|
||||
|
||||
const rows = await database.delete(aiKnowledgeEntries).where(eq(aiKnowledgeEntries.id, id)).returning();
|
||||
const row = rows[0];
|
||||
if (!row) {
|
||||
return { success: false, reason: "知识库删除失败", status: 500 };
|
||||
}
|
||||
return { success: true, data: { entry: toKnowledgeEntry(row) } };
|
||||
}
|
||||
|
||||
export async function retrieveKnowledge(
|
||||
organizationId: string,
|
||||
query: string,
|
||||
options?: { limit?: number },
|
||||
): Promise<ServiceResult<{ results: KnowledgeRetrievalResult[] }>> {
|
||||
const embeddingsResult = createEmbeddings();
|
||||
if (!embeddingsResult.success) {
|
||||
return { success: false, reason: embeddingsResult.reason, status: 500 };
|
||||
}
|
||||
|
||||
const [embedding] = await embeddingsResult.embeddings.embedDocuments([query]);
|
||||
if (!embedding) {
|
||||
return { success: false, reason: "知识库向量生成失败", status: 500 };
|
||||
}
|
||||
|
||||
const database = getDatabase();
|
||||
const distance = sql<number>`${aiKnowledgeChunks.embedding} <=> ${JSON.stringify(embedding)}::vector`;
|
||||
const rows = await database
|
||||
.select({
|
||||
entryId: aiKnowledgeChunks.entryId,
|
||||
chunkId: aiKnowledgeChunks.id,
|
||||
title: aiKnowledgeChunks.sourceTitle,
|
||||
category: aiKnowledgeChunks.sourceCategory,
|
||||
content: aiKnowledgeChunks.content,
|
||||
distance,
|
||||
})
|
||||
.from(aiKnowledgeChunks)
|
||||
.innerJoin(aiKnowledgeEntries, eq(aiKnowledgeEntries.id, aiKnowledgeChunks.entryId))
|
||||
.where(getEnabledChunkWhere(organizationId))
|
||||
.orderBy(distance)
|
||||
.limit(options?.limit ?? DEFAULT_RETRIEVAL_LIMIT);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
results: rows.map((row) => ({
|
||||
entryId: row.entryId,
|
||||
chunkId: row.chunkId,
|
||||
title: row.title,
|
||||
category: row.category,
|
||||
content: row.content,
|
||||
score: 1 - row.distance,
|
||||
})),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeKnowledgeScope(scope: AiKnowledgeScope, organizationId?: string): string {
|
||||
return scope === "platform" ? "platform" : `organization:${organizationId ?? ""}`;
|
||||
}
|
||||
Reference in New Issue
Block a user