test: harden AI analysis path
This commit is contained in:
495
modules/ai/server/analysis.test.ts
Normal file
495
modules/ai/server/analysis.test.ts
Normal file
@@ -0,0 +1,495 @@
|
||||
import { ChatOpenAI } from "@langchain/openai";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { AiRuntimeConfigResult } from "@/modules/ai/server/config";
|
||||
import { getAiRuntimeConfig } from "@/modules/ai/server/config";
|
||||
import type { ElderAiResidentContext } from "@/modules/ai/server/elder-context";
|
||||
import { buildElderAiContext } from "@/modules/ai/server/elder-context";
|
||||
import { generateElderAiAnalysis, listElderAiAnalyses } from "@/modules/ai/server/analysis";
|
||||
import { retrieveKnowledge } from "@/modules/ai/server/knowledge";
|
||||
import type { AiCitation, ElderAiAnalysisOutput } from "@/modules/ai/types";
|
||||
import { recordAuditLog } from "@/modules/core/server/audit";
|
||||
import type { AppDatabase } from "@/modules/core/server/db";
|
||||
import { getDatabase } from "@/modules/core/server/db";
|
||||
import type { AuthContext, Permission } from "@/modules/core/types";
|
||||
|
||||
const chatMocks = vi.hoisted(() => ({
|
||||
invoke: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("server-only", () => ({}));
|
||||
|
||||
vi.mock("@langchain/openai", () => ({
|
||||
ChatOpenAI: vi.fn(function MockChatOpenAI() {
|
||||
return {
|
||||
invoke: chatMocks.invoke,
|
||||
};
|
||||
}),
|
||||
}));
|
||||
|
||||
vi.mock("@/modules/ai/server/config", () => ({
|
||||
getAiRuntimeConfig: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/modules/ai/server/elder-context", () => ({
|
||||
buildElderAiContext: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/modules/ai/server/knowledge", () => ({
|
||||
retrieveKnowledge: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/modules/core/server/audit", () => ({
|
||||
recordAuditLog: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/modules/core/server/db", () => ({
|
||||
getDatabase: vi.fn(),
|
||||
}));
|
||||
|
||||
type AnalysisStatus = "completed" | "failed";
|
||||
|
||||
type AnalysisRow = {
|
||||
id: string;
|
||||
organizationId: string;
|
||||
elderId: string;
|
||||
actorAccountId: string | null;
|
||||
status: AnalysisStatus;
|
||||
dataScopes: string[];
|
||||
resultJson: unknown;
|
||||
citationsJson: unknown;
|
||||
modelSummaryJson: unknown;
|
||||
errorCategory: string;
|
||||
errorReason: string;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
};
|
||||
|
||||
type AnalysisInsertPayload = {
|
||||
organizationId?: unknown;
|
||||
elderId?: unknown;
|
||||
actorAccountId?: unknown;
|
||||
status?: unknown;
|
||||
dataScopes?: unknown;
|
||||
resultJson?: unknown;
|
||||
citationsJson?: unknown;
|
||||
modelSummaryJson?: unknown;
|
||||
errorCategory?: unknown;
|
||||
errorReason?: unknown;
|
||||
};
|
||||
|
||||
type AnalysisDatabaseFake = {
|
||||
insertedValues: AnalysisInsertPayload[];
|
||||
insert: ReturnlessMock;
|
||||
select: ReturnlessMock;
|
||||
};
|
||||
|
||||
type ReturnlessMock = ReturnlessFunction & {
|
||||
mock: unknown;
|
||||
};
|
||||
|
||||
type ReturnlessFunction = (...args: unknown[]) => unknown;
|
||||
|
||||
const runtimeConfig: AiRuntimeConfigResult = {
|
||||
success: true,
|
||||
config: {
|
||||
baseUrl: "https://ai.example.test/v1",
|
||||
apiKey: "test-api-key",
|
||||
chatModel: "gpt-test",
|
||||
embeddingModel: "embedding-test",
|
||||
embeddingDimensions: 1536,
|
||||
},
|
||||
};
|
||||
|
||||
const account: AuthContext["account"] = {
|
||||
id: "account-1",
|
||||
name: "Nurse Admin",
|
||||
email: "admin@example.com",
|
||||
avatarUrl: "",
|
||||
role: "org_admin",
|
||||
status: "active",
|
||||
createdAt: "2026-07-02T00:00:00.000Z",
|
||||
updatedAt: "2026-07-02T00:00:00.000Z",
|
||||
};
|
||||
|
||||
const organization: NonNullable<AuthContext["organization"]> = {
|
||||
id: "org-1",
|
||||
name: "TeaCare Home",
|
||||
slug: "teacare",
|
||||
status: "active",
|
||||
registrationEnabled: true,
|
||||
oidcEnabled: false,
|
||||
oidcIssuerUrl: "",
|
||||
oidcClientId: "",
|
||||
oidcHasClientSecret: false,
|
||||
oidcScopes: "",
|
||||
oidcRedirectUri: "",
|
||||
oidcAvatarClaim: "",
|
||||
oidcAutoProvision: false,
|
||||
createdAt: "2026-07-02T00:00:00.000Z",
|
||||
updatedAt: "2026-07-02T00:00:00.000Z",
|
||||
};
|
||||
|
||||
const residentCitation: AiCitation = {
|
||||
id: "resident-1",
|
||||
sourceType: "resident_context",
|
||||
sourceId: "elder-1",
|
||||
title: "Resident risk notes",
|
||||
excerpt: "Night wandering and prior fall history.",
|
||||
};
|
||||
|
||||
const unusedResidentCitation: AiCitation = {
|
||||
id: "resident-2",
|
||||
sourceType: "resident_context",
|
||||
sourceId: "elder-1-vitals",
|
||||
title: "Recent vitals",
|
||||
excerpt: "Blood pressure readings are stable.",
|
||||
};
|
||||
|
||||
const knowledgeCitation = {
|
||||
entryId: "entry-1",
|
||||
chunkId: "chunk-1",
|
||||
title: "Fall prevention protocol",
|
||||
category: "Safety",
|
||||
content: "Keep walkways clear and increase observation after night wandering.",
|
||||
score: 0.89,
|
||||
};
|
||||
|
||||
const organizationContext: NonNullable<AuthContext["organization"]> = organization;
|
||||
|
||||
function createAuthContext(permissions: Permission[] = ["ai:read", "elder:read", "health:read", "knowledge:read"]): AuthContext {
|
||||
return {
|
||||
account,
|
||||
organization: organizationContext,
|
||||
organizations: [
|
||||
{
|
||||
id: organizationContext.id,
|
||||
name: organizationContext.name,
|
||||
slug: organizationContext.slug,
|
||||
status: organizationContext.status,
|
||||
registrationEnabled: organizationContext.registrationEnabled,
|
||||
oidcEnabled: organizationContext.oidcEnabled,
|
||||
roleLabel: "Admin",
|
||||
isActive: true,
|
||||
},
|
||||
],
|
||||
membership: {
|
||||
id: "membership-1",
|
||||
accountId: account.id,
|
||||
organizationId: organizationContext.id,
|
||||
roleId: "role-1",
|
||||
roleKey: "org_admin",
|
||||
roleLabel: "Admin",
|
||||
status: "active",
|
||||
createdAt: "2026-07-02T00:00:00.000Z",
|
||||
updatedAt: "2026-07-02T00:00:00.000Z",
|
||||
},
|
||||
permissions,
|
||||
session: {
|
||||
id: "session-1",
|
||||
accountId: account.id,
|
||||
activeOrganizationId: organizationContext.id,
|
||||
createdAt: "2026-07-02T00:00:00.000Z",
|
||||
expiresAt: "2026-07-09T00:00:00.000Z",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createResidentContext(citations: AiCitation[] = [residentCitation]): ElderAiResidentContext {
|
||||
return {
|
||||
organizationId: "org-1",
|
||||
elderId: "elder-1",
|
||||
elderName: "王阿姨",
|
||||
dataScopes: ["elder", "health"],
|
||||
sections: ["Resident profile", "Health profile"],
|
||||
citations,
|
||||
promptContext: "Resident profile includes night wandering and prior fall history.",
|
||||
};
|
||||
}
|
||||
|
||||
function createModelOutput(overrides: Partial<ElderAiAnalysisOutput> = {}): ElderAiAnalysisOutput {
|
||||
return {
|
||||
overallRiskLevel: "medium",
|
||||
summary: "Resident has elevated fall risk at night.",
|
||||
keyFindings: [
|
||||
{
|
||||
category: "fall-risk",
|
||||
severity: "warning",
|
||||
evidence: "Night wandering and prior fall history are present.",
|
||||
citationIds: ["resident-1"],
|
||||
},
|
||||
],
|
||||
recommendations: [
|
||||
{
|
||||
title: "Increase night checks",
|
||||
priority: "high",
|
||||
rationale: "Additional observation reduces missed night wandering events.",
|
||||
suggestedNextStep: "Add a night-shift round note for the next care review.",
|
||||
citationIds: ["resident-1"],
|
||||
},
|
||||
],
|
||||
dataGaps: [],
|
||||
citations: [residentCitation],
|
||||
confidence: 0.74,
|
||||
modelSummary: {
|
||||
provider: "openai-compatible",
|
||||
chatModel: "gpt-test",
|
||||
embeddingModel: "embedding-test",
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createAnalysisRow(values: AnalysisInsertPayload, index: number): AnalysisRow {
|
||||
const status = values.status === "completed" || values.status === "failed" ? values.status : "failed";
|
||||
const dataScopes = Array.isArray(values.dataScopes) ? values.dataScopes.filter((scope): scope is string => typeof scope === "string") : [];
|
||||
return {
|
||||
id: `analysis-${index + 1}`,
|
||||
organizationId: typeof values.organizationId === "string" ? values.organizationId : "org-1",
|
||||
elderId: typeof values.elderId === "string" ? values.elderId : "elder-1",
|
||||
actorAccountId: typeof values.actorAccountId === "string" ? values.actorAccountId : null,
|
||||
status,
|
||||
dataScopes,
|
||||
resultJson: values.resultJson ?? null,
|
||||
citationsJson: values.citationsJson ?? [],
|
||||
modelSummaryJson: values.modelSummaryJson ?? {},
|
||||
errorCategory: typeof values.errorCategory === "string" ? values.errorCategory : "",
|
||||
errorReason: typeof values.errorReason === "string" ? values.errorReason : "",
|
||||
createdAt: new Date(`2026-07-02T0${index}:00:00.000Z`),
|
||||
updatedAt: new Date(`2026-07-02T0${index}:00:00.000Z`),
|
||||
};
|
||||
}
|
||||
|
||||
function createDatabaseFake(selectRows: AnalysisRow[] = []): AnalysisDatabaseFake {
|
||||
const insertedValues: AnalysisInsertPayload[] = [];
|
||||
const insert = vi.fn(() => ({
|
||||
values: vi.fn((values: AnalysisInsertPayload) => {
|
||||
insertedValues.push(values);
|
||||
return {
|
||||
returning: vi.fn(async () => [createAnalysisRow(values, insertedValues.length)]),
|
||||
};
|
||||
}),
|
||||
}));
|
||||
const select = vi.fn(() => ({
|
||||
from: vi.fn(() => ({
|
||||
where: vi.fn(() => ({
|
||||
orderBy: vi.fn(() => ({
|
||||
limit: vi.fn(async () => selectRows),
|
||||
})),
|
||||
})),
|
||||
})),
|
||||
}));
|
||||
|
||||
return { insertedValues, insert, select };
|
||||
}
|
||||
|
||||
function useDatabase(fake: AnalysisDatabaseFake): void {
|
||||
vi.mocked(getDatabase).mockReturnValue(fake as unknown as AppDatabase);
|
||||
}
|
||||
|
||||
function mockSuccessfulContext(citations?: AiCitation[]): void {
|
||||
vi.mocked(buildElderAiContext).mockResolvedValue({
|
||||
success: true,
|
||||
context: createResidentContext(citations),
|
||||
});
|
||||
}
|
||||
|
||||
function mockSuccessfulKnowledge(): void {
|
||||
vi.mocked(retrieveKnowledge).mockResolvedValue({
|
||||
success: true,
|
||||
data: { results: [] },
|
||||
});
|
||||
}
|
||||
|
||||
function modelMessage(output: unknown): { content: string } {
|
||||
return { content: JSON.stringify(output) };
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.mocked(getAiRuntimeConfig).mockReturnValue(runtimeConfig);
|
||||
mockSuccessfulContext();
|
||||
mockSuccessfulKnowledge();
|
||||
chatMocks.invoke.mockResolvedValue(modelMessage(createModelOutput()));
|
||||
useDatabase(createDatabaseFake());
|
||||
});
|
||||
|
||||
describe("elder AI analysis service", () => {
|
||||
it("saves sanitized failed history when AI runtime config is missing", async () => {
|
||||
const database = createDatabaseFake();
|
||||
useDatabase(database);
|
||||
vi.mocked(getAiRuntimeConfig).mockReturnValue({ success: false, reason: "AI_API_KEY 未配置" });
|
||||
|
||||
const result = await generateElderAiAnalysis(createAuthContext(), "elder-1");
|
||||
|
||||
expect(result).toEqual({ success: false, reason: "AI_API_KEY 未配置", status: 500 });
|
||||
expect(database.insertedValues).toEqual([
|
||||
expect.objectContaining({
|
||||
organizationId: "org-1",
|
||||
elderId: "elder-1",
|
||||
actorAccountId: "account-1",
|
||||
status: "failed",
|
||||
dataScopes: ["elder"],
|
||||
errorCategory: "missing_config",
|
||||
errorReason: "AI 服务未配置",
|
||||
}),
|
||||
]);
|
||||
expect(database.insertedValues[0]).not.toHaveProperty("resultJson");
|
||||
expect(buildElderAiContext).not.toHaveBeenCalled();
|
||||
expect(ChatOpenAI).not.toHaveBeenCalled();
|
||||
expect(recordAuditLog).toHaveBeenCalledWith(expect.objectContaining({ result: "failure", reason: "AI 服务未配置" }));
|
||||
});
|
||||
|
||||
it("maps provider errors to sanitized failed history without persisting raw prompts or provider details", async () => {
|
||||
const database = createDatabaseFake();
|
||||
useDatabase(database);
|
||||
chatMocks.invoke.mockRejectedValue(new Error("provider exploded with sk-live-secret and full resident prompt"));
|
||||
|
||||
const result = await generateElderAiAnalysis(createAuthContext(), "elder-1");
|
||||
|
||||
expect(result).toEqual({ success: false, reason: "AI 服务暂时不可用", status: 502 });
|
||||
expect(database.insertedValues).toEqual([
|
||||
expect.objectContaining({
|
||||
status: "failed",
|
||||
dataScopes: ["elder", "health"],
|
||||
errorCategory: "provider_error",
|
||||
errorReason: "AI 服务暂时不可用",
|
||||
}),
|
||||
]);
|
||||
const persistedFailure = JSON.stringify(database.insertedValues[0]);
|
||||
expect(persistedFailure).not.toContain("sk-live-secret");
|
||||
expect(persistedFailure).not.toContain("full resident prompt");
|
||||
expect(persistedFailure).not.toContain("Night wandering");
|
||||
expect(recordAuditLog).toHaveBeenCalledWith(expect.objectContaining({ result: "failure", reason: "AI 服务暂时不可用" }));
|
||||
});
|
||||
|
||||
it("saves schema validation failures when the model output cannot be parsed into the analysis contract", async () => {
|
||||
const database = createDatabaseFake();
|
||||
useDatabase(database);
|
||||
chatMocks.invoke.mockResolvedValue(modelMessage({
|
||||
...createModelOutput(),
|
||||
overallRiskLevel: "urgent",
|
||||
}));
|
||||
|
||||
const result = await generateElderAiAnalysis(createAuthContext(), "elder-1");
|
||||
|
||||
expect(result).toEqual({ success: false, reason: "AI 返回结构不符合固定分析格式", status: 502 });
|
||||
expect(database.insertedValues).toEqual([
|
||||
expect.objectContaining({
|
||||
status: "failed",
|
||||
errorCategory: "schema_validation_failed",
|
||||
errorReason: "AI 返回结构不符合固定分析格式",
|
||||
}),
|
||||
]);
|
||||
expect(database.insertedValues[0]).not.toHaveProperty("resultJson");
|
||||
});
|
||||
|
||||
it("degrades analysis when knowledge retrieval fails and records the retrieval audit failure", async () => {
|
||||
const database = createDatabaseFake();
|
||||
useDatabase(database);
|
||||
vi.mocked(retrieveKnowledge).mockResolvedValue({ success: false, reason: "vector store unavailable", status: 500 });
|
||||
chatMocks.invoke.mockResolvedValue(modelMessage(createModelOutput({ dataGaps: ["No gait score available."] })));
|
||||
|
||||
const result = await generateElderAiAnalysis(createAuthContext(), "elder-1");
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(recordAuditLog).toHaveBeenCalledWith(expect.objectContaining({
|
||||
action: "ai.knowledge.retrieve",
|
||||
result: "failure",
|
||||
reason: "知识库检索失败",
|
||||
}));
|
||||
expect(recordAuditLog).toHaveBeenCalledWith(expect.objectContaining({
|
||||
action: "ai.elder_analysis.generate",
|
||||
result: "success",
|
||||
}));
|
||||
expect(database.insertedValues[0]).toEqual(expect.objectContaining({
|
||||
status: "completed",
|
||||
dataScopes: ["elder", "health"],
|
||||
}));
|
||||
const resultJson = database.insertedValues[0]?.resultJson;
|
||||
expect(resultJson).toEqual(expect.objectContaining({
|
||||
dataGaps: ["No gait score available.", "知识库检索不可用，分析未使用内部知识库"],
|
||||
}));
|
||||
});
|
||||
|
||||
it("redacts analysis history when the caller lacks a scope permission stored on the history row", async () => {
|
||||
const completedRow = createAnalysisRow({
|
||||
organizationId: "org-1",
|
||||
elderId: "elder-1",
|
||||
actorAccountId: "account-1",
|
||||
status: "completed",
|
||||
dataScopes: ["elder", "health"],
|
||||
resultJson: createModelOutput(),
|
||||
citationsJson: [residentCitation],
|
||||
modelSummaryJson: createModelOutput().modelSummary,
|
||||
}, 0);
|
||||
useDatabase(createDatabaseFake([completedRow]));
|
||||
|
||||
const result = await listElderAiAnalyses(createAuthContext(["ai:read", "elder:read"]), "elder-1");
|
||||
|
||||
expect(result).toEqual({
|
||||
success: true,
|
||||
data: {
|
||||
history: [
|
||||
{
|
||||
id: "analysis-1",
|
||||
elderId: "elder-1",
|
||||
status: "completed",
|
||||
dataScopes: ["elder", "health"],
|
||||
createdAt: "2026-07-02T00:00:00.000Z",
|
||||
restricted: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects model output that cites IDs outside resident and knowledge citations", async () => {
|
||||
const database = createDatabaseFake();
|
||||
useDatabase(database);
|
||||
chatMocks.invoke.mockResolvedValue(modelMessage(createModelOutput({
|
||||
keyFindings: [
|
||||
{
|
||||
category: "fall-risk",
|
||||
severity: "warning",
|
||||
evidence: "The model references an unavailable source.",
|
||||
citationIds: ["hallucinated-source"],
|
||||
},
|
||||
],
|
||||
citations: [],
|
||||
})));
|
||||
|
||||
const result = await generateElderAiAnalysis(createAuthContext(), "elder-1");
|
||||
|
||||
expect(result).toEqual({ success: false, reason: "AI 返回结构不符合固定分析格式", status: 502 });
|
||||
expect(database.insertedValues).toEqual([
|
||||
expect.objectContaining({
|
||||
status: "failed",
|
||||
errorCategory: "schema_validation_failed",
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("persists only allowed citations actually referenced by findings or recommendations", async () => {
|
||||
const database = createDatabaseFake();
|
||||
useDatabase(database);
|
||||
mockSuccessfulContext([residentCitation, unusedResidentCitation]);
|
||||
vi.mocked(retrieveKnowledge).mockResolvedValue({
|
||||
success: true,
|
||||
data: { results: [knowledgeCitation] },
|
||||
});
|
||||
chatMocks.invoke.mockResolvedValue(modelMessage(createModelOutput({
|
||||
citations: [residentCitation],
|
||||
dataGaps: [],
|
||||
})));
|
||||
|
||||
const result = await generateElderAiAnalysis(createAuthContext(), "elder-1");
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
const resultJson = database.insertedValues[0]?.resultJson;
|
||||
expect(resultJson).toEqual(expect.objectContaining({
|
||||
citations: [residentCitation],
|
||||
}));
|
||||
expect(database.insertedValues[0]?.citationsJson).toEqual([residentCitation]);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user