import { describe, expect, it } from "bun:test";
import {
findResearchQuestion,
isPublicWebSearch,
observedWebSourceUrls,
publicResearchFailureReason,
publicResearchResultFailure,
researchQuestionDefinition,
researchQuestionSnapshot,
} from "./research-question";
import { parse } from "@chopin/dialect/parse";
import { sourceHash } from "../plan/service";
import type { JobExecution } from "./registry";
import type { ResearchQuestionInput } from "./research-question";
import type { DocumentTarget } from "../plan/service";
import type { BackgroundJob } from "../storage/model";
const ID = "01K0N4TR8K7JGM4R1J7PW4R8YJ";
const SOURCE = `\n\nWhat changed in the API?\n\n\n`;
const QUESTION = "What changed in the API?";
const QUESTION_HASH = researchQuestionSnapshot(SOURCE, ID)!.questionHash;
function target(source = SOURCE, revision = 3): DocumentTarget {
return { channelId: "channel", revision, source, sourceHash: sourceHash(source) };
}
function execution(
value: DocumentTarget,
progress: JobExecution["progress"] = async () => {},
): JobExecution {
let now = new Date();
let input = {
questionId: ID,
question: QUESTION,
questionHash: QUESTION_HASH,
revision: value.revision,
};
let job: BackgroundJob = {
id: "job",
channelId: value.channelId,
type: "research-question",
version: 1,
origin: "user",
targetKey: `research-question:${ID}`,
targetGeneration: 1,
idempotencyKey: "request",
fingerprint: "fingerprint",
input,
state: "running",
revision: 2,
attempts: 1,
failures: 0,
claimGeneration: 1,
claimOwner: "worker",
claimBinding: undefined,
claimExpiresAt: new Date(now.getTime() + 60_000),
availableAt: now,
reason: undefined,
progress: [],
createdAt: now,
updatedAt: now,
};
return {
job,
input,
credential: {
kind: "active-planner",
token: "ghu_owner",
ownerSessionId: "session",
ownerGeneration: 2,
credentialRevision: 3,
expiresAt: new Date(now.getTime() + 60_000),
authorize: async () => true,
},
signal: new AbortController().signal,
deadline: new Date(now.getTime() + 60_000),
progress,
};
}
describe("research question job", () => {
it("extracts only public source URLs directly present in web-search output", () => {
let rich = JSON.stringify({
text: {
value: "Answer",
annotations: [{
url_citation: { title: "Rich", url: "https://rich.example/source" },
}],
},
});
let urls = observedWebSourceUrls({
citableSources: [{ url: "https://citable.example/source" }],
structuredContent: {
text: { annotations: [{ url_citation: { url: "https://structured.example/source" } }] },
},
content: rich,
detailedContent: "[Detailed](https://detailed.example/source)",
contents: [{ type: "text", text: "Source: " }, {
type: "resource_link",
uri: "https://resource.example/source",
}, {
type: "resource",
resource: { uri: "https://embedded.example/source", text: rich },
}],
mcpMeta: {
url_citation: { url: "https://metadata-only.example/source" },
},
});
expect(new Set(urls)).toEqual(
new Set([
"https://citable.example/source",
"https://structured.example/source",
"https://rich.example/source",
"https://resource.example/source",
"https://embedded.example/source",
]),
);
});
it("rejects unsafe and metadata-only web-search URLs", () => {
let urls = observedWebSourceUrls({
structuredContent: {
text: {
annotations: [
{ url_citation: { url: "http://insecure.example/source" } },
{ url_citation: { url: "https://user:secret@example.com/source" } },
{ url_citation: { url: "https://example.com:8443/source" } },
{ url_citation: { url: "https://127.0.0.1/source" } },
{ url_citation: { url: "https://10.0.0.1/source" } },
{ url_citation: { url: "https://100.64.0.1/source" } },
{ url_citation: { url: "https://198.18.0.1/source" } },
{ url_citation: { url: "https://app.localhost/source" } },
{ url_citation: { url: "https://safe.example/source" } },
],
},
},
content: [
"",
"",
"",
"",
"",
"",
"",
"",
"",
"A bare mention is not provenance: https://bare.example/source",
].join(" "),
mcpMeta: { url_citation: { url: "https://metadata-only.example/source" } },
});
expect(urls).toEqual(["https://safe.example/source"]);
});
it("bounds web-search provenance extraction", () => {
let citableSources = Array.from({ length: 100 }, (_, index) => ({
url: `https://source-${index}.example/item`,
}));
expect(observedWebSourceUrls({ citableSources })).toHaveLength(64);
expect(observedWebSourceUrls({
citableSources: Array.from({ length: 5_000 }, () => ({ url: "not a URL" })),
})).toEqual([]);
});
it("classifies unavailable public web search without exposing provider errors", () => {
expect(publicResearchFailureReason(new Error("Background worker MCP readiness timed out.")))
.toBe("web-search-unavailable");
expect(publicResearchFailureReason(new Error("Background worker capability audit failed.")))
.toBe("web-search-unavailable");
expect(publicResearchFailureReason(new Error("provider failed")))
.toBe("public-research-failed");
});
it("distinguishes public search and result failures", () => {
let base = {
webCalls: 1,
webSuccesses: 1,
webFailures: 0,
hostedCalls: 0,
hostedCompleted: 0,
resultInvalid: false,
webSearchDenied: false,
};
expect(publicResearchResultFailure(undefined, new Set(), { ...base, webCalls: 0 }))
.toBe("web-search-not-invoked");
expect(publicResearchResultFailure(undefined, new Set(), {
...base,
webSuccesses: 0,
webFailures: 1,
})).toBe("web-search-failed");
expect(publicResearchResultFailure(undefined, new Set(), base))
.toBe("research-result-missing");
expect(publicResearchResultFailure(undefined, new Set(), { ...base, resultInvalid: true }))
.toBe("research-result-invalid");
expect(publicResearchResultFailure(undefined, new Set(), { ...base, webSearchDenied: true }))
.toBe("research-permission-denied");
expect(publicResearchResultFailure({ findings: [], sources: [] }, new Set(), {
...base,
webSuccesses: 0,
webFailures: 1,
})).toBe("web-search-failed");
expect(publicResearchResultFailure({ findings: [], sources: [] }, new Set(), base))
.toBeUndefined();
let evidence = {
findings: ["A finding"],
sources: [{ title: "Source", url: "https://example.com/source" }],
};
expect(publicResearchResultFailure(evidence, new Set(), base))
.toBe("research-sources-unverifiable");
expect(publicResearchResultFailure(evidence, new Set(), {
...base,
hostedCalls: 1,
hostedCompleted: 1,
})).toBe("hosted-search-sources-unverifiable");
expect(publicResearchResultFailure(undefined, new Set(), {
...base,
webSuccesses: 0,
hostedCalls: 1,
hostedCompleted: 0,
})).toBe("web-search-failed");
expect(publicResearchResultFailure(evidence, new Set(["https://example.com/other"]), base))
.toBe("research-source-mismatch");
expect(publicResearchResultFailure(evidence, new Set(["https://example.com/source"]), base))
.toBeUndefined();
});
it("extracts the uniquely identified question prose", () => {
expect(findResearchQuestion(parse(SOURCE), ID)).toBe(QUESTION);
expect(findResearchQuestion(parse(SOURCE), "01K0N4V4E7Y6P4MJ5WD8XZF3B2"))
.toBeUndefined();
});
it("recognizes only the permission-visible built-in web search", () => {
expect(isPublicWebSearch({
mcpServerName: "github-mcp-server",
mcpToolName: "web_search",
})).toBe(true);
expect(isPublicWebSearch({ mcpServerName: "github", mcpToolName: "web_search" })).toBe(false);
expect(isPublicWebSearch({ mcpServerName: "github-mcp-server", mcpToolName: "get_file" }))
.toBe(false);
});
it("keeps public and private research inputs separated before synthesis", async () => {
let current = target();
let calls: string[] = [];
let progress: string[] = [];
let definition = researchQuestionDefinition({
config: { agent: true, model: "research-model" },
current: async () => current,
commitCurrent: async (_channel, _id, _hash, _revision, _sourceHash, commit) => {
await commit();
return true;
},
engines: {
public: async (_execution, question) => {
calls.push(`public:${question}`);
return {
findings: ["A public finding"],
sources: [{ title: "Source", url: "https://example.com/source" }],
};
},
private: async (_execution, question, source) => {
calls.push(`private:${question}:${source === SOURCE}`);
return { findings: ["A private finding"] };
},
synthesize: async (_execution, question, publicValue, privateValue) => {
calls.push(
`synthesis:${question}:${publicValue.findings.length}:${privateValue.findings.length}`,
);
return {
title: "API research",
summary: "Combined report",
findings: [{ text: "Finding", sourceUrls: ["https://example.com/source"] }],
caveats: [],
};
},
},
});
expect(definition.limits.maxAiCredits).toBe(90);
let artifact = await definition.execute(execution(current, async (stage, state) => {
progress.push(`${stage}:${state}`);
}));
expect(calls).toEqual([
`public:${QUESTION}`,
`private:${QUESTION}:true`,
`synthesis:${QUESTION}:1:1`,
]);
expect(progress).toEqual([
"public-web:started",
"public-web:completed",
"private-document:started",
"private-document:completed",
"report-synthesis:started",
"report-synthesis:completed",
]);
expect(artifact).toMatchObject({
questionId: ID,
questionHash: QUESTION_HASH,
documentRevision: 3,
documentSourceHash: current.sourceHash,
model: "research-model",
sources: [{ url: "https://example.com/source" }],
report: { summary: "Combined report" },
});
});
it("rejects changed questions and unknown synthesized citations", async () => {
let current = target(
`\n\nA different question?\n\n\n`,
4,
);
let definition = researchQuestionDefinition({
config: { agent: true, model: "research-model" },
current: async () => current,
commitCurrent: async () => false,
engines: {
public: async () => ({ findings: [], sources: [] }),
private: async () => ({ findings: [] }),
synthesize: async () => ({
title: "Report",
summary: "Summary",
findings: [],
caveats: [],
}),
},
});
await expect(definition.execute(execution(target()))).rejects.toThrow("changed");
let progress: string[] = [];
let valid = execution(target(), async (stage, state) => {
progress.push(`${stage}:${state}`);
});
current = target();
let citing = researchQuestionDefinition({
config: { agent: true, model: "research-model" },
current: async () => current,
commitCurrent: async () => false,
engines: {
public: async () => ({ findings: [], sources: [] }),
private: async () => ({ findings: [] }),
synthesize: async () => ({
title: "Report",
summary: "Summary",
findings: [{ text: "Claim", sourceUrls: ["https://unknown.example/source"] }],
caveats: [],
}),
},
});
await expect(citing.execute(valid)).rejects.toMatchObject({
progressReason: "source-validation-failed",
});
expect(progress.at(-1)).toBe("report-synthesis:started");
expect(progress).not.toContain("report-synthesis:completed");
});
it("validates the immutable question snapshot", () => {
let definition = researchQuestionDefinition({
config: { agent: true, model: "research-model" },
current: async () => target(),
commitCurrent: async () => false,
engines: {
public: async () => ({ findings: [], sources: [] }),
private: async () => ({ findings: [] }),
synthesize: async () => ({
title: "Report",
summary: "Summary",
findings: [],
caveats: [],
}),
},
});
expect(() =>
definition.input.parse({
questionId: ID,
question: QUESTION,
questionHash: "bad-hash",
revision: 3,
})
).toThrow("hash");
});
});