365 lines
14 KiB
JavaScript
365 lines
14 KiB
JavaScript
|
// 上传附件的本地预处理层：MiniMax 负责感知，DeepSeek 只消费结构化证据。
|
|||
|
|
const crypto = require("crypto");
|
|||
|
|
const { gatewayConfig } = require("./multimodal-config");
|
|||
|
|
|
|||
|
|
const cache = new Map();
|
|||
|
|
const CACHE_TTL_MS = 10 * 60 * 1000;
|
|||
|
|
const MAX_IMAGES_PER_REQUEST = 8;
|
|||
|
|
const MAX_IMAGE_VALUE_LENGTH = 12 * 1024 * 1024;
|
|||
|
|
const MAX_BINARY_BYTES = 25 * 1024 * 1024;
|
|||
|
|
const MAX_PDF_PAGES = 20;
|
|||
|
|
const MAX_PDF_TEXT_LENGTH = 32000;
|
|||
|
|
|
|||
|
|
const SYSTEM_PROMPT = [
|
|||
|
|
"你是附件理解服务。只根据附件可见内容回答，不要猜测。",
|
|||
|
|
"返回严格 JSON，不要 Markdown：",
|
|||
|
|
'{"summary":"简短概括","items":[{"kind":"ocr|scene|ui|chart","text":"证据","confidence":0.0}]}',
|
|||
|
|
"OCR 保留原文；图表需给出数据、单位与趋势；无法确认时降低 confidence。",
|
|||
|
|
].join("\n");
|
|||
|
|
|
|||
|
|
function sha256(value) {
|
|||
|
|
return crypto.createHash("sha256").update(String(value || "")).digest("hex");
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function textFromPart(part) {
|
|||
|
|
if (!part || typeof part !== "object") return "";
|
|||
|
|
return String(part.text || part.content || "").trim();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function urlFromPart(part) {
|
|||
|
|
if (!part || typeof part !== "object") return "";
|
|||
|
|
const inputImage = part.input_image && typeof part.input_image === "object" ? part.input_image : {};
|
|||
|
|
const inputFile = part.input_file && typeof part.input_file === "object" ? part.input_file : {};
|
|||
|
|
const inputAudio = part.input_audio && typeof part.input_audio === "object" ? part.input_audio : {};
|
|||
|
|
const candidate =
|
|||
|
|
(part.image_url && (part.image_url.url || part.image_url)) ||
|
|||
|
|
(part.file_url && (part.file_url.url || part.file_url)) ||
|
|||
|
|
(part.audio_url && (part.audio_url.url || part.audio_url)) ||
|
|||
|
|
inputImage.image_url ||
|
|||
|
|
inputImage.url ||
|
|||
|
|
inputImage.data ||
|
|||
|
|
inputFile.file_url ||
|
|||
|
|
inputFile.file_data ||
|
|||
|
|
inputFile.url ||
|
|||
|
|
inputFile.data ||
|
|||
|
|
inputAudio.audio_url ||
|
|||
|
|
inputAudio.url ||
|
|||
|
|
inputAudio.data ||
|
|||
|
|
part.url ||
|
|||
|
|
part.data ||
|
|||
|
|
part.file_data ||
|
|||
|
|
"";
|
|||
|
|
return typeof candidate === "string" ? candidate.trim() : "";
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function kindFromPart(part, value) {
|
|||
|
|
const type = String((part && part.type) || "").toLowerCase();
|
|||
|
|
const name = String((part && (part.filename || part.name)) || "").toLowerCase();
|
|||
|
|
const mime = String((part && (part.mime_type || part.mimeType)) || "").toLowerCase();
|
|||
|
|
if (type.includes("image") || /^data:image\//i.test(value) || /\.(png|jpe?g|gif|webp|bmp)$/i.test(name)) return "image";
|
|||
|
|
if (type.includes("audio") || /^data:audio\//i.test(value) || /\.(mp3|wav|m4a|aac|ogg|webm)$/i.test(name)) return "audio";
|
|||
|
|
if (type.includes("file") || type.includes("pdf") || mime.includes("pdf") || /\.pdf$/i.test(name)) return "document";
|
|||
|
|
return "";
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function collectAttachments(request) {
|
|||
|
|
const attachments = [];
|
|||
|
|
const input = request && request.input;
|
|||
|
|
if (!Array.isArray(input)) return attachments;
|
|||
|
|
for (const message of input) {
|
|||
|
|
if (!message || !Array.isArray(message.content)) continue;
|
|||
|
|
for (const part of message.content) {
|
|||
|
|
const value = urlFromPart(part);
|
|||
|
|
const kind = kindFromPart(part, value);
|
|||
|
|
if (!kind) continue;
|
|||
|
|
const nestedFile = part.input_file && typeof part.input_file === "object" ? part.input_file : {};
|
|||
|
|
const nestedAudio = part.input_audio && typeof part.input_audio === "object" ? part.input_audio : {};
|
|||
|
|
const format = String(nestedAudio.format || "").trim().toLowerCase();
|
|||
|
|
const name = String(
|
|||
|
|
part.filename ||
|
|||
|
|
part.name ||
|
|||
|
|
nestedFile.filename ||
|
|||
|
|
nestedFile.name ||
|
|||
|
|
(kind === "audio" && format ? `audio.${format}` : "")
|
|||
|
|
);
|
|||
|
|
attachments.push({
|
|||
|
|
id: `${kind}:${sha256(value || `${name}:${textFromPart(part)}`)}`,
|
|||
|
|
kind,
|
|||
|
|
value,
|
|||
|
|
name,
|
|||
|
|
mimeType: String(part.mime_type || part.mimeType || nestedFile.mime_type || nestedFile.mimeType || ""),
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return attachments;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function extractContentText(payload) {
|
|||
|
|
const message = payload && payload.choices && payload.choices[0] && payload.choices[0].message;
|
|||
|
|
const content = message && message.content;
|
|||
|
|
if (typeof content === "string") return content;
|
|||
|
|
if (Array.isArray(content)) {
|
|||
|
|
return content.map((part) => (typeof part === "string" ? part : (part && (part.text || part.content)) || "")).join("");
|
|||
|
|
}
|
|||
|
|
return "";
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function parseEvidence(text) {
|
|||
|
|
const raw = String(text || "").trim();
|
|||
|
|
const fenced = raw.replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/i, "");
|
|||
|
|
try {
|
|||
|
|
const parsed = JSON.parse(fenced);
|
|||
|
|
return {
|
|||
|
|
summary: String(parsed.summary || "").trim(),
|
|||
|
|
items: Array.isArray(parsed.items)
|
|||
|
|
? parsed.items.slice(0, 40).map((item) => ({
|
|||
|
|
kind: String((item && item.kind) || "scene"),
|
|||
|
|
text: String((item && item.text) || "").trim(),
|
|||
|
|
confidence: Number.isFinite(Number(item && item.confidence)) ? Number(item.confidence) : null,
|
|||
|
|
})).filter((item) => item.text)
|
|||
|
|
: [],
|
|||
|
|
};
|
|||
|
|
} catch {
|
|||
|
|
return { summary: raw.slice(0, 6000), items: [] };
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function chatEndpoint(baseUrl) {
|
|||
|
|
const base = String(baseUrl || "").replace(/\/+$/, "");
|
|||
|
|
if (/\/chat\/completions$/i.test(base)) return base;
|
|||
|
|
if (/\/v\d+$/i.test(base)) return `${base}/chat/completions`;
|
|||
|
|
return `${base}/v1/chat/completions`;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function transcriptionEndpoint(baseUrl) {
|
|||
|
|
const base = String(baseUrl || "").replace(/\/+$/, "");
|
|||
|
|
if (/\/v\d+$/i.test(base)) return `${base}/audio/transcriptions`;
|
|||
|
|
return `${base}/v1/audio/transcriptions`;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function binaryFromAsset(asset, requestFetch) {
|
|||
|
|
const value = String(asset.value || "").trim();
|
|||
|
|
if (!value) throw new Error("附件没有可读取的数据或 URL");
|
|||
|
|
if (/^data:/i.test(value)) {
|
|||
|
|
const comma = value.indexOf(",");
|
|||
|
|
if (comma < 0) throw new Error("附件 data URL 格式无效");
|
|||
|
|
const meta = value.slice(0, comma);
|
|||
|
|
const body = value.slice(comma + 1);
|
|||
|
|
const buffer = /;base64/i.test(meta)
|
|||
|
|
? Buffer.from(body, "base64")
|
|||
|
|
: Buffer.from(decodeURIComponent(body), "utf-8");
|
|||
|
|
if (buffer.length > MAX_BINARY_BYTES) throw new Error("附件超过 25MB 限制");
|
|||
|
|
return buffer;
|
|||
|
|
}
|
|||
|
|
if (/^https?:\/\//i.test(value)) {
|
|||
|
|
const response = await requestFetch(value, { headers: { "user-agent": "codex-market" } });
|
|||
|
|
if (!response.ok) throw new Error(`读取附件失败（HTTP ${response.status}）`);
|
|||
|
|
const size = Number(response.headers && response.headers.get && response.headers.get("content-length"));
|
|||
|
|
if (Number.isFinite(size) && size > MAX_BINARY_BYTES) throw new Error("附件超过 25MB 限制");
|
|||
|
|
const buffer = Buffer.from(await response.arrayBuffer());
|
|||
|
|
if (buffer.length > MAX_BINARY_BYTES) throw new Error("附件超过 25MB 限制");
|
|||
|
|
return buffer;
|
|||
|
|
}
|
|||
|
|
const buffer = Buffer.from(value, "base64");
|
|||
|
|
if (!buffer.length || buffer.length > MAX_BINARY_BYTES) throw new Error("附件内容无效或超过 25MB 限制");
|
|||
|
|
return buffer;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function understandPdf(asset, requestFetch) {
|
|||
|
|
const data = await binaryFromAsset(asset, requestFetch);
|
|||
|
|
const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs");
|
|||
|
|
const task = pdfjs.getDocument({ data: new Uint8Array(data), isEvalSupported: false });
|
|||
|
|
const document = await task.promise;
|
|||
|
|
const items = [];
|
|||
|
|
for (let pageNumber = 1; pageNumber <= Math.min(document.numPages, MAX_PDF_PAGES); pageNumber += 1) {
|
|||
|
|
const page = await document.getPage(pageNumber);
|
|||
|
|
const content = await page.getTextContent();
|
|||
|
|
const text = content.items.map((item) => String(item.str || "")).join(" ").replace(/\s+/g, " ").trim();
|
|||
|
|
if (text) items.push({ kind: "document", text: text.slice(0, MAX_PDF_TEXT_LENGTH), page: pageNumber, confidence: 1 });
|
|||
|
|
}
|
|||
|
|
if (!items.length) {
|
|||
|
|
return {
|
|||
|
|
summary: "PDF 未提取到可复制文本，可能是扫描件；需要后续 OCR/页面视觉解析。",
|
|||
|
|
items: [],
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
return {
|
|||
|
|
summary: `PDF 共 ${document.numPages} 页，已提取 ${items.length} 页文本。`,
|
|||
|
|
items,
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function audioMimeType(asset) {
|
|||
|
|
if (asset.mimeType) return asset.mimeType;
|
|||
|
|
const name = String(asset.name || "").toLowerCase();
|
|||
|
|
if (name.endsWith(".wav")) return "audio/wav";
|
|||
|
|
if (name.endsWith(".mp3")) return "audio/mpeg";
|
|||
|
|
if (name.endsWith(".m4a")) return "audio/mp4";
|
|||
|
|
if (name.endsWith(".aac")) return "audio/aac";
|
|||
|
|
if (name.endsWith(".ogg")) return "audio/ogg";
|
|||
|
|
return "audio/webm";
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function understandAudio(asset, config, requestFetch) {
|
|||
|
|
const data = await binaryFromAsset(asset, requestFetch);
|
|||
|
|
const mimeType = audioMimeType(asset);
|
|||
|
|
const filename = asset.name || "audio.webm";
|
|||
|
|
const form = new FormData();
|
|||
|
|
form.append("model", config.sttModel);
|
|||
|
|
form.append("file", new Blob([data], { type: mimeType }), filename);
|
|||
|
|
const response = await requestFetch(transcriptionEndpoint(config.baseUrl), {
|
|||
|
|
method: "POST",
|
|||
|
|
headers: {
|
|||
|
|
authorization: `Bearer ${config.key}`,
|
|||
|
|
"user-agent": "codex-market",
|
|||
|
|
},
|
|||
|
|
body: form,
|
|||
|
|
});
|
|||
|
|
if (!response.ok) {
|
|||
|
|
const detail = (await response.text().catch(() => "")).slice(0, 300);
|
|||
|
|
throw new Error(`音频转写失败（HTTP ${response.status}${detail ? `：${detail}` : ""}）`);
|
|||
|
|
}
|
|||
|
|
const result = await response.json();
|
|||
|
|
const segments = Array.isArray(result.segments) ? result.segments : [];
|
|||
|
|
const items = segments.map((segment) => ({
|
|||
|
|
kind: "transcript",
|
|||
|
|
text: String(segment.text || "").trim(),
|
|||
|
|
start_ms: Number.isFinite(Number(segment.start)) ? Math.round(Number(segment.start) * 1000) : null,
|
|||
|
|
end_ms: Number.isFinite(Number(segment.end)) ? Math.round(Number(segment.end) * 1000) : null,
|
|||
|
|
confidence: null,
|
|||
|
|
})).filter((item) => item.text);
|
|||
|
|
return {
|
|||
|
|
summary: String(result.text || items.map((item) => item.text).join(" ")).slice(0, MAX_PDF_TEXT_LENGTH),
|
|||
|
|
items,
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function understandImages(images, instruction, config, requestFetch) {
|
|||
|
|
const content = [
|
|||
|
|
{ type: "text", text: `用户问题：${String(instruction || "请识别附件内容").slice(0, 4000)}` },
|
|||
|
|
...images.map((asset) => ({ type: "image_url", image_url: { url: asset.value } })),
|
|||
|
|
];
|
|||
|
|
const response = await requestFetch(chatEndpoint(config.baseUrl), {
|
|||
|
|
method: "POST",
|
|||
|
|
headers: {
|
|||
|
|
"content-type": "application/json",
|
|||
|
|
authorization: `Bearer ${config.key}`,
|
|||
|
|
"user-agent": "codex-market",
|
|||
|
|
},
|
|||
|
|
body: JSON.stringify({
|
|||
|
|
model: config.model,
|
|||
|
|
temperature: 0.1,
|
|||
|
|
max_tokens: 1800,
|
|||
|
|
stream: false,
|
|||
|
|
messages: [
|
|||
|
|
{ role: "system", content: SYSTEM_PROMPT },
|
|||
|
|
{ role: "user", content },
|
|||
|
|
],
|
|||
|
|
}),
|
|||
|
|
});
|
|||
|
|
if (!response.ok) {
|
|||
|
|
const detail = (await response.text().catch(() => "")).slice(0, 300);
|
|||
|
|
throw new Error(`MiniMax 识别失败（HTTP ${response.status}${detail ? `：${detail}` : ""}）`);
|
|||
|
|
}
|
|||
|
|
return parseEvidence(extractContentText(await response.json()));
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function evidenceText(evidence) {
|
|||
|
|
return `[TD 多模态识别证据]\n${JSON.stringify(evidence)}`;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function appendEvidence(request, evidence) {
|
|||
|
|
if (!Array.isArray(request.input)) return;
|
|||
|
|
let target = null;
|
|||
|
|
for (let i = request.input.length - 1; i >= 0; i -= 1) {
|
|||
|
|
const item = request.input[i];
|
|||
|
|
if (item && item.type === "message" && item.role === "user" && Array.isArray(item.content)) {
|
|||
|
|
target = item;
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if (!target) {
|
|||
|
|
target = { type: "message", role: "user", content: [] };
|
|||
|
|
request.input.push(target);
|
|||
|
|
}
|
|||
|
|
target.content.push({ type: "input_text", text: evidenceText(evidence) });
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function cleanupCache(now) {
|
|||
|
|
for (const [key, value] of cache) if (now - value.at > CACHE_TTL_MS) cache.delete(key);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function enrichResponsesRequest(request, { instruction = "", fetchImpl = global.fetch } = {}) {
|
|||
|
|
const config = gatewayConfig();
|
|||
|
|
const attachments = collectAttachments(request);
|
|||
|
|
if (!attachments.length) return { processed: false, attachments: [] };
|
|||
|
|
const imageAssets = attachments
|
|||
|
|
.filter((asset) => asset.kind === "image" && asset.value && asset.value.length <= MAX_IMAGE_VALUE_LENGTH)
|
|||
|
|
.slice(0, MAX_IMAGES_PER_REQUEST);
|
|||
|
|
const evidence = {
|
|||
|
|
version: 1,
|
|||
|
|
provider: "minimax",
|
|||
|
|
modality: attachments.length > 1 ? "mixed" : attachments[0].kind,
|
|||
|
|
summary: "",
|
|||
|
|
items: [],
|
|||
|
|
unsupported: [],
|
|||
|
|
};
|
|||
|
|
const summaries = [];
|
|||
|
|
|
|||
|
|
if (imageAssets.length) {
|
|||
|
|
if (!config.key) {
|
|||
|
|
summaries.push("已检测到图片附件，但免费多模态网关尚未注入访问密钥，不能识别图片内容。不要声称已查看或识别该图片。");
|
|||
|
|
} else {
|
|||
|
|
cleanupCache(Date.now());
|
|||
|
|
const cacheKey = sha256(`${config.model}:${instruction}:${imageAssets.map((asset) => asset.id).join(",")}`);
|
|||
|
|
const cached = cache.get(cacheKey);
|
|||
|
|
if (cached) {
|
|||
|
|
summaries.push(cached.value.summary);
|
|||
|
|
evidence.items.push(...cached.value.items);
|
|||
|
|
} else {
|
|||
|
|
try {
|
|||
|
|
const understood = await understandImages(imageAssets, instruction, config, fetchImpl);
|
|||
|
|
summaries.push(understood.summary);
|
|||
|
|
evidence.items.push(...understood.items);
|
|||
|
|
cache.set(cacheKey, { at: Date.now(), value: understood });
|
|||
|
|
} catch (error) {
|
|||
|
|
summaries.push(String((error && error.message) || "MiniMax 图片识别失败"));
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
for (const asset of attachments.filter((item) => item.kind === "document")) {
|
|||
|
|
try {
|
|||
|
|
const understood = await understandPdf(asset, fetchImpl);
|
|||
|
|
summaries.push(understood.summary);
|
|||
|
|
evidence.items.push(...understood.items);
|
|||
|
|
} catch (error) {
|
|||
|
|
evidence.unsupported.push({ kind: "document", name: asset.name || "未命名附件" });
|
|||
|
|
summaries.push(String((error && error.message) || "PDF 解析失败"));
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
for (const asset of attachments.filter((item) => item.kind === "audio")) {
|
|||
|
|
if (!config.key || !config.sttModel) {
|
|||
|
|
evidence.unsupported.push({ kind: "audio", name: asset.name || "未命名附件" });
|
|||
|
|
summaries.push("音频已检测到，但免费多模态网关尚未配置音频转写模型。");
|
|||
|
|
continue;
|
|||
|
|
}
|
|||
|
|
try {
|
|||
|
|
const understood = await understandAudio(asset, config, fetchImpl);
|
|||
|
|
summaries.push(understood.summary);
|
|||
|
|
evidence.items.push(...understood.items);
|
|||
|
|
} catch (error) {
|
|||
|
|
summaries.push(String((error && error.message) || "音频转写失败"));
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
evidence.summary = summaries.filter(Boolean).join("\n").slice(0, MAX_PDF_TEXT_LENGTH);
|
|||
|
|
appendEvidence(request, evidence);
|
|||
|
|
return { processed: true, attachments, evidence };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
module.exports = {
|
|||
|
|
collectAttachments,
|
|||
|
|
enrichResponsesRequest,
|
|||
|
|
parseEvidence,
|
|||
|
|
};
|