// 上传附件的本地预处理层:MiniMax 负责感知,DeepSeek 只消费结构化证据。 const crypto = require("crypto"); const { gatewayConfig } = require("./multimodal-config"); const cache = new Map(); const CACHE_TTL_MS = 10 * 60 * 1000; const MAX_IMAGES_PER_REQUEST = 8; const MAX_IMAGE_VALUE_LENGTH = 12 * 1024 * 1024; const MAX_BINARY_BYTES = 25 * 1024 * 1024; const MAX_PDF_PAGES = 20; const MAX_PDF_TEXT_LENGTH = 32000; const SYSTEM_PROMPT = [ "你是附件理解服务。只根据附件可见内容回答,不要猜测。", "返回严格 JSON,不要 Markdown:", '{"summary":"简短概括","items":[{"kind":"ocr|scene|ui|chart","text":"证据","confidence":0.0}]}', "OCR 保留原文;图表需给出数据、单位与趋势;无法确认时降低 confidence。", ].join("\n"); function sha256(value) { return crypto.createHash("sha256").update(String(value || "")).digest("hex"); } function textFromPart(part) { if (!part || typeof part !== "object") return ""; return String(part.text || part.content || "").trim(); } function urlFromPart(part) { if (!part || typeof part !== "object") return ""; const inputImage = part.input_image && typeof part.input_image === "object" ? part.input_image : {}; const inputFile = part.input_file && typeof part.input_file === "object" ? part.input_file : {}; const inputAudio = part.input_audio && typeof part.input_audio === "object" ? part.input_audio : {}; const candidate = (part.image_url && (part.image_url.url || part.image_url)) || (part.file_url && (part.file_url.url || part.file_url)) || (part.audio_url && (part.audio_url.url || part.audio_url)) || inputImage.image_url || inputImage.url || inputImage.data || inputFile.file_url || inputFile.file_data || inputFile.url || inputFile.data || inputAudio.audio_url || inputAudio.url || inputAudio.data || part.url || part.data || part.file_data || ""; return typeof candidate === "string" ? candidate.trim() : ""; } function kindFromPart(part, value) { const type = String((part && part.type) || "").toLowerCase(); const name = String((part && (part.filename || part.name)) || "").toLowerCase(); const mime = String((part && (part.mime_type || part.mimeType)) || "").toLowerCase(); if (type.includes("image") || /^data:image\//i.test(value) || /\.(png|jpe?g|gif|webp|bmp)$/i.test(name)) return "image"; if (type.includes("audio") || /^data:audio\//i.test(value) || /\.(mp3|wav|m4a|aac|ogg|webm)$/i.test(name)) return "audio"; if (type.includes("file") || type.includes("pdf") || mime.includes("pdf") || /\.pdf$/i.test(name)) return "document"; return ""; } function collectAttachments(request) { const attachments = []; const input = request && request.input; if (!Array.isArray(input)) return attachments; for (const message of input) { if (!message || !Array.isArray(message.content)) continue; for (const part of message.content) { const value = urlFromPart(part); const kind = kindFromPart(part, value); if (!kind) continue; const nestedFile = part.input_file && typeof part.input_file === "object" ? part.input_file : {}; const nestedAudio = part.input_audio && typeof part.input_audio === "object" ? part.input_audio : {}; const format = String(nestedAudio.format || "").trim().toLowerCase(); const name = String( part.filename || part.name || nestedFile.filename || nestedFile.name || (kind === "audio" && format ? `audio.${format}` : "") ); attachments.push({ id: `${kind}:${sha256(value || `${name}:${textFromPart(part)}`)}`, kind, value, name, mimeType: String(part.mime_type || part.mimeType || nestedFile.mime_type || nestedFile.mimeType || ""), }); } } return attachments; } function extractContentText(payload) { const message = payload && payload.choices && payload.choices[0] && payload.choices[0].message; const content = message && message.content; if (typeof content === "string") return content; if (Array.isArray(content)) { return content.map((part) => (typeof part === "string" ? part : (part && (part.text || part.content)) || "")).join(""); } return ""; } function parseEvidence(text) { const raw = String(text || "").trim(); const fenced = raw.replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/i, ""); try { const parsed = JSON.parse(fenced); return { summary: String(parsed.summary || "").trim(), items: Array.isArray(parsed.items) ? parsed.items.slice(0, 40).map((item) => ({ kind: String((item && item.kind) || "scene"), text: String((item && item.text) || "").trim(), confidence: Number.isFinite(Number(item && item.confidence)) ? Number(item.confidence) : null, })).filter((item) => item.text) : [], }; } catch { return { summary: raw.slice(0, 6000), items: [] }; } } function chatEndpoint(baseUrl) { const base = String(baseUrl || "").replace(/\/+$/, ""); if (/\/chat\/completions$/i.test(base)) return base; if (/\/v\d+$/i.test(base)) return `${base}/chat/completions`; return `${base}/v1/chat/completions`; } function transcriptionEndpoint(baseUrl) { const base = String(baseUrl || "").replace(/\/+$/, ""); if (/\/v\d+$/i.test(base)) return `${base}/audio/transcriptions`; return `${base}/v1/audio/transcriptions`; } async function binaryFromAsset(asset, requestFetch) { const value = String(asset.value || "").trim(); if (!value) throw new Error("附件没有可读取的数据或 URL"); if (/^data:/i.test(value)) { const comma = value.indexOf(","); if (comma < 0) throw new Error("附件 data URL 格式无效"); const meta = value.slice(0, comma); const body = value.slice(comma + 1); const buffer = /;base64/i.test(meta) ? Buffer.from(body, "base64") : Buffer.from(decodeURIComponent(body), "utf-8"); if (buffer.length > MAX_BINARY_BYTES) throw new Error("附件超过 25MB 限制"); return buffer; } if (/^https?:\/\//i.test(value)) { const response = await requestFetch(value, { headers: { "user-agent": "codex-market" } }); if (!response.ok) throw new Error(`读取附件失败(HTTP ${response.status})`); const size = Number(response.headers && response.headers.get && response.headers.get("content-length")); if (Number.isFinite(size) && size > MAX_BINARY_BYTES) throw new Error("附件超过 25MB 限制"); const buffer = Buffer.from(await response.arrayBuffer()); if (buffer.length > MAX_BINARY_BYTES) throw new Error("附件超过 25MB 限制"); return buffer; } const buffer = Buffer.from(value, "base64"); if (!buffer.length || buffer.length > MAX_BINARY_BYTES) throw new Error("附件内容无效或超过 25MB 限制"); return buffer; } async function understandPdf(asset, requestFetch) { const data = await binaryFromAsset(asset, requestFetch); const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs"); const task = pdfjs.getDocument({ data: new Uint8Array(data), isEvalSupported: false }); const document = await task.promise; const items = []; for (let pageNumber = 1; pageNumber <= Math.min(document.numPages, MAX_PDF_PAGES); pageNumber += 1) { const page = await document.getPage(pageNumber); const content = await page.getTextContent(); const text = content.items.map((item) => String(item.str || "")).join(" ").replace(/\s+/g, " ").trim(); if (text) items.push({ kind: "document", text: text.slice(0, MAX_PDF_TEXT_LENGTH), page: pageNumber, confidence: 1 }); } if (!items.length) { return { summary: "PDF 未提取到可复制文本,可能是扫描件;需要后续 OCR/页面视觉解析。", items: [], }; } return { summary: `PDF 共 ${document.numPages} 页,已提取 ${items.length} 页文本。`, items, }; } function audioMimeType(asset) { if (asset.mimeType) return asset.mimeType; const name = String(asset.name || "").toLowerCase(); if (name.endsWith(".wav")) return "audio/wav"; if (name.endsWith(".mp3")) return "audio/mpeg"; if (name.endsWith(".m4a")) return "audio/mp4"; if (name.endsWith(".aac")) return "audio/aac"; if (name.endsWith(".ogg")) return "audio/ogg"; return "audio/webm"; } async function understandAudio(asset, config, requestFetch) { const data = await binaryFromAsset(asset, requestFetch); const mimeType = audioMimeType(asset); const filename = asset.name || "audio.webm"; const form = new FormData(); form.append("model", config.sttModel); form.append("file", new Blob([data], { type: mimeType }), filename); const response = await requestFetch(transcriptionEndpoint(config.baseUrl), { method: "POST", headers: { authorization: `Bearer ${config.key}`, "user-agent": "codex-market", }, body: form, }); if (!response.ok) { const detail = (await response.text().catch(() => "")).slice(0, 300); throw new Error(`音频转写失败(HTTP ${response.status}${detail ? `:${detail}` : ""})`); } const result = await response.json(); const segments = Array.isArray(result.segments) ? result.segments : []; const items = segments.map((segment) => ({ kind: "transcript", text: String(segment.text || "").trim(), start_ms: Number.isFinite(Number(segment.start)) ? Math.round(Number(segment.start) * 1000) : null, end_ms: Number.isFinite(Number(segment.end)) ? Math.round(Number(segment.end) * 1000) : null, confidence: null, })).filter((item) => item.text); return { summary: String(result.text || items.map((item) => item.text).join(" ")).slice(0, MAX_PDF_TEXT_LENGTH), items, }; } async function understandImages(images, instruction, config, requestFetch) { const content = [ { type: "text", text: `用户问题:${String(instruction || "请识别附件内容").slice(0, 4000)}` }, ...images.map((asset) => ({ type: "image_url", image_url: { url: asset.value } })), ]; const response = await requestFetch(chatEndpoint(config.baseUrl), { method: "POST", headers: { "content-type": "application/json", authorization: `Bearer ${config.key}`, "user-agent": "codex-market", }, body: JSON.stringify({ model: config.model, temperature: 0.1, max_tokens: 1800, stream: false, messages: [ { role: "system", content: SYSTEM_PROMPT }, { role: "user", content }, ], }), }); if (!response.ok) { const detail = (await response.text().catch(() => "")).slice(0, 300); throw new Error(`MiniMax 识别失败(HTTP ${response.status}${detail ? `:${detail}` : ""})`); } return parseEvidence(extractContentText(await response.json())); } function evidenceText(evidence) { return `[TD 多模态识别证据]\n${JSON.stringify(evidence)}`; } function appendEvidence(request, evidence) { if (!Array.isArray(request.input)) return; let target = null; for (let i = request.input.length - 1; i >= 0; i -= 1) { const item = request.input[i]; if (item && item.type === "message" && item.role === "user" && Array.isArray(item.content)) { target = item; break; } } if (!target) { target = { type: "message", role: "user", content: [] }; request.input.push(target); } target.content.push({ type: "input_text", text: evidenceText(evidence) }); } function cleanupCache(now) { for (const [key, value] of cache) if (now - value.at > CACHE_TTL_MS) cache.delete(key); } async function enrichResponsesRequest(request, { instruction = "", fetchImpl = global.fetch } = {}) { const config = gatewayConfig(); const attachments = collectAttachments(request); if (!attachments.length) return { processed: false, attachments: [] }; const imageAssets = attachments .filter((asset) => asset.kind === "image" && asset.value && asset.value.length <= MAX_IMAGE_VALUE_LENGTH) .slice(0, MAX_IMAGES_PER_REQUEST); const evidence = { version: 1, provider: "minimax", modality: attachments.length > 1 ? "mixed" : attachments[0].kind, summary: "", items: [], unsupported: [], }; const summaries = []; if (imageAssets.length) { if (!config.key) { summaries.push("已检测到图片附件,但免费多模态网关尚未注入访问密钥,不能识别图片内容。不要声称已查看或识别该图片。"); } else { cleanupCache(Date.now()); const cacheKey = sha256(`${config.model}:${instruction}:${imageAssets.map((asset) => asset.id).join(",")}`); const cached = cache.get(cacheKey); if (cached) { summaries.push(cached.value.summary); evidence.items.push(...cached.value.items); } else { try { const understood = await understandImages(imageAssets, instruction, config, fetchImpl); summaries.push(understood.summary); evidence.items.push(...understood.items); cache.set(cacheKey, { at: Date.now(), value: understood }); } catch (error) { summaries.push(String((error && error.message) || "MiniMax 图片识别失败")); } } } } for (const asset of attachments.filter((item) => item.kind === "document")) { try { const understood = await understandPdf(asset, fetchImpl); summaries.push(understood.summary); evidence.items.push(...understood.items); } catch (error) { evidence.unsupported.push({ kind: "document", name: asset.name || "未命名附件" }); summaries.push(String((error && error.message) || "PDF 解析失败")); } } for (const asset of attachments.filter((item) => item.kind === "audio")) { if (!config.key || !config.sttModel) { evidence.unsupported.push({ kind: "audio", name: asset.name || "未命名附件" }); summaries.push("音频已检测到,但免费多模态网关尚未配置音频转写模型。"); continue; } try { const understood = await understandAudio(asset, config, fetchImpl); summaries.push(understood.summary); evidence.items.push(...understood.items); } catch (error) { summaries.push(String((error && error.message) || "音频转写失败")); } } evidence.summary = summaries.filter(Boolean).join("\n").slice(0, MAX_PDF_TEXT_LENGTH); appendEvidence(request, evidence); return { processed: true, attachments, evidence }; } module.exports = { collectAttachments, enrichResponsesRequest, parseEvidence, };