Files
codex-market-app/electron/lib/voice_cdp.js
T
mac 02fb553c9e feat: enhance session sync and add voice input feature
- 增强会话同步:确保所有会话完全同步到新账号
  - 强制更新 model_provider 字段
  - 添加 session_index.jsonl 重建功能
  - 确保数据库 UPSERT 逻辑正确处理所有情况
- 新增语音输入功能(voice_cdp.js, voice_inject.js)
- 测试脚本和增强模块

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-16 19:16:45 +08:00

618 lines
20 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// voice_cdp.js —— Codex 语音输入注入:主进程侧。
// 用 CDP(Chrome DevTools Protocol)连接带 --remote-debugging-port 的 Codex 桌面版,
// 往其页面注入麦克风按钮脚本,录音经 binding 回传 → 调中转站 Whisper 转写 → 文字回填 composer。
// 不改 Codex 源码、不写补丁文件。无第三方依赖:CDP over WebSocket 用 Node 内置 net 手写最小 WS 客户端
// (Electron 34 主进程的 Node 无全局 WebSocket);HTTP 用全局 fetch;multipart 用全局 FormData/Blob。
//
// 机制参考 CodexPlusPlus 的 bridge.rs:Runtime.evaluate / Page.addScriptToEvaluateOnNewDocument /
// Runtime.addBinding + Runtime.bindingCalled。
const { spawn, execFile } = require("child_process");
const os = require("os");
const path = require("path");
const net = require("net");
const crypto = require("crypto");
function dlog(...a) { try { console.log("[voice]", ...a); } catch {} }
const router = require("./router");
const providerSync = require("./provider_sync");
const { INJECT_SCRIPT } = require("./voice_inject");
const DEFAULT_PORT = 19222; // Codex 调试端口(避开 Chrome 默认 9222)
const BINDING_NAME = "__voiceBridge"; // 页面 → 主进程 的 CDP binding
const CDP_HTTP_TIMEOUT = 3000; // GET /json 超时
const READY_POLL_INTERVAL = 500; // 轮询就绪间隔
const READY_POLL_MAX_MS = 20000; // 轮询就绪最长等待
const RECORD_MAX_MS = 60000; // 单次录音上限(注入脚本侧也限)
// —— 状态机 ——
// unknown / checking / not_running / running_no_port / restarting / connecting /
// injecting / connected / disconnected / error
let state = {
phase: "unknown",
port: DEFAULT_PORT,
codexPath: "",
lastError: "",
target: null, // { id, url, title }
};
let session = null; // 当前 CdpSession
let onStatusCb = null;
function snapshot() {
const { phase, port, codexPath, lastError, model, target } = state;
return { phase, port, codexPath, lastError, model, target };
}
function setPhase(phase, extra = {}) {
state = { ...state, phase, ...extra };
if (onStatusCb) {
try {
onStatusCb(snapshot());
} catch {}
}
}
// —— 探测 Codex 可执行路径 ——
function codexExecutable() {
const home = os.homedir();
if (process.platform === "darwin") {
const cands = [
path.join(home, "Applications/ChatGPT.app/Contents/MacOS/ChatGPT"),
"/Applications/ChatGPT.app/Contents/MacOS/ChatGPT",
"/Applications/Codex.app/Contents/MacOS/Codex",
];
for (const p of cands) {
try {
if (require("fs").existsSync(p)) return p;
} catch {}
}
return "";
}
if (process.platform === "win32") {
const local = process.env.LOCALAPPDATA || "";
const pf = process.env["ProgramFiles"] || "C:\\Program Files";
const cands = [
local && path.join(local, "Programs\\ChatGPT\\ChatGPT.exe"),
path.join(pf, "ChatGPT\\ChatGPT.exe"),
].filter(Boolean);
for (const p of cands) {
try {
if (require("fs").existsSync(p)) return p;
} catch {}
}
return "";
}
return "";
}
// —— HTTP:GET /json 取 targets ——
async function listTargets(port) {
const ctrl = new AbortController();
const t = setTimeout(() => ctrl.abort(), CDP_HTTP_TIMEOUT);
try {
const res = await fetch(`http://127.0.0.1:${port}/json`, { signal: ctrl.signal });
if (!res.ok) return [];
const arr = await res.json();
return Array.isArray(arr) ? arr : [];
} catch {
return [];
} finally {
clearTimeout(t);
}
}
// —— 挑可注入的 Codex 主页面 target ——
// Codex 可能有多个 page:主对话页 app://-/index.html,以及 avatar-overlay / global-dictation
// 等带 ?initialRoute= 的子路由覆盖层。优先选「无 query 的主入口」,次选普通 codex 页,排除覆盖层。
function pickTarget(targets) {
let main = null; // url 恰为 app://-/index.html(无 query)
let fallback = null; // 其它非覆盖层 codex 页
for (const t of targets) {
if (t.type !== "page") continue;
const ws = t.webSocketDebuggerUrl;
if (!ws) continue;
const url = (t.url || "").toLowerCase();
const title = (t.title || "").toLowerCase();
// 覆盖层:带 initialRoute 的子路由(avatar-overlay / global-dictation 等)
const isOverlayRoute = url.includes("initialroute=");
const isCodex =
url.startsWith("app://-/index.html") || url.includes("index.html") || title.includes("codex");
if (!isCodex) continue;
if (isOverlayRoute) continue; // 覆盖层不注入
if (url === "app://-/index.html" || url === "app://-/index.html/") {
main = main || t;
} else {
fallback = fallback || t;
}
}
return main || fallback;
}
// —— 最小 WebSocket 客户端(Node 内置 net,无需依赖;仅支持 CDP 需要的文本帧)——
// 解析 ws://host:port/path,TCP 连接 → HTTP 升级握手(带 Sec-WebSocket-Key)→ 文本帧收发。
// 客户端发送必须掩码;CDP 服务端发送不掩码。帧可能分片到达,用缓冲区拼。
function wsConnect(wsUrl) {
const u = new URL(wsUrl);
const host = u.hostname;
const port = u.port || (u.protocol === "wss:" ? "443" : "80");
const pathStr = u.pathname + u.search;
const key = crypto.randomBytes(16).toString("base64");
const socket = net.connect({ host, port: Number(port) });
const events = { open: [], message: [], close: [], error: [] };
const emit = (name, arg) => (events[name] || []).forEach((cb) => cb(arg));
// —— 发送一帧文本(客户端必须掩码)——
function sendFrame(str) {
const payload = Buffer.from(str, "utf8");
const len = payload.length;
// opcode 1 = 文本;FIN=1
let header;
const mask = crypto.randomBytes(4);
if (len < 126) {
header = Buffer.alloc(6);
header[0] = 0x81;
header[1] = 0x80 | len;
mask.copy(header, 2);
} else if (len < 65536) {
header = Buffer.alloc(8);
header[0] = 0x81;
header[1] = 0x80 | 126;
header.writeUInt16BE(len, 2);
mask.copy(header, 4);
} else {
header = Buffer.alloc(14);
header[0] = 0x81;
header[1] = 0x80 | 127;
header.writeUInt32BE(0, 2);
header.writeUInt32BE(len, 6);
mask.copy(header, 10);
}
const masked = Buffer.alloc(len);
for (let i = 0; i < len; i++) masked[i] = payload[i] ^ mask[i & 3];
socket.write(Buffer.concat([header, masked]));
}
// —— 接收缓冲 + 帧解析 ——
let buf = Buffer.alloc(0);
let handshakeDone = false;
function parseFrames() {
while (true) {
if (buf.length < 2) return;
const b0 = buf[0];
const b1 = buf[1];
const opcode = b0 & 0x0f;
const masked = (b1 & 0x80) !== 0;
let payloadLen = b1 & 0x7f;
let offset = 2;
if (payloadLen === 126) {
if (buf.length < 4) return;
payloadLen = buf.readUInt16BE(2);
offset = 4;
} else if (payloadLen === 127) {
if (buf.length < 10) return;
// 高 32 位忽略(CDP 帧不会那么大)
payloadLen = Number(buf.readUInt32BE(6));
offset = 10;
}
let maskKey = null;
if (masked) {
if (buf.length < offset + 4) return;
maskKey = buf.slice(offset, offset + 4);
offset += 4;
}
if (buf.length < offset + payloadLen) return;
let payload = buf.slice(offset, offset + payloadLen);
if (masked) {
const un = Buffer.alloc(payload.length);
for (let i = 0; i < payload.length; i++) un[i] = payload[i] ^ maskKey[i & 3];
payload = un;
}
buf = buf.slice(offset + payloadLen);
// 只处理文本(1)/ping(9)/pong(10)/关闭(8)
if (opcode === 0x8) {
// 关闭帧
emit("close");
try {
socket.end();
} catch {}
return;
} else if (opcode === 0x9) {
// ping → 回 pong(无掩码服务端不要求,但回时按服务端→客户端本可不掩码;此处回文本帧不规范,CDP 不发 ping)
continue;
} else if (opcode === 0xa) {
continue; // pong,忽略
} else {
// 文本帧(含分片首帧 opcode 0 也按文本拼;CDP 单帧文本足够,简单按单帧处理)
emit("message", payload.toString("utf8"));
}
}
}
socket.on("connect", () => {
const req =
`GET ${pathStr} HTTP/1.1\r\n` +
`Host: ${host}:${port}\r\n` +
`Upgrade: websocket\r\n` +
`Connection: Upgrade\r\n` +
`Sec-WebSocket-Key: ${key}\r\n` +
`Sec-WebSocket-Version: 13\r\n\r\n`;
socket.write(req);
});
socket.on("data", (chunk) => {
buf = Buffer.concat([buf, chunk]);
if (!handshakeDone) {
const idx = buf.indexOf("\r\n\r\n");
if (idx < 0) return;
const head = buf.slice(0, idx).toString("utf8");
buf = buf.slice(idx + 4);
if (!/101/.test(head.split("\r\n")[0])) {
emit("error", new Error("WS 握手失败: " + head.split("\r\n")[0]));
try {
socket.destroy();
} catch {}
return;
}
handshakeDone = true;
emit("open");
parseFrames();
} else {
parseFrames();
}
});
socket.on("close", () => emit("close"));
socket.on("error", (e) => emit("error", e));
return {
send: sendFrame,
close: () => {
try {
socket.destroy();
} catch {}
},
on: (name, cb) => {
(events[name] = events[name] || []).push(cb);
},
_socket: socket,
};
}
// —— CDP WebSocket 会话 ——
function openSession(wsUrl, onBindingCalled) {
const pending = new Map(); // id → {resolve, reject}
const handlers = new Map(); // method → Set<cb>
let nextId = 1;
let closed = false;
const ws = wsConnect(wsUrl);
function send(method, params = {}) {
return new Promise((resolve, reject) => {
if (closed) return reject(new Error("CDP 会话已关闭"));
const id = nextId++;
pending.set(id, { resolve, reject });
ws.send(JSON.stringify({ id, method, params }));
});
}
function on(method, cb) {
if (!handlers.has(method)) handlers.set(method, new Set());
handlers.get(method).add(cb);
}
function close() {
closed = true;
try {
ws.close();
} catch {}
}
ws.on("message", (data) => {
let msg;
try {
msg = JSON.parse(data);
} catch {
return;
}
if (msg.id != null) {
const p = pending.get(msg.id);
if (!p) return;
pending.delete(msg.id);
if (msg.error) p.reject(new Error(msg.error.message || "CDP error"));
else p.resolve(msg.result);
return;
}
if (msg.method) {
if (msg.method === "Runtime.bindingCalled" && onBindingCalled) {
try {
onBindingCalled(msg.params);
} catch {}
}
const set = handlers.get(msg.method);
if (set) for (const cb of set) cb(msg.params);
}
});
ws.on("close", () => {
closed = true;
for (const { reject } of pending.values()) {
try {
reject(new Error("CDP 会话关闭"));
} catch {}
}
pending.clear();
});
return new Promise((resolve, reject) => {
ws.on("open", () => resolve({ send, on, close, ws }));
ws.on("error", (e) => reject(new Error("CDP 连接失败:" + (e && e.message ? e.message : "ws error"))));
});
}
// —— 转写:中转站 /v1/audio/transcriptions(用 sttGroup:勾选了 STT 的分组)——
async function runStt(audioB64, mime) {
const g = router.sttGroup();
if (!g || !g.baseUrl || !g.key) throw new Error("未配置支持语音转文字的中转分组(在路由中转里勾选「支持语音转文字」)");
const base = (g.baseUrl || "").replace(/\/+$/, "");
const model = g.sttModel || "whisper-1";
const buf = Buffer.from(audioB64, "base64");
const ext = mime && mime.includes("webm") ? "webm" : mime && mime.includes("mp4") ? "mp4" : "wav";
const form = new FormData();
form.append("file", new Blob([buf], { type: mime || "audio/webm" }), `voice.${ext}`);
form.append("model", model);
form.append("response_format", "json");
const res = await fetch(base + "/v1/audio/transcriptions", {
method: "POST",
headers: { Authorization: "Bearer " + g.key },
body: form,
});
dlog("转写请求 →", base + "/v1/audio/transcriptions", "model=" + model, "size=" + buf.length, "HTTP " + res.status);
if (!res.ok) {
const detail = await res.text().catch(() => "");
throw new Error(`转写接口 HTTP ${res.status}:${detail.slice(0, 300)}`);
}
const j = await res.json();
return (j && (j.text || "")).trim();
}
// —— 主进程收到页面 binding 回传:处理转写并把文字送回页面 ——
async function handleBinding(params) {
const name = params && params.name;
if (name !== BINDING_NAME) return;
let payload;
try {
payload = JSON.parse(params.payload);
} catch {
return;
}
if (!payload || payload.kind !== "transcribe") return;
const { id, audioB64, mime } = payload;
dlog("binding 收到转写请求 id=" + id, "mime=" + mime, "audioB64长度=" + (audioB64?audioB64.length:0));
setPhase("transcribing");
let text = "";
let err = "";
try {
text = await runStt(audioB64, mime);
dlog("转写结果:", JSON.stringify(text).slice(0, 120));
} catch (e) {
err = String((e && e.message) || e);
dlog("转写失败:", err);
}
setPhase(connectedPhase());
// 把结果回调给页面
const expr = `window.__voiceInjectHandle && window.__voiceInjectHandle(${JSON.stringify(
{ id, text, err }
)})`;
try {
if (session) await session.send("Runtime.evaluate", { expression: expr, awaitPromise: false });
} catch {}
}
function connectedPhase() {
return session ? "connected" : "disconnected";
}
// —— 注入:addBinding + addScriptToEvaluateOnNewDocument + 立即 evaluate ——
async function inject(sessionObj) {
await sessionObj.send("Runtime.enable");
try {
await sessionObj.send("Runtime.removeBinding", { name: BINDING_NAME });
} catch {}
await sessionObj.send("Runtime.addBinding", { name: BINDING_NAME });
await sessionObj.send("Page.enable");
await sessionObj.send("Page.addScriptToEvaluateOnNewDocument", { source: INJECT_SCRIPT });
await sessionObj.send("Runtime.evaluate", { expression: INJECT_SCRIPT, awaitPromise: false });
}
// —— 连接到已带端口的 Codex ——
async function connectToTarget(port) {
const targets = await listTargets(port);
const tgt = pickTarget(targets);
if (!tgt) return false;
state.target = { id: tgt.id, url: tgt.url, title: tgt.title };
setPhase("connecting");
session = await openSession(tgt.webSocketDebuggerUrl, handleBinding);
session.on("Runtime.bindingCalled", handleBinding);
session.ws.on("close", () => {
session = null;
setPhase("disconnected");
});
setPhase("injecting");
await inject(session);
setPhase("connected", { target: state.target });
return true;
}
// —— 轮询直到出现 page target(重启后用)——
async function waitForReady(port) {
const t0 = Date.now();
while (Date.now() - t0 < READY_POLL_MAX_MS) {
const targets = await listTargets(port);
const tgt = pickTarget(targets);
if (tgt) return tgt;
await new Promise((r) => setTimeout(r, READY_POLL_INTERVAL));
}
return null;
}
// —— 启动接管:杀当前 Codex → 带端口重启 ——
function killCodex() {
return new Promise((resolve) => {
if (process.platform === "darwin") {
execFile("osascript", ["-e", 'tell application "ChatGPT" to quit'], () => {
// 兜底:osascript 没杀掉就 pkill
execFile("pgrep", ["-f", "ChatGPT.app/Contents/MacOS/ChatGPT"], (_e, out) => {
if (out && out.trim()) {
execFile("pkill", ["-f", "ChatGPT.app/Contents/MacOS/ChatGPT"], () =>
setTimeout(resolve, 800)
);
} else resolve();
});
});
} else if (process.platform === "win32") {
execFile("taskkill", ["/IM", "ChatGPT.exe", "/F"], () =>
setTimeout(resolve, 800)
);
} else resolve();
});
}
function spawnCodex(exePath, port) {
const child = spawn(exePath, [`--remote-debugging-port=${port}`], {
detached: true,
stdio: "ignore",
});
try {
child.unref();
} catch {}
}
// —— 对外:status / connect / disconnect ——
function status() {
return snapshot();
}
function onStatus(cb) {
onStatusCb = cb;
}
// connect(onStatusCb):流程见计划。forceRestart=true 时直接重启(前端已确认)。
async function connect(cb, opts = {}) {
if (cb) onStatusCb = cb;
const port = opts.port || DEFAULT_PORT;
state.port = port;
setPhase("checking");
// 1. 端口已开?
const ok = await connectToTarget(port);
if (ok) return snapshot();
// 2. 判断 Codex 是否在跑
const running = providerSync.codexRunning ? providerSync.codexRunning() : false;
if (!running && !opts.forceRestart) {
// 没在跑也没开端口:直接尝试拉起带端口的实例(无需杀进程)
const exe = codexExecutable();
if (!exe) {
setPhase("error", { lastError: "未找到 Codex 可执行文件" });
return snapshot();
}
state.codexPath = exe;
setPhase("restarting", { codexPath: exe });
spawnCodex(exe, port);
const tgt = await waitForReady(port);
if (!tgt) {
setPhase("error", { lastError: "Codex 启动后未开放调试端口,请重试" });
return snapshot();
}
const ok2 = await connectToTarget(port);
if (!ok2) setPhase("error", { lastError: "已启动但未找到可注入页面" });
return snapshot();
}
// 3. 在跑但没端口:需用户在前端确认后 forceRestart 重启
if (running && !opts.forceRestart) {
state.codexPath = codexExecutable();
setPhase("running_no_port", { codexPath: state.codexPath });
return snapshot(); // 前端据此弹"重启确认"
}
// 4. forceRestart:杀进程 → 重启
const exe = codexExecutable();
if (!exe) {
setPhase("error", { lastError: "未找到 Codex 可执行文件" });
return snapshot();
}
state.codexPath = exe;
setPhase("restarting");
await killCodex();
spawnCodex(exe, port);
const tgt = await waitForReady(port);
if (!tgt) {
setPhase("error", { lastError: "重启 Codex 后未开放调试端口" });
return snapshot();
}
const ok3 = await connectToTarget(port);
if (!ok3) setPhase("error", { lastError: "已启动但未找到可注入页面" });
return snapshot();
}
function disconnect() {
if (session) {
try {
session.close();
} catch {}
session = null;
}
setPhase("disconnected");
return snapshot();
}
// 当前语音转写配置(来自路由里勾选 STT 的分组),供前端只读展示。
function sttInfo() {
const g = router.sttGroup();
if (!g) return { ready: false, groupName: "", model: "" };
return {
ready: true,
groupName: g.name || "",
model: g.sttModel || "whisper-1",
baseUrl: (g.baseUrl || "").replace(/\/+$/, ""),
};
}
// 诊断:用每个非默认分组的真 key 探测 /v1/audio/transcriptions 是否存在。
// 404 = 路径不存在(不支持STT);400/401/4xx-body非404/200 = 路径存在(支持,鉴权/参数问题另算)。
async function probeStt() {
const groups = router.usableGroups ? router.usableGroups() : [];
const out = [];
const body = Buffer.from([0x1a, 0x45, 0xdf, 0xa3]); // 假 webm 头,只为触发路径判断
for (const g of groups) {
const base = (g.baseUrl || "").replace(/\/+$/, "");
if (!base || !g.key) { out.push({ name: g.name, baseUrl: base, status: 0, note: "缺地址或密钥" }); continue; }
try {
const form = new FormData();
form.append("file", new Blob([body], { type: "audio/webm" }), "v.webm");
form.append("model", g.sttModel || "whisper-1");
const res = await fetch(base + "/v1/audio/transcriptions", {
method: "POST",
headers: { Authorization: "Bearer " + g.key },
body: form,
});
const text = await res.text().catch(() => "");
const notFound = res.status === 404 || /404 page not found|not found/i.test(text.slice(0, 60));
out.push({
name: g.name,
baseUrl: base,
status: res.status,
supported: !notFound,
note: text.slice(0, 120).replace(/\s+/g, " "),
});
} catch (e) {
out.push({ name: g.name, baseUrl: base, status: 0, supported: false, note: String((e && e.message) || e).slice(0, 120) });
}
}
return out;
}
module.exports = {
status,
connect,
disconnect,
onStatus,
sttInfo,
probeStt,
// 供临时实测脚本复用
_listTargets: listTargets,
_pickTarget: pickTarget,
_openSession: openSession,
_inject: inject,
DEFAULT_PORT,
};