diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index 03a1a501a01..9f116728c34 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -576,10 +576,21 @@ Payload shapes the broker validates on `admin/add`: - Discord (`kind: "discord"`): `{ guildId: string, channelId: string, driverBotToken: string, sutBotToken: string, sutApplicationId: string }`. - WhatsApp (`kind: "whatsapp"`): `{ driverPhoneE164: string, sutPhoneE164: string, driverAuthArchiveBase64: string, sutAuthArchiveBase64: string, groupJid?: string }` - phone numbers must be distinct E.164 strings. -For visual real-user Telegram proof, prefer `pnpm qa:telegram-user:crabbox -- --text /status`. -It uses one Convex `telegram-user` lease for both the TDLib CLI driver and the -Telegram Desktop witness, captures a Crabbox recording plus motion-trimmed -video/GIF artifacts, and releases the lease on shutdown. +For visual real-user Telegram proof, prefer a held Crabbox session: + +```bash +pnpm qa:telegram-user:crabbox -- start --tdlib-url http://artifacts.openclaw.ai/tdlib-v1.8.0-linux-x64.tgz --output-dir .artifacts/qa-e2e/telegram-user-crabbox/pr-review +pnpm qa:telegram-user:crabbox -- send --session .artifacts/qa-e2e/telegram-user-crabbox/pr-review/session.json --text /status +pnpm qa:telegram-user:crabbox -- finish --session .artifacts/qa-e2e/telegram-user-crabbox/pr-review/session.json +``` + +`start` holds one exclusive Convex `telegram-user` lease for both the TDLib CLI +driver and Telegram Desktop witness, starts desktop recording, and leaves the +Crabbox alive for arbitrary agent-driven repro steps. Agents can use `send`, +`run`, `screenshot`, and `status` until they are satisfied, then `finish` +collects the screenshot, video, motion-trimmed video/GIF, TDLib probe outputs, +and logs before releasing the credential. The default `probe` command remains a +one-command shorthand for quick `/status` smoke checks. Slack lanes can also use the pool. Slack payload shape checks currently live in the Slack QA runner rather than the broker; use `{ channelId: string, driverBotToken: string, sutBotToken: string, sutAppToken: string }`, with a Slack channel id like `Cxxxxxxxxxx`. See [Setting up the Slack workspace](#setting-up-the-slack-workspace) for app and scope provisioning. diff --git a/docs/help/testing.md b/docs/help/testing.md index ef507bdb77b..338cb70db0e 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -452,19 +452,42 @@ node --import tsx scripts/e2e/telegram-user-credential.ts release --lease-file " Use the restored Desktop profile with `Telegram -workdir "$tmp/desktop"` when a visual recording is needed. In local operator environments, `scripts/e2e/telegram-user-credential.ts` reads `~/.codex/skills/custom/telegram-e2e-bot-to-bot/convex.local.env` by default if process env vars are absent. +Agent-driven Crabbox session: + +```bash +pnpm qa:telegram-user:crabbox -- start \ + --tdlib-url http://artifacts.openclaw.ai/tdlib-v1.8.0-linux-x64.tgz \ + --output-dir .artifacts/qa-e2e/telegram-user-crabbox/pr-review +pnpm qa:telegram-user:crabbox -- send \ + --session .artifacts/qa-e2e/telegram-user-crabbox/pr-review/session.json \ + --text /status +pnpm qa:telegram-user:crabbox -- finish \ + --session .artifacts/qa-e2e/telegram-user-crabbox/pr-review/session.json +``` + +`start` leases the `telegram-user` credential, restores the same account into +TDLib and Telegram Desktop on a Crabbox Linux desktop, starts a local mock SUT +gateway from the current checkout, opens the visible Telegram chat, starts +desktop recording, and writes a private `session.json`. While the session is +alive, an agent can keep testing until satisfied: + +- `send --session --text ` sends through the real TDLib user and waits for the SUT reply. +- `run --session -- ` runs an arbitrary command on the Crabbox and saves its output, for example `bash -lc 'source /tmp/openclaw-telegram-user-crabbox/env.sh && python3 /tmp/openclaw-telegram-user-crabbox/user-driver.py transcript --limit 20 --json'`. +- `screenshot --session ` captures the current visible desktop. +- `status --session ` prints the lease and WebVNC command. +- `finish --session ` stops the recorder, captures screenshot/video/motion-trim artifacts, releases the Convex credential, stops local SUT processes, and stops the Crabbox lease unless `--keep-box` is passed. + One-command Crabbox proof: ```bash pnpm qa:telegram-user:crabbox -- --text /status ``` -That command leases the `telegram-user` credential, restores the same account -into TDLib and Telegram Desktop on a Crabbox Linux desktop, starts a local mock -SUT gateway from the current checkout, sends the command as the real QA user, -records the visible Telegram Desktop session, trims the recording to the motion -window, writes artifacts under `.artifacts/qa-e2e/telegram-user-crabbox/`, then -releases the credential and stops the box. Use `--id ` to reuse a warm -desktop lease, `--keep-box` to keep VNC open after failure, +The default `probe` command is shorthand for one start/send/finish cycle. Use +it for a quick `/status` smoke. Use the session commands for PR review, +bug-reproduction work, or any case where the agent needs minutes of arbitrary +experimentation before deciding the proof is complete. Use `--id ` to +reuse a warm desktop lease, `--keep-box` to keep VNC open after finish, `--desktop-chat-title ` to pick the visible chat, and `--tdlib-url ` when using a prebaked Linux `libtdjson.so` archive instead of building TDLib on a fresh box. The runner verifies `--tdlib-url` with `--tdlib-sha256 ` or, diff --git a/qa/convex-credential-broker/README.md b/qa/convex-credential-broker/README.md index cdeda7a6a5a..7e5b3242c5d 100644 --- a/qa/convex-credential-broker/README.md +++ b/qa/convex-credential-broker/README.md @@ -161,6 +161,10 @@ credential for both the TDLib CLI driver and the Telegram Desktop visual witness - non-empty `desktopTdataArchiveBase64` - `desktopTdataArchiveSha256` as a SHA-256 hex string +Long-running agent sessions should acquire this lease once, keep it for the +whole Crabbox review/repro session, then release it from the same session file. +Do not run parallel `telegram-user` jobs against the burner account. + For `kind: "discord"`, broker `admin/add` validates that payload includes: - `guildId` as a Discord snowflake string diff --git a/scripts/e2e/telegram-user-crabbox-proof.ts b/scripts/e2e/telegram-user-crabbox-proof.ts index 89c9236b997..55627d920d7 100644 --- a/scripts/e2e/telegram-user-crabbox-proof.ts +++ b/scripts/e2e/telegram-user-crabbox-proof.ts @@ -23,6 +23,7 @@ type CrabboxInspect = { }; type Options = { + command: "finish" | "probe" | "run" | "screenshot" | "send" | "start" | "status"; crabboxBin: string; desktopChatTitle: string; dryRun: boolean; @@ -36,6 +37,8 @@ type Options = { outputDir: string; provider: string; recordSeconds: number; + remoteCommand: string[]; + sessionFile?: string; sutUsername?: string; target: string; tdlibSha256?: string; @@ -64,6 +67,43 @@ type LocalSut = { gatewayLog: string; }; +type SessionFile = { + command: "telegram-user-crabbox-session"; + createdAt: string; + crabbox: { + createdLease: boolean; + id: string; + inspect: CrabboxInspect; + provider: string; + target: string; + }; + credential: { + groupId: string; + leaseFile: string; + sutUsername: string; + testerUserId: string; + testerUsername: string; + }; + localRoot: string; + localSut: { + gatewayLog: string; + gatewayPid: number; + mockLog: string; + mockPid: number; + requestLog: string; + stateDir: string; + tempRoot: string; + workspace: string; + }; + outputDir: string; + recorder: { + log: string; + pidFile: string; + remoteVideo: string; + }; + remoteRoot: string; +}; + const DEFAULT_SKILL_DIR = "~/.codex/skills/custom/telegram-e2e-bot-to-bot"; const DEFAULT_CONVEX_ENV_FILE = `${DEFAULT_SKILL_DIR}/convex.local.env`; const DEFAULT_USER_DRIVER = `${DEFAULT_SKILL_DIR}/scripts/user-driver.py`; @@ -73,7 +113,13 @@ const REMOTE_ROOT = "/tmp/openclaw-telegram-user-crabbox"; function usageText() { return [ "Usage:", - " node --import tsx scripts/e2e/telegram-user-crabbox-proof.ts [--text /status] [--expect OpenClaw]", + " node --import tsx scripts/e2e/telegram-user-crabbox-proof.ts [probe] [--text /status] [--expect OpenClaw]", + " node --import tsx scripts/e2e/telegram-user-crabbox-proof.ts start [--tdlib-url ]", + " node --import tsx scripts/e2e/telegram-user-crabbox-proof.ts send --session --text ", + " node --import tsx scripts/e2e/telegram-user-crabbox-proof.ts run --session -- ", + " node --import tsx scripts/e2e/telegram-user-crabbox-proof.ts screenshot --session ", + " node --import tsx scripts/e2e/telegram-user-crabbox-proof.ts status --session ", + " node --import tsx scripts/e2e/telegram-user-crabbox-proof.ts finish --session ", "", "Useful options:", " --desktop-chat-title Telegram Desktop chat to select before recording.", @@ -81,6 +127,7 @@ function usageText() { " --keep-box Leave the Crabbox lease running for VNC debugging.", " --output-dir Artifact directory under the repo.", " --record-seconds Desktop video duration. Default: 35.", + " --session Session file from start. Default: /session.json.", " --tdlib-sha256 Expected SHA-256 for --tdlib-url. Defaults to .sha256.", " --tdlib-url Linux tdlib archive containing libtdjson.so.", " --dry-run Validate local inputs and print the plan.", @@ -116,8 +163,11 @@ function parsePositiveInteger(value: string, label: string) { function parseArgs(argv: string[]): Options { argv = argv[0] === "--" ? argv.slice(1) : argv; + const commands = new Set(["finish", "probe", "run", "screenshot", "send", "start", "status"]); + const command = commands.has(argv[0] ?? "") ? (argv.shift() as Options["command"]) : "probe"; const stamp = new Date().toISOString().replace(/[:.]/gu, "-"); const opts: Options = { + command, crabboxBin: trimToValue(process.env.OPENCLAW_TELEGRAM_USER_CRABBOX_BIN) ?? "crabbox", desktopChatTitle: trimToValue(process.env.OPENCLAW_TELEGRAM_USER_DESKTOP_CHAT_TITLE) ?? "OpenClaw Testing", @@ -130,12 +180,18 @@ function parseArgs(argv: string[]): Options { outputDir: path.join(DEFAULT_OUTPUT_ROOT, stamp), provider: process.env.OPENCLAW_TELEGRAM_USER_CRABBOX_PROVIDER?.trim() || "aws", recordSeconds: 35, + remoteCommand: [], target: "linux", text: "/status", timeoutMs: 90_000, ttl: "120m", userDriverScript: DEFAULT_USER_DRIVER, }; + const commandSeparator = argv.indexOf("--"); + if (command === "run" && commandSeparator >= 0) { + opts.remoteCommand = argv.slice(commandSeparator + 1); + argv = argv.slice(0, commandSeparator); + } let expectWasPassed = false; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; @@ -177,6 +233,8 @@ function parseArgs(argv: string[]): Options { opts.provider = readValue(); } else if (arg === "--record-seconds") { opts.recordSeconds = parsePositiveInteger(readValue(), "--record-seconds"); + } else if (arg === "--session") { + opts.sessionFile = readValue(); } else if (arg === "--sut-username") { opts.sutUsername = readValue().replace(/^@/u, ""); } else if (arg === "--target") { @@ -200,6 +258,12 @@ function parseArgs(argv: string[]): Options { throw new Error(`Unknown argument: ${arg}`); } } + if (command === "run" && opts.remoteCommand.length === 0) { + throw new Error("run requires a remote command after --."); + } + if (["finish", "run", "screenshot", "send", "status"].includes(command) && !opts.sessionFile) { + throw new Error(`${command} requires --session.`); + } return opts; } @@ -250,6 +314,10 @@ function optionalString(source: JsonObject, key: string) { return typeof value === "string" && value.trim() ? value.trim() : undefined; } +function shellQuote(value: string) { + return `'${value.replaceAll("'", "'\\''")}'`; +} + function runCommand(params: { args: string[]; command: string; @@ -377,6 +445,53 @@ function killTree(child: ChildProcess | undefined) { } } +function killPidTree(pid: number | undefined) { + if (!pid) { + return; + } + try { + process.kill(-pid, "SIGTERM"); + } catch { + try { + process.kill(pid, "SIGTERM"); + } catch { + return; + } + } +} + +function spawnDaemon(params: { + args: string[]; + command: string; + cwd: string; + env: NodeJS.ProcessEnv; + logPath: string; +}) { + const log = fs.openSync(params.logPath, "a"); + const child = spawn(params.command, params.args, { + cwd: params.cwd, + detached: true, + env: params.env, + stdio: ["ignore", log, log], + }); + child.unref(); + fs.closeSync(log); + return child.pid; +} + +async function waitForLog(logPath: string, pattern: RegExp, label: string, timeoutMs: number) { + const started = Date.now(); + while (Date.now() - started < timeoutMs) { + const text = fs.existsSync(logPath) ? fs.readFileSync(logPath, "utf8") : ""; + if (pattern.test(text)) { + return; + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + const text = fs.existsSync(logPath) ? fs.readFileSync(logPath, "utf8") : ""; + throw new Error(`${label} did not become ready within ${timeoutMs}ms\n${text.slice(-4000)}`); +} + async function telegram(token: string, method: string, body: JsonObject = {}) { const response = await fetch(`https://api.telegram.org/bot${token}/${method}`, { method: "POST", @@ -568,6 +683,65 @@ async function startLocalSut(params: { }; } +async function startLocalSutDaemon(params: { + gatewayPort: number; + groupId: string; + mockPort: number; + outputDir: string; + sutToken: string; + testerId: string; + repoRoot: string; +}) { + const drained = await drainSutUpdates(params.sutToken); + const config = writeSutConfig(params); + const requestLog = path.join(params.outputDir, "mock-openai-requests.ndjson"); + const mockLog = path.join(params.outputDir, "mock-openai.log"); + const gatewayLog = path.join(params.outputDir, "gateway.log"); + const mockPid = spawnDaemon({ + command: "node", + args: ["scripts/e2e/mock-openai-server.mjs"], + cwd: params.repoRoot, + env: { + ...process.env, + MOCK_PORT: String(params.mockPort), + MOCK_REQUEST_LOG: requestLog, + SUCCESS_MARKER: "OPENCLAW_E2E_OK", + }, + logPath: mockLog, + }); + if (!mockPid) { + throw new Error("mock-openai did not start."); + } + await waitForLog(mockLog, /mock-openai listening/u, "mock-openai", 10_000); + + const gatewayPid = spawnDaemon({ + command: "pnpm", + args: ["openclaw", "gateway", "--port", String(params.gatewayPort)], + cwd: params.repoRoot, + env: { + ...process.env, + OPENAI_API_KEY: "sk-openclaw-e2e-mock", + OPENCLAW_CONFIG_PATH: config.configPath, + OPENCLAW_STATE_DIR: config.stateDir, + TELEGRAM_BOT_TOKEN: params.sutToken, + }, + logPath: gatewayLog, + }); + if (!gatewayPid) { + throw new Error("gateway did not start."); + } + await waitForLog(gatewayLog, /\[gateway\] ready/u, "gateway", 60_000); + return { + ...config, + drained, + gatewayLog, + gatewayPid, + mockLog, + mockPid, + requestLog, + }; +} + function extractLeaseId(output: string) { return output.match(/\b(?:cbx_[a-f0-9]+|tbx_[A-Za-z0-9_-]+)\b/u)?.[0]; } @@ -694,7 +868,7 @@ tdlib_url=${tdlibUrl} mkdir -p "$root" tar -xzf "$root/state.tgz" -C "$root" sudo apt-get update -y -sudo DEBIAN_FRONTEND=noninteractive apt-get install -y curl git cmake g++ make zlib1g-dev libssl-dev python3 ffmpeg scrot xz-utils tar wmctrl xdotool libopengl0 libxcb-cursor0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-shape0 libxcb-xfixes0 libxcb-xinerama0 libxkbcommon-x11-0 >/tmp/openclaw-telegram-apt.log +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y curl git cmake g++ make zlib1g-dev libssl-dev python3 ffmpeg scrot xz-utils tar wmctrl xdotool x11-utils libopengl0 libxcb-cursor0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-shape0 libxcb-xfixes0 libxcb-xinerama0 libxkbcommon-x11-0 >/tmp/openclaw-telegram-apt.log if ! command -v python3 >/dev/null 2>&1; then echo "python3 is required" >&2 exit 127 @@ -773,6 +947,7 @@ sleep 1 function renderRemoteProbe(params: { expect: string[]; + outputPath?: string; sutUsername: string; text: string; timeoutMs: number; @@ -784,7 +959,7 @@ function renderRemoteProbe(params: { "--timeout-ms", String(params.timeoutMs), "--output", - `${REMOTE_ROOT}/probe.json`, + params.outputPath ?? `${REMOTE_ROOT}/probe.json`, "--json", ]; for (const expected of params.expect) { @@ -913,10 +1088,22 @@ function summarizeProbe(probePath: string) { const probe = readJsonFile(probePath); const reply = probe.reply; const sent = probe.sent; + const messageId = (value: unknown) => { + if (!value || typeof value !== "object") { + return undefined; + } + if ("messageId" in value) { + return value.messageId; + } + if ("id" in value) { + return value.id; + } + return undefined; + }; return { ok: probe.ok === true, - replyMessageId: reply && typeof reply === "object" && "id" in reply ? reply.id : undefined, - sentMessageId: sent && typeof sent === "object" && "id" in sent ? sent.id : undefined, + replyMessageId: messageId(reply), + sentMessageId: messageId(sent), }; } @@ -953,6 +1140,416 @@ function writeReport(params: { return reportPath; } +function sessionPath(root: string, opts: Options, outputDir: string) { + return opts.sessionFile + ? resolveRepoPath(root, opts.sessionFile) + : path.join(outputDir, "session.json"); +} + +function writeSession(pathname: string, session: SessionFile) { + fs.mkdirSync(path.dirname(pathname), { recursive: true }); + fs.writeFileSync(pathname, `${JSON.stringify(session, null, 2)}\n`, { mode: 0o600 }); + fs.chmodSync(pathname, 0o600); +} + +function readSession(root: string, opts: Options, outputDir: string) { + const pathname = sessionPath(root, opts, outputDir); + if (!fs.existsSync(pathname)) { + throw new Error(`Missing session file: ${path.relative(root, pathname)}`); + } + const session = readJsonFile(pathname) as SessionFile; + if (session.command !== "telegram-user-crabbox-session") { + throw new Error(`Invalid Telegram Crabbox session file: ${path.relative(root, pathname)}`); + } + return { + path: pathname, + session, + }; +} + +async function writeRemoteSessionScripts(params: { + inspect: CrabboxInspect; + localRoot: string; + opts: Options; + root: string; + stateArchive: string; + sutUsername: string; +}) { + const setupScript = path.join(params.localRoot, "remote-setup.sh"); + const launchScript = path.join(params.localRoot, "launch-desktop.sh"); + const selectChatScript = path.join(params.localRoot, "select-desktop-chat.sh"); + await writeExecutable( + setupScript, + renderRemoteSetup({ tdlibSha256: params.opts.tdlibSha256, tdlibUrl: params.opts.tdlibUrl }), + ); + await writeExecutable(launchScript, renderLaunchDesktop()); + await writeExecutable( + selectChatScript, + renderSelectDesktopChat({ chatTitle: params.opts.desktopChatTitle }), + ); + + await sshRun(params.root, params.inspect, `rm -rf ${REMOTE_ROOT} && mkdir -p ${REMOTE_ROOT}`); + await scpToRemote(params.root, params.inspect, params.stateArchive, `${REMOTE_ROOT}/state.tgz`); + await scpToRemote(params.root, params.inspect, setupScript, `${REMOTE_ROOT}/remote-setup.sh`); + await scpToRemote(params.root, params.inspect, launchScript, `${REMOTE_ROOT}/launch-desktop.sh`); + await scpToRemote( + params.root, + params.inspect, + selectChatScript, + `${REMOTE_ROOT}/select-desktop-chat.sh`, + ); + await sshRun(params.root, params.inspect, `bash ${REMOTE_ROOT}/remote-setup.sh`); + await sshRun(params.root, params.inspect, `bash ${REMOTE_ROOT}/launch-desktop.sh`); + await sshRun(params.root, params.inspect, `bash ${REMOTE_ROOT}/select-desktop-chat.sh`); + await sshRun( + params.root, + params.inspect, + `cat >${REMOTE_ROOT}/env.sh <<'EOF' +export TELEGRAM_USER_DRIVER_STATE_DIR=${REMOTE_ROOT}/user-driver +export TELEGRAM_USER_DRIVER_SUT_USERNAME=${params.sutUsername} +EOF +`, + ); +} + +async function startRemoteRecording(root: string, inspect: CrabboxInspect) { + const command = `set -euo pipefail +export DISPLAY="\${DISPLAY:-:99}" +root=${REMOTE_ROOT} +video="$root/session.mp4" +log="$root/ffmpeg.log" +pid_file="$root/ffmpeg.pid" +rm -f "$video" "$log" "$pid_file" +size="$(xdpyinfo | awk '/dimensions:/ {print $2; exit}')" +nohup ffmpeg -y -hide_banner -loglevel warning -f x11grab -framerate 15 -video_size "$size" -i "$DISPLAY" -pix_fmt yuv420p "$video" >"$log" 2>&1 & +echo $! >"$pid_file"`; + await sshRun(root, inspect, command); + return { + log: `${REMOTE_ROOT}/ffmpeg.log`, + pidFile: `${REMOTE_ROOT}/ffmpeg.pid`, + remoteVideo: `${REMOTE_ROOT}/session.mp4`, + }; +} + +async function stopRemoteRecording(root: string, inspect: CrabboxInspect, session: SessionFile) { + await sshRun( + root, + inspect, + `set -euo pipefail +pid_file=${JSON.stringify(session.recorder.pidFile)} +if [ -s "$pid_file" ]; then + pid="$(cat "$pid_file")" + kill -INT "$pid" >/dev/null 2>&1 || true + for _ in $(seq 1 20); do + kill -0 "$pid" >/dev/null 2>&1 || exit 0 + sleep 0.5 + done + kill -TERM "$pid" >/dev/null 2>&1 || true +fi`, + ); +} + +async function startSession(root: string, opts: Options, outputDir: string) { + const localRoot = path.join(outputDir, ".session"); + fs.rmSync(localRoot, { force: true, recursive: true }); + fs.mkdirSync(localRoot, { mode: 0o700, recursive: true }); + + const convexEnvFile = expandHome(opts.envFile ?? DEFAULT_CONVEX_ENV_FILE); + const hasConvexEnv = + trimToValue(process.env.OPENCLAW_QA_CONVEX_SITE_URL) && + trimToValue(process.env.OPENCLAW_QA_CONVEX_SECRET_CI); + if (!hasConvexEnv && !fs.existsSync(convexEnvFile)) { + throw new Error(`Missing Convex env file: ${opts.envFile ?? DEFAULT_CONVEX_ENV_FILE}`); + } + await runCommand({ command: opts.crabboxBin, args: ["--version"], cwd: root }); + if (opts.dryRun) { + return { + command: "telegram-user-crabbox-session", + outputDir, + provider: opts.provider, + target: opts.target, + tdlibSha256: opts.tdlibSha256, + tdlibUrl: opts.tdlibUrl, + }; + } + + const credential = await leaseCredential({ localRoot, opts, root }); + const sut = opts.sutUsername + ? { id: "", username: opts.sutUsername } + : await sutIdentity(credential.sutToken); + const stateArchive = await prepareRemoteState({ localRoot, opts, root }); + let leaseId = opts.leaseId; + let createdLease = false; + if (!leaseId) { + leaseId = await warmupCrabbox(opts, root); + createdLease = true; + } + const inspect = await inspectCrabbox(opts, root, leaseId); + let localSut: Awaited> | undefined; + try { + await writeRemoteSessionScripts({ + inspect, + localRoot, + opts, + root, + stateArchive, + sutUsername: sut.username, + }); + localSut = await startLocalSutDaemon({ + gatewayPort: opts.gatewayPort, + groupId: credential.groupId, + mockPort: opts.mockPort, + outputDir, + repoRoot: root, + sutToken: credential.sutToken, + testerId: credential.testerUserId, + }); + const recorder = await startRemoteRecording(root, inspect); + const session: SessionFile = { + command: "telegram-user-crabbox-session", + createdAt: new Date().toISOString(), + crabbox: { + createdLease, + id: leaseId, + inspect, + provider: opts.provider, + target: opts.target, + }, + credential: { + groupId: credential.groupId, + leaseFile: credential.leaseFile, + sutUsername: sut.username, + testerUserId: credential.testerUserId, + testerUsername: credential.testerUsername, + }, + localRoot, + localSut, + outputDir, + recorder, + remoteRoot: REMOTE_ROOT, + }; + const pathname = sessionPath(root, opts, outputDir); + writeSession(pathname, session); + return { + session: path.relative(root, pathname), + status: "pass", + telegram: { + groupId: credential.groupId, + sutUsername: sut.username, + testerUserId: credential.testerUserId, + testerUsername: credential.testerUsername, + }, + webvnc: `${opts.crabboxBin} webvnc --provider ${opts.provider} --target ${opts.target} --id ${leaseId} --open`, + commands: { + send: `pnpm qa:telegram-user:crabbox -- send --session ${path.relative(root, pathname)} --text '/status'`, + run: `pnpm qa:telegram-user:crabbox -- run --session ${path.relative(root, pathname)} -- bash -lc 'source ${REMOTE_ROOT}/env.sh && python3 ${REMOTE_ROOT}/user-driver.py transcript --limit 20 --json'`, + finish: `pnpm qa:telegram-user:crabbox -- finish --session ${path.relative(root, pathname)}`, + }, + }; + } catch (error) { + killPidTree(localSut?.gatewayPid); + killPidTree(localSut?.mockPid); + await releaseCredential(root, opts, credential.leaseFile).catch(() => {}); + if (leaseId && createdLease) { + await stopCrabbox(root, opts, leaseId).catch(() => {}); + } + throw error; + } +} + +async function sendSessionProbe(root: string, opts: Options, outputDir: string) { + const { session } = readSession(root, opts, outputDir); + const stamp = new Date().toISOString().replace(/[:.]/gu, "-"); + const targetText = buildTargetText(opts.text, session.credential.sutUsername); + const remoteProbe = `${REMOTE_ROOT}/probe-${stamp}.json`; + const probeScript = path.join(session.localRoot, `remote-probe-${stamp}.sh`); + await writeExecutable( + probeScript, + renderRemoteProbe({ + expect: opts.expect, + outputPath: remoteProbe, + sutUsername: session.credential.sutUsername, + text: targetText, + timeoutMs: opts.timeoutMs, + }), + ); + await scpToRemote(root, session.crabbox.inspect, probeScript, `${REMOTE_ROOT}/remote-probe.sh`); + await sshRun(root, session.crabbox.inspect, `bash ${REMOTE_ROOT}/remote-probe.sh`); + const localProbe = path.join(session.outputDir, `probe-${stamp}.json`); + await scpFromRemote(root, session.crabbox.inspect, remoteProbe, localProbe); + return { + probe: path.relative(root, localProbe), + status: "pass", + summary: summarizeProbe(localProbe), + text: targetText, + }; +} + +async function runSessionCommand(root: string, opts: Options, outputDir: string) { + const { session } = readSession(root, opts, outputDir); + const command = opts.remoteCommand.map(shellQuote).join(" "); + const result = await sshRun(root, session.crabbox.inspect, command); + const logPath = path.join( + session.outputDir, + `remote-command-${new Date().toISOString().replace(/[:.]/gu, "-")}.log`, + ); + fs.writeFileSync(logPath, `${result.stdout}${result.stderr}`); + return { command: opts.remoteCommand, log: path.relative(root, logPath), status: "pass" }; +} + +async function screenshotSession(root: string, opts: Options, outputDir: string) { + const { session } = readSession(root, opts, outputDir); + const screenshotPath = path.join( + session.outputDir, + `telegram-user-crabbox-${new Date().toISOString().replace(/[:.]/gu, "-")}.png`, + ); + await runCommand({ + command: opts.crabboxBin, + args: [ + "screenshot", + "--provider", + session.crabbox.provider, + "--target", + session.crabbox.target, + "--id", + session.crabbox.id, + "--output", + screenshotPath, + ], + cwd: root, + stdio: "inherit", + }); + return { screenshot: path.relative(root, screenshotPath), status: "pass" }; +} + +async function statusSession(root: string, opts: Options, outputDir: string) { + const { path: pathname, session } = readSession(root, opts, outputDir); + const inspect = await inspectCrabbox(opts, root, session.crabbox.id); + return { + crabbox: { + id: session.crabbox.id, + slug: inspect.slug, + state: inspect.state, + }, + session: path.relative(root, pathname), + status: "pass", + webvnc: `${opts.crabboxBin} webvnc --provider ${session.crabbox.provider} --target ${session.crabbox.target} --id ${session.crabbox.id} --open`, + }; +} + +async function finishSession(root: string, opts: Options, outputDir: string) { + const { path: pathname, session } = readSession(root, opts, outputDir); + const summary: JsonObject = { + artifacts: {}, + finishedAt: new Date().toISOString(), + session: path.relative(root, pathname), + startedAt: session.createdAt, + status: "fail", + }; + const videoPath = path.join(session.outputDir, "telegram-user-crabbox-session.mp4"); + const motionVideoPath = path.join(session.outputDir, "telegram-user-crabbox-session-motion.mp4"); + const motionGifPath = path.join(session.outputDir, "telegram-user-crabbox-session-motion.gif"); + const screenshotPath = path.join(session.outputDir, "telegram-user-crabbox-session.png"); + const desktopLogPath = path.join(session.outputDir, "telegram-desktop.log"); + const statusPath = path.join(session.outputDir, "status.json"); + const ffmpegLogPath = path.join(session.outputDir, "ffmpeg.log"); + try { + await stopRemoteRecording(root, session.crabbox.inspect, session); + await scpFromRemote(root, session.crabbox.inspect, session.recorder.remoteVideo, videoPath); + await scpFromRemote( + root, + session.crabbox.inspect, + `${REMOTE_ROOT}/telegram-desktop.log`, + desktopLogPath, + ).catch(() => {}); + await scpFromRemote( + root, + session.crabbox.inspect, + `${REMOTE_ROOT}/status.json`, + statusPath, + ).catch(() => {}); + await scpFromRemote(root, session.crabbox.inspect, session.recorder.log, ffmpegLogPath).catch( + () => {}, + ); + const preview = await runCommand({ + command: opts.crabboxBin, + args: [ + "media", + "preview", + "--input", + videoPath, + "--output", + motionGifPath, + "--trimmed-video-output", + motionVideoPath, + "--json", + ], + cwd: root, + stdio: "inherit", + }); + summary.mediaPreview = JSON.parse(preview.stdout) as JsonObject; + await runCommand({ + command: opts.crabboxBin, + args: [ + "screenshot", + "--provider", + session.crabbox.provider, + "--target", + session.crabbox.target, + "--id", + session.crabbox.id, + "--output", + screenshotPath, + ], + cwd: root, + stdio: "inherit", + }); + summary.artifacts = { + desktopLog: path.relative(root, desktopLogPath), + ffmpegLog: path.relative(root, ffmpegLogPath), + previewGif: path.relative(root, motionGifPath), + screenshot: path.relative(root, screenshotPath), + status: path.relative(root, statusPath), + trimmedVideo: path.relative(root, motionVideoPath), + video: path.relative(root, videoPath), + }; + summary.status = "pass"; + } finally { + killPidTree(session.localSut.gatewayPid); + killPidTree(session.localSut.mockPid); + await releaseCredential(root, opts, session.credential.leaseFile).catch((error: unknown) => { + summary.credentialReleaseError = error instanceof Error ? error.message : String(error); + }); + if (session.crabbox.createdLease && !opts.keepBox) { + await stopCrabbox(root, opts, session.crabbox.id).catch((error: unknown) => { + summary.crabboxStopError = error instanceof Error ? error.message : String(error); + }); + } + if (opts.keepBox) { + summary.keepBox = true; + summary.webvnc = `${opts.crabboxBin} webvnc --provider ${session.crabbox.provider} --target ${session.crabbox.target} --id ${session.crabbox.id} --open`; + } + fs.rmSync(session.localRoot, { force: true, recursive: true }); + const summaryPath = path.join(session.outputDir, "telegram-user-crabbox-session-summary.json"); + fs.writeFileSync(summaryPath, `${JSON.stringify(summary, null, 2)}\n`); + const reportPath = writeReport({ + motionGifPath, + motionVideoPath, + outputDir: session.outputDir, + screenshotPath, + status: summary.status === "pass" ? "pass" : "fail", + summaryPath, + videoPath, + }); + summary.report = path.relative(root, reportPath); + fs.writeFileSync(summaryPath, `${JSON.stringify(summary, null, 2)}\n`); + console.log(JSON.stringify({ reportPath, status: summary.status, summaryPath }, null, 2)); + } + if (summary.status !== "pass") { + process.exitCode = 1; + } +} + async function main() { const opts = parseArgs(process.argv.slice(2)); const root = repoRoot(); @@ -960,6 +1557,31 @@ async function main() { fs.mkdirSync(outputDir, { recursive: true }); opts.outputDir = outputDir; + if (opts.command === "start") { + console.log(JSON.stringify(await startSession(root, opts, outputDir), null, 2)); + return; + } + if (opts.command === "send") { + console.log(JSON.stringify(await sendSessionProbe(root, opts, outputDir), null, 2)); + return; + } + if (opts.command === "run") { + console.log(JSON.stringify(await runSessionCommand(root, opts, outputDir), null, 2)); + return; + } + if (opts.command === "screenshot") { + console.log(JSON.stringify(await screenshotSession(root, opts, outputDir), null, 2)); + return; + } + if (opts.command === "status") { + console.log(JSON.stringify(await statusSession(root, opts, outputDir), null, 2)); + return; + } + if (opts.command === "finish") { + await finishSession(root, opts, outputDir); + return; + } + const localRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-telegram-crabbox-")); const summary: JsonObject = { artifacts: {},