mirror of
https://github.com/moltbot/moltbot.git
synced 2026-05-13 15:47:28 +00:00
Fixes Copilot image understanding by exchanging OAuth tokens for Copilot API tokens, routing Copilot Gemini image requests through Chat Completions, and sending the prompt in user content with Copilot vision headers. Real behavior proof: - Old Responses route with real Copilot key reproduced `400 model gemini-3.1-pro-preview does not support Responses API`. - Fixed route with the same real Copilot key returned `Cat`. - Final CLI live smoke returned `ok: true` and `text: Cat` for `github-copilot/gemini-3.1-pro-preview`. Verification: - pnpm test src/media-understanding/image.test.ts extensions/github-copilot/models.test.ts extensions/github-copilot/stream.test.ts src/agents/pi-hooks/compaction-safeguard.test.ts -- --reporter=verbose - pnpm check:changed via Blacksmith Testbox tbx_01krgt56pqmft8txekt017wke6, Actions run https://github.com/openclaw/openclaw/actions/runs/25803926150, exit 0. Refs #80393, #80442. Co-authored-by: Yang Haoyu <150496764+afunnyhy@users.noreply.github.com>
158 lines
5.0 KiB
TypeScript
158 lines
5.0 KiB
TypeScript
import type { StreamFn } from "@earendil-works/pi-agent-core";
|
|
import type { Context } from "@earendil-works/pi-ai";
|
|
import type { ProviderWrapStreamFnContext } from "openclaw/plugin-sdk/plugin-entry";
|
|
import { buildCopilotIdeHeaders, COPILOT_INTEGRATION_ID } from "openclaw/plugin-sdk/provider-auth";
|
|
import {
|
|
applyAnthropicEphemeralCacheControlMarkers,
|
|
streamWithPayloadPatch,
|
|
} from "openclaw/plugin-sdk/provider-stream-shared";
|
|
import { rewriteCopilotResponsePayloadConnectionBoundIds } from "./connection-bound-ids.js";
|
|
|
|
type StreamOptions = Parameters<StreamFn>[2];
|
|
|
|
function containsCopilotContentType(value: unknown, type: string): boolean {
|
|
if (Array.isArray(value)) {
|
|
return value.some((item) => containsCopilotContentType(item, type));
|
|
}
|
|
if (!value || typeof value !== "object") {
|
|
return false;
|
|
}
|
|
const entry = value as { type?: unknown; content?: unknown };
|
|
return entry.type === type || containsCopilotContentType(entry.content, type);
|
|
}
|
|
|
|
function inferCopilotInitiator(messages: Context["messages"]): "agent" | "user" {
|
|
const last = messages[messages.length - 1];
|
|
if (!last) {
|
|
return "user";
|
|
}
|
|
if (last.role === "user" && containsCopilotContentType(last.content, "tool_result")) {
|
|
return "agent";
|
|
}
|
|
return last.role === "user" ? "user" : "agent";
|
|
}
|
|
|
|
export function hasCopilotVisionInput(messages: Context["messages"]): boolean {
|
|
return messages.some((message) => {
|
|
if (message.role === "user" && Array.isArray(message.content)) {
|
|
return message.content.some((item) => containsCopilotContentType(item, "image"));
|
|
}
|
|
if (message.role === "toolResult" && Array.isArray(message.content)) {
|
|
return message.content.some((item) => containsCopilotContentType(item, "image"));
|
|
}
|
|
return false;
|
|
});
|
|
}
|
|
|
|
export function buildCopilotDynamicHeaders(params: {
|
|
messages: Context["messages"];
|
|
hasImages: boolean;
|
|
}): Record<string, string> {
|
|
return {
|
|
...buildCopilotIdeHeaders(),
|
|
"Copilot-Integration-Id": COPILOT_INTEGRATION_ID,
|
|
"Openai-Organization": "github-copilot",
|
|
"x-initiator": inferCopilotInitiator(params.messages),
|
|
...(params.hasImages ? { "Copilot-Vision-Request": "true" } : {}),
|
|
};
|
|
}
|
|
|
|
function patchOnPayloadResult(result: unknown): unknown {
|
|
if (result && typeof result === "object" && "then" in result) {
|
|
return Promise.resolve(result).then((next) => {
|
|
rewriteCopilotResponsePayloadConnectionBoundIds(next);
|
|
return next;
|
|
});
|
|
}
|
|
rewriteCopilotResponsePayloadConnectionBoundIds(result);
|
|
return result;
|
|
}
|
|
|
|
function buildCopilotRequestHeaders(
|
|
context: Parameters<StreamFn>[1],
|
|
headers: Record<string, string> | undefined,
|
|
): Record<string, string> {
|
|
return {
|
|
...buildCopilotDynamicHeaders({
|
|
messages: context.messages,
|
|
hasImages: hasCopilotVisionInput(context.messages),
|
|
}),
|
|
...headers,
|
|
};
|
|
}
|
|
|
|
export function wrapCopilotAnthropicStream(
|
|
baseStreamFn: StreamFn | undefined,
|
|
): StreamFn | undefined {
|
|
if (!baseStreamFn) {
|
|
return undefined;
|
|
}
|
|
const underlying = baseStreamFn;
|
|
return (model, context, options) => {
|
|
if (model.provider !== "github-copilot" || model.api !== "anthropic-messages") {
|
|
return underlying(model, context, options);
|
|
}
|
|
|
|
return streamWithPayloadPatch(
|
|
underlying,
|
|
model,
|
|
context,
|
|
{
|
|
...options,
|
|
headers: buildCopilotRequestHeaders(context, options?.headers),
|
|
},
|
|
applyAnthropicEphemeralCacheControlMarkers,
|
|
);
|
|
};
|
|
}
|
|
|
|
export function wrapCopilotOpenAIResponsesStream(
|
|
baseStreamFn: StreamFn | undefined,
|
|
): StreamFn | undefined {
|
|
if (!baseStreamFn) {
|
|
return undefined;
|
|
}
|
|
const underlying = baseStreamFn;
|
|
return (model, context, options) => {
|
|
if (model.provider !== "github-copilot" || model.api !== "openai-responses") {
|
|
return underlying(model, context, options);
|
|
}
|
|
|
|
const originalOnPayload = options?.onPayload;
|
|
const wrappedOptions: StreamOptions = {
|
|
...options,
|
|
headers: buildCopilotRequestHeaders(context, options?.headers),
|
|
onPayload: (payload, payloadModel) => {
|
|
rewriteCopilotResponsePayloadConnectionBoundIds(payload);
|
|
return patchOnPayloadResult(originalOnPayload?.(payload, payloadModel));
|
|
},
|
|
};
|
|
return underlying(model, context, wrappedOptions);
|
|
};
|
|
}
|
|
|
|
export function wrapCopilotOpenAICompletionsStream(
|
|
baseStreamFn: StreamFn | undefined,
|
|
): StreamFn | undefined {
|
|
if (!baseStreamFn) {
|
|
return undefined;
|
|
}
|
|
const underlying = baseStreamFn;
|
|
return (model, context, options) => {
|
|
if (model.provider !== "github-copilot" || model.api !== "openai-completions") {
|
|
return underlying(model, context, options);
|
|
}
|
|
|
|
return underlying(model, context, {
|
|
...options,
|
|
headers: buildCopilotRequestHeaders(context, options?.headers),
|
|
});
|
|
};
|
|
}
|
|
|
|
export function wrapCopilotProviderStream(ctx: ProviderWrapStreamFnContext): StreamFn | undefined {
|
|
return wrapCopilotOpenAICompletionsStream(
|
|
wrapCopilotOpenAIResponsesStream(wrapCopilotAnthropicStream(ctx.streamFn)),
|
|
);
|
|
}
|