test(release): harden live release validation

This commit is contained in:
Peter Steinberger
2026-05-10 00:50:13 +01:00
parent 6f9f3bc1ec
commit d5a1792a66
4 changed files with 40 additions and 19 deletions

View File

@@ -320,7 +320,7 @@ describeLive("openai plugin live", () => {
const collapsedText = text.replace(/[\s-]+/g, "");
expect(text.length).toBeGreaterThan(0);
expect(collapsedText).toContain("speech");
expect(collapsedText).toContain("check");
expect(collapsedText).toMatch(/(?:check|okay|ok|transcription)/);
}, 45_000);
it("opens OpenAI realtime STT before sending audio", async () => {

View File

@@ -178,7 +178,7 @@ send_channels_flow() {
# a stable no-op smoke path when the config starts empty.
wait_for_log "Where will the Gateway run?" 120
send $'\r' 0.6
wait_for_log "Configure/link" 120
wait_for_log "Channel setup" 120
send $'\e[B\r' 0.8
# Keep stdin open until wizard exits.
send "" 2.0

View File

@@ -273,6 +273,14 @@ describe("isProviderUnavailableErrorMessage", () => {
isProviderUnavailableErrorMessage("provider returned error: 502 Internal Server Error"),
).toBe(true);
});
it("matches xAI temporary capacity errors", () => {
expect(
isProviderUnavailableErrorMessage(
"Service temporarily unavailable. The model is at capacity and currently cannot serve this request. Please try again later.",
),
).toBe(true);
});
});
function isChatGPTUsageLimitErrorMessage(raw: string): boolean {
@@ -314,6 +322,7 @@ function isProviderUnavailableErrorMessage(raw: string): boolean {
msg.includes("upstream provider unavailable") ||
msg.includes("upstream error from google") ||
msg.includes("temporarily rate-limited upstream") ||
(msg.includes("service temporarily unavailable") && msg.includes("capacity")) ||
msg.includes("unable to access non-serverless model") ||
msg.includes("create and start a new dedicated endpoint") ||
msg.includes("no available capacity was found for the model") ||

View File

@@ -6,12 +6,16 @@ import {
extractNonEmptyAssistantText,
isLiveTestEnabled,
} from "./live-test-helpers.js";
import { isBillingErrorMessage } from "./pi-embedded-helpers/failover-matches.js";
import {
isBillingErrorMessage,
isOverloadedErrorMessage,
} from "./pi-embedded-helpers/failover-matches.js";
import { applyExtraParamsToAgent } from "./pi-embedded-runner.js";
import { createWebSearchTool } from "./tools/web-search.js";
const XAI_KEY = process.env.XAI_API_KEY ?? "";
const LIVE = isLiveTestEnabled(["XAI_LIVE_TEST"]);
const XAI_COMPLETE_LIVE_TIMEOUT_MS = 90_000;
const XAI_WEB_SEARCH_LIVE_TIMEOUT_SECONDS = 60;
const describeLive = LIVE && XAI_KEY ? describe : describe.skip;
@@ -47,6 +51,10 @@ async function runXaiLiveCase(label: string, run: () => Promise<void>): Promise<
console.warn(`[xai:live] skip ${label}: billing drift: ${message}`);
return;
}
if (isOverloadedErrorMessage(message)) {
console.warn(`[xai:live] skip ${label}: temporary provider capacity: ${message}`);
return;
}
if (message.includes("web_search is disabled or no provider is available")) {
console.warn(`[xai:live] skip ${label}: web_search unavailable in this environment`);
return;
@@ -68,23 +76,27 @@ async function collectDoneMessage(
}
describeLive("xai live", () => {
it("returns assistant text for Grok 4.3", async () => {
await runXaiLiveCase("complete", async () => {
const model = requireLiveValue(resolveLiveXaiModel(), "xAI model");
const res = await completeSimple(
model,
{
messages: createSingleUserPromptMessage(),
},
{
apiKey: XAI_KEY,
maxTokens: 64,
},
);
it(
"returns assistant text for Grok 4.3",
async () => {
await runXaiLiveCase("complete", async () => {
const model = requireLiveValue(resolveLiveXaiModel(), "xAI model");
const res = await completeSimple(
model,
{
messages: createSingleUserPromptMessage(),
},
{
apiKey: XAI_KEY,
maxTokens: 64,
},
);
expect(extractNonEmptyAssistantText(res.content).length).toBeGreaterThan(0);
});
}, 30_000);
expect(extractNonEmptyAssistantText(res.content).length).toBeGreaterThan(0);
});
},
XAI_COMPLETE_LIVE_TIMEOUT_MS,
);
it("sends wrapped xAI tool payloads live", async () => {
await runXaiLiveCase("tool-call", async () => {