Assess model fit

/**
 * Which models is this machine likely to run — asked before downloading any
 * weights. Nothing is fetched, loaded or reserved.
 *
 * The header prints the evidence the answer rests on: `system-memory` on a
 * CPU-only or integrated-GPU host, `device-memory` on a discrete card,
 * `device-budget` for the per-process allowance Windows grants on one. Which
 * of those applies depends on where the engine would put the model.
 *
 * `unknown` is a real answer, not an error: the evidence does not support a
 * call either way. Show it as "can't say", never as "no".
 */
import { assessModelFit, QWEN3_600M_INST_Q4, QWEN3_1_7B_INST_Q4, QWEN3_4B_INST_Q4_K_M, QWEN3_8B_INST_Q4_K_M, QWEN3_8_27B_MULTIMODAL_UD_Q8_K_XL } from '@qvac/sdk';
// Sizes the assessment the way it sizes the load: the estimator reads the
// context out of the config `loadModel` would be given.
const CONTEXT_TOKENS = 8192;
// A ladder ending well past what a laptop has, so one screen shows every verdict.
const CANDIDATES = [
    QWEN3_600M_INST_Q4,
    QWEN3_1_7B_INST_Q4,
    QWEN3_4B_INST_Q4_K_M,
    QWEN3_8B_INST_Q4_K_M,
    QWEN3_8_27B_MULTIMODAL_UD_Q8_K_XL
];
const VERDICT_MARK = {
    'likely-fits': '✔',
    'likely-too-large': '✖',
    unknown: '?'
};
function gib(bytes) {
    return `${(bytes / 1024 ** 3).toFixed(2)} GiB`;
}
try {
    const result = await assessModelFit({
        models: CANDIDATES.map((model) => ({
            modelSrc: model,
            modelType: 'llamacpp-completion',
            modelConfig: { ctx_size: CONTEXT_TOKENS }
        })),
        // Declared for aggregation only, not a scheduling instruction: 'sequential'
        // counts the largest operation peak, 'concurrent' counts one per model.
        execution: 'sequential',
        policy: 'interactive-v1'
    });
    console.log(`▸ Budget basis: ${result.basis}`);
    if (result.budget) {
        console.log(`    ${gib(result.budget.availableAfterReserveBytes)} budget` +
            ` (${gib(result.budget.totalBytes)} total,` +
            ` ${gib(result.budget.usedBytes)} in use,` +
            ` ${gib(result.budget.availableBytes)} free,` +
            ` ${gib(result.budget.reservedBytes)} held back)`);
    }
    // Says which device the verdicts assume, when the model would run on a GPU.
    const placement = result.assumptions.find((line) => line.includes('assumed to execute on'));
    if (placement)
        console.log(`▸ ${placement}`);
    console.log(`\n▸ At ${CONTEXT_TOKENS} tokens of context`);
    for (const [index, model] of result.models.entries()) {
        const mark = VERDICT_MARK[model.verdict];
        const size = gib(CANDIDATES[index].expectedSize).padStart(9);
        // A calibrated row has a two-sided estimate; a computed-only row has the floor.
        const needs = model.estimate
            ? `needs ${gib(model.estimate.upperBoundBytes)}`
            : model.evidence === 'computed-only' && model.floorBytes
                ? `at least ${gib(model.floorBytes)}`
                : '';
        console.log(`  ${mark} ${model.name.padEnd(46)} ${size} on disk  ${needs}`);
        // Present on every `unknown`, and worth surfacing: it names what is missing.
        if (model.verdict === 'unknown') {
            for (const reason of model.reasons)
                console.log(`      ${reason}`);
        }
    }
    console.log(`\n▸ All five together, sequentially: ${result.verdict}`);
    console.log('▸ Advisory only — this does not gate loadModel or reserve anything.');
    process.exit(0);
}
catch (error) {
    console.error('✖', error);
    process.exit(1);
}

/**
 * Advisory model fit check.
 *
 * Before a load, the SDK runs the fitter belonging to the engine that would
 * run it and projects whether the exact configuration it is about to load will
 * fit in device memory. Desktop runs it in a disposable Bare child; mobile,
 * which cannot spawn one, runs it on a worker thread.
 *
 * The result is ADVISORY. It never blocks a load. `does-not-fit` is logged and
 * the ordinary load path runs unchanged. Crashes, timeouts, malformed
 * responses, unsupported configurations, and internal errors all resolve to
 * "no evidence" and are equally non-blocking. No verdict changes the load;
 * `getLoadedModelInfo` returns it as `fitProbe`.
 *
 * The verdict is emitted on the SDK server log stream, not to stdout, so this
 * example subscribes to `loggingStream({ id: SDK_LOG_ID })` and reprints the
 * `[advisory-fit:…]` lines.
 *
 * ---------------------------------------------------------------------------
 * What fits on this machine (Apple M4 Pro, 24 GiB unified memory)
 * ---------------------------------------------------------------------------
 *
 * Measured at the default 1024 MiB margin. The fitter budgets against what the
 * machine can actually keep resident
 * (total − wired − compressor: 17.4 GiB on this machine at idle), not the raw
 * RAM figure.
 *
 *   PROJECTED TO FIT — all layers on GPU
 *     Qwen3.5 0.8B  Q4_K_M   0.5 GiB  @   4k ctx
 *     gpt-oss-20B   Q4_K_M  10.8 GiB  @  32k ctx
 *     gte-large     fp16     0.6 GiB  (embedding, context pinned to 512)
 *
 *   PROJECTED NOT TO FIT — with `gpu_layers: 99` pinned
 *     gpt-oss-20B   Q4_K_M  10.8 GiB  @ 128k ctx with an f32 KV cache ← below
 *
 * These rows were measured with `gpu_layers` pinned at 99. Loads no longer pin
 * the layer count, and the `does-not-fit` row has not been re-measured
 * unpinned.
 *
 * gpt-oss-20B at 128k context FITS with the default KV cache and DOES NOT FIT
 * once `cache-type-k`/`cache-type-v` are set to `f32`. Same model, same
 * context, same machine — the verdict tracks the configuration, not the file
 * size.
 *
 * Two boundaries worth understanding when reading verdicts:
 *
 * 1. The verdict answers for a PLACEMENT, not a model. With `gpu_layers`
 *    unset, the fitter may move layers to the CPU side; setting it pins the
 *    layer count, and `does-not-fit` then means "not at this placement".
 * 2. A `does-not-fit` configuration can still RUN on macOS when the OS
 *    compresses and pages hard enough — and a `fits` configuration right at
 *    the boundary can still fail at first decode under memory pressure.
 *    Prediction cannot separate those cases from a snapshot; the addon-side
 *    probe decode (QVAC-24114) is the runtime check that catches the
 *    remainder. The verdict here is the honest working-set budget, and the
 *    measured failure modes punish over-committing, so treat `does-not-fit`
 *    as "expect degradation or decode failure", not "the load will error".
 */
import { completion, loadModel, unloadModel, loggingStream, SDK_LOG_ID, QWEN3_5_0_8B_MULTIMODAL_Q4_K_M, GPT_OSS_20B_INST_Q4_K_M } from '@qvac/sdk';
// The oversized load below is expected to be reported as `does-not-fit` and
// then attempted anyway, because the check is advisory. It really does try to
// allocate ~11 GiB, so it stays opt-in.
const ATTEMPT_OVERSIZED = process.env['QVAC_FIT_DEMO_ATTEMPT_OVERSIZED'] === '1';
// Reprint the worker's advisory verdicts. They arrive on the SDK server log
// stream; everything else on that stream is filtered out to keep this readable.
//
// Called once per phase rather than once for the process: a subscription
// currently stops delivering after any `unloadModel`, so a single one would go
// silent before the second verdict. Resubscribing after the unload works.
function watchVerdicts() {
    void (async () => {
        for await (const log of loggingStream({ id: SDK_LOG_ID })) {
            if (log.message.includes('[advisory-fit:')) {
                console.log(`▸ [${log.level.toUpperCase()}] ${log.message}`);
            }
        }
    })().catch(() => {
        // Stream terminated — normal on shutdown.
    });
}
watchVerdicts();
try {
    // 1. A load the fitter projects to fit. The verdict carries the plan it
    //    projected: resolved context, offloaded layers, and GPU device count.
    console.log('▸ Loading Qwen3.5 0.8B @ 4k — expected verdict: projected to fit');
    const smallModelId = await loadModel({
        modelSrc: QWEN3_5_0_8B_MULTIMODAL_Q4_K_M,
        modelConfig: { ctx_size: 4096 }
    });
    console.log(`▸ Loaded ${smallModelId}\n`);
    const result = completion({
        modelId: smallModelId,
        history: [{ role: 'user', content: 'Say hello in five words.' }],
        stream: false,
        generationParams: { predict: 48 }
    });
    const final = await result.final;
    console.log(`▸ Completion still works normally: ${final.contentText.trim().slice(0, 120)}\n`);
    // Unloaded before the next phase, so the second verdict is measured on an
    // idle machine and stays comparable to the fixture tables above. Leaving it
    // loaded would shift the verdict: the check reserves resident weight bytes
    // through the fit margin, and the fitter also sees system-wide wired memory.
    await unloadModel({ modelId: smallModelId, clearStorage: false });
    watchVerdicts();
    // 2. A load the fitter projects NOT to fit. The point of this example is that
    //    the SDK reports the verdict and then loads anyway — the check is
    //    evidence, not admission control.
    if (!ATTEMPT_OVERSIZED) {
        console.log('▸ Skipping the oversized gpt-oss-20B load.');
        console.log('▸ Set QVAC_FIT_DEMO_ATTEMPT_OVERSIZED=1 to let it run and watch the');
        console.log('  load proceed past a `does-not-fit` verdict (allocates ~11 GiB).');
    }
    else {
        console.log('▸ Loading gpt-oss-20B @ 128k with an f32 KV cache');
        console.log('▸ Expected verdict: projected NOT to fit');
        console.log('▸ The load is attempted regardless. That is the fail-open contract:');
        console.log('  the verdict is evidence, not admission control.\n');
        const bigModelId = await loadModel({
            modelSrc: GPT_OSS_20B_INST_Q4_K_M,
            modelConfig: {
                ctx_size: 131072,
                'cache-type-k': 'f32',
                'cache-type-v': 'f32'
            }
        });
        console.log(`▸ Load returned ${bigModelId} — the advisory verdict did not block it`);
        // Loading is not the same as being usable: a model can load and then fail
        // at decode time. Gemma 4 31B does exactly that on this machine. So run a
        // real completion and report throughput rather than trusting the load.
        try {
            const check = completion({
                modelId: bigModelId,
                history: [{ role: 'user', content: 'Name three colours. Answer briefly.' }],
                stream: false,
                generationParams: { predict: 40 }
            });
            const checkFinal = await check.final;
            console.log(`▸ ...and it actually runs: ${checkFinal.stats?.tokensPerSecond?.toFixed(1) ?? '?'} tok/s ` +
                `— the OS compressed and paged its way past the working-set budget`);
        }
        catch (inferenceError) {
            console.log(`▸ ...but it cannot run: ${inferenceError instanceof Error ? inferenceError.message : String(inferenceError)}`);
            console.log('▸ The verdict was right, and `loadModel` succeeding did not mean usable.');
        }
        await unloadModel({ modelId: bigModelId, clearStorage: false });
    }
}
catch (error) {
    // A failing load here is the native loader's own error, not the fit check.
    // The check never throws and never converts a verdict into a load failure.
    console.error('✖', error instanceof Error ? error.message : error);
    process.exitCode = 1;
}
// The log subscription is an open stream and would otherwise keep the process
// alive after the work is done.
process.exit(process.exitCode ?? 0);