Text generation

/**
 * Event-driven completion — demonstrates the unified `CompletionEvent` stream.
 *
 * `completion()` returns a `CompletionRun` with two primary surfaces:
 *
 *  - `events`  — an `AsyncIterable<CompletionEvent>` of ordered, typed events
 *                (`contentDelta`, `thinkingDelta`, `toolCall`, `toolError`,
 *                 `completionStats`, `completionDone`, `rawDelta`).
 *  - `final`   — a `Promise<CompletionFinal>` that resolves once the stream
 *                ends, providing aggregated `contentText`, `thinkingText`,
 *                `toolCalls`, `stats`, and `raw.fullText`.
 *
 * Set `captureThinking: true` to attempt best-effort `<think>` block parsing
 * into dedicated `thinkingDelta` events. `final.raw.fullText` keeps the exact
 * model output.
 */
import { completion, loadModel, unloadModel, QWEN3_600M_INST_Q4 } from '@qvac/sdk';
try {
    const modelId = await loadModel({
        modelSrc: QWEN3_600M_INST_Q4,
        modelConfig: { ctx_size: 4096 },
        onProgress: (p) => {
            const mb = (n) => (n / 1e6).toFixed(1);
            const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
            process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
            if (p.percentage >= 100)
                process.stderr.write('\n');
        }
    });
    console.log(`▸ Model loaded: ${modelId}`);
    const result = completion({
        modelId,
        history: [{ role: 'user', content: 'Explain quantum computing in 2 sentences' }],
        stream: true,
        captureThinking: true
    });
    for await (const event of result.events) {
        handleEvent(event);
    }
    const final = await result.final;
    console.log('\n▸ Final result');
    console.log(`▸ Content: ${final.contentText}`);
    if (final.thinkingText) {
        console.log(`▸ Thinking: ${final.thinkingText}`);
    }
    if (final.stats) {
        console.log(`▸ ${final.stats.tokensPerSecond?.toFixed(1)} tok/s`);
    }
    if (final.toolCalls.length > 0) {
        console.log(`▸ Tool calls: ${final.toolCalls.map((c) => c.name).join(', ')}`);
    }
    if (final.stopReason) {
        console.log(`▸ Stop reason: ${final.stopReason}`);
    }
    console.log(`▸ Raw output length: ${final.raw.fullText.length} chars`);
    await unloadModel({ modelId, clearStorage: false });
}
catch (error) {
    console.error('✖', error);
    process.exit(1);
}
function handleEvent(event) {
    switch (event.type) {
        case 'contentDelta':
            process.stdout.write(event.text);
            break;
        case 'thinkingDelta':
            process.stdout.write(`\x1b[2m${event.text}\x1b[0m`);
            break;
        case 'toolCall':
            console.log(`\n▸ Tool: ${event.call.name}(${JSON.stringify(event.call.arguments)})`);
            break;
        case 'toolError':
            console.log(`\n✖ Tool error [${event.error.code}]: ${event.error.message}`);
            break;
        case 'completionStats':
            console.log(`\n▸ ${event.stats.tokensPerSecond?.toFixed(1)} tok/s`);
            break;
        case 'completionDone':
            if (event.stopReason === 'error' && 'error' in event) {
                console.log(`\n✖ ${event.error.message}`);
            }
            break;
        case 'rawDelta':
            break;
    }
}

/**
 * Multi-job continuous batching — concurrent completions on one loaded LLM.
 *
 * Unlike `batchCompletion()` (which bundles many prompts into ONE call), this
 * shows independent `completion()` calls fired concurrently against the SAME
 * loaded model. When the model is loaded with `modelConfig.parallel >= 2`, the
 * SDK admits up to `parallel` completions at once and the llama.cpp addon
 * decodes their sequences together — so a burst of prompts finishes in roughly
 * the time of the slowest one, not the sum of all of them.
 *
 * Each `completion()` returns an independent `CompletionRun`:
 *  - `events`     — an `AsyncIterable` of streamed events for that call.
 *  - `final`      — a `Promise<CompletionFinal>` with THIS call's own
 *                   `contentText` and `stats` (its own tokens/sec, TTFT).
 *                   `stats.avgConcurrentSeq` stays model-level (how busy the
 *                   shared backend was).
 *  - `requestId`  — cancel just this call with `cancel({ requestId })`; peers
 *                   keep decoding.
 *
 * Completions and batches share the model's `parallel` admission cap: with
 * `parallel` in flight, admitting one more waits FIFO until a slot frees.
 *
 * Run from packages/sdk:
 *   bun run examples/multi-job-completion.ts
 */
import { completion, cancel, loadModel, unloadModel, InferenceCancelledError, LLAMA_3_2_1B_INST_Q4_0 } from '@qvac/sdk';
const PROMPTS = [
    { id: 'cherry', ask: 'Reply with only the word CHERRY.' },
    { id: 'banana', ask: 'Reply with only the word BANANA.' },
    { id: 'grape', ask: 'Reply with only the word GRAPE.' },
    { id: 'lemon', ask: 'Reply with only the word LEMON.' }
];
try {
    // `parallel: 4` opens the concurrent decode slots continuous batching needs.
    const modelId = await loadModel({
        modelSrc: LLAMA_3_2_1B_INST_Q4_0,
        modelType: 'llm',
        modelConfig: { ctx_size: 4096, parallel: 4 },
        onProgress: (p) => {
            const mb = (n) => (n / 1e6).toFixed(1);
            const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
            process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
            if (p.percentage >= 100)
                process.stderr.write('\n');
        }
    });
    console.log(`▸ Model loaded: ${modelId}`);
    // ---- Concurrent completions. Fire every call before awaiting any, so all
    // four are admitted and decode together (not one-after-another). ----
    console.log('\n▸ Firing 4 completions concurrently:');
    const started = Date.now();
    const runs = PROMPTS.map(({ ask }) => completion({
        modelId,
        history: [{ role: 'user', content: ask }],
        generationParams: { temp: 0, seed: 42, predict: 16 },
        stream: true
    }));
    const outputs = await Promise.all(runs.map(async (run, i) => {
        const final = await run.final;
        return {
            id: PROMPTS[i].id,
            text: final.contentText.replace(/\s+/g, ' ').trim(),
            stats: final.stats
        };
    }));
    const elapsed = Date.now() - started;
    for (const { id, text, stats } of outputs) {
        const tps = stats?.tokensPerSecond?.toFixed(1) ?? 'n/a';
        console.log(`  ▸ ${id}: "${text}"  (own ${tps} tok/s)`);
    }
    // Report the engine's own measure of overlap rather than inferring it from
    // wall-clock: avgConcurrentSeq > 1 means the addon decoded sequences together.
    const peakConcurrent = Math.max(...outputs.map((o) => o.stats?.avgConcurrentSeq ?? 0));
    console.log(`\n▸ All 4 finished in ${elapsed} ms; engine avg concurrent sequences: ${peakConcurrent.toFixed(1)} (> 1 = decoded together).`);
    // ---- Per-request cancel isolation. Fire two long runs; cancel only the
    // first by its requestId. The second must keep decoding past the cancel. ----
    console.log('\n▸ Per-request cancel: cancelling one of two in-flight completions:');
    const doomed = completion({
        modelId,
        history: [{ role: 'user', content: 'Write a long story about an otter.' }],
        generationParams: { temp: 0, seed: 1, predict: 256 },
        stream: true
    });
    const survivor = completion({
        modelId,
        history: [{ role: 'user', content: 'Write a long story about a melon.' }],
        generationParams: { temp: 0, seed: 42, predict: 256 },
        stream: true
    });
    // Consume the survivor's stream directly so we can count tokens produced AFTER
    // the cancel is acknowledged — a short survivor could otherwise finish before
    // the cancel lands and pass even under a model-wide cancel.
    const survivorTokens = survivor.events[Symbol.asyncIterator]();
    // Wait until BOTH runs are actually streaming before cancelling, so the cancel
    // hits a genuinely in-flight peer, not a pre-admission drop.
    await Promise.all([doomed.events[Symbol.asyncIterator]().next(), survivorTokens.next()]);
    await cancel({ requestId: doomed.requestId });
    let doomedCancelled = false;
    try {
        await doomed.final;
    }
    catch (err) {
        doomedCancelled = err instanceof InferenceCancelledError;
    }
    // Drain what the survivor emits from here on: a per-request cancel leaves it
    // decoding to completion, a model-wide cancel would have stopped it.
    let tokensAfterCancel = 0;
    for (let next = await survivorTokens.next(); !next.done; next = await survivorTokens.next()) {
        tokensAfterCancel++;
    }
    const survivorText = (await survivor.final).contentText.replace(/\s+/g, ' ').trim();
    console.log(`  ▸ cancelled run -> ${doomedCancelled ? 'cancelled' : 'NOT cancelled'}`);
    console.log(`  ▸ peer kept decoding -> +${tokensAfterCancel} tokens after the cancel ack`);
    await unloadModel({ modelId, clearStorage: false });
    // A zero exit must prove concurrent scheduling, typed cancellation, and survivor
    // isolation: the first burst reported multiple resident sequences, the doomed
    // run cancelled, and its peer produced tokens AFTER the cancel landed.
    if (peakConcurrent <= 1 || !doomedCancelled || tokensAfterCancel === 0) {
        console.error(`✖ multi-job behavior not demonstrated (peakConcurrent=${peakConcurrent.toFixed(2)}, doomedCancelled=${doomedCancelled}, tokensAfterCancel=${tokensAfterCancel}, survivorText="${survivorText}")`);
        process.exit(1);
    }
    process.exit(0);
}
catch (error) {
    console.error('✖', error);
    process.exit(1);
}

import { completion, loadModel, unloadModel, QWEN3_1_7B_INST_Q4 } from '@qvac/sdk';
import { tools, toolSchemas, mockExecute } from './shared';
try {
    const modelId = await loadModel({
        modelSrc: QWEN3_1_7B_INST_Q4,
        modelConfig: {
            ctx_size: 4096,
            tools: true
        },
        onProgress: (p) => {
            const mb = (n) => (n / 1e6).toFixed(1);
            const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
            process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
            if (p.percentage >= 100)
                process.stderr.write('\n');
        }
    });
    console.log(`▸ Model loaded successfully! Model ID: ${modelId}`);
    const history = [
        {
            role: 'system',
            content: 'You are a helpful assistant that can use tools to get the weather and horoscope.'
        },
        {
            role: 'user',
            content: "What's the weather in Tokyo and my horoscope for Aquarius?"
        }
    ];
    console.log('\n▸ AI Response:');
    console.log('▸ (Streaming with tool definitions in prompt)\n');
    const result = completion({ modelId, history, stream: true, tools });
    const tokensTask = (async () => {
        for await (const token of result.tokenStream) {
            process.stdout.write(token);
        }
    })();
    const toolsTask = (async () => {
        for await (const evt of result.toolCallStream) {
            console.log(`\n\n▸ Tool Call Detected: ${evt.call.name}(${JSON.stringify(evt.call.arguments)})`);
            console.log(`▸ ID: ${evt.call.id}`);
        }
    })();
    await Promise.all([tokensTask, toolsTask]);
    const stats = await result.stats;
    const toolCalls = await result.toolCalls;
    console.log('\n\n▸ Parsed Tool Calls:');
    if (toolCalls.length > 0) {
        for (const call of toolCalls) {
            console.log(`▸ ${call.name}(${JSON.stringify(call.arguments)})`);
            const schema = toolSchemas[call.name];
            if (schema) {
                const validated = schema.safeParse(call.arguments);
                if (validated.success) {
                    console.log(`▸ Arguments validated with Zod`);
                }
                else {
                    console.log(`✖ Validation failed:`, validated.error);
                }
            }
        }
    }
    else {
        console.log('▸ No tool calls detected in response');
    }
    console.log('\n▸ Performance Stats:', stats);
    if (toolCalls.length > 0) {
        console.log('\n\n▸ Simulating Tool Execution...');
        const toolResults = toolCalls.map((call) => {
            const result = mockExecute(call.name, call.arguments);
            console.log(`▸ ${call.name}: ${result}`);
            return { toolCallId: call.id, result };
        });
        history.push({
            role: 'assistant',
            content: await result.text
        });
        for (const toolResult of toolResults) {
            history.push({
                role: 'tool',
                content: toolResult.result
            });
        }
        console.log('\n\n▸ Follow-up Response with Tool Results:');
        const followUpResult = completion({
            modelId,
            history,
            stream: true,
            tools
        });
        for await (const token of followUpResult.tokenStream) {
            process.stdout.write(token);
        }
        const followUpStats = await followUpResult.stats;
        console.log('\n\n▸ Follow-up Stats:', followUpStats);
    }
    console.log('\n\n▸ Completed!');
    await unloadModel({ modelId, clearStorage: false });
}
catch (error) {
    console.error('✖', error);
    process.exit(1);
}

"""Worker-orchestrated tool loop — mirrors the SDK's server-side tool loop
(beyond the client-side loop in completion_tools.py).

`completion_orchestrate()` hands the multi-turn tool loop to the WORKER: when
the model asks for a tool, the worker emits a callback frame, this client runs
the registered `handler`, writes the result back, and generation continues —
no manual history stitching. Every tool needs a `handler` (it runs code on
this machine, so only a local worker may orchestrate).

Advanced path: `completion_orchestrate` lives in `tetherto.qvac_sdk._completion` rather than
the flat `tetherto.qvac_sdk` surface. The public tool API is `completion(tools=...)` (the
client-side loop in completion_tools.py), matching the JS SDK; worker
orchestration has no JS client wrapper yet.

RUN: python examples/completion_orchestrate.py
"""

from __future__ import annotations

import asyncio
import sys

from tetherto.qvac_sdk import Client, load_model, unload_model
from tetherto.qvac_sdk._completion import completion_orchestrate
from tetherto.qvac_sdk.models import QWEN3_1_7B_INST_Q4


def print_progress(p) -> None:
    """Print model download progress; pass as `on_progress=` to `load_model`."""
    line = (
        f"▸ Downloading {p.percentage:.0f}% "
        f"({p.downloaded / 1e6:.1f}/{p.total / 1e6:.1f} MB)"
    )
    print(line, end="\r" if sys.stderr.isatty() else "\n", file=sys.stderr)
    if p.percentage >= 100:
        print(file=sys.stderr)


async def get_weather(arguments) -> str:
    return f"The weather in {arguments.get('location')} is 22°C and sunny."


async def get_horoscope(arguments) -> str:
    return f"{arguments.get('sign')}: today favours careful validation."


TOOLS = [
    {
        "name": "get_weather",
        "description": "Get the current weather for a location",
        "parameters": {
            "type": "object",
            "properties": {"location": {"type": "string"}},
            "required": ["location"],
        },
        "handler": get_weather,
    },
    {
        "name": "get_horoscope",
        "description": "Get the horoscope for a star sign",
        "parameters": {
            "type": "object",
            "properties": {"sign": {"type": "string"}},
            "required": ["sign"],
        },
        "handler": get_horoscope,
    },
]


async def main() -> int:
    async with Client() as client:
        t = client.transport
        try:
            model_id = await load_model(
                t,
                model_src=QWEN3_1_7B_INST_Q4,
                model_config={"ctx_size": 4096, "tools": True},
                on_progress=print_progress,
            )
            print(f"▸ Model loaded: {model_id}\n")

            run = completion_orchestrate(
                t,
                model_id=model_id,
                history=[
                    {
                        "role": "user",
                        "content": "What's the weather in Tokyo and my horoscope for Aquarius?",
                    }
                ],
                tools=TOOLS,
                max_tool_turns=4,
            )

            async for event in run.events:
                if event.type == "contentDelta":
                    sys.stdout.write(event.text)
                    sys.stdout.flush()

            final = await run.final
            print("\n\n▸ Final answer:", final.content_text)

            await unload_model(t, model_id)
        except Exception as error:
            print(f"✖ {error}", file=sys.stderr)
            return 1
    return 0


if __name__ == "__main__":
    sys.exit(asyncio.run(main()))

/**
 * MCP DuckDuckGo Search Example
 *
 * A web search example using DuckDuckGo - no API key required!
 * The server provides tools to search the web and get answers.
 *
 * Prerequisites:
 * - Install MCP SDK: bun add @modelcontextprotocol/sdk
 *
 * Run with: bun run examples/mcp-websearch.ts
 */
import { completion, loadModel, unloadModel, QWEN3_1_7B_INST_Q4 } from '@/index';
// MCP SDK is a user-installed optional dependency
// Install with: bun add @modelcontextprotocol/sdk
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
function parseSearchResults(mcpResult) {
    try {
        const result = mcpResult;
        // Extract text content from MCP response
        const textContent = result.content?.find((c) => c.type === 'text');
        if (!textContent?.text) {
            return JSON.stringify(mcpResult);
        }
        // Parse the JSON array of search results
        const rawResults = JSON.parse(textContent.text);
        // Extract just the useful fields (title, url, snippet)
        const cleanResults = rawResults.slice(0, 5).map((r) => ({
            title: r.title ?? 'Unknown',
            url: r.url ?? '',
            snippet: r.snippet ?? ''
        }));
        // Format as concise text for LLM
        return cleanResults
            .map((r, i) => `[${i + 1}] ${r.title}\n    URL: ${r.url}\n    ${r.snippet}`)
            .join('\n\n');
    }
    catch {
        // If parsing fails, return a truncated version
        const str = typeof mcpResult === 'string' ? mcpResult : JSON.stringify(mcpResult);
        return str.slice(0, 2000);
    }
}
let mcpClient = null;
try {
    console.log('▸ MCP DuckDuckGo Search Example\n');
    // ============================================================
    // STEP 1: Connect to DuckDuckGo MCP server
    // ============================================================
    console.log('▸ Starting DuckDuckGo MCP server...');
    mcpClient = new Client({
        name: 'qvac-ddg-example',
        version: '1.0.0'
    });
    const transport = new StdioClientTransport({
        command: 'npx',
        args: ['-y', '@oevortex/ddg_search']
    });
    await mcpClient.connect(transport);
    console.log('▸ MCP server connected\n');
    // ============================================================
    // STEP 2: Load model
    // ============================================================
    console.log('▸ Loading model...');
    const modelId = await loadModel({
        modelSrc: QWEN3_1_7B_INST_Q4,
        modelConfig: {
            ctx_size: 4096,
            tools: true
        },
        onProgress: (p) => {
            const mb = (n) => (n / 1e6).toFixed(1);
            const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
            process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
            if (p.percentage >= 100)
                process.stderr.write('\n');
        }
    });
    console.log(`\n▸ Model loaded\n`);
    // ============================================================
    // STEP 3: Ask AI to search the web (with MCP client)
    // ============================================================
    const history = [
        {
            role: 'system',
            content: `You are a helpful assistant with access to web search.
Use the search tool when you need current information.
Always cite your sources with the URL.`
        },
        {
            role: 'user',
            content: 'What is the current weather in New York City?'
        }
    ];
    console.log('▸ Asking AI to search the web...\n');
    console.log('▸ AI Response:');
    // Pass MCP client directly to completion - tools are adapted internally!
    const result = completion({
        modelId,
        history,
        stream: true,
        mcp: [{ client: mcpClient, includeResources: false }]
    });
    for await (const token of result.tokenStream) {
        process.stdout.write(token);
    }
    const toolCalls = await result.toolCalls;
    console.log('\n');
    // ============================================================
    // STEP 4: Execute tool calls using call() - automatic MCP routing!
    // ============================================================
    if (toolCalls.length > 0) {
        console.log('▸ Executing search...\n');
        const toolResults = [];
        for (const toolCall of toolCalls) {
            console.log(`▸ ${toolCall.name}(${JSON.stringify(toolCall.arguments)})`);
            if (!toolCall.invoke) {
                console.log(`▸ No handler found for tool "${toolCall.name}"`);
                continue;
            }
            // Use invoke() - automatically routes to the correct MCP client!
            const mcpResult = await toolCall.invoke();
            // Parse and clean up the search results
            const cleanResult = parseSearchResults(mcpResult);
            console.log(`▸ Got search results:`);
            console.log(cleanResult
                .split('\n')
                .map((l) => `      ${l}`)
                .join('\n'));
            console.log();
            toolResults.push({ id: toolCall.id, result: cleanResult });
        }
        // ============================================================
        // STEP 5: Continue with search results
        // ============================================================
        console.log('▸ Getting AI response with search results...\n');
        history.push({
            role: 'assistant',
            content: await result.text
        });
        for (const tr of toolResults) {
            history.push({
                role: 'tool',
                content: tr.result
            });
        }
        console.log('▸ Final Response:');
        const finalResult = completion({
            modelId,
            history,
            stream: true,
            mcp: [{ client: mcpClient, includeResources: false }]
        });
        for await (const token of finalResult.tokenStream) {
            process.stdout.write(token);
        }
        console.log('\n');
    }
    // ============================================================
    // Cleanup
    // ============================================================
    console.log('▸ Cleaning up...');
    await unloadModel({ modelId, clearStorage: false });
    console.log('▸ Done\n');
    console.log('▸ Example completed!');
    process.exit(0);
}
catch (error) {
    console.error('✖', error);
    process.exit(1);
}
finally {
    if (mcpClient) {
        try {
            await mcpClient.close();
        }
        catch {
            // Ignore close errors
        }
    }
}

import { completion, LLAMA_3_2_1B_INST_Q4_0, loadModel, unloadModel, VERBOSITY } from '@qvac/sdk';
try {
    // Load the model
    const modelId = await loadModel({
        modelSrc: LLAMA_3_2_1B_INST_Q4_0,
        modelConfig: {
            device: 'gpu',
            ctx_size: 2048,
            verbosity: VERBOSITY.ERROR
        }
    });
    console.log('▸ Testing KV Cache functionality...\n');
    // First conversation with auto-keyed cache enabled
    console.log('▸ First conversation (building cache for the next turn):');
    const history1 = [{ role: 'user', content: 'What is the capital of France?' }];
    const result1 = completion({
        modelId,
        history: history1,
        stream: true,
        kvCache: true
    }); // kvCache = true
    for await (const token of result1.tokenStream) {
        process.stdout.write(token);
    }
    const final1 = await result1.final;
    const stats1 = final1.stats;
    console.log(`\n▸ First completion stats: ${JSON.stringify(stats1)}\n`);
    // Continue conversation (should reuse the completed first-turn cache).
    console.log('▸ Continuing conversation (reusing previous turn cache):');
    const history2 = [
        { role: 'user', content: 'What is the capital of France?' },
        {
            role: 'assistant',
            content: final1.cacheableAssistantContent ?? final1.contentText
        },
        { role: 'user', content: 'What about Germany?' }
    ];
    // Auto-keyed caching should:
    // 1. Find the cache saved after turn 1 under [user, assistant]
    // 2. Load that cache and process only the new "What about Germany?" user turn
    // 3. Save the updated cache and rename it to include the new assistant response
    const result2 = completion({
        modelId,
        history: history2,
        stream: true,
        kvCache: true
    }); // kvCache = true
    for await (const token of result2.tokenStream) {
        process.stdout.write(token);
    }
    const stats2 = await result2.stats;
    console.log(`\n▸ Second completion stats: ${JSON.stringify(stats2)}\n`);
    // Compare with non-cached version
    console.log('▸ Same conversation without cache:');
    const result3 = completion({
        modelId,
        history: history2,
        stream: true,
        kvCache: false
    }); // kvCache = false
    for await (const token of result3.tokenStream) {
        process.stdout.write(token);
    }
    const stats3 = await result3.stats;
    console.log(`\n▸ Non-cached completion stats: ${JSON.stringify(stats3)}\n`);
    console.log('▸ KV Cache test completed!');
    await unloadModel({ modelId, clearStorage: false });
}
catch (error) {
    console.error('✖', error);
    process.exit(1);
}