diff --git a/Makefile b/Makefile index 5605f4b03..aa2f9a2a9 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,6 @@ install-ios-rust-targets: dev: install-and-build yarn download:bin make build-mlx-server-if-exists - make build-foundation-models-server-if-exists make build-cli-dev yarn dev @@ -117,7 +116,6 @@ endif yarn copy:assets:tauri yarn build:icon yarn build:mlx-server - make build-foundation-models-server-if-exists make build-cli cargo test --manifest-path src-tauri/Cargo.toml --no-default-features --features test-tauri -- --test-threads=1 cargo test --manifest-path src-tauri/plugins/tauri-plugin-hardware/Cargo.toml @@ -174,40 +172,6 @@ else @echo "Skipping MLX server build (macOS only)" endif -# Build Apple Foundation Models server (macOS 26+ only) - always builds -build-foundation-models-server: -ifeq ($(shell uname -s),Darwin) - @echo "Building Foundation Models server for macOS 26+..." - cd src-tauri/plugins/tauri-plugin-foundation-models/swift-server && swift build -c release - @echo "Copying foundation-models-server binary..." - @cp src-tauri/plugins/tauri-plugin-foundation-models/swift-server/.build/release/foundation-models-server src-tauri/resources/bin/foundation-models-server - @chmod +x src-tauri/resources/bin/foundation-models-server - @echo "Foundation Models server built and copied successfully" - @echo "Checking for code signing identity..." - @SIGNING_IDENTITY=$$(security find-identity -v -p codesigning | grep "Developer ID Application" | head -1 | sed 's/.*"\(.*\)".*/\1/'); \ - if [ -n "$$SIGNING_IDENTITY" ]; then \ - echo "Signing foundation-models-server with identity: $$SIGNING_IDENTITY"; \ - codesign --force --options runtime --timestamp --sign "$$SIGNING_IDENTITY" src-tauri/resources/bin/foundation-models-server; \ - echo "Code signing completed successfully"; \ - else \ - echo "Warning: No Developer ID Application identity found. Skipping code signing."; \ - fi -else - @echo "Skipping Foundation Models server build (macOS only)" -endif - -# Build Foundation Models server only if not already present (for dev) -build-foundation-models-server-if-exists: -ifeq ($(shell uname -s),Darwin) - @if [ -f "src-tauri/resources/bin/foundation-models-server" ]; then \ - echo "Foundation Models server already exists at src-tauri/resources/bin/foundation-models-server, skipping build..."; \ - else \ - make build-foundation-models-server; \ - fi -else - @echo "Skipping Foundation Models server build (macOS only)" -endif - # Build jan CLI (release, platform-aware) → src-tauri/resources/bin/jan[.exe] build-cli: ifeq ($(shell uname -s),Darwin) diff --git a/README.md b/README.md index 9f0b3261b..58da60730 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ Download from [jan.ai](https://jan.ai/) or [GitHub Releases](https://github.com/ ## Features - **Local AI Models**: Download and run LLMs (Llama, Gemma, Qwen, GPT-oss etc.) from HuggingFace -- **Cloud Integration**: Connect to GPT models via OpenAI, Claude models via Anthropic, Mistral, Groq, and others +- **Cloud Integration**: Connect to GPT models via OpenAI, Claude models via Anthropic, Mistral, Groq, MiniMax, and others - **Custom Assistants**: Create specialized AI assistants for your tasks - **OpenAI-Compatible API**: Local server at `localhost:1337` for other applications - **Model Context Protocol**: MCP integration for agentic capabilities diff --git a/docs/src/pages/changelog/2026-03-23-jan-remote-models-context-cap.mdx b/docs/src/pages/changelog/2026-03-23-jan-remote-models-context-cap.mdx new file mode 100644 index 000000000..094a9a14a --- /dev/null +++ b/docs/src/pages/changelog/2026-03-23-jan-remote-models-context-cap.mdx @@ -0,0 +1,33 @@ +--- +title: "Jan v0.7.9: CLI on Windows, Smarter Context Management & Safer Data Location Management" +version: 0.7.9 +description: "Jan v0.7.9 fetches the latest models during onboarding, caps context length to avoid high RAM usage, fixes CLI on Windows, and improves data location management." +date: 2026-03-23 +--- + + +import ChangelogHeader from "@/components/Changelog/ChangelogHeader" +import { Callout } from 'nextra/components' + + + +# Highlights + +**Remove OpenClaw Integration** + +We recognize that the OpenClaw integration has been causing issues for some users. OpenClaw is a popular agent that has received significant attention from the community. However, it is a resource-heavy product that consumes a large amount of memory and tokens, and can introduce critical security vulnerabilities if not carefully handled in a local AI environment. Therefore, we have decided to remove OpenClaw from Jan and shift our focus toward exploring and building a simpler, safer, and more practical agent experience for Jan users in the near future. + +- Jan now remotely fetches the latest models during onboarding, ready for upcoming new Jan models +- Context length is now automatically capped and increased reasonably, avoiding excessively large context sizes that caused high RAM usage +- Fixed CLI path installation on Windows +- Fixed changing Jan data location +- Safely remove Jan data folder with a registered list + +--- + +Update your Jan or [download the latest](https://jan.ai/). + +For the complete list of changes, see the [GitHub release notes](https://github.com/janhq/jan/releases/tag/v0.7.9) diff --git a/extensions/foundation-models-extension/package.json b/extensions/foundation-models-extension/package.json index 5ccd7ae34..03a022879 100644 --- a/extensions/foundation-models-extension/package.json +++ b/extensions/foundation-models-extension/package.json @@ -22,7 +22,6 @@ "@janhq/core": "../../core/package.tgz", "@janhq/tauri-plugin-foundation-models-api": "link:../../src-tauri/plugins/tauri-plugin-foundation-models", "@tauri-apps/api": "2.8.0", - "@tauri-apps/plugin-http": "2.5.0", "@tauri-apps/plugin-log": "^2.6.0" }, "engines": { diff --git a/extensions/foundation-models-extension/src/index.ts b/extensions/foundation-models-extension/src/index.ts index 9110b2154..04f295b00 100644 --- a/extensions/foundation-models-extension/src/index.ts +++ b/extensions/foundation-models-extension/src/index.ts @@ -6,13 +6,10 @@ * connection or external API key is required. * * Architecture: - * Jan extension (TypeScript) → Tauri plugin (Rust) → foundation-models-server (Swift) - * ↓ - * Apple FoundationModels.framework + * Jan extension (TypeScript) → Tauri plugin (Rust / fm-rs) → FoundationModels.framework * - * The extension spawns a lightweight OpenAI-compatible HTTP server (`foundation-models-server`) - * that wraps the system Foundation Models API. Chat requests are proxied through that - * local server, keeping the same pattern used by the MLX and llama.cpp engines. + * The Tauri plugin calls Apple's FoundationModels framework directly via Rust + * FFI bindings (fm-rs), eliminating the need for a separate HTTP server process. */ import { @@ -28,30 +25,22 @@ import { } from '@janhq/core' import { info, warn, error as logError } from '@tauri-apps/plugin-log' -import { invoke } from '@tauri-apps/api/core' +import { listen, type UnlistenFn } from '@tauri-apps/api/event' import { - loadFoundationModelsServer, - unloadFoundationModelsServer, - isFoundationModelsProcessRunning, - getFoundationModelsRandomPort, - findFoundationModelsSession, + loadFoundationModels, + unloadFoundationModels, + isFoundationModelsLoaded, + foundationModelsChatCompletion, + foundationModelsChatCompletionStream, + abortFoundationModelsStream, checkFoundationModelsAvailability, } from '@janhq/tauri-plugin-foundation-models-api' // ─── Constants ─────────────────────────────────────────────────────────────── -/** The stable model ID used throughout Jan for the Apple on-device model. */ const APPLE_MODEL_ID = 'apple/on-device' - -/** Display name shown in the Jan UI. */ const APPLE_MODEL_NAME = 'Apple On-Device Model' -/** Shared API secret used to authorise requests to the local server. */ -const API_SECRET = 'JanFoundationModels' - -/** Seconds to wait for the server binary to become ready. */ -const SERVER_STARTUP_TIMEOUT = 60 - // ─── Logger ────────────────────────────────────────────────────────────────── const logger = { @@ -75,16 +64,13 @@ export default class FoundationModelsExtension extends AIEngine { readonly provider: string = 'foundation-models' readonly providerId: string = 'foundation-models' - /** Seconds before a streaming request is considered timed out. */ timeout: number = 300 // ── Lifecycle ────────────────────────────────────────────────────────────── override async onLoad(): Promise { - super.onLoad() // registers into EngineManager + super.onLoad() - // Check device eligibility and silently remove ourselves if not supported. - // This prevents the provider from appearing in the UI on ineligible devices. try { const availability = await checkFoundationModelsAvailability() if (availability !== 'available') { @@ -101,7 +87,7 @@ export default class FoundationModelsExtension extends AIEngine { } override async onUnload(): Promise { - // Clean-up is handled by the Tauri plugin on app exit. + // Cleanup handled by the Tauri plugin on app exit. } // ── Model catalogue ──────────────────────────────────────────────────────── @@ -141,50 +127,31 @@ export default class FoundationModelsExtension extends AIEngine { ) } - // Return existing session if already running - const existing = await findFoundationModelsSession() - if (existing) { - logger.info('Foundation Models server already running on port', existing.port) - return this.toSessionInfo(existing) + const alreadyLoaded = await isFoundationModelsLoaded() + if (alreadyLoaded) { + logger.info('Foundation Models already loaded') + return this.toSessionInfo(modelId) } - const port = await getFoundationModelsRandomPort() - const apiKey = await this.generateApiKey(port) - - logger.info('Starting Foundation Models server on port', port) + logger.info('Loading Foundation Models...') try { - const session = await loadFoundationModelsServer( - APPLE_MODEL_ID, - port, - apiKey, - SERVER_STARTUP_TIMEOUT - ) - logger.info('Foundation Models server started, PID', session.pid) - return this.toSessionInfo(session) + await loadFoundationModels(modelId) + logger.info('Foundation Models loaded successfully') + return this.toSessionInfo(modelId) } catch (err) { - logger.error('Failed to start Foundation Models server:', err) + logger.error('Failed to load Foundation Models:', err) throw err } } - override async unload(modelId: string): Promise { - const session = await findFoundationModelsSession() - if (!session) { - logger.warn('No active Foundation Models session to unload') - return { success: false, error: 'No active session found' } - } - + override async unload(_modelId: string): Promise { try { - const result = await unloadFoundationModelsServer(session.pid) - if (result.success) { - logger.info('Foundation Models server unloaded successfully') - } else { - logger.warn('Failed to unload Foundation Models server:', result.error) - } - return result + await unloadFoundationModels() + logger.info('Foundation Models unloaded successfully') + return { success: true } } catch (err) { - logger.error('Error unloading Foundation Models server:', err) + logger.error('Error unloading Foundation Models:', err) return { success: false, error: String(err) } } } @@ -195,138 +162,100 @@ export default class FoundationModelsExtension extends AIEngine { opts: chatCompletionRequest, abortController?: AbortController ): Promise> { - const session = await findFoundationModelsSession() - if (!session) { + const loaded = await isFoundationModelsLoaded() + if (!loaded) { throw new Error( 'Apple Foundation Model is not loaded. Please load the model first.' ) } - // Verify the server process is still alive - const alive = await isFoundationModelsProcessRunning(session.pid) - if (!alive) { - throw new Error( - 'Apple Foundation Model server has crashed. Please reload the model.' - ) - } - - // Health check - try { - await fetch(`http://localhost:${session.port}/health`) - } catch { - throw new Error( - 'Apple Foundation Model server is not responding. Please reload the model.' - ) - } - - const url = `http://localhost:${session.port}/v1/chat/completions` - const headers: HeadersInit = { - 'Content-Type': 'application/json', - Authorization: `Bearer ${session.api_key}`, - } const body = JSON.stringify(opts) if (opts.stream) { - return this.handleStreamingResponse(url, headers, body, abortController) + return this.handleStreamingChat(body, abortController) } - const response = await fetch(url, { - method: 'POST', - headers, - body, - signal: abortController?.signal, - }) - - if (!response.ok) { - const errData = await response.json().catch(() => null) - throw new Error( - `Foundation Models API request failed (${response.status}): ${JSON.stringify(errData)}` - ) - } - - return (await response.json()) as chatCompletion + const result = await foundationModelsChatCompletion(body) + return JSON.parse(result) as chatCompletion } - private async *handleStreamingResponse( - url: string, - headers: HeadersInit, + private async *handleStreamingChat( body: string, abortController?: AbortController ): AsyncIterable { - const combinedController = new AbortController() - const timeoutId = setTimeout( - () => combinedController.abort(new Error('Request timed out')), - this.timeout * 1000 - ) + const requestId = crypto.randomUUID() + const chunks: chatCompletionChunk[] = [] + let done = false + let streamError: Error | null = null + let resolver: (() => void) | null = null - if (abortController?.signal) { - if (abortController.signal.aborted) { - combinedController.abort(abortController.signal.reason) - } else { - abortController.signal.addEventListener( - 'abort', - () => combinedController.abort(abortController.signal.reason), - { once: true } - ) - } - } - - const response = await fetch(url, { - method: 'POST', - headers, - body, - signal: combinedController.signal, - }).finally(() => clearTimeout(timeoutId)) - - if (!response.ok) { - const errData = await response.json().catch(() => null) - throw new Error( - `Foundation Models streaming request failed (${response.status}): ${JSON.stringify(errData)}` - ) - } - - if (!response.body) { - throw new Error('Response body is null') - } - - const reader = response.body.getReader() - const decoder = new TextDecoder('utf-8') - let buffer = '' - - try { - while (true) { - const { done, value } = await reader.read() - if (done) break - - buffer += decoder.decode(value, { stream: true }) - const lines = buffer.split('\n') - buffer = lines.pop() ?? '' - - for (const line of lines) { - const trimmed = line.trim() - if (!trimmed || trimmed === 'data: [DONE]') continue - - if (trimmed.startsWith('data: ')) { - try { - const data = JSON.parse(trimmed.slice(6)) as chatCompletionChunk - yield data - } catch (e) { - logger.error('Error parsing Foundation Models stream JSON:', e) - throw e - } - } else if (trimmed.startsWith('error: ')) { - const errObj = JSON.parse(trimmed.slice(7)) - throw new Error(errObj.message ?? 'Unknown streaming error') + const unlisten: UnlistenFn = await listen( + `foundation-models-stream-${requestId}`, + (event) => { + const payload = event.payload as { + data?: string + done?: boolean + error?: string + } + if (payload.done) { + done = true + resolver?.() + } else if (payload.error) { + streamError = new Error(payload.error) + resolver?.() + } else if (payload.data) { + try { + chunks.push(JSON.parse(payload.data) as chatCompletionChunk) + } catch (e) { + logger.error('Error parsing Foundation Models stream JSON:', e) } + resolver?.() } } + ) + + foundationModelsChatCompletionStream(body, requestId).catch((err) => { + streamError = + err instanceof Error ? err : new Error(String(err)) + resolver?.() + }) + + if (abortController?.signal) { + const onAbort = () => { + abortFoundationModelsStream(requestId).catch(() => {}) + } + if (abortController.signal.aborted) { + onAbort() + } else { + abortController.signal.addEventListener('abort', onAbort, { once: true }) + } + } + + try { + while (!done) { + if (streamError) throw streamError + + while (chunks.length > 0) { + yield chunks.shift()! + } + + if (done) break + if (streamError) throw streamError + + await new Promise((r) => { + resolver = r + }) + } + + while (chunks.length > 0) { + yield chunks.shift()! + } } finally { - reader.releaseLock() + unlisten() } } // ── Unsupported operations ───────────────────────────────────────────────── - // Foundation Models are built into the OS — there are no files to manage. override async delete(_modelId: string): Promise { throw new Error( @@ -351,45 +280,24 @@ export default class FoundationModelsExtension extends AIEngine { } override async getLoadedModels(): Promise { - const session = await findFoundationModelsSession() - return session ? [APPLE_MODEL_ID] : [] + const loaded = await isFoundationModelsLoaded() + return loaded ? [APPLE_MODEL_ID] : [] } override async isToolSupported(_modelId: string): Promise { - // The Foundation Models framework supports function calling. return true } // ── Helpers ──────────────────────────────────────────────────────────────── - /** - * Derive a per-session API key from the shared secret and port number. - * Uses the same HMAC-SHA256 approach as the llamacpp extension so the - * Tauri `generate_api_key` command can be reused. - */ - private async generateApiKey(port: number): Promise { - return invoke('plugin:llamacpp|generate_api_key', { - modelId: APPLE_MODEL_ID + port, - apiSecret: API_SECRET, - }) - } - - /** - * Map the plugin SessionInfo shape to the core SessionInfo shape. - */ - private toSessionInfo(session: { - pid: number - port: number - model_id: string - api_key: string - }): SessionInfo { + private toSessionInfo(modelId: string): SessionInfo { return { - pid: session.pid, - port: session.port, - model_id: session.model_id, + pid: 0, + port: 0, + model_id: modelId, model_path: '', is_embedding: false, - api_key: session.api_key, + api_key: '', } } } diff --git a/foundation-models-server/Package.swift b/foundation-models-server/Package.swift deleted file mode 100644 index d899a6925..000000000 --- a/foundation-models-server/Package.swift +++ /dev/null @@ -1,30 +0,0 @@ -// swift-tools-version: 6.2 - -import PackageDescription - -let package = Package( - name: "foundation-models-server", - platforms: [ - .macOS(.v26) - ], - products: [ - .executable(name: "foundation-models-server", targets: ["FoundationModelsServer"]) - ], - dependencies: [ - .package(url: "https://github.com/apple/swift-argument-parser", from: "1.7.0"), - .package(url: "https://github.com/hummingbird-project/hummingbird", from: "2.19.0"), - ], - targets: [ - .executableTarget( - name: "FoundationModelsServer", - dependencies: [ - .product(name: "ArgumentParser", package: "swift-argument-parser"), - .product(name: "Hummingbird", package: "hummingbird"), - ], - path: "Sources/FoundationModelsServer", - swiftSettings: [ - .swiftLanguageMode(.v6) - ] - ) - ] -) diff --git a/foundation-models-server/README.md b/foundation-models-server/README.md deleted file mode 100644 index 3d840b72c..000000000 --- a/foundation-models-server/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# foundation-models-server - -A lightweight OpenAI-compatible HTTP server that wraps Apple's Foundation Models framework, enabling Jan to use on-device Apple Intelligence models on macOS 26+. - -## Requirements - -- macOS 26 (Tahoe) or later -- Apple Silicon Mac with Apple Intelligence enabled -- Xcode 26 or later - -## Building - -```bash -swift build -c release -``` - -The binary will be at `.build/release/foundation-models-server`. - -## Usage - -```bash -# Check availability -foundation-models-server --check - -# Start server on default port -foundation-models-server --port 8080 - -# Start server with API key -foundation-models-server --port 8080 --api-key -``` - -## API - -The server exposes an OpenAI-compatible API: - -- `GET /health` — health check -- `GET /v1/models` — lists the `apple/on-device` model -- `POST /v1/chat/completions` — chat completions (streaming and non-streaming) - -The model ID is always `apple/on-device`. diff --git a/foundation-models-server/Sources/FoundationModelsServer/FoundationModelsServerCommand.swift b/foundation-models-server/Sources/FoundationModelsServer/FoundationModelsServerCommand.swift deleted file mode 100644 index 36d2d5ec8..000000000 --- a/foundation-models-server/Sources/FoundationModelsServer/FoundationModelsServerCommand.swift +++ /dev/null @@ -1,78 +0,0 @@ -import ArgumentParser -import Foundation -import Hummingbird -import FoundationModels - -@main -struct FoundationModelsServerCommand: AsyncParsableCommand { - static let configuration = CommandConfiguration( - commandName: "foundation-models-server", - abstract: "Apple Foundation Models inference server with OpenAI-compatible API" - ) - - @Option(name: .long, help: "Port to listen on") - var port: Int = 8080 - - @Option(name: .long, help: "API key for authentication (optional)") - var apiKey: String = "" - - @Flag(name: .long, help: "Check availability and exit with status 0 if available") - var check: Bool = false - - func run() async throws { - let availability = SystemLanguageModel.default.availability - - // In --check mode, always print a machine-readable status token and exit 0. - // Callers (e.g. the Tauri plugin) parse this string to decide visibility. - if check { - switch availability { - case .available: - print("available") - case .unavailable(.deviceNotEligible): - print("notEligible") - case .unavailable(.appleIntelligenceNotEnabled): - print("appleIntelligenceNotEnabled") - case .unavailable(.modelNotReady): - print("modelNotReady") - default: - print("unavailable") - } - return - } - - guard case .available = availability else { - let reason: String - switch availability { - case .unavailable(.deviceNotEligible): - reason = "Device is not eligible for Apple Intelligence" - case .unavailable(.appleIntelligenceNotEnabled): - reason = "Apple Intelligence is not enabled in System Settings" - case .unavailable(.modelNotReady): - reason = "Foundation model is downloading or not yet ready" - default: - reason = "Foundation model is unavailable on this system" - } - fputs("[foundation-models] ERROR: \(reason)\n", stderr) - throw ExitCode(1) - } - - log("[foundation-models] Foundation Models Server starting...") - log("[foundation-models] Port: \(port)") - - let server = FoundationModelsHTTPServer( - modelId: "apple/on-device", - apiKey: apiKey - ) - - let router = server.buildRouter() - let app = Application( - router: router, - configuration: .init(address: .hostname("127.0.0.1", port: port)) - ) - - log("[foundation-models] http server listening on http://127.0.0.1:\(port)") - log("[foundation-models] server is listening on 127.0.0.1:\(port)") - - try await app.run() - } -} diff --git a/foundation-models-server/Sources/FoundationModelsServer/Logger.swift b/foundation-models-server/Sources/FoundationModelsServer/Logger.swift deleted file mode 100644 index 34bb17517..000000000 --- a/foundation-models-server/Sources/FoundationModelsServer/Logger.swift +++ /dev/null @@ -1,6 +0,0 @@ -import Foundation - -func log(_ message: String) { - print(message) - fflush(stdout) -} diff --git a/foundation-models-server/Sources/FoundationModelsServer/Server.swift b/foundation-models-server/Sources/FoundationModelsServer/Server.swift deleted file mode 100644 index b14eb3574..000000000 --- a/foundation-models-server/Sources/FoundationModelsServer/Server.swift +++ /dev/null @@ -1,299 +0,0 @@ -import Foundation -import Hummingbird -import FoundationModels - -/// HTTP server exposing an OpenAI-compatible API backed by Apple Foundation Models -struct FoundationModelsHTTPServer: Sendable { - let modelId: String - let apiKey: String - - func buildRouter() -> Router { - let router = Router() - - // Health check - router.get("/health") { _, _ in - let response = HealthResponse(status: "ok") - return try encodeJSONResponse(response) - } - - // List available models - router.get("/v1/models") { _, _ in - let response = ModelsListResponse( - object: "list", - data: [ - ModelData( - id: self.modelId, - object: "model", - created: currentTimestamp(), - owned_by: "apple" - ) - ] - ) - return try encodeJSONResponse(response) - } - - // Chat completions (OpenAI-compatible) - router.post("/v1/chat/completions") { request, _ in - // Validate API key when configured - if !self.apiKey.isEmpty { - let authHeader = request.headers[.authorization] - guard authHeader == "Bearer \(self.apiKey)" else { - let errorResp = ErrorResponse( - error: ErrorDetail( - message: "Unauthorized: invalid or missing API key", - type: "authentication_error", - code: "unauthorized" - ) - ) - return try Response( - status: .unauthorized, - headers: [.contentType: "application/json"], - body: .init(byteBuffer: encodeJSONBuffer(errorResp)) - ) - } - } - - let body = try await request.body.collect(upTo: 10 * 1024 * 1024) - let chatRequest: ChatCompletionRequest - do { - chatRequest = try JSONDecoder().decode(ChatCompletionRequest.self, from: body) - } catch { - let errorResp = ErrorResponse( - error: ErrorDetail( - message: "Invalid request body: \(error.localizedDescription)", - type: "invalid_request_error", - code: nil - ) - ) - return try Response( - status: .badRequest, - headers: [.contentType: "application/json"], - body: .init(byteBuffer: encodeJSONBuffer(errorResp)) - ) - } - let isStreaming = chatRequest.stream ?? false - - log("[foundation-models] Request: messages=\(chatRequest.messages.count), stream=\(isStreaming)") - - if isStreaming { - return try await self.handleStreamingRequest(chatRequest) - } else { - return try await self.handleNonStreamingRequest(chatRequest) - } - } - - return router - } - - // MARK: - Non-streaming - - private func handleNonStreamingRequest(_ chatRequest: ChatCompletionRequest) async throws -> Response { - let session = buildSession(from: chatRequest.messages) - let lastUserMessage = extractLastUserMessage(from: chatRequest.messages) - - let response = try await session.respond(to: lastUserMessage) - let content = response.content - - let completionResponse = ChatCompletionResponse( - id: "chatcmpl-\(UUID().uuidString)", - object: "chat.completion", - created: currentTimestamp(), - model: modelId, - choices: [ - ChatCompletionChoice( - index: 0, - message: ChatResponseMessage(role: "assistant", content: content), - finish_reason: "stop" - ) - ], - usage: UsageInfo( - prompt_tokens: 0, - completion_tokens: 0, - total_tokens: 0 - ) - ) - - return try encodeJSONResponse(completionResponse) - } - - // MARK: - Streaming - - private func handleStreamingRequest(_ chatRequest: ChatCompletionRequest) async throws -> Response { - let requestId = "chatcmpl-\(UUID().uuidString)" - let created = currentTimestamp() - let modelId = self.modelId - let messages = chatRequest.messages - - let (stream, continuation) = AsyncStream.makeStream() - - let task = Task { [self] in - do { - let session = self.buildSession(from: messages) - let lastUserMessage = self.extractLastUserMessage(from: messages) - - let roleDelta = ChatCompletionChunk( - id: requestId, - object: "chat.completion.chunk", - created: created, - model: modelId, - choices: [ - ChunkChoice( - index: 0, - delta: DeltaContent(role: "assistant", content: nil), - finish_reason: nil - ) - ] - ) - if let buffer = encodeSSEBuffer(roleDelta) { - continuation.yield(buffer) - } - - var previousText = "" - for try await snapshot in session.streamResponse(to: lastUserMessage) { - let currentText = snapshot.content - let delta = String(currentText.dropFirst(previousText.count)) - previousText = currentText - - if delta.isEmpty { continue } - - let chunk = ChatCompletionChunk( - id: requestId, - object: "chat.completion.chunk", - created: created, - model: modelId, - choices: [ - ChunkChoice( - index: 0, - delta: DeltaContent(role: nil, content: delta), - finish_reason: nil - ) - ] - ) - if let buffer = encodeSSEBuffer(chunk) { - continuation.yield(buffer) - } - } - - // Send stop chunk - let stopChunk = ChatCompletionChunk( - id: requestId, - object: "chat.completion.chunk", - created: created, - model: modelId, - choices: [ - ChunkChoice( - index: 0, - delta: DeltaContent(role: nil, content: nil), - finish_reason: "stop" - ) - ] - ) - if let buffer = encodeSSEBuffer(stopChunk) { - continuation.yield(buffer) - } - - // SSE terminator - var doneBuffer = ByteBufferAllocator().buffer(capacity: 16) - doneBuffer.writeString("data: [DONE]\n\n") - continuation.yield(doneBuffer) - } catch { - log("[foundation-models] Streaming error: \(error.localizedDescription)") - var errBuffer = ByteBufferAllocator().buffer(capacity: 256) - errBuffer.writeString("error: {\"message\":\"\(error.localizedDescription)\"}\n\n") - continuation.yield(errBuffer) - } - continuation.finish() - } - - // Cancel the generation task when the client disconnects - continuation.onTermination = { @Sendable _ in - log("[foundation-models] SSE continuation terminated by client disconnect") - task.cancel() - } - - return Response( - status: .ok, - headers: [ - .contentType: "text/event-stream", - .cacheControl: "no-cache", - .init("X-Accel-Buffering")!: "no" - ], - body: .init(asyncSequence: stream) - ) - } - - // MARK: - Session Construction - - /// Build a `LanguageModelSession` from the OpenAI message list. - /// - /// System messages become the session instructions. - /// Prior user/assistant turns are serialised into the instructions block so - /// the model has full conversation context without re-running inference. - /// (The Foundation Models `Transcript` API is not used for history injection - /// because it is designed for observing an already-live session's state, not - /// for priming a fresh one with arbitrary history.) - private func buildSession(from messages: [ChatMessage]) -> LanguageModelSession { - let systemContent = messages.first(where: { $0.role == "system" })?.content ?? "" - let nonSystem = messages.filter { $0.role != "system" } - let history = nonSystem.dropLast() // all turns except the last user message - - var instructionsText: String - if systemContent.isEmpty { - instructionsText = "You are a helpful assistant." - } else { - instructionsText = systemContent - } - - // Append prior turns so the model understands conversation context - if !history.isEmpty { - instructionsText += "\n\n[Previous conversation]\n" - for msg in history { - let label = msg.role == "assistant" ? "Assistant" : "User" - instructionsText += "\(label): \(msg.content ?? "")\n" - } - instructionsText += "[End of previous conversation]" - } - - return LanguageModelSession(instructions: instructionsText) - } - - private func extractLastUserMessage(from messages: [ChatMessage]) -> String { - let nonSystem = messages.filter { $0.role != "system" } - return nonSystem.last?.content ?? "" - } -} - -// MARK: - Helpers - -private func currentTimestamp() -> Int { - Int(Date().timeIntervalSince1970) -} - -private func encodeJSONResponse(_ value: T) throws -> Response { - let data = try JSONEncoder().encode(value) - var buffer = ByteBufferAllocator().buffer(capacity: data.count) - buffer.writeBytes(data) - return Response( - status: .ok, - headers: [.contentType: "application/json"], - body: .init(byteBuffer: buffer) - ) -} - -private func encodeJSONBuffer(_ value: T) -> ByteBuffer { - let data = (try? JSONEncoder().encode(value)) ?? Data() - var buffer = ByteBufferAllocator().buffer(capacity: data.count) - buffer.writeBytes(data) - return buffer -} - -private func encodeSSEBuffer(_ value: T) -> ByteBuffer? { - guard let json = try? JSONEncoder().encode(value), - let jsonString = String(data: json, encoding: .utf8) else { - return nil - } - let line = "data: \(jsonString)\n\n" - var buffer = ByteBufferAllocator().buffer(capacity: line.utf8.count) - buffer.writeString(line) - return buffer -} diff --git a/foundation-models-server/Sources/FoundationModelsServer/Types.swift b/foundation-models-server/Sources/FoundationModelsServer/Types.swift deleted file mode 100644 index 86dc656f7..000000000 --- a/foundation-models-server/Sources/FoundationModelsServer/Types.swift +++ /dev/null @@ -1,98 +0,0 @@ -import Foundation - -// MARK: - OpenAI Request Types - -struct ChatCompletionRequest: Codable, Sendable { - let model: String - let messages: [ChatMessage] - var temperature: Double? - var top_p: Double? - var max_tokens: Int? - var n_predict: Int? - var stream: Bool? - var stop: [String]? -} - -struct ChatMessage: Codable, Sendable { - let role: String - let content: String? -} - -// MARK: - OpenAI Response Types - -struct ChatCompletionResponse: Codable, Sendable { - let id: String - let object: String - let created: Int - let model: String - let choices: [ChatCompletionChoice] - let usage: UsageInfo -} - -struct ChatCompletionChoice: Codable, Sendable { - let index: Int - let message: ChatResponseMessage - let finish_reason: String -} - -struct ChatResponseMessage: Codable, Sendable { - let role: String - let content: String -} - -struct UsageInfo: Codable, Sendable { - let prompt_tokens: Int - let completion_tokens: Int - let total_tokens: Int -} - -// MARK: - Streaming Types - -struct ChatCompletionChunk: Codable, Sendable { - let id: String - let object: String - let created: Int - let model: String - let choices: [ChunkChoice] -} - -struct ChunkChoice: Codable, Sendable { - let index: Int - let delta: DeltaContent - let finish_reason: String? -} - -struct DeltaContent: Codable, Sendable { - let role: String? - let content: String? -} - -// MARK: - Model List Types - -struct ModelsListResponse: Codable, Sendable { - let object: String - let data: [ModelData] -} - -struct ModelData: Codable, Sendable { - let id: String - let object: String - let created: Int - let owned_by: String -} - -// MARK: - Health / Error Types - -struct HealthResponse: Codable, Sendable { - let status: String -} - -struct ErrorDetail: Codable, Sendable { - let message: String - let type: String - let code: String? -} - -struct ErrorResponse: Codable, Sendable { - let error: ErrorDetail -} diff --git a/package.json b/package.json index 842da4b7c..8169361dd 100644 --- a/package.json +++ b/package.json @@ -31,11 +31,10 @@ "download:lib": "node ./scripts/download-lib.mjs", "download:bin": "node ./scripts/download-bin.mjs", "build:mlx-server": "make build-mlx-server", - "build:foundation-models-server": "make build-foundation-models-server", "build:cli": "make build-cli", "build:tauri:win32": "yarn download:bin && yarn build:cli && yarn tauri build", "build:tauri:linux": "yarn download:bin && yarn build:cli && NO_STRIP=1 ./src-tauri/build-utils/shim-linuxdeploy.sh yarn tauri build && ./src-tauri/build-utils/buildAppImage.sh", - "build:tauri:darwin": "yarn download:bin && yarn build:mlx-server && yarn build:foundation-models-server && yarn build:cli && yarn tauri build --target universal-apple-darwin", + "build:tauri:darwin": "yarn download:bin && yarn build:mlx-server && yarn build:cli && yarn tauri build --target universal-apple-darwin", "build:tauri": "yarn build:icon && yarn copy:assets:tauri && run-script-os", "build:tauri:plugin:api": "cd src-tauri/plugins && yarn install && yarn workspaces foreach -Apt run build", "build:icon": "tauri icon ./src-tauri/icons/icon.png", diff --git a/src-tauri/.gitignore b/src-tauri/.gitignore index 02bc782bf..96c958f28 100644 --- a/src-tauri/.gitignore +++ b/src-tauri/.gitignore @@ -5,4 +5,7 @@ /gen/android binaries !binaries/download.sh -!binaries/download.bat \ No newline at end of file +!binaries/download.bat + +# Autogenerated plugin permissions +plugins/*/permissions/autogenerated/ \ No newline at end of file diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 82c6781af..c3fd9b89f 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -1819,6 +1819,16 @@ dependencies = [ "spin", ] +[[package]] +name = "fm-rs" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ba96ac90cf1955a708bc53f2a2730a042609fe0836c5989b1d4424f471e868" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "fnv" version = "1.0.7" @@ -6505,15 +6515,15 @@ dependencies = [ name = "tauri-plugin-foundation-models" version = "0.6.599" dependencies = [ - "jan-utils", + "fm-rs", "log", - "nix", "serde", - "sysinfo", + "serde_json", "tauri", "tauri-plugin", "thiserror 2.0.17", "tokio", + "uuid", ] [[package]] diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 9bfad4126..d0700aba7 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -47,6 +47,7 @@ mobile = [ "tauri/wry", "dep:sqlx", ] +# foundation-models excluded from test-tauri - framework not available in CI test-tauri = [ "tauri/wry", "tauri/x11", @@ -54,12 +55,15 @@ test-tauri = [ "tauri/macos-private-api", "tauri/tray-icon", "tauri/test", - "desktop", + "deep-link", + "hardware", + "mlx", ] cli = ["dep:env_logger", "dep:dialoguer", "dep:indicatif", "dep:console"] [build-dependencies] tauri-build = { version = "2.0.2", features = [] } +serde_json = "1.0" [dependencies] dirs = "6.0.0" diff --git a/src-tauri/build.rs b/src-tauri/build.rs index d773e7a94..639182a1f 100644 --- a/src-tauri/build.rs +++ b/src-tauri/build.rs @@ -1,4 +1,32 @@ fn main() { #[cfg(not(feature = "cli"))] - tauri_build::build() + { + tauri_build::build(); + } + + #[cfg(target_os = "macos")] + { + println!("cargo:rustc-link-arg=-Wl,-rpath,/usr/lib/swift"); + + if let Ok(output) = std::process::Command::new("xcrun") + .args(["--toolchain", "default", "--find", "swift"]) + .output() + { + let swift_path = String::from_utf8_lossy(&output.stdout) + .trim() + .to_string(); + if let Some(toolchain) = std::path::Path::new(&swift_path) + .parent() + .and_then(|p| p.parent()) + { + let lib_path = toolchain.join("lib/swift/macosx"); + if lib_path.exists() { + println!( + "cargo:rustc-link-arg=-Wl,-rpath,{}", + lib_path.display() + ); + } + } + } + } } diff --git a/src-tauri/capabilities/default.json b/src-tauri/capabilities/default.json index fbfb3bde6..b6a4eba7f 100644 --- a/src-tauri/capabilities/default.json +++ b/src-tauri/capabilities/default.json @@ -30,7 +30,6 @@ "deep-link:default", "llamacpp:default", "mlx:default", - "foundation-models:default", "updater:default", "updater:allow-check", { diff --git a/src-tauri/capabilities/desktop.json b/src-tauri/capabilities/desktop.json index a4df1696d..cc49c6687 100644 --- a/src-tauri/capabilities/desktop.json +++ b/src-tauri/capabilities/desktop.json @@ -28,7 +28,6 @@ "rag:default", "llamacpp:default", "mlx:default", - "foundation-models:default", "deep-link:default", "hardware:default", { diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/Cargo.toml b/src-tauri/plugins/tauri-plugin-foundation-models/Cargo.toml index fc53de1ef..a50bc8996 100644 --- a/src-tauri/plugins/tauri-plugin-foundation-models/Cargo.toml +++ b/src-tauri/plugins/tauri-plugin-foundation-models/Cargo.toml @@ -2,7 +2,7 @@ name = "tauri-plugin-foundation-models" version = "0.6.599" authors = ["Jan "] -description = "Tauri plugin for managing Apple Foundation Models server processes on macOS 26+" +description = "Tauri plugin for Apple Foundation Models inference via fm-rs on macOS 26+" license = "MIT" repository = "https://github.com/janhq/jan" edition = "2021" @@ -13,15 +13,14 @@ links = "tauri-plugin-foundation-models" [dependencies] log = "0.4" serde = { version = "1.0", features = ["derive"] } -sysinfo = "0.34.2" +serde_json = "1.0" tauri = { version = "2.5.0", default-features = false, features = [] } thiserror = "2.0.12" tokio = { version = "1", features = ["full"] } -jan-utils = { path = "../../utils" } +uuid = { version = "1", features = ["v4"] } -# Unix-specific dependencies (macOS) -[target.'cfg(unix)'.dependencies] -nix = { version = "=0.30.1", features = ["signal", "process"] } +[target.'cfg(all(target_os = "macos", target_arch = "aarch64"))'.dependencies] +fm-rs = "0.1" [build-dependencies] tauri-plugin = { version = "2.3.1", features = ["build"] } diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/build.rs b/src-tauri/plugins/tauri-plugin-foundation-models/build.rs index 0667d6b3d..1ed06b1e9 100644 --- a/src-tauri/plugins/tauri-plugin-foundation-models/build.rs +++ b/src-tauri/plugins/tauri-plugin-foundation-models/build.rs @@ -1,13 +1,12 @@ const COMMANDS: &[&str] = &[ "cleanup_foundation_models_processes", - "load_foundation_models_server", - "unload_foundation_models_server", - "is_foundation_models_process_running", - "get_foundation_models_random_port", - "find_foundation_models_session", - "get_foundation_models_loaded", - "get_foundation_models_all_sessions", "check_foundation_models_availability", + "load_foundation_models", + "unload_foundation_models", + "is_foundation_models_loaded", + "foundation_models_chat_completion", + "foundation_models_chat_completion_stream", + "abort_foundation_models_stream", ]; fn main() { diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/guest-js/index.ts b/src-tauri/plugins/tauri-plugin-foundation-models/guest-js/index.ts index 014b019de..badd47224 100644 --- a/src-tauri/plugins/tauri-plugin-foundation-models/guest-js/index.ts +++ b/src-tauri/plugins/tauri-plugin-foundation-models/guest-js/index.ts @@ -1,64 +1,41 @@ import { invoke } from '@tauri-apps/api/core' -import { SessionInfo, UnloadResult } from './types' -export { SessionInfo, UnloadResult } from './types' +export { StreamEvent } from './types' -export async function loadFoundationModelsServer( - modelId: string, - port: number, - apiKey: string, - timeout: number = 60 -): Promise { - return await invoke('plugin:foundation-models|load_foundation_models_server', { - modelId, - port, - apiKey, - timeout, - }) +export async function checkFoundationModelsAvailability(): Promise { + return await invoke('plugin:foundation-models|check_foundation_models_availability') } -export async function unloadFoundationModelsServer( - pid: number -): Promise { - return await invoke('plugin:foundation-models|unload_foundation_models_server', { pid }) +export async function loadFoundationModels(modelId: string): Promise { + return await invoke('plugin:foundation-models|load_foundation_models', { modelId }) } -export async function isFoundationModelsProcessRunning( - pid: number -): Promise { - return await invoke('plugin:foundation-models|is_foundation_models_process_running', { pid }) -} - -export async function getFoundationModelsRandomPort(): Promise { - return await invoke('plugin:foundation-models|get_foundation_models_random_port') -} - -export async function findFoundationModelsSession(): Promise { - return await invoke('plugin:foundation-models|find_foundation_models_session') +export async function unloadFoundationModels(): Promise { + return await invoke('plugin:foundation-models|unload_foundation_models') } export async function isFoundationModelsLoaded(): Promise { - return await invoke('plugin:foundation-models|get_foundation_models_loaded') + return await invoke('plugin:foundation-models|is_foundation_models_loaded') } -export async function getAllFoundationModelsSessions(): Promise { - return await invoke('plugin:foundation-models|get_foundation_models_all_sessions') +export async function foundationModelsChatCompletion(body: string): Promise { + return await invoke('plugin:foundation-models|foundation_models_chat_completion', { body }) +} + +export async function foundationModelsChatCompletionStream( + body: string, + requestId: string +): Promise { + return await invoke('plugin:foundation-models|foundation_models_chat_completion_stream', { + body, + requestId, + }) +} + +export async function abortFoundationModelsStream(requestId: string): Promise { + return await invoke('plugin:foundation-models|abort_foundation_models_stream', { requestId }) } export async function cleanupFoundationModelsProcesses(): Promise { return await invoke('plugin:foundation-models|cleanup_foundation_models_processes') } - -/** - * Run `foundation-models-server --check` and return a machine-readable - * availability token. Possible values: - * - `"available"` — device is eligible and ready - * - `"notEligible"` — device does not support Apple Intelligence - * - `"appleIntelligenceNotEnabled"` — Apple Intelligence disabled in Settings - * - `"modelNotReady"` — model is still downloading - * - `"unavailable"` — other unavailability reason - * - `"binaryNotFound"` — server binary was not bundled (non-macOS build) - */ -export async function checkFoundationModelsAvailability(): Promise { - return await invoke('plugin:foundation-models|check_foundation_models_availability') -} diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/guest-js/types.ts b/src-tauri/plugins/tauri-plugin-foundation-models/guest-js/types.ts index 2d290125d..d150c29e3 100644 --- a/src-tauri/plugins/tauri-plugin-foundation-models/guest-js/types.ts +++ b/src-tauri/plugins/tauri-plugin-foundation-models/guest-js/types.ts @@ -1,11 +1,5 @@ -export interface SessionInfo { - pid: number - port: number - model_id: string - api_key: string -} - -export interface UnloadResult { - success: boolean +export interface StreamEvent { + data?: string + done?: boolean error?: string } diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/check_foundation_models_availability.toml b/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/check_foundation_models_availability.toml deleted file mode 100644 index 7782714ac..000000000 --- a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/check_foundation_models_availability.toml +++ /dev/null @@ -1,13 +0,0 @@ -# Automatically generated - DO NOT EDIT! - -"$schema" = "../../schemas/schema.json" - -[[permission]] -identifier = "allow-check-foundation-models-availability" -description = "Enables the check_foundation_models_availability command without any pre-configured scope." -commands.allow = ["check_foundation_models_availability"] - -[[permission]] -identifier = "deny-check-foundation-models-availability" -description = "Denies the check_foundation_models_availability command without any pre-configured scope." -commands.deny = ["check_foundation_models_availability"] diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/cleanup_foundation_models_processes.toml b/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/cleanup_foundation_models_processes.toml deleted file mode 100644 index e8b16dcca..000000000 --- a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/cleanup_foundation_models_processes.toml +++ /dev/null @@ -1,13 +0,0 @@ -# Automatically generated - DO NOT EDIT! - -"$schema" = "../../schemas/schema.json" - -[[permission]] -identifier = "allow-cleanup-foundation-models-processes" -description = "Enables the cleanup_foundation_models_processes command without any pre-configured scope." -commands.allow = ["cleanup_foundation_models_processes"] - -[[permission]] -identifier = "deny-cleanup-foundation-models-processes" -description = "Denies the cleanup_foundation_models_processes command without any pre-configured scope." -commands.deny = ["cleanup_foundation_models_processes"] diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/find_foundation_models_session.toml b/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/find_foundation_models_session.toml deleted file mode 100644 index 5f5da8609..000000000 --- a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/find_foundation_models_session.toml +++ /dev/null @@ -1,13 +0,0 @@ -# Automatically generated - DO NOT EDIT! - -"$schema" = "../../schemas/schema.json" - -[[permission]] -identifier = "allow-find-foundation-models-session" -description = "Enables the find_foundation_models_session command without any pre-configured scope." -commands.allow = ["find_foundation_models_session"] - -[[permission]] -identifier = "deny-find-foundation-models-session" -description = "Denies the find_foundation_models_session command without any pre-configured scope." -commands.deny = ["find_foundation_models_session"] diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/get_foundation_models_all_sessions.toml b/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/get_foundation_models_all_sessions.toml deleted file mode 100644 index 276159846..000000000 --- a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/get_foundation_models_all_sessions.toml +++ /dev/null @@ -1,13 +0,0 @@ -# Automatically generated - DO NOT EDIT! - -"$schema" = "../../schemas/schema.json" - -[[permission]] -identifier = "allow-get-foundation-models-all-sessions" -description = "Enables the get_foundation_models_all_sessions command without any pre-configured scope." -commands.allow = ["get_foundation_models_all_sessions"] - -[[permission]] -identifier = "deny-get-foundation-models-all-sessions" -description = "Denies the get_foundation_models_all_sessions command without any pre-configured scope." -commands.deny = ["get_foundation_models_all_sessions"] diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/get_foundation_models_loaded.toml b/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/get_foundation_models_loaded.toml deleted file mode 100644 index 99d2c0b96..000000000 --- a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/get_foundation_models_loaded.toml +++ /dev/null @@ -1,13 +0,0 @@ -# Automatically generated - DO NOT EDIT! - -"$schema" = "../../schemas/schema.json" - -[[permission]] -identifier = "allow-get-foundation-models-loaded" -description = "Enables the get_foundation_models_loaded command without any pre-configured scope." -commands.allow = ["get_foundation_models_loaded"] - -[[permission]] -identifier = "deny-get-foundation-models-loaded" -description = "Denies the get_foundation_models_loaded command without any pre-configured scope." -commands.deny = ["get_foundation_models_loaded"] diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/get_foundation_models_random_port.toml b/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/get_foundation_models_random_port.toml deleted file mode 100644 index eb310b423..000000000 --- a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/get_foundation_models_random_port.toml +++ /dev/null @@ -1,13 +0,0 @@ -# Automatically generated - DO NOT EDIT! - -"$schema" = "../../schemas/schema.json" - -[[permission]] -identifier = "allow-get-foundation-models-random-port" -description = "Enables the get_foundation_models_random_port command without any pre-configured scope." -commands.allow = ["get_foundation_models_random_port"] - -[[permission]] -identifier = "deny-get-foundation-models-random-port" -description = "Denies the get_foundation_models_random_port command without any pre-configured scope." -commands.deny = ["get_foundation_models_random_port"] diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/is_foundation_models_process_running.toml b/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/is_foundation_models_process_running.toml deleted file mode 100644 index e6d4d98e1..000000000 --- a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/is_foundation_models_process_running.toml +++ /dev/null @@ -1,13 +0,0 @@ -# Automatically generated - DO NOT EDIT! - -"$schema" = "../../schemas/schema.json" - -[[permission]] -identifier = "allow-is-foundation-models-process-running" -description = "Enables the is_foundation_models_process_running command without any pre-configured scope." -commands.allow = ["is_foundation_models_process_running"] - -[[permission]] -identifier = "deny-is-foundation-models-process-running" -description = "Denies the is_foundation_models_process_running command without any pre-configured scope." -commands.deny = ["is_foundation_models_process_running"] diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/load_foundation_models_server.toml b/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/load_foundation_models_server.toml deleted file mode 100644 index 83ebad058..000000000 --- a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/load_foundation_models_server.toml +++ /dev/null @@ -1,13 +0,0 @@ -# Automatically generated - DO NOT EDIT! - -"$schema" = "../../schemas/schema.json" - -[[permission]] -identifier = "allow-load-foundation-models-server" -description = "Enables the load_foundation_models_server command without any pre-configured scope." -commands.allow = ["load_foundation_models_server"] - -[[permission]] -identifier = "deny-load-foundation-models-server" -description = "Denies the load_foundation_models_server command without any pre-configured scope." -commands.deny = ["load_foundation_models_server"] diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/unload_foundation_models_server.toml b/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/unload_foundation_models_server.toml deleted file mode 100644 index ba2859681..000000000 --- a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/commands/unload_foundation_models_server.toml +++ /dev/null @@ -1,13 +0,0 @@ -# Automatically generated - DO NOT EDIT! - -"$schema" = "../../schemas/schema.json" - -[[permission]] -identifier = "allow-unload-foundation-models-server" -description = "Enables the unload_foundation_models_server command without any pre-configured scope." -commands.allow = ["unload_foundation_models_server"] - -[[permission]] -identifier = "deny-unload-foundation-models-server" -description = "Denies the unload_foundation_models_server command without any pre-configured scope." -commands.deny = ["unload_foundation_models_server"] diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/reference.md b/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/reference.md deleted file mode 100644 index 3d12ef964..000000000 --- a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/autogenerated/reference.md +++ /dev/null @@ -1,259 +0,0 @@ -## Default Permission - -Default permissions for the Foundation Models plugin - -#### This default permission set includes the following: - -- `allow-cleanup-foundation-models-processes` -- `allow-load-foundation-models-server` -- `allow-unload-foundation-models-server` -- `allow-is-foundation-models-process-running` -- `allow-get-foundation-models-random-port` -- `allow-find-foundation-models-session` -- `allow-get-foundation-models-loaded` -- `allow-get-foundation-models-all-sessions` -- `allow-check-foundation-models-availability` - -## Permission Table - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
IdentifierDescription
- -`foundation-models:allow-check-foundation-models-availability` - - - -Enables the check_foundation_models_availability command without any pre-configured scope. - -
- -`foundation-models:deny-check-foundation-models-availability` - - - -Denies the check_foundation_models_availability command without any pre-configured scope. - -
- -`foundation-models:allow-cleanup-foundation-models-processes` - - - -Enables the cleanup_foundation_models_processes command without any pre-configured scope. - -
- -`foundation-models:deny-cleanup-foundation-models-processes` - - - -Denies the cleanup_foundation_models_processes command without any pre-configured scope. - -
- -`foundation-models:allow-find-foundation-models-session` - - - -Enables the find_foundation_models_session command without any pre-configured scope. - -
- -`foundation-models:deny-find-foundation-models-session` - - - -Denies the find_foundation_models_session command without any pre-configured scope. - -
- -`foundation-models:allow-get-foundation-models-all-sessions` - - - -Enables the get_foundation_models_all_sessions command without any pre-configured scope. - -
- -`foundation-models:deny-get-foundation-models-all-sessions` - - - -Denies the get_foundation_models_all_sessions command without any pre-configured scope. - -
- -`foundation-models:allow-get-foundation-models-loaded` - - - -Enables the get_foundation_models_loaded command without any pre-configured scope. - -
- -`foundation-models:deny-get-foundation-models-loaded` - - - -Denies the get_foundation_models_loaded command without any pre-configured scope. - -
- -`foundation-models:allow-get-foundation-models-random-port` - - - -Enables the get_foundation_models_random_port command without any pre-configured scope. - -
- -`foundation-models:deny-get-foundation-models-random-port` - - - -Denies the get_foundation_models_random_port command without any pre-configured scope. - -
- -`foundation-models:allow-is-foundation-models-process-running` - - - -Enables the is_foundation_models_process_running command without any pre-configured scope. - -
- -`foundation-models:deny-is-foundation-models-process-running` - - - -Denies the is_foundation_models_process_running command without any pre-configured scope. - -
- -`foundation-models:allow-load-foundation-models-server` - - - -Enables the load_foundation_models_server command without any pre-configured scope. - -
- -`foundation-models:deny-load-foundation-models-server` - - - -Denies the load_foundation_models_server command without any pre-configured scope. - -
- -`foundation-models:allow-unload-foundation-models-server` - - - -Enables the unload_foundation_models_server command without any pre-configured scope. - -
- -`foundation-models:deny-unload-foundation-models-server` - - - -Denies the unload_foundation_models_server command without any pre-configured scope. - -
diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/default.toml b/src-tauri/plugins/tauri-plugin-foundation-models/permissions/default.toml index 51bfddcb8..f2cd967d0 100644 --- a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/default.toml +++ b/src-tauri/plugins/tauri-plugin-foundation-models/permissions/default.toml @@ -2,12 +2,11 @@ description = "Default permissions for the Foundation Models plugin" permissions = [ "allow-cleanup-foundation-models-processes", - "allow-load-foundation-models-server", - "allow-unload-foundation-models-server", - "allow-is-foundation-models-process-running", - "allow-get-foundation-models-random-port", - "allow-find-foundation-models-session", - "allow-get-foundation-models-loaded", - "allow-get-foundation-models-all-sessions", "allow-check-foundation-models-availability", + "allow-load-foundation-models", + "allow-unload-foundation-models", + "allow-is-foundation-models-loaded", + "allow-foundation-models-chat-completion", + "allow-foundation-models-chat-completion-stream", + "allow-abort-foundation-models-stream", ] diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/schemas/schema.json b/src-tauri/plugins/tauri-plugin-foundation-models/permissions/schemas/schema.json deleted file mode 100644 index 858d8817e..000000000 --- a/src-tauri/plugins/tauri-plugin-foundation-models/permissions/schemas/schema.json +++ /dev/null @@ -1,414 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "PermissionFile", - "description": "Permission file that can define a default permission, a set of permissions or a list of inlined permissions.", - "type": "object", - "properties": { - "default": { - "description": "The default permission set for the plugin", - "anyOf": [ - { - "$ref": "#/definitions/DefaultPermission" - }, - { - "type": "null" - } - ] - }, - "set": { - "description": "A list of permissions sets defined", - "type": "array", - "items": { - "$ref": "#/definitions/PermissionSet" - } - }, - "permission": { - "description": "A list of inlined permissions", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Permission" - } - } - }, - "definitions": { - "DefaultPermission": { - "description": "The default permission set of the plugin.\n\nWorks similarly to a permission with the \"default\" identifier.", - "type": "object", - "required": [ - "permissions" - ], - "properties": { - "version": { - "description": "The version of the permission.", - "type": [ - "integer", - "null" - ], - "format": "uint64", - "minimum": 1.0 - }, - "description": { - "description": "Human-readable description of what the permission does. Tauri convention is to use `

` headings in markdown content for Tauri documentation generation purposes.", - "type": [ - "string", - "null" - ] - }, - "permissions": { - "description": "All permissions this set contains.", - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "PermissionSet": { - "description": "A set of direct permissions grouped together under a new name.", - "type": "object", - "required": [ - "description", - "identifier", - "permissions" - ], - "properties": { - "identifier": { - "description": "A unique identifier for the permission.", - "type": "string" - }, - "description": { - "description": "Human-readable description of what the permission does.", - "type": "string" - }, - "permissions": { - "description": "All permissions this set contains.", - "type": "array", - "items": { - "$ref": "#/definitions/PermissionKind" - } - } - } - }, - "Permission": { - "description": "Descriptions of explicit privileges of commands.\n\nIt can enable commands to be accessible in the frontend of the application.\n\nIf the scope is defined it can be used to fine grain control the access of individual or multiple commands.", - "type": "object", - "required": [ - "identifier" - ], - "properties": { - "version": { - "description": "The version of the permission.", - "type": [ - "integer", - "null" - ], - "format": "uint64", - "minimum": 1.0 - }, - "identifier": { - "description": "A unique identifier for the permission.", - "type": "string" - }, - "description": { - "description": "Human-readable description of what the permission does. Tauri internal convention is to use `

` headings in markdown content for Tauri documentation generation purposes.", - "type": [ - "string", - "null" - ] - }, - "commands": { - "description": "Allowed or denied commands when using this permission.", - "default": { - "allow": [], - "deny": [] - }, - "allOf": [ - { - "$ref": "#/definitions/Commands" - } - ] - }, - "scope": { - "description": "Allowed or denied scoped when using this permission.", - "allOf": [ - { - "$ref": "#/definitions/Scopes" - } - ] - }, - "platforms": { - "description": "Target platforms this permission applies. By default all platforms are affected by this permission.", - "type": [ - "array", - "null" - ], - "items": { - "$ref": "#/definitions/Target" - } - } - } - }, - "Commands": { - "description": "Allowed and denied commands inside a permission.\n\nIf two commands clash inside of `allow` and `deny`, it should be denied by default.", - "type": "object", - "properties": { - "allow": { - "description": "Allowed command.", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "deny": { - "description": "Denied command, which takes priority.", - "default": [], - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "Scopes": { - "description": "An argument for fine grained behavior control of Tauri commands.\n\nIt can be of any serde serializable type and is used to allow or prevent certain actions inside a Tauri command. The configured scope is passed to the command and will be enforced by the command implementation.\n\n## Example\n\n```json { \"allow\": [{ \"path\": \"$HOME/**\" }], \"deny\": [{ \"path\": \"$HOME/secret.txt\" }] } ```", - "type": "object", - "properties": { - "allow": { - "description": "Data that defines what is allowed by the scope.", - "type": [ - "array", - "null" - ], - "items": { - "$ref": "#/definitions/Value" - } - }, - "deny": { - "description": "Data that defines what is denied by the scope. This should be prioritized by validation logic.", - "type": [ - "array", - "null" - ], - "items": { - "$ref": "#/definitions/Value" - } - } - } - }, - "Value": { - "description": "All supported ACL values.", - "anyOf": [ - { - "description": "Represents a null JSON value.", - "type": "null" - }, - { - "description": "Represents a [`bool`].", - "type": "boolean" - }, - { - "description": "Represents a valid ACL [`Number`].", - "allOf": [ - { - "$ref": "#/definitions/Number" - } - ] - }, - { - "description": "Represents a [`String`].", - "type": "string" - }, - { - "description": "Represents a list of other [`Value`]s.", - "type": "array", - "items": { - "$ref": "#/definitions/Value" - } - }, - { - "description": "Represents a map of [`String`] keys to [`Value`]s.", - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/Value" - } - } - ] - }, - "Number": { - "description": "A valid ACL number.", - "anyOf": [ - { - "description": "Represents an [`i64`].", - "type": "integer", - "format": "int64" - }, - { - "description": "Represents a [`f64`].", - "type": "number", - "format": "double" - } - ] - }, - "Target": { - "description": "Platform target.", - "oneOf": [ - { - "description": "MacOS.", - "type": "string", - "enum": [ - "macOS" - ] - }, - { - "description": "Windows.", - "type": "string", - "enum": [ - "windows" - ] - }, - { - "description": "Linux.", - "type": "string", - "enum": [ - "linux" - ] - }, - { - "description": "Android.", - "type": "string", - "enum": [ - "android" - ] - }, - { - "description": "iOS.", - "type": "string", - "enum": [ - "iOS" - ] - } - ] - }, - "PermissionKind": { - "type": "string", - "oneOf": [ - { - "description": "Enables the check_foundation_models_availability command without any pre-configured scope.", - "type": "string", - "const": "allow-check-foundation-models-availability", - "markdownDescription": "Enables the check_foundation_models_availability command without any pre-configured scope." - }, - { - "description": "Denies the check_foundation_models_availability command without any pre-configured scope.", - "type": "string", - "const": "deny-check-foundation-models-availability", - "markdownDescription": "Denies the check_foundation_models_availability command without any pre-configured scope." - }, - { - "description": "Enables the cleanup_foundation_models_processes command without any pre-configured scope.", - "type": "string", - "const": "allow-cleanup-foundation-models-processes", - "markdownDescription": "Enables the cleanup_foundation_models_processes command without any pre-configured scope." - }, - { - "description": "Denies the cleanup_foundation_models_processes command without any pre-configured scope.", - "type": "string", - "const": "deny-cleanup-foundation-models-processes", - "markdownDescription": "Denies the cleanup_foundation_models_processes command without any pre-configured scope." - }, - { - "description": "Enables the find_foundation_models_session command without any pre-configured scope.", - "type": "string", - "const": "allow-find-foundation-models-session", - "markdownDescription": "Enables the find_foundation_models_session command without any pre-configured scope." - }, - { - "description": "Denies the find_foundation_models_session command without any pre-configured scope.", - "type": "string", - "const": "deny-find-foundation-models-session", - "markdownDescription": "Denies the find_foundation_models_session command without any pre-configured scope." - }, - { - "description": "Enables the get_foundation_models_all_sessions command without any pre-configured scope.", - "type": "string", - "const": "allow-get-foundation-models-all-sessions", - "markdownDescription": "Enables the get_foundation_models_all_sessions command without any pre-configured scope." - }, - { - "description": "Denies the get_foundation_models_all_sessions command without any pre-configured scope.", - "type": "string", - "const": "deny-get-foundation-models-all-sessions", - "markdownDescription": "Denies the get_foundation_models_all_sessions command without any pre-configured scope." - }, - { - "description": "Enables the get_foundation_models_loaded command without any pre-configured scope.", - "type": "string", - "const": "allow-get-foundation-models-loaded", - "markdownDescription": "Enables the get_foundation_models_loaded command without any pre-configured scope." - }, - { - "description": "Denies the get_foundation_models_loaded command without any pre-configured scope.", - "type": "string", - "const": "deny-get-foundation-models-loaded", - "markdownDescription": "Denies the get_foundation_models_loaded command without any pre-configured scope." - }, - { - "description": "Enables the get_foundation_models_random_port command without any pre-configured scope.", - "type": "string", - "const": "allow-get-foundation-models-random-port", - "markdownDescription": "Enables the get_foundation_models_random_port command without any pre-configured scope." - }, - { - "description": "Denies the get_foundation_models_random_port command without any pre-configured scope.", - "type": "string", - "const": "deny-get-foundation-models-random-port", - "markdownDescription": "Denies the get_foundation_models_random_port command without any pre-configured scope." - }, - { - "description": "Enables the is_foundation_models_process_running command without any pre-configured scope.", - "type": "string", - "const": "allow-is-foundation-models-process-running", - "markdownDescription": "Enables the is_foundation_models_process_running command without any pre-configured scope." - }, - { - "description": "Denies the is_foundation_models_process_running command without any pre-configured scope.", - "type": "string", - "const": "deny-is-foundation-models-process-running", - "markdownDescription": "Denies the is_foundation_models_process_running command without any pre-configured scope." - }, - { - "description": "Enables the load_foundation_models_server command without any pre-configured scope.", - "type": "string", - "const": "allow-load-foundation-models-server", - "markdownDescription": "Enables the load_foundation_models_server command without any pre-configured scope." - }, - { - "description": "Denies the load_foundation_models_server command without any pre-configured scope.", - "type": "string", - "const": "deny-load-foundation-models-server", - "markdownDescription": "Denies the load_foundation_models_server command without any pre-configured scope." - }, - { - "description": "Enables the unload_foundation_models_server command without any pre-configured scope.", - "type": "string", - "const": "allow-unload-foundation-models-server", - "markdownDescription": "Enables the unload_foundation_models_server command without any pre-configured scope." - }, - { - "description": "Denies the unload_foundation_models_server command without any pre-configured scope.", - "type": "string", - "const": "deny-unload-foundation-models-server", - "markdownDescription": "Denies the unload_foundation_models_server command without any pre-configured scope." - }, - { - "description": "Default permissions for the Foundation Models plugin\n#### This default permission set includes:\n\n- `allow-cleanup-foundation-models-processes`\n- `allow-load-foundation-models-server`\n- `allow-unload-foundation-models-server`\n- `allow-is-foundation-models-process-running`\n- `allow-get-foundation-models-random-port`\n- `allow-find-foundation-models-session`\n- `allow-get-foundation-models-loaded`\n- `allow-get-foundation-models-all-sessions`\n- `allow-check-foundation-models-availability`", - "type": "string", - "const": "default", - "markdownDescription": "Default permissions for the Foundation Models plugin\n#### This default permission set includes:\n\n- `allow-cleanup-foundation-models-processes`\n- `allow-load-foundation-models-server`\n- `allow-unload-foundation-models-server`\n- `allow-is-foundation-models-process-running`\n- `allow-get-foundation-models-random-port`\n- `allow-find-foundation-models-session`\n- `allow-get-foundation-models-loaded`\n- `allow-get-foundation-models-all-sessions`\n- `allow-check-foundation-models-availability`" - } - ] - } - } -} \ No newline at end of file diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/src/cleanup.rs b/src-tauri/plugins/tauri-plugin-foundation-models/src/cleanup.rs index 425dceca2..9f844b5e9 100644 --- a/src-tauri/plugins/tauri-plugin-foundation-models/src/cleanup.rs +++ b/src-tauri/plugins/tauri-plugin-foundation-models/src/cleanup.rs @@ -9,56 +9,11 @@ pub async fn cleanup_processes(app_handle: &tauri::AppHandle) { } }; - let mut map = app_state.sessions.lock().await; - let pids: Vec = map.keys().cloned().collect(); - - for pid in pids { - if let Some(session) = map.remove(&pid) { - let mut child = session.child; - - #[cfg(unix)] - { - use nix::sys::signal::{kill, Signal}; - use nix::unistd::Pid; - use tokio::time::{timeout, Duration}; - - if let Some(raw_pid) = child.id() { - let raw_pid = raw_pid as i32; - log::info!( - "Sending SIGTERM to Foundation Models PID {} during shutdown", - raw_pid - ); - let _ = kill(Pid::from_raw(raw_pid), Signal::SIGTERM); - - match timeout(Duration::from_secs(2), child.wait()).await { - Ok(Ok(status)) => log::info!( - "Foundation Models process {} exited gracefully: {}", - raw_pid, - status - ), - Ok(Err(e)) => log::error!( - "Error waiting after SIGTERM for Foundation Models PID {}: {}", - raw_pid, - e - ), - Err(_) => { - log::warn!( - "SIGTERM timed out for Foundation Models PID {}; sending SIGKILL", - raw_pid - ); - let _ = kill(Pid::from_raw(raw_pid), Signal::SIGKILL); - let _ = child.wait().await; - } - } - } - } - - #[cfg(not(unix))] - { - let _ = child.kill().await; - } - } + *app_state.loaded.lock().await = false; + if let Ok(mut tokens) = app_state.cancel_tokens.lock() { + tokens.clear(); } + log::info!("Foundation Models state cleaned up"); } #[tauri::command] diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/src/commands.rs b/src-tauri/plugins/tauri-plugin-foundation-models/src/commands.rs index eb114a25e..cd9bfc5a7 100644 --- a/src-tauri/plugins/tauri-plugin-foundation-models/src/commands.rs +++ b/src-tauri/plugins/tauri-plugin-foundation-models/src/commands.rs @@ -1,298 +1,488 @@ -use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::process::Stdio; -use std::time::Duration; -use tauri::{Manager, Runtime, State}; -use tokio::io::{AsyncBufReadExt, BufReader}; -use tokio::time::Instant; +use serde::{Deserialize, Serialize}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use tauri::{Emitter, Manager, Runtime, State}; -use crate::error::{ErrorCode, FoundationModelsError, ServerError, ServerResult}; -use crate::process::{ - find_active_session, get_random_available_port, is_process_running_by_pid, -}; -use crate::state::{FoundationModelsBackendSession, FoundationModelsState, SessionInfo}; +use crate::error::FoundationModelsError; +use crate::state::FoundationModelsState; -#[cfg(unix)] -use crate::process::graceful_terminate_process; +const MODEL_ID: &str = "apple/on-device"; -#[derive(serde::Serialize, serde::Deserialize)] -pub struct UnloadResult { - pub success: bool, - pub error: Option, +// ─── OpenAI-compatible types ──────────────────────────────────────────────── + +#[derive(Debug, Deserialize)] +struct ChatCompletionRequest { + #[allow(dead_code)] + model: Option, + messages: Vec, + temperature: Option, + #[allow(dead_code)] + top_p: Option, + max_tokens: Option, + #[allow(dead_code)] + stream: Option, + #[allow(dead_code)] + stop: Option>, } -/// Start the Foundation Models server binary. +#[derive(Debug, Deserialize)] +struct ChatMessage { + role: String, + content: Option, +} + +#[derive(Debug, Serialize)] +struct ChatCompletionResponse { + id: String, + object: String, + created: u64, + model: String, + choices: Vec, + usage: UsageInfo, +} + +#[derive(Debug, Serialize)] +struct ChatCompletionChoice { + index: u32, + message: ChatResponseMessage, + finish_reason: String, +} + +#[derive(Debug, Serialize)] +struct ChatResponseMessage { + role: String, + content: String, +} + +#[derive(Debug, Serialize)] +struct UsageInfo { + prompt_tokens: u32, + completion_tokens: u32, + total_tokens: u32, +} + +#[derive(Debug, Serialize)] +struct ChatCompletionChunk { + id: String, + object: String, + created: u64, + model: String, + choices: Vec, +} + +#[derive(Debug, Serialize)] +struct ChunkChoice { + index: u32, + delta: DeltaContent, + finish_reason: Option, +} + +#[derive(Debug, Serialize)] +struct DeltaContent { + role: Option, + content: Option, +} + +fn current_timestamp() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +/// Build session instructions from the OpenAI message list. /// -/// The binary is located at `binary_path` and is started with -/// the given `port` and optional `api_key`. Success is detected -/// by watching stdout for the ready signal within `timeout` seconds. -pub async fn load_foundation_models_server_impl( - sessions_arc: std::sync::Arc>>, - binary_path: &Path, - model_id: String, - port: u16, - api_key: String, - timeout: u64, -) -> ServerResult { - let bin_path = PathBuf::from(binary_path); - if !bin_path.exists() { - return Err(FoundationModelsError::new( - ErrorCode::BinaryNotFound, - format!( - "foundation-models-server binary not found at: {}", - binary_path.display() - ), - None, - ) - .into()); +/// System messages become the instruction text. Prior user/assistant turns +/// are serialised into the instructions block so the model has full +/// conversation context (matching the previous Swift server behaviour). +fn build_instructions(messages: &[ChatMessage]) -> String { + let system_content = messages + .iter() + .find(|m| m.role == "system") + .and_then(|m| m.content.as_deref()) + .unwrap_or(""); + + let non_system: Vec<&ChatMessage> = messages.iter().filter(|m| m.role != "system").collect(); + let history = if non_system.len() > 1 { + &non_system[..non_system.len() - 1] + } else { + &[] + }; + + let mut instructions = if system_content.is_empty() { + "You are a helpful assistant.".to_string() + } else { + system_content.to_string() + }; + + if !history.is_empty() { + instructions.push_str("\n\n[Previous conversation]\n"); + for msg in history { + let label = if msg.role == "assistant" { + "Assistant" + } else { + "User" + }; + instructions.push_str(&format!( + "{}: {}\n", + label, + msg.content.as_deref().unwrap_or("") + )); + } + instructions.push_str("[End of previous conversation]"); } - let mut args = vec!["--port".to_string(), port.to_string()]; - if !api_key.is_empty() { - args.push("--api-key".to_string()); - args.push(api_key.clone()); - } + instructions +} - log::info!("Launching Foundation Models server: {:?} {:?}", bin_path, args); +fn extract_last_user_message(messages: &[ChatMessage]) -> String { + messages + .iter() + .filter(|m| m.role != "system") + .last() + .and_then(|m| m.content.clone()) + .unwrap_or_default() +} - let mut child = tokio::process::Command::new(&bin_path) - .args(&args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|e| ServerError::Io(e))?; +// ─── Commands ─────────────────────────────────────────────────────────────── - let pid = child.id().unwrap_or(0) as i32; - let stdout = child.stdout.take().expect("stdout not captured"); - let stderr = child.stderr.take().expect("stderr not captured"); +#[tauri::command] +pub async fn check_foundation_models_availability( + _app_handle: tauri::AppHandle, +) -> Result { + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] + { + let result = tokio::task::spawn_blocking(|| { + let model = match fm_rs::SystemLanguageModel::new() { + Ok(m) => m, + Err(_) => return "unavailable".to_string(), + }; - // Watch stderr for error messages - let (stderr_tx, mut stderr_rx) = tokio::sync::mpsc::channel::(64); - tokio::spawn(async move { - let mut reader = BufReader::new(stderr).lines(); - while let Ok(Some(line)) = reader.next_line().await { - log::error!("[foundation-models stderr] {}", line); - let _ = stderr_tx.send(line).await; - } - }); - - // Watch stdout for the readiness signal - let (ready_tx, mut ready_rx) = tokio::sync::mpsc::channel::(1); - tokio::spawn(async move { - let mut reader = BufReader::new(stdout).lines(); - while let Ok(Some(line)) = reader.next_line().await { - log::info!("[foundation-models] {}", line); - if line.contains("server is listening on") || line.contains("http server listening") { - let _ = ready_tx.send(true).await; - } - } - }); - - let deadline = Instant::now() + Duration::from_secs(timeout); - - loop { - tokio::select! { - _ = tokio::time::sleep_until(deadline) => { - log::error!("Foundation Models server startup timed out after {}s", timeout); - // Terminate the process - #[cfg(unix)] - graceful_terminate_process(&mut child).await; - #[cfg(not(unix))] - let _ = child.kill().await; - - return Err(FoundationModelsError::new( - ErrorCode::ServerStartTimedOut, - format!( - "Foundation Models server did not become ready within {} seconds.", - timeout - ), - None, - ) - .into()); + if model.is_available() { + return "available".to_string(); } - ready = ready_rx.recv() => { - if ready == Some(true) { - log::info!("Foundation Models server ready on port {}", port); - let session_info = SessionInfo { - pid, - port: port as i32, - model_id: model_id.clone(), - api_key: api_key.clone(), - }; - let backend_session = FoundationModelsBackendSession { - child, - info: session_info.clone(), - }; - sessions_arc.lock().await.insert(pid, backend_session); - return Ok(session_info); - } - } - - stderr_line = stderr_rx.recv() => { - if let Some(line) = stderr_line { - if line.contains("[foundation-models] ERROR:") { - // The availability check failed; terminate immediately - #[cfg(unix)] - graceful_terminate_process(&mut child).await; - #[cfg(not(unix))] - let _ = child.kill().await; - - return Err(FoundationModelsError::from_stderr(&line).into()); + match model.ensure_available() { + Ok(()) => "available".to_string(), + Err(e) => { + let msg = e.to_string().to_lowercase(); + if msg.contains("not eligible") || msg.contains("eligible") { + "notEligible".to_string() + } else if msg.contains("not enabled") || msg.contains("intelligence") { + "appleIntelligenceNotEnabled".to_string() + } else if msg.contains("not ready") || msg.contains("ready") { + "modelNotReady".to_string() + } else { + "unavailable".to_string() } } } - - status = child.wait() => { - // Process exited prematurely - let code = match status { - Ok(s) => s.code().unwrap_or(-1), - Err(_) => -1, - }; - log::error!("Foundation Models server exited prematurely with code {}", code); - return Err(FoundationModelsError::new( - ErrorCode::ServerStartFailed, - format!( - "Foundation Models server exited with code {} before becoming ready. \ - Ensure Apple Intelligence is enabled in System Settings.", - code - ), - None, - ) - .into()); - } - } - } -} - -// ─── Tauri commands ────────────────────────────────────────────────────────── - -#[tauri::command] -pub async fn load_foundation_models_server( - app_handle: tauri::AppHandle, - model_id: String, - port: u16, - api_key: String, - timeout: u64, -) -> Result { - let state: State = app_handle.state(); - - let resource_dir = app_handle - .path() - .resource_dir() - .map_err(ServerError::Tauri)?; - let binary_path = resource_dir.join("resources/bin/foundation-models-server"); - - load_foundation_models_server_impl( - state.sessions.clone(), - &binary_path, - model_id, - port, - api_key, - timeout, - ) - .await -} - -#[tauri::command] -pub async fn unload_foundation_models_server( - app_handle: tauri::AppHandle, - pid: i32, -) -> Result { - let state: State = app_handle.state(); - let mut map = state.sessions.lock().await; - - if let Some(session) = map.remove(&pid) { - let mut child = session.child; - #[cfg(unix)] - graceful_terminate_process(&mut child).await; - #[cfg(not(unix))] - let _ = child.kill().await; - - log::info!("Successfully unloaded Foundation Models server PID {}", pid); - Ok(UnloadResult { - success: true, - error: None, }) - } else { - Ok(UnloadResult { - success: false, - error: Some(format!("No active Foundation Models session found for PID {}", pid)), - }) - } -} - -#[tauri::command] -pub async fn is_foundation_models_process_running( - app_handle: tauri::AppHandle, - pid: i32, -) -> Result { - is_process_running_by_pid(app_handle, pid).await -} - -#[tauri::command] -pub async fn get_foundation_models_random_port( - app_handle: tauri::AppHandle, -) -> Result { - get_random_available_port(app_handle).await -} - -#[tauri::command] -pub async fn find_foundation_models_session( - app_handle: tauri::AppHandle, -) -> Result, String> { - Ok(find_active_session(app_handle).await) -} - -#[tauri::command] -pub async fn get_foundation_models_loaded( - app_handle: tauri::AppHandle, -) -> Result { - Ok(find_active_session(app_handle).await.is_some()) -} - -#[tauri::command] -pub async fn get_foundation_models_all_sessions( - app_handle: tauri::AppHandle, -) -> Result, String> { - let state: State = app_handle.state(); - let map = state.sessions.lock().await; - Ok(map.values().map(|s| s.info.clone()).collect()) -} - -/// Run the server binary with `--check` and return a machine-readable -/// availability token: `"available"`, `"notEligible"`, -/// `"appleIntelligenceNotEnabled"`, `"modelNotReady"`, `"unavailable"`, -/// or `"binaryNotFound"` if the binary is missing. -/// -/// Always returns `Ok` — the caller decides what to do with the status. -#[tauri::command] -pub async fn check_foundation_models_availability( - app_handle: tauri::AppHandle, -) -> Result { - let resource_dir = app_handle - .path() - .resource_dir() - .map_err(|e| e.to_string())?; - let binary_path = resource_dir.join("resources/bin/foundation-models-server"); - - if !binary_path.exists() { - return Ok("binaryNotFound".to_string()); - } - - let output = tokio::process::Command::new(&binary_path) - .arg("--check") - .output() .await .map_err(|e| e.to_string())?; + Ok(result) + } - let status = String::from_utf8_lossy(&output.stdout) - .trim() - .to_string(); - - if status.is_empty() { + #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))] + { Ok("unavailable".to_string()) - } else { - Ok(status) } } + +#[tauri::command] +pub async fn load_foundation_models( + app_handle: tauri::AppHandle, + _model_id: String, +) -> Result<(), FoundationModelsError> { + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] + { + tokio::task::spawn_blocking(|| { + let model = fm_rs::SystemLanguageModel::new() + .map_err(|e| FoundationModelsError::unavailable(e.to_string()))?; + model + .ensure_available() + .map_err(|e| FoundationModelsError::unavailable(e.to_string()))?; + Ok::<(), FoundationModelsError>(()) + }) + .await + .map_err(|e| FoundationModelsError::internal_error(e.to_string()))??; + + let state: State = app_handle.state(); + *state.loaded.lock().await = true; + log::info!("Foundation Models loaded successfully"); + Ok(()) + } + + #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))] + { + let _ = app_handle; + Err(FoundationModelsError::unavailable( + "Foundation Models are only available on macOS 26+ with Apple Silicon".into(), + )) + } +} + +#[tauri::command] +pub async fn unload_foundation_models( + app_handle: tauri::AppHandle, +) -> Result<(), String> { + let state: State = app_handle.state(); + *state.loaded.lock().await = false; + state + .cancel_tokens + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clear(); + log::info!("Foundation Models unloaded"); + Ok(()) +} + +#[tauri::command] +pub async fn is_foundation_models_loaded( + app_handle: tauri::AppHandle, +) -> Result { + let state: State = app_handle.state(); + let loaded = *state.loaded.lock().await; + Ok(loaded) +} + +#[tauri::command] +pub async fn foundation_models_chat_completion( + app_handle: tauri::AppHandle, + body: String, +) -> Result { + { + let state: State = app_handle.state(); + if !*state.loaded.lock().await { + return Err(FoundationModelsError::not_loaded()); + } + } + + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] + { + let result = tokio::task::spawn_blocking(move || { + let request: ChatCompletionRequest = serde_json::from_str(&body) + .map_err(|e| FoundationModelsError::invalid_request(e.to_string()))?; + + let model = fm_rs::SystemLanguageModel::new() + .map_err(|e| FoundationModelsError::inference_error(e.to_string()))?; + + let instructions = build_instructions(&request.messages); + let last_message = extract_last_user_message(&request.messages); + + let session = fm_rs::Session::with_instructions(&model, &instructions) + .map_err(|e| FoundationModelsError::inference_error(e.to_string()))?; + + let mut opts = fm_rs::GenerationOptions::builder(); + if let Some(temp) = request.temperature { + opts = opts.temperature(temp); + } + if let Some(max) = request.max_tokens { + opts = opts.max_response_tokens(max); + } + let opts = opts.build(); + + let response = session + .respond(&last_message, &opts) + .map_err(|e| FoundationModelsError::inference_error(e.to_string()))?; + + let completion = ChatCompletionResponse { + id: format!("chatcmpl-{}", uuid::Uuid::new_v4()), + object: "chat.completion".to_string(), + created: current_timestamp(), + model: MODEL_ID.to_string(), + choices: vec![ChatCompletionChoice { + index: 0, + message: ChatResponseMessage { + role: "assistant".to_string(), + content: response.content().to_string(), + }, + finish_reason: "stop".to_string(), + }], + usage: UsageInfo { + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + }, + }; + + serde_json::to_string(&completion) + .map_err(|e| FoundationModelsError::internal_error(e.to_string())) + }) + .await + .map_err(|e| FoundationModelsError::internal_error(e.to_string()))??; + + Ok(result) + } + + #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))] + { + Err(FoundationModelsError::unavailable( + "Foundation Models are only available on macOS 26+ with Apple Silicon".into(), + )) + } +} + +#[tauri::command] +pub async fn foundation_models_chat_completion_stream( + app_handle: tauri::AppHandle, + body: String, + request_id: String, +) -> Result<(), FoundationModelsError> { + { + let state: State = app_handle.state(); + if !*state.loaded.lock().await { + return Err(FoundationModelsError::not_loaded()); + } + } + + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] + { + let state: State = app_handle.state(); + let cancel_tokens = state.cancel_tokens.clone(); + let event_name = format!("foundation-models-stream-{}", request_id); + let handle = app_handle.clone(); + + tokio::task::spawn_blocking(move || -> Result<(), FoundationModelsError> { + let request: ChatCompletionRequest = serde_json::from_str(&body) + .map_err(|e| FoundationModelsError::invalid_request(e.to_string()))?; + + let model = fm_rs::SystemLanguageModel::new() + .map_err(|e| FoundationModelsError::inference_error(e.to_string()))?; + + let instructions = build_instructions(&request.messages); + let last_message = extract_last_user_message(&request.messages); + + let session = fm_rs::Session::with_instructions(&model, &instructions) + .map_err(|e| FoundationModelsError::inference_error(e.to_string()))?; + + let mut opts = fm_rs::GenerationOptions::builder(); + if let Some(temp) = request.temperature { + opts = opts.temperature(temp); + } + if let Some(max) = request.max_tokens { + opts = opts.max_response_tokens(max); + } + let opts = opts.build(); + + let chunk_id = format!("chatcmpl-{}", uuid::Uuid::new_v4()); + let created = current_timestamp(); + let model_id = MODEL_ID.to_string(); + + // Emit role chunk + let role_chunk = ChatCompletionChunk { + id: chunk_id.clone(), + object: "chat.completion.chunk".to_string(), + created, + model: model_id.clone(), + choices: vec![ChunkChoice { + index: 0, + delta: DeltaContent { + role: Some("assistant".to_string()), + content: None, + }, + finish_reason: None, + }], + }; + if let Ok(json) = serde_json::to_string(&role_chunk) { + let _ = handle.emit(&event_name, serde_json::json!({ "data": json })); + } + + let cancelled = Arc::new(AtomicBool::new(false)); + let cancelled_ref = cancelled.clone(); + let cancel_tokens_ref = cancel_tokens.clone(); + let request_id_ref = request_id.clone(); + let event_name_ref = event_name.clone(); + let handle_ref = handle.clone(); + let chunk_id_ref = chunk_id.clone(); + let model_id_ref = model_id.clone(); + + let stream_result = + session.stream_response(&last_message, &opts, move |chunk_text: &str| { + if cancelled_ref.load(Ordering::Relaxed) { + return; + } + + if let Ok(tokens) = cancel_tokens_ref.lock() { + if tokens.contains(&request_id_ref) { + cancelled_ref.store(true, Ordering::Relaxed); + return; + } + } + + let chunk = ChatCompletionChunk { + id: chunk_id_ref.clone(), + object: "chat.completion.chunk".to_string(), + created, + model: model_id_ref.clone(), + choices: vec![ChunkChoice { + index: 0, + delta: DeltaContent { + role: None, + content: Some(chunk_text.to_string()), + }, + finish_reason: None, + }], + }; + if let Ok(json) = serde_json::to_string(&chunk) { + let _ = handle_ref.emit(&event_name_ref, serde_json::json!({ "data": json })); + } + }); + + // Emit stop chunk + let stop_chunk = ChatCompletionChunk { + id: chunk_id, + object: "chat.completion.chunk".to_string(), + created, + model: model_id, + choices: vec![ChunkChoice { + index: 0, + delta: DeltaContent { + role: None, + content: None, + }, + finish_reason: Some("stop".to_string()), + }], + }; + if let Ok(json) = serde_json::to_string(&stop_chunk) { + let _ = handle.emit(&event_name, serde_json::json!({ "data": json })); + } + + // Signal completion + let _ = handle.emit(&event_name, serde_json::json!({ "done": true })); + + // Clean up cancel token + if let Ok(mut tokens) = cancel_tokens.lock() { + tokens.remove(&request_id); + } + + stream_result + .map(|_| ()) + .map_err(|e| FoundationModelsError::inference_error(e.to_string())) + }) + .await + .map_err(|e| FoundationModelsError::internal_error(e.to_string()))??; + + Ok(()) + } + + #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))] + { + let _ = (body, request_id); + Err(FoundationModelsError::unavailable( + "Foundation Models are only available on macOS 26+ with Apple Silicon".into(), + )) + } +} + +#[tauri::command] +pub async fn abort_foundation_models_stream( + app_handle: tauri::AppHandle, + request_id: String, +) -> Result<(), String> { + let state: State = app_handle.state(); + if let Ok(mut tokens) = state.cancel_tokens.lock() { + tokens.insert(request_id); + } + Ok(()) +} diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/src/error.rs b/src-tauri/plugins/tauri-plugin-foundation-models/src/error.rs index 651cc7251..f726eb47e 100644 --- a/src-tauri/plugins/tauri-plugin-foundation-models/src/error.rs +++ b/src-tauri/plugins/tauri-plugin-foundation-models/src/error.rs @@ -3,12 +3,10 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum ErrorCode { - BinaryNotFound, - FoundationModelsUnavailable, - ServerStartFailed, - ServerStartTimedOut, - ProcessError, - IoError, + NotLoaded, + Unavailable, + InvalidRequest, + InferenceError, InternalError, } @@ -22,86 +20,43 @@ pub struct FoundationModelsError { } impl FoundationModelsError { - pub fn new(code: ErrorCode, message: String, details: Option) -> Self { + pub fn not_loaded() -> Self { Self { - code, - message, - details, + code: ErrorCode::NotLoaded, + message: "Foundation Models are not loaded. Please load the model first.".into(), + details: None, } } - /// Interpret stderr output from the server binary and produce a descriptive error. - pub fn from_stderr(stderr: &str) -> Self { - let lower = stderr.to_lowercase(); - - if lower.contains("device is not eligible") - || lower.contains("devicenoteligible") - { - return Self::new( - ErrorCode::FoundationModelsUnavailable, - "This device is not eligible for Apple Intelligence.".into(), - Some(stderr.into()), - ); + pub fn unavailable(details: String) -> Self { + Self { + code: ErrorCode::Unavailable, + message: "Foundation Models are not available on this device.".into(), + details: Some(details), } + } - if lower.contains("apple intelligence is not enabled") - || lower.contains("appleintelligencenotenabled") - { - return Self::new( - ErrorCode::FoundationModelsUnavailable, - "Apple Intelligence is not enabled. Please enable it in System Settings → Apple Intelligence & Siri.".into(), - Some(stderr.into()), - ); + pub fn invalid_request(details: String) -> Self { + Self { + code: ErrorCode::InvalidRequest, + message: "Invalid request.".into(), + details: Some(details), } + } - if lower.contains("model not ready") || lower.contains("modelnotready") { - return Self::new( - ErrorCode::FoundationModelsUnavailable, - "The Foundation Model is still downloading or not yet ready. Please wait and try again.".into(), - Some(stderr.into()), - ); + pub fn inference_error(details: String) -> Self { + Self { + code: ErrorCode::InferenceError, + message: "An error occurred during inference.".into(), + details: Some(details), } + } - Self::new( - ErrorCode::ProcessError, - "The Foundation Models server encountered an unexpected error.".into(), - Some(stderr.into()), - ) + pub fn internal_error(details: String) -> Self { + Self { + code: ErrorCode::InternalError, + message: "An internal error occurred.".into(), + details: Some(details), + } } } - -#[derive(Debug, thiserror::Error)] -pub enum ServerError { - #[error(transparent)] - FoundationModels(#[from] FoundationModelsError), - - #[error("IO error: {0}")] - Io(#[from] std::io::Error), - - #[error("Tauri error: {0}")] - Tauri(#[from] tauri::Error), -} - -impl serde::Serialize for ServerError { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - let err = match self { - ServerError::FoundationModels(e) => e.clone(), - ServerError::Io(e) => FoundationModelsError::new( - ErrorCode::IoError, - "An input/output error occurred.".into(), - Some(e.to_string()), - ), - ServerError::Tauri(e) => FoundationModelsError::new( - ErrorCode::InternalError, - "A Tauri internal error occurred.".into(), - Some(e.to_string()), - ), - }; - err.serialize(serializer) - } -} - -pub type ServerResult = Result; diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/src/lib.rs b/src-tauri/plugins/tauri-plugin-foundation-models/src/lib.rs index 2a07185d9..a42976bb1 100644 --- a/src-tauri/plugins/tauri-plugin-foundation-models/src/lib.rs +++ b/src-tauri/plugins/tauri-plugin-foundation-models/src/lib.rs @@ -6,25 +6,22 @@ use tauri::{ pub mod cleanup; mod commands; mod error; -mod process; pub mod state; pub use cleanup::cleanup_processes; pub use state::FoundationModelsState; -/// Initializes the Foundation Models plugin. pub fn init() -> TauriPlugin { Builder::new("foundation-models") .invoke_handler(tauri::generate_handler![ cleanup::cleanup_foundation_models_processes, - commands::load_foundation_models_server, - commands::unload_foundation_models_server, - commands::is_foundation_models_process_running, - commands::get_foundation_models_random_port, - commands::find_foundation_models_session, - commands::get_foundation_models_loaded, - commands::get_foundation_models_all_sessions, commands::check_foundation_models_availability, + commands::load_foundation_models, + commands::unload_foundation_models, + commands::is_foundation_models_loaded, + commands::foundation_models_chat_completion, + commands::foundation_models_chat_completion_stream, + commands::abort_foundation_models_stream, ]) .setup(|app, _api| { app.manage(state::FoundationModelsState::new()); diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/src/process.rs b/src-tauri/plugins/tauri-plugin-foundation-models/src/process.rs deleted file mode 100644 index 7dab56f08..000000000 --- a/src-tauri/plugins/tauri-plugin-foundation-models/src/process.rs +++ /dev/null @@ -1,102 +0,0 @@ -use std::collections::HashSet; -use sysinfo::{Pid, System}; -use tauri::{Manager, Runtime, State}; - -use crate::state::{FoundationModelsState, SessionInfo}; -use jan_utils::generate_random_port; - -/// Returns true if the process with the given PID is still running. -/// Removes the session from state if the process has exited. -pub async fn is_process_running_by_pid( - app_handle: tauri::AppHandle, - pid: i32, -) -> Result { - let mut system = System::new(); - system.refresh_processes(sysinfo::ProcessesToUpdate::All, true); - let alive = system.process(Pid::from(pid as usize)).is_some(); - - if !alive { - let state: State = app_handle.state(); - let mut map = state.sessions.lock().await; - map.remove(&pid); - } - - Ok(alive) -} - -/// Returns a random available port that is not used by any active session. -pub async fn get_random_available_port( - app_handle: tauri::AppHandle, -) -> Result { - let state: State = app_handle.state(); - let map = state.sessions.lock().await; - - let used_ports: HashSet = map - .values() - .filter_map(|s| { - if s.info.port > 0 && s.info.port <= u16::MAX as i32 { - Some(s.info.port as u16) - } else { - None - } - }) - .collect(); - - drop(map); - generate_random_port(&used_ports) -} - -/// Returns the SessionInfo for the only expected active session (if any). -pub async fn find_active_session( - app_handle: tauri::AppHandle, -) -> Option { - let state: State = app_handle.state(); - let map = state.sessions.lock().await; - map.values().next().map(|s| s.info.clone()) -} - -/// Gracefully terminate a process on Unix (macOS). -#[cfg(unix)] -pub async fn graceful_terminate_process(child: &mut tokio::process::Child) { - use nix::sys::signal::{kill, Signal}; - use nix::unistd::Pid; - use std::time::Duration; - use tokio::time::timeout; - - if let Some(raw_pid) = child.id() { - let raw_pid = raw_pid as i32; - log::info!( - "Sending SIGTERM to Foundation Models PID {}", - raw_pid - ); - let _ = kill(Pid::from_raw(raw_pid), Signal::SIGTERM); - - match timeout(Duration::from_secs(5), child.wait()).await { - Ok(Ok(status)) => log::info!( - "Foundation Models process {} exited gracefully: {}", - raw_pid, - status - ), - Ok(Err(e)) => log::error!( - "Error waiting after SIGTERM for Foundation Models PID {}: {}", - raw_pid, - e - ), - Err(_) => { - log::warn!( - "SIGTERM timed out for Foundation Models PID {}; sending SIGKILL", - raw_pid - ); - let _ = kill(Pid::from_raw(raw_pid), Signal::SIGKILL); - match child.wait().await { - Ok(s) => log::info!("Force-killed Foundation Models process: {}", s), - Err(e) => log::error!( - "Error waiting after SIGKILL for Foundation Models PID {}: {}", - raw_pid, - e - ), - } - } - } - } -} diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/src/state.rs b/src-tauri/plugins/tauri-plugin-foundation-models/src/state.rs index 216c02682..1f952f2ff 100644 --- a/src-tauri/plugins/tauri-plugin-foundation-models/src/state.rs +++ b/src-tauri/plugins/tauri-plugin-foundation-models/src/state.rs @@ -1,32 +1,19 @@ -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::collections::HashSet; use std::sync::Arc; -use tokio::process::Child; use tokio::sync::Mutex; -/// Session information for a running Foundation Models server instance -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SessionInfo { - pub pid: i32, - pub port: i32, - pub model_id: String, - pub api_key: String, -} - -pub struct FoundationModelsBackendSession { - pub child: Child, - pub info: SessionInfo, -} - -/// Plugin state — tracks all active server processes keyed by PID pub struct FoundationModelsState { - pub sessions: Arc>>, + pub loaded: Arc>, + /// Request IDs that have been signalled for cancellation. + /// Checked by the streaming callback to stop emitting events. + pub cancel_tokens: Arc>>, } impl Default for FoundationModelsState { fn default() -> Self { Self { - sessions: Arc::new(Mutex::new(HashMap::new())), + loaded: Arc::new(Mutex::new(false)), + cancel_tokens: Arc::new(std::sync::Mutex::new(HashSet::new())), } } } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index c190e591b..5e8733184 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -387,7 +387,7 @@ pub fn run() { { use tauri_plugin_foundation_models::cleanup_processes; cleanup_processes(&app_handle).await; - log::info!("Foundation Models processes cleaned up successfully"); + log::info!("Foundation Models state cleaned up successfully"); } log::info!("App cleanup completed"); diff --git a/src-tauri/tauri.macos.conf.json b/src-tauri/tauri.macos.conf.json index fc43f3f26..b9af1ca13 100644 --- a/src-tauri/tauri.macos.conf.json +++ b/src-tauri/tauri.macos.conf.json @@ -30,7 +30,7 @@ }, "bundle": { "targets": ["app", "dmg"], - "resources": ["resources/pre-install/**/*", "resources/LICENSE", "resources/bin/mlx-server", "resources/bin/mlx-swift_Cmlx.bundle", "resources/bin/foundation-models-server", "resources/bin/jan-cli"], + "resources": ["resources/pre-install/**/*", "resources/LICENSE", "resources/bin/mlx-server", "resources/bin/mlx-swift_Cmlx.bundle", "resources/bin/jan-cli"], "externalBin": ["resources/bin/bun", "resources/bin/uv"], "macOS": { "entitlements": "./Entitlements.plist" diff --git a/web-app/public/images/model-provider/apple-intelligence.svg b/web-app/public/images/model-provider/apple-intelligence.svg new file mode 100644 index 000000000..35f0cd9f4 --- /dev/null +++ b/web-app/public/images/model-provider/apple-intelligence.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/web-app/public/images/model-provider/minimax.svg b/web-app/public/images/model-provider/minimax.svg new file mode 100644 index 000000000..3771665f5 --- /dev/null +++ b/web-app/public/images/model-provider/minimax.svg @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/web-app/src/components/ai-elements/tool.tsx b/web-app/src/components/ai-elements/tool.tsx index 71206951e..93b7748a5 100644 --- a/web-app/src/components/ai-elements/tool.tsx +++ b/web-app/src/components/ai-elements/tool.tsx @@ -108,7 +108,7 @@ export const ToolHeader = memo( return ( diff --git a/web-app/src/constants/models.ts b/web-app/src/constants/models.ts index e087d1329..c6dbdfbaf 100644 --- a/web-app/src/constants/models.ts +++ b/web-app/src/constants/models.ts @@ -108,6 +108,15 @@ export const providerModels = { supportsToolCalls: ['sonar', 'sonar-pro', 'sonar-reasoning-pro'], supportsN: true, }, + minimax: { + models: ['MiniMax-M2.7', 'MiniMax-M2.7-highspeed', 'MiniMax-M2.5', 'MiniMax-M2.5-highspeed'], + supportsCompletion: true, + supportsStreaming: ['MiniMax-M2.7', 'MiniMax-M2.7-highspeed', 'MiniMax-M2.5', 'MiniMax-M2.5-highspeed'], + supportsJSON: [], + supportsImages: [], + supportsToolCalls: ['MiniMax-M2.7', 'MiniMax-M2.7-highspeed', 'MiniMax-M2.5', 'MiniMax-M2.5-highspeed'], + supportsN: true, + }, openrouter: { models: true, supportsCompletion: true, diff --git a/web-app/src/constants/providers.ts b/web-app/src/constants/providers.ts index fcfc187eb..518f4f080 100644 --- a/web-app/src/constants/providers.ts +++ b/web-app/src/constants/providers.ts @@ -325,6 +325,69 @@ export const predefinedProviders = [ ], models: [], }, + { + active: true, + api_key: '', + base_url: 'https://api.minimax.io/v1', + explore_models_url: 'https://platform.minimax.io/docs/api-reference/text-openai-api', + provider: 'minimax', + settings: [ + { + key: 'api-key', + title: 'API Key', + description: + "The MiniMax API uses API keys for authentication. Visit your [API Keys](https://platform.minimax.io/user-center/basic-information/interface-key) page to retrieve the API key you'll use in your requests.", + controller_type: 'input', + controller_props: { + placeholder: 'Insert API Key', + value: '', + type: 'password', + input_actions: ['unobscure', 'copy'], + }, + }, + { + key: 'base-url', + title: 'Base URL', + description: + 'The base endpoint to use. Use `https://api.minimax.io/v1` for global access or `https://api.minimaxi.com/v1` for users in China. See the [MiniMax API documentation](https://platform.minimax.io/docs/api-reference/text-openai-api) for more information.', + controller_type: 'input', + controller_props: { + placeholder: 'https://api.minimax.io/v1', + value: 'https://api.minimax.io/v1', + }, + }, + ], + models: [ + { + id: 'MiniMax-M2.7', + name: 'MiniMax-M2.7', + version: '1.0', + description: 'Latest flagship model with enhanced reasoning and coding.', + capabilities: ['completion', 'tools'], + }, + { + id: 'MiniMax-M2.7-highspeed', + name: 'MiniMax-M2.7-highspeed', + version: '1.0', + description: 'High-speed version of M2.7 for low-latency scenarios.', + capabilities: ['completion', 'tools'], + }, + { + id: 'MiniMax-M2.5', + name: 'MiniMax-M2.5', + version: '1.0', + description: 'Peak Performance. Ultimate Value. Master the Complex. 204K context window.', + capabilities: ['completion', 'tools'], + }, + { + id: 'MiniMax-M2.5-highspeed', + name: 'MiniMax-M2.5-highspeed', + version: '1.0', + description: 'Same performance, faster and more agile. 204K context window.', + capabilities: ['completion', 'tools'], + }, + ], + }, { active: true, api_key: '', diff --git a/web-app/src/containers/DropdownModelProvider.tsx b/web-app/src/containers/DropdownModelProvider.tsx index 306337632..e9ffce8c0 100644 --- a/web-app/src/containers/DropdownModelProvider.tsx +++ b/web-app/src/containers/DropdownModelProvider.tsx @@ -261,6 +261,7 @@ const DropdownModelProvider = memo(function DropdownModelProvider({ providers.forEach((provider) => { if (!provider.active) return + if (provider.provider === 'foundation-models') return provider.models.forEach((modelItem) => { // Skip embedding models - they can't be used for chat @@ -342,7 +343,7 @@ const DropdownModelProvider = memo(function DropdownModelProvider({ // When not searching, show all active providers (even without models) // Sort: local first, then providers with API keys or custom with models, then others, alphabetically const activeProviders = providers - .filter((p) => p.active) + .filter((p) => p.active && p.provider !== 'foundation-models') .sort((a, b) => { const aIsLocal = a.provider === 'llamacpp' || a.provider === 'mlx' const bIsLocal = b.provider === 'llamacpp' || b.provider === 'mlx' diff --git a/web-app/src/containers/SettingsMenu.tsx b/web-app/src/containers/SettingsMenu.tsx index 3f9d96c45..7a6c1ea3b 100644 --- a/web-app/src/containers/SettingsMenu.tsx +++ b/web-app/src/containers/SettingsMenu.tsx @@ -58,12 +58,14 @@ const SettingsMenu = () => { const activeProviders = providers.filter((provider) => { if (!provider.active) return false if (!IS_MACOS && provider.provider === 'mlx') return false + if (provider.provider === 'foundation-models') return false return true }) const hiddenProviders = providers.filter((provider) => { if (provider.active) return false if (!IS_MACOS && provider.provider === 'mlx') return false + if (provider.provider === 'foundation-models') return false return true }) diff --git a/web-app/src/lib/__tests__/model-factory.test.ts b/web-app/src/lib/__tests__/model-factory.test.ts index fa2c0e617..698b4eb1a 100644 --- a/web-app/src/lib/__tests__/model-factory.test.ts +++ b/web-app/src/lib/__tests__/model-factory.test.ts @@ -139,6 +139,21 @@ describe('ModelFactory', () => { expect(model.type).toBe('openai-compatible') }) + it('should create an OpenAI-compatible model for minimax provider', async () => { + const provider: ProviderObject = { + provider: 'minimax', + api_key: 'test-api-key', + base_url: 'https://api.minimax.io/v1', + models: [], + settings: [], + active: true, + } + + const model = await ModelFactory.createModel('MiniMax-M2.7', provider) + expect(model).toBeDefined() + expect(model.type).toBe('openai-compatible') + }) + it('should handle custom headers for OpenAI-compatible providers', async () => { const provider: ProviderObject = { provider: 'custom', @@ -207,7 +222,7 @@ describe('ModelFactory', () => { await expect( ModelFactory.createModel('apple/on-device', foundationModelsProvider) ).rejects.toThrow( - 'The Foundation Models server binary is missing. Please reinstall Jan.' + 'Apple Foundation Models are currently unavailable on this device.' ) }) @@ -221,27 +236,35 @@ describe('ModelFactory', () => { ) }) - it('should throw when available but no session is found after start', async () => { + it('should throw when available but model is not loaded after start', async () => { mockedInvoke .mockResolvedValueOnce('available') // check_foundation_models_availability - .mockResolvedValueOnce(null) // find_foundation_models_session + .mockResolvedValueOnce(false) // is_foundation_models_loaded await expect( ModelFactory.createModel('apple/on-device', foundationModelsProvider) ).rejects.toThrow( - 'No running Foundation Models session. The server may have failed to start' + 'No running Foundation Models session. The model may have failed to load — please check the logs.' + ) + + expect(mockStartModel).toHaveBeenCalledWith( + foundationModelsProvider, + 'apple/on-device' + ) + expect(mockedInvoke).toHaveBeenCalledWith( + 'plugin:foundation-models|check_foundation_models_availability', + {} + ) + expect(mockedInvoke).toHaveBeenCalledWith( + 'plugin:foundation-models|is_foundation_models_loaded', + {} ) }) - it('should create a model when available and session exists', async () => { + it('should create a model when available and model is loaded', async () => { mockedInvoke .mockResolvedValueOnce('available') // check_foundation_models_availability - .mockResolvedValueOnce({ // find_foundation_models_session - pid: 12345, - port: 9876, - model_id: 'apple/on-device', - api_key: 'test-session-key', - }) + .mockResolvedValueOnce(true) // is_foundation_models_loaded const model = await ModelFactory.createModel( 'apple/on-device', @@ -249,12 +272,16 @@ describe('ModelFactory', () => { ) expect(model).toBeDefined() + expect(mockStartModel).toHaveBeenCalledWith( + foundationModelsProvider, + 'apple/on-device' + ) expect(mockedInvoke).toHaveBeenCalledWith( 'plugin:foundation-models|check_foundation_models_availability', {} ) expect(mockedInvoke).toHaveBeenCalledWith( - 'plugin:foundation-models|find_foundation_models_session', + 'plugin:foundation-models|is_foundation_models_loaded', {} ) }) diff --git a/web-app/src/lib/__tests__/utils.test.ts b/web-app/src/lib/__tests__/utils.test.ts index 57634bbf7..e46437c8a 100644 --- a/web-app/src/lib/__tests__/utils.test.ts +++ b/web-app/src/lib/__tests__/utils.test.ts @@ -20,6 +20,9 @@ describe('getProviderLogo', () => { expect(getProviderLogo('openai')).toBe('/images/model-provider/openai.svg') expect(getProviderLogo('gemini')).toBe('/images/model-provider/gemini.svg') expect(getProviderLogo('nvidia')).toBe('/images/model-provider/nvidia.svg') + expect(getProviderLogo('foundation-models')).toBe( + '/images/model-provider/apple-intelligence.svg' + ) }) it('returns undefined for unknown providers', () => { @@ -35,6 +38,7 @@ describe('getProviderTitle', () => { expect(getProviderTitle('openrouter')).toBe('OpenRouter') expect(getProviderTitle('gemini')).toBe('Gemini') expect(getProviderTitle('nvidia')).toBe('NVIDIA NIM') + expect(getProviderTitle('foundation-models')).toBe('Apple Intelligence') }) it('capitalizes first letter for unknown providers', () => { diff --git a/web-app/src/lib/model-factory.ts b/web-app/src/lib/model-factory.ts index 7a03ce356..c6f166293 100644 --- a/web-app/src/lib/model-factory.ts +++ b/web-app/src/lib/model-factory.ts @@ -142,6 +142,95 @@ function createCustomFetch( } } +/** + * Custom fetch for Foundation Models that routes through Tauri IPC + * instead of HTTP, emulating an OpenAI-compatible fetch interface. + */ +function createFoundationModelsFetch( + parameters: Record +): typeof globalThis.fetch { + return async ( + _input: RequestInfo | URL, + init?: RequestInit + ): Promise => { + const rawBody = init?.body ? JSON.parse(init.body as string) : {} + const body = { ...rawBody, ...parameters } + const isStreaming = body.stream === true + + if (!isStreaming) { + const result = await invoke( + 'plugin:foundation-models|foundation_models_chat_completion', + { body: JSON.stringify(body) } + ) + return new Response(result, { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + } + + const requestId = crypto.randomUUID() + const { listen } = await import('@tauri-apps/api/event') + + let unlistenFn: (() => void) | null = null + const stream = new ReadableStream({ + async start(controller) { + const encoder = new TextEncoder() + + unlistenFn = await listen( + `foundation-models-stream-${requestId}`, + (event: { payload: { data?: string; done?: boolean; error?: string } }) => { + const payload = event.payload + if (payload.done) { + controller.enqueue(encoder.encode('data: [DONE]\n\n')) + try { + controller.close() + } catch { + /* already closed */ + } + unlistenFn?.() + } else if (payload.error) { + try { + controller.error(new Error(payload.error)) + } catch { + /* already errored */ + } + unlistenFn?.() + } else if (payload.data) { + controller.enqueue( + encoder.encode(`data: ${payload.data}\n\n`) + ) + } + } + ) + + invoke( + 'plugin:foundation-models|foundation_models_chat_completion_stream', + { body: JSON.stringify(body), requestId } + ).catch((err) => { + try { + controller.error(err) + } catch { + /* already errored */ + } + unlistenFn?.() + }) + }, + cancel() { + unlistenFn?.() + invoke( + 'plugin:foundation-models|abort_foundation_models_stream', + { requestId } + ).catch(() => {}) + }, + }) + + return new Response(stream, { + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + }) + } +} + /** * Factory for creating language models based on provider type. * Supports native AI SDK providers (Anthropic, Google) and OpenAI-compatible providers. @@ -183,6 +272,7 @@ export class ModelFactory { case 'cohere': case 'perplexity': case 'moonshot': + case 'minimax': return this.createOpenAICompatibleModel(modelId, provider) case 'xai': @@ -345,8 +435,8 @@ export class ModelFactory { } /** - * Create a Foundation Models model (Apple on-device) by starting the local - * Swift server via the Tauri plugin and connecting over localhost. + * Create a Foundation Models model (Apple on-device) via direct Tauri IPC + * to the fm-rs Rust bindings — no HTTP server involved. */ private static async createFoundationModelsModel( modelId: string, @@ -366,8 +456,6 @@ export class ModelFactory { 'Apple Intelligence is not enabled. Please enable it in System Settings > Apple Intelligence & Siri.', modelNotReady: 'The Apple on-device model is still preparing. Please wait and try again shortly.', - binaryNotFound: - 'The Foundation Models server binary is missing. Please reinstall Jan.', unavailable: 'Apple Foundation Models are currently unavailable on this device.', } @@ -390,30 +478,24 @@ export class ModelFactory { } } - const sessionInfo = await invoke( - 'plugin:foundation-models|find_foundation_models_session', + const loaded = await invoke( + 'plugin:foundation-models|is_foundation_models_loaded', {} ) - if (!sessionInfo) { + if (!loaded) { throw new Error( - 'No running Foundation Models session. The server may have failed to start — please check the logs.' + 'No running Foundation Models session. The model may have failed to load — please check the logs.' ) } - const baseUrl = `http://localhost:${sessionInfo.port}` - const authHeaders = { - Authorization: `Bearer ${sessionInfo.api_key}`, - Origin: 'tauri://localhost', - } - - const customFetch = createCustomFetch(httpFetch, parameters) + const customFetch = createFoundationModelsFetch(parameters) const model = new OpenAICompatibleChatLanguageModel(modelId, { provider: 'foundation-models', - headers: () => authHeaders, - url: ({ path }) => new URL(`${baseUrl}/v1${path}`).toString(), - fetch: customFetch, + headers: () => ({}), + url: ({ path }) => `foundation-models://local/v1${path}`, + fetch: customFetch as typeof httpFetch, metadataExtractor: providerMetadataExtractor, }) diff --git a/web-app/src/lib/utils.ts b/web-app/src/lib/utils.ts index 1dd1504ae..d4d269cc4 100644 --- a/web-app/src/lib/utils.ts +++ b/web-app/src/lib/utils.ts @@ -72,6 +72,8 @@ export function getProviderLogo(provider: string) { return '/images/model-provider/llamacpp.svg' case 'mlx': return '/images/model-provider/mlx.png' + case 'foundation-models': + return '/images/model-provider/apple-intelligence.svg' case 'anthropic': return '/images/model-provider/anthropic.svg' case 'huggingface': @@ -92,6 +94,8 @@ export function getProviderLogo(provider: string) { return '/images/model-provider/azure.svg' case 'xai': return '/images/model-provider/xai.svg' + case 'minimax': + return '/images/model-provider/minimax.svg' case 'nvidia': return '/images/model-provider/nvidia.svg' default: @@ -107,6 +111,8 @@ export const getProviderTitle = (provider: string) => { return 'Llama.cpp' case 'mlx': return 'MLX' + case 'foundation-models': + return 'Apple Intelligence' case 'openai': return 'OpenAI' case 'openrouter': @@ -117,6 +123,8 @@ export const getProviderTitle = (provider: string) => { return 'Hugging Face' case 'xai': return 'xAI' + case 'minimax': + return 'MiniMax' case 'nvidia': return 'NVIDIA NIM' default: diff --git a/web-app/src/services/providers/tauri.ts b/web-app/src/services/providers/tauri.ts index 5ecc1560f..af78099e0 100644 --- a/web-app/src/services/providers/tauri.ts +++ b/web-app/src/services/providers/tauri.ts @@ -99,6 +99,7 @@ export class TauriProvidersService extends DefaultProvidersService { id: model.id, model: model.id, name: model.name, + displayName: model.name, description: model.description, capabilities, embedding: model.embedding, // Preserve embedding flag for filtering in UI