From 80ae5518493914da6aace9e146e81c501a6494e5 Mon Sep 17 00:00:00 2001 From: Evgenii Utkin Date: Wed, 18 Mar 2026 16:23:10 +0100 Subject: [PATCH 01/10] fix: improve Windows support in Makefile by standardizing OS detection --- Makefile | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 10ed25fd2..ffcf8bcec 100644 --- a/Makefile +++ b/Makefile @@ -6,15 +6,22 @@ REPORT_PORTAL_PROJECT_NAME ?= "" REPORT_PORTAL_LAUNCH_NAME ?= "Jan App" REPORT_PORTAL_DESCRIPTION ?= "Jan App report" +# Detect OS +ifeq ($(OS),Windows_NT) + DETECTED_OS := Windows +else + DETECTED_OS := $(shell uname -s) +endif + # Default target, does nothing all: @echo "Specify a target to run" # Installs yarn dependencies and builds core and extensions install-and-build: -ifeq ($(OS),Windows_NT) +ifeq ($(DETECTED_OS),Windows) echo "skip" -else ifeq ($(shell uname -s),Linux) +else ifeq ($(DETECTED_OS),Linux) chmod +x src-tauri/build-utils/* endif yarn install @@ -24,7 +31,7 @@ endif # Install required Rust targets for macOS universal builds install-rust-targets: -ifeq ($(shell uname -s),Darwin) +ifeq ($(DETECTED_OS),Darwin) @echo "Detected macOS, installing universal build targets..." rustup target add x86_64-apple-darwin rustup target add aarch64-apple-darwin @@ -89,7 +96,7 @@ dev-android: install-and-build install-android-rust-targets dev-ios: install-and-build install-ios-rust-targets @echo "Setting up iOS development environment..." -ifeq ($(shell uname -s),Darwin) +ifeq ($(DETECTED_OS),Darwin) @if [ ! -d "src-tauri/gen/ios" ]; then \ echo "iOS app not initialized. Initializing..."; \ yarn tauri ios init; \ @@ -111,7 +118,7 @@ lint: install-and-build # Testing test: lint install-rust-targets yarn download:bin -ifeq ($(OS),Windows_NT) +ifeq ($(DETECTED_OS),Windows) endif yarn test yarn copy:assets:tauri @@ -126,7 +133,7 @@ endif # Build MLX server (macOS Apple Silicon only) - always builds build-mlx-server: -ifeq ($(shell uname -s),Darwin) +ifeq ($(DETECTED_OS),Darwin) @echo "Building MLX server for Apple Silicon..." cd mlx-server && swift build -c release @echo "Copying build products..." @@ -164,7 +171,7 @@ endif # Build MLX server only if not already present (for dev) build-mlx-server-if-exists: -ifeq ($(shell uname -s),Darwin) +ifeq ($(DETECTED_OS),Darwin) @if [ -f "src-tauri/resources/bin/mlx-server" ]; then \ echo "MLX server already exists at src-tauri/resources/bin/mlx-server, skipping build..."; \ else \ @@ -176,7 +183,7 @@ endif # Build Apple Foundation Models server (macOS 26+ only) - always builds build-foundation-models-server: -ifeq ($(shell uname -s),Darwin) +ifeq ($(DETECTED_OS),Darwin) @echo "Building Foundation Models server for macOS 26+..." cd foundation-models-server && swift build -c release @echo "Copying foundation-models-server binary..." @@ -198,7 +205,7 @@ endif # Build Foundation Models server only if not already present (for dev) build-foundation-models-server-if-exists: -ifeq ($(shell uname -s),Darwin) +ifeq ($(DETECTED_OS),Darwin) @if [ -f "src-tauri/resources/bin/foundation-models-server" ]; then \ echo "Foundation Models server already exists at src-tauri/resources/bin/foundation-models-server, skipping build..."; \ else \ @@ -210,7 +217,7 @@ endif # Build jan CLI (release, platform-aware) → src-tauri/resources/bin/jan[.exe] build-cli: -ifeq ($(shell uname -s),Darwin) +ifeq ($(DETECTED_OS),Darwin) cd src-tauri && cargo build --release --features cli --bin jan-cli --target aarch64-apple-darwin cd src-tauri && cargo build --release --features cli --bin jan-cli --target x86_64-apple-darwin lipo -create \ @@ -231,7 +238,7 @@ ifeq ($(shell uname -s),Darwin) fi cp src-tauri/resources/bin/jan-cli src-tauri/target/universal-apple-darwin/release/jan-cli -else ifeq ($(OS),Windows_NT) +else ifeq ($(DETECTED_OS),Windows) cd src-tauri && cargo build --release --features cli --bin jan-cli cp src-tauri/target/release/jan-cli.exe src-tauri/resources/bin/jan-cli.exe else @@ -250,7 +257,7 @@ build: install-and-build install-rust-targets yarn build clean: -ifeq ($(OS),Windows_NT) +ifeq ($(DETECTED_OS),Windows) -powershell -Command "Get-ChildItem -Path . -Include node_modules, .next, dist, build, out, .turbo, .yarn -Recurse -Directory | Remove-Item -Recurse -Force" -powershell -Command "Get-ChildItem -Path . -Include package-lock.json, tsconfig.tsbuildinfo -Recurse -File | Remove-Item -Recurse -Force" -powershell -Command "Remove-Item -Recurse -Force ./pre-install/*.tgz" @@ -259,7 +266,7 @@ ifeq ($(OS),Windows_NT) -powershell -Command "Remove-Item -Recurse -Force ./src-tauri/resources" -powershell -Command "Remove-Item -Recurse -Force ./src-tauri/target" -powershell -Command "if (Test-Path \"$($env:USERPROFILE)\jan\extensions\") { Remove-Item -Path \"$($env:USERPROFILE)\jan\extensions\" -Recurse -Force }" -else ifeq ($(shell uname -s),Linux) +else ifeq ($(DETECTED_OS),Linux) find . -name "node_modules" -type d -prune -exec rm -rf '{}' + find . -name ".next" -type d -exec rm -rf '{}' + find . -name "dist" -type d -exec rm -rf '{}' + From 0cf831a044fcd38b819ee503f0395553f5347423 Mon Sep 17 00:00:00 2001 From: Evgenii Utkin Date: Wed, 18 Mar 2026 16:23:10 +0100 Subject: [PATCH 02/10] fix: improve Windows support in Makefile by standardizing OS detection --- Makefile | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 10ed25fd2..f9365e561 100644 --- a/Makefile +++ b/Makefile @@ -6,15 +6,22 @@ REPORT_PORTAL_PROJECT_NAME ?= "" REPORT_PORTAL_LAUNCH_NAME ?= "Jan App" REPORT_PORTAL_DESCRIPTION ?= "Jan App report" +# Detect OS +ifeq ($(OS),Windows_NT) + DETECTED_OS := Windows +else + DETECTED_OS := $(shell uname -s) +endif + # Default target, does nothing all: @echo "Specify a target to run" # Installs yarn dependencies and builds core and extensions install-and-build: -ifeq ($(OS),Windows_NT) +ifeq ($(DETECTED_OS),Windows) echo "skip" -else ifeq ($(shell uname -s),Linux) +else ifeq ($(DETECTED_OS),Linux) chmod +x src-tauri/build-utils/* endif yarn install @@ -24,7 +31,7 @@ endif # Install required Rust targets for macOS universal builds install-rust-targets: -ifeq ($(shell uname -s),Darwin) +ifeq ($(DETECTED_OS),Darwin) @echo "Detected macOS, installing universal build targets..." rustup target add x86_64-apple-darwin rustup target add aarch64-apple-darwin @@ -89,7 +96,7 @@ dev-android: install-and-build install-android-rust-targets dev-ios: install-and-build install-ios-rust-targets @echo "Setting up iOS development environment..." -ifeq ($(shell uname -s),Darwin) +ifeq ($(DETECTED_OS),Darwin) @if [ ! -d "src-tauri/gen/ios" ]; then \ echo "iOS app not initialized. Initializing..."; \ yarn tauri ios init; \ @@ -111,7 +118,7 @@ lint: install-and-build # Testing test: lint install-rust-targets yarn download:bin -ifeq ($(OS),Windows_NT) +ifeq ($(DETECTED_OS),Windows) endif yarn test yarn copy:assets:tauri @@ -126,7 +133,7 @@ endif # Build MLX server (macOS Apple Silicon only) - always builds build-mlx-server: -ifeq ($(shell uname -s),Darwin) +ifeq ($(DETECTED_OS),Darwin) @echo "Building MLX server for Apple Silicon..." cd mlx-server && swift build -c release @echo "Copying build products..." @@ -164,7 +171,7 @@ endif # Build MLX server only if not already present (for dev) build-mlx-server-if-exists: -ifeq ($(shell uname -s),Darwin) +ifeq ($(DETECTED_OS),Darwin) @if [ -f "src-tauri/resources/bin/mlx-server" ]; then \ echo "MLX server already exists at src-tauri/resources/bin/mlx-server, skipping build..."; \ else \ @@ -176,7 +183,7 @@ endif # Build Apple Foundation Models server (macOS 26+ only) - always builds build-foundation-models-server: -ifeq ($(shell uname -s),Darwin) +ifeq ($(DETECTED_OS),Darwin) @echo "Building Foundation Models server for macOS 26+..." cd foundation-models-server && swift build -c release @echo "Copying foundation-models-server binary..." @@ -198,7 +205,7 @@ endif # Build Foundation Models server only if not already present (for dev) build-foundation-models-server-if-exists: -ifeq ($(shell uname -s),Darwin) +ifeq ($(DETECTED_OS),Darwin) @if [ -f "src-tauri/resources/bin/foundation-models-server" ]; then \ echo "Foundation Models server already exists at src-tauri/resources/bin/foundation-models-server, skipping build..."; \ else \ @@ -210,7 +217,7 @@ endif # Build jan CLI (release, platform-aware) → src-tauri/resources/bin/jan[.exe] build-cli: -ifeq ($(shell uname -s),Darwin) +ifeq ($(DETECTED_OS),Darwin) cd src-tauri && cargo build --release --features cli --bin jan-cli --target aarch64-apple-darwin cd src-tauri && cargo build --release --features cli --bin jan-cli --target x86_64-apple-darwin lipo -create \ @@ -231,7 +238,7 @@ ifeq ($(shell uname -s),Darwin) fi cp src-tauri/resources/bin/jan-cli src-tauri/target/universal-apple-darwin/release/jan-cli -else ifeq ($(OS),Windows_NT) +else ifeq ($(DETECTED_OS),Windows) cd src-tauri && cargo build --release --features cli --bin jan-cli cp src-tauri/target/release/jan-cli.exe src-tauri/resources/bin/jan-cli.exe else @@ -241,7 +248,11 @@ endif # Debug build for local dev (faster, native arch only) build-cli-dev: +ifeq ($(DETECTED_OS),Windows) + mkdir src-tauri/resources/bin +else mkdir -p src-tauri/resources/bin +endif cd src-tauri && cargo build --features cli --bin jan-cli install -m755 src-tauri/target/debug/jan-cli src-tauri/resources/bin/jan-cli @@ -250,7 +261,7 @@ build: install-and-build install-rust-targets yarn build clean: -ifeq ($(OS),Windows_NT) +ifeq ($(DETECTED_OS),Windows) -powershell -Command "Get-ChildItem -Path . -Include node_modules, .next, dist, build, out, .turbo, .yarn -Recurse -Directory | Remove-Item -Recurse -Force" -powershell -Command "Get-ChildItem -Path . -Include package-lock.json, tsconfig.tsbuildinfo -Recurse -File | Remove-Item -Recurse -Force" -powershell -Command "Remove-Item -Recurse -Force ./pre-install/*.tgz" @@ -259,7 +270,7 @@ ifeq ($(OS),Windows_NT) -powershell -Command "Remove-Item -Recurse -Force ./src-tauri/resources" -powershell -Command "Remove-Item -Recurse -Force ./src-tauri/target" -powershell -Command "if (Test-Path \"$($env:USERPROFILE)\jan\extensions\") { Remove-Item -Path \"$($env:USERPROFILE)\jan\extensions\" -Recurse -Force }" -else ifeq ($(shell uname -s),Linux) +else ifeq ($(DETECTED_OS),Linux) find . -name "node_modules" -type d -prune -exec rm -rf '{}' + find . -name ".next" -type d -exec rm -rf '{}' + find . -name "dist" -type d -exec rm -rf '{}' + From ab12f7463f4c88b023e11fa88638f498142db3dc Mon Sep 17 00:00:00 2001 From: Vanalite Date: Thu, 19 Mar 2026 14:03:38 +0700 Subject: [PATCH 03/10] chore: disable foundation model for RC --- .gitignore | 5 + Makefile | 4 +- extensions/yarn.lock | 34 ++ .../swift-server/Package.swift | 30 ++ .../swift-server/README.md | 40 +++ .../FoundationModelsServerCommand.swift | 78 +++++ .../FoundationModelsServer/Logger.swift | 6 + .../FoundationModelsServer/Server.swift | 299 ++++++++++++++++++ .../FoundationModelsServer/Types.swift | 98 ++++++ web-app/src/services/providers/tauri.ts | 3 + 10 files changed, 595 insertions(+), 2 deletions(-) create mode 100644 src-tauri/plugins/tauri-plugin-foundation-models/swift-server/Package.swift create mode 100644 src-tauri/plugins/tauri-plugin-foundation-models/swift-server/README.md create mode 100644 src-tauri/plugins/tauri-plugin-foundation-models/swift-server/Sources/FoundationModelsServer/FoundationModelsServerCommand.swift create mode 100644 src-tauri/plugins/tauri-plugin-foundation-models/swift-server/Sources/FoundationModelsServer/Logger.swift create mode 100644 src-tauri/plugins/tauri-plugin-foundation-models/swift-server/Sources/FoundationModelsServer/Server.swift create mode 100644 src-tauri/plugins/tauri-plugin-foundation-models/swift-server/Sources/FoundationModelsServer/Types.swift diff --git a/.gitignore b/.gitignore index ca807a51f..cd948c22c 100644 --- a/.gitignore +++ b/.gitignore @@ -54,6 +54,11 @@ docs/.next/ **/yarn-error.log* **/pnpm-debug.log* +## Swift Package Manager (Foundation Models server) +src-tauri/plugins/tauri-plugin-foundation-models/swift-server/.build/ +src-tauri/plugins/tauri-plugin-foundation-models/swift-server/.swiftpm/ +src-tauri/plugins/tauri-plugin-foundation-models/swift-server/Package.resolved + ## cargo target Cargo.lock diff --git a/Makefile b/Makefile index 10ed25fd2..5605f4b03 100644 --- a/Makefile +++ b/Makefile @@ -178,9 +178,9 @@ endif build-foundation-models-server: ifeq ($(shell uname -s),Darwin) @echo "Building Foundation Models server for macOS 26+..." - cd foundation-models-server && swift build -c release + cd src-tauri/plugins/tauri-plugin-foundation-models/swift-server && swift build -c release @echo "Copying foundation-models-server binary..." - @cp foundation-models-server/.build/release/foundation-models-server src-tauri/resources/bin/foundation-models-server + @cp src-tauri/plugins/tauri-plugin-foundation-models/swift-server/.build/release/foundation-models-server src-tauri/resources/bin/foundation-models-server @chmod +x src-tauri/resources/bin/foundation-models-server @echo "Foundation Models server built and copied successfully" @echo "Checking for code signing identity..." diff --git a/extensions/yarn.lock b/extensions/yarn.lock index a05a6e928..90a482738 100644 --- a/extensions/yarn.lock +++ b/extensions/yarn.lock @@ -376,6 +376,18 @@ __metadata: languageName: node linkType: hard +"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Ffoundation-models-extension%40workspace%3Afoundation-models-extension": + version: 0.1.10 + resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=6c8a42&locator=%40janhq%2Ffoundation-models-extension%40workspace%3Afoundation-models-extension" + dependencies: + rxjs: "npm:^7.8.1" + ulidx: "npm:^2.3.0" + peerDependencies: + react: 19.0.0 + checksum: 10c0/c80736877d9b0d9498d76588a1e92664f467a84265aee7cb6be497b89ed4b1e1c9379c12686c4676bb438eb268ba97caae607b01686e3dc0d485933a4ab69af7 + languageName: node + linkType: hard + "@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Fllamacpp-extension%40workspace%3Allamacpp-extension": version: 0.1.10 resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=6c8a42&locator=%40janhq%2Fllamacpp-extension%40workspace%3Allamacpp-extension" @@ -439,6 +451,22 @@ __metadata: languageName: unknown linkType: soft +"@janhq/foundation-models-extension@workspace:foundation-models-extension": + version: 0.0.0-use.local + resolution: "@janhq/foundation-models-extension@workspace:foundation-models-extension" + dependencies: + "@janhq/core": ../../core/package.tgz + "@janhq/tauri-plugin-foundation-models-api": "link:../../src-tauri/plugins/tauri-plugin-foundation-models" + "@tauri-apps/api": "npm:2.8.0" + "@tauri-apps/plugin-http": "npm:2.5.0" + "@tauri-apps/plugin-log": "npm:^2.6.0" + cpx: "npm:1.5.0" + rimraf: "npm:3.0.2" + rolldown: "npm:1.0.0-beta.1" + typescript: "npm:5.9.2" + languageName: unknown + linkType: soft + "@janhq/llamacpp-extension@workspace:llamacpp-extension": version: 0.0.0-use.local resolution: "@janhq/llamacpp-extension@workspace:llamacpp-extension" @@ -493,6 +521,12 @@ __metadata: languageName: unknown linkType: soft +"@janhq/tauri-plugin-foundation-models-api@link:../../src-tauri/plugins/tauri-plugin-foundation-models::locator=%40janhq%2Ffoundation-models-extension%40workspace%3Afoundation-models-extension": + version: 0.0.0-use.local + resolution: "@janhq/tauri-plugin-foundation-models-api@link:../../src-tauri/plugins/tauri-plugin-foundation-models::locator=%40janhq%2Ffoundation-models-extension%40workspace%3Afoundation-models-extension" + languageName: node + linkType: soft + "@janhq/tauri-plugin-hardware-api@link:../../src-tauri/plugins/tauri-plugin-hardware::locator=%40janhq%2Fllamacpp-extension%40workspace%3Allamacpp-extension": version: 0.0.0-use.local resolution: "@janhq/tauri-plugin-hardware-api@link:../../src-tauri/plugins/tauri-plugin-hardware::locator=%40janhq%2Fllamacpp-extension%40workspace%3Allamacpp-extension" diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/swift-server/Package.swift b/src-tauri/plugins/tauri-plugin-foundation-models/swift-server/Package.swift new file mode 100644 index 000000000..d899a6925 --- /dev/null +++ b/src-tauri/plugins/tauri-plugin-foundation-models/swift-server/Package.swift @@ -0,0 +1,30 @@ +// swift-tools-version: 6.2 + +import PackageDescription + +let package = Package( + name: "foundation-models-server", + platforms: [ + .macOS(.v26) + ], + products: [ + .executable(name: "foundation-models-server", targets: ["FoundationModelsServer"]) + ], + dependencies: [ + .package(url: "https://github.com/apple/swift-argument-parser", from: "1.7.0"), + .package(url: "https://github.com/hummingbird-project/hummingbird", from: "2.19.0"), + ], + targets: [ + .executableTarget( + name: "FoundationModelsServer", + dependencies: [ + .product(name: "ArgumentParser", package: "swift-argument-parser"), + .product(name: "Hummingbird", package: "hummingbird"), + ], + path: "Sources/FoundationModelsServer", + swiftSettings: [ + .swiftLanguageMode(.v6) + ] + ) + ] +) diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/swift-server/README.md b/src-tauri/plugins/tauri-plugin-foundation-models/swift-server/README.md new file mode 100644 index 000000000..3d840b72c --- /dev/null +++ b/src-tauri/plugins/tauri-plugin-foundation-models/swift-server/README.md @@ -0,0 +1,40 @@ +# foundation-models-server + +A lightweight OpenAI-compatible HTTP server that wraps Apple's Foundation Models framework, enabling Jan to use on-device Apple Intelligence models on macOS 26+. + +## Requirements + +- macOS 26 (Tahoe) or later +- Apple Silicon Mac with Apple Intelligence enabled +- Xcode 26 or later + +## Building + +```bash +swift build -c release +``` + +The binary will be at `.build/release/foundation-models-server`. + +## Usage + +```bash +# Check availability +foundation-models-server --check + +# Start server on default port +foundation-models-server --port 8080 + +# Start server with API key +foundation-models-server --port 8080 --api-key +``` + +## API + +The server exposes an OpenAI-compatible API: + +- `GET /health` — health check +- `GET /v1/models` — lists the `apple/on-device` model +- `POST /v1/chat/completions` — chat completions (streaming and non-streaming) + +The model ID is always `apple/on-device`. diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/swift-server/Sources/FoundationModelsServer/FoundationModelsServerCommand.swift b/src-tauri/plugins/tauri-plugin-foundation-models/swift-server/Sources/FoundationModelsServer/FoundationModelsServerCommand.swift new file mode 100644 index 000000000..36d2d5ec8 --- /dev/null +++ b/src-tauri/plugins/tauri-plugin-foundation-models/swift-server/Sources/FoundationModelsServer/FoundationModelsServerCommand.swift @@ -0,0 +1,78 @@ +import ArgumentParser +import Foundation +import Hummingbird +import FoundationModels + +@main +struct FoundationModelsServerCommand: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "foundation-models-server", + abstract: "Apple Foundation Models inference server with OpenAI-compatible API" + ) + + @Option(name: .long, help: "Port to listen on") + var port: Int = 8080 + + @Option(name: .long, help: "API key for authentication (optional)") + var apiKey: String = "" + + @Flag(name: .long, help: "Check availability and exit with status 0 if available") + var check: Bool = false + + func run() async throws { + let availability = SystemLanguageModel.default.availability + + // In --check mode, always print a machine-readable status token and exit 0. + // Callers (e.g. the Tauri plugin) parse this string to decide visibility. + if check { + switch availability { + case .available: + print("available") + case .unavailable(.deviceNotEligible): + print("notEligible") + case .unavailable(.appleIntelligenceNotEnabled): + print("appleIntelligenceNotEnabled") + case .unavailable(.modelNotReady): + print("modelNotReady") + default: + print("unavailable") + } + return + } + + guard case .available = availability else { + let reason: String + switch availability { + case .unavailable(.deviceNotEligible): + reason = "Device is not eligible for Apple Intelligence" + case .unavailable(.appleIntelligenceNotEnabled): + reason = "Apple Intelligence is not enabled in System Settings" + case .unavailable(.modelNotReady): + reason = "Foundation model is downloading or not yet ready" + default: + reason = "Foundation model is unavailable on this system" + } + fputs("[foundation-models] ERROR: \(reason)\n", stderr) + throw ExitCode(1) + } + + log("[foundation-models] Foundation Models Server starting...") + log("[foundation-models] Port: \(port)") + + let server = FoundationModelsHTTPServer( + modelId: "apple/on-device", + apiKey: apiKey + ) + + let router = server.buildRouter() + let app = Application( + router: router, + configuration: .init(address: .hostname("127.0.0.1", port: port)) + ) + + log("[foundation-models] http server listening on http://127.0.0.1:\(port)") + log("[foundation-models] server is listening on 127.0.0.1:\(port)") + + try await app.run() + } +} diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/swift-server/Sources/FoundationModelsServer/Logger.swift b/src-tauri/plugins/tauri-plugin-foundation-models/swift-server/Sources/FoundationModelsServer/Logger.swift new file mode 100644 index 000000000..34bb17517 --- /dev/null +++ b/src-tauri/plugins/tauri-plugin-foundation-models/swift-server/Sources/FoundationModelsServer/Logger.swift @@ -0,0 +1,6 @@ +import Foundation + +func log(_ message: String) { + print(message) + fflush(stdout) +} diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/swift-server/Sources/FoundationModelsServer/Server.swift b/src-tauri/plugins/tauri-plugin-foundation-models/swift-server/Sources/FoundationModelsServer/Server.swift new file mode 100644 index 000000000..b14eb3574 --- /dev/null +++ b/src-tauri/plugins/tauri-plugin-foundation-models/swift-server/Sources/FoundationModelsServer/Server.swift @@ -0,0 +1,299 @@ +import Foundation +import Hummingbird +import FoundationModels + +/// HTTP server exposing an OpenAI-compatible API backed by Apple Foundation Models +struct FoundationModelsHTTPServer: Sendable { + let modelId: String + let apiKey: String + + func buildRouter() -> Router { + let router = Router() + + // Health check + router.get("/health") { _, _ in + let response = HealthResponse(status: "ok") + return try encodeJSONResponse(response) + } + + // List available models + router.get("/v1/models") { _, _ in + let response = ModelsListResponse( + object: "list", + data: [ + ModelData( + id: self.modelId, + object: "model", + created: currentTimestamp(), + owned_by: "apple" + ) + ] + ) + return try encodeJSONResponse(response) + } + + // Chat completions (OpenAI-compatible) + router.post("/v1/chat/completions") { request, _ in + // Validate API key when configured + if !self.apiKey.isEmpty { + let authHeader = request.headers[.authorization] + guard authHeader == "Bearer \(self.apiKey)" else { + let errorResp = ErrorResponse( + error: ErrorDetail( + message: "Unauthorized: invalid or missing API key", + type: "authentication_error", + code: "unauthorized" + ) + ) + return try Response( + status: .unauthorized, + headers: [.contentType: "application/json"], + body: .init(byteBuffer: encodeJSONBuffer(errorResp)) + ) + } + } + + let body = try await request.body.collect(upTo: 10 * 1024 * 1024) + let chatRequest: ChatCompletionRequest + do { + chatRequest = try JSONDecoder().decode(ChatCompletionRequest.self, from: body) + } catch { + let errorResp = ErrorResponse( + error: ErrorDetail( + message: "Invalid request body: \(error.localizedDescription)", + type: "invalid_request_error", + code: nil + ) + ) + return try Response( + status: .badRequest, + headers: [.contentType: "application/json"], + body: .init(byteBuffer: encodeJSONBuffer(errorResp)) + ) + } + let isStreaming = chatRequest.stream ?? false + + log("[foundation-models] Request: messages=\(chatRequest.messages.count), stream=\(isStreaming)") + + if isStreaming { + return try await self.handleStreamingRequest(chatRequest) + } else { + return try await self.handleNonStreamingRequest(chatRequest) + } + } + + return router + } + + // MARK: - Non-streaming + + private func handleNonStreamingRequest(_ chatRequest: ChatCompletionRequest) async throws -> Response { + let session = buildSession(from: chatRequest.messages) + let lastUserMessage = extractLastUserMessage(from: chatRequest.messages) + + let response = try await session.respond(to: lastUserMessage) + let content = response.content + + let completionResponse = ChatCompletionResponse( + id: "chatcmpl-\(UUID().uuidString)", + object: "chat.completion", + created: currentTimestamp(), + model: modelId, + choices: [ + ChatCompletionChoice( + index: 0, + message: ChatResponseMessage(role: "assistant", content: content), + finish_reason: "stop" + ) + ], + usage: UsageInfo( + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0 + ) + ) + + return try encodeJSONResponse(completionResponse) + } + + // MARK: - Streaming + + private func handleStreamingRequest(_ chatRequest: ChatCompletionRequest) async throws -> Response { + let requestId = "chatcmpl-\(UUID().uuidString)" + let created = currentTimestamp() + let modelId = self.modelId + let messages = chatRequest.messages + + let (stream, continuation) = AsyncStream.makeStream() + + let task = Task { [self] in + do { + let session = self.buildSession(from: messages) + let lastUserMessage = self.extractLastUserMessage(from: messages) + + let roleDelta = ChatCompletionChunk( + id: requestId, + object: "chat.completion.chunk", + created: created, + model: modelId, + choices: [ + ChunkChoice( + index: 0, + delta: DeltaContent(role: "assistant", content: nil), + finish_reason: nil + ) + ] + ) + if let buffer = encodeSSEBuffer(roleDelta) { + continuation.yield(buffer) + } + + var previousText = "" + for try await snapshot in session.streamResponse(to: lastUserMessage) { + let currentText = snapshot.content + let delta = String(currentText.dropFirst(previousText.count)) + previousText = currentText + + if delta.isEmpty { continue } + + let chunk = ChatCompletionChunk( + id: requestId, + object: "chat.completion.chunk", + created: created, + model: modelId, + choices: [ + ChunkChoice( + index: 0, + delta: DeltaContent(role: nil, content: delta), + finish_reason: nil + ) + ] + ) + if let buffer = encodeSSEBuffer(chunk) { + continuation.yield(buffer) + } + } + + // Send stop chunk + let stopChunk = ChatCompletionChunk( + id: requestId, + object: "chat.completion.chunk", + created: created, + model: modelId, + choices: [ + ChunkChoice( + index: 0, + delta: DeltaContent(role: nil, content: nil), + finish_reason: "stop" + ) + ] + ) + if let buffer = encodeSSEBuffer(stopChunk) { + continuation.yield(buffer) + } + + // SSE terminator + var doneBuffer = ByteBufferAllocator().buffer(capacity: 16) + doneBuffer.writeString("data: [DONE]\n\n") + continuation.yield(doneBuffer) + } catch { + log("[foundation-models] Streaming error: \(error.localizedDescription)") + var errBuffer = ByteBufferAllocator().buffer(capacity: 256) + errBuffer.writeString("error: {\"message\":\"\(error.localizedDescription)\"}\n\n") + continuation.yield(errBuffer) + } + continuation.finish() + } + + // Cancel the generation task when the client disconnects + continuation.onTermination = { @Sendable _ in + log("[foundation-models] SSE continuation terminated by client disconnect") + task.cancel() + } + + return Response( + status: .ok, + headers: [ + .contentType: "text/event-stream", + .cacheControl: "no-cache", + .init("X-Accel-Buffering")!: "no" + ], + body: .init(asyncSequence: stream) + ) + } + + // MARK: - Session Construction + + /// Build a `LanguageModelSession` from the OpenAI message list. + /// + /// System messages become the session instructions. + /// Prior user/assistant turns are serialised into the instructions block so + /// the model has full conversation context without re-running inference. + /// (The Foundation Models `Transcript` API is not used for history injection + /// because it is designed for observing an already-live session's state, not + /// for priming a fresh one with arbitrary history.) + private func buildSession(from messages: [ChatMessage]) -> LanguageModelSession { + let systemContent = messages.first(where: { $0.role == "system" })?.content ?? "" + let nonSystem = messages.filter { $0.role != "system" } + let history = nonSystem.dropLast() // all turns except the last user message + + var instructionsText: String + if systemContent.isEmpty { + instructionsText = "You are a helpful assistant." + } else { + instructionsText = systemContent + } + + // Append prior turns so the model understands conversation context + if !history.isEmpty { + instructionsText += "\n\n[Previous conversation]\n" + for msg in history { + let label = msg.role == "assistant" ? "Assistant" : "User" + instructionsText += "\(label): \(msg.content ?? "")\n" + } + instructionsText += "[End of previous conversation]" + } + + return LanguageModelSession(instructions: instructionsText) + } + + private func extractLastUserMessage(from messages: [ChatMessage]) -> String { + let nonSystem = messages.filter { $0.role != "system" } + return nonSystem.last?.content ?? "" + } +} + +// MARK: - Helpers + +private func currentTimestamp() -> Int { + Int(Date().timeIntervalSince1970) +} + +private func encodeJSONResponse(_ value: T) throws -> Response { + let data = try JSONEncoder().encode(value) + var buffer = ByteBufferAllocator().buffer(capacity: data.count) + buffer.writeBytes(data) + return Response( + status: .ok, + headers: [.contentType: "application/json"], + body: .init(byteBuffer: buffer) + ) +} + +private func encodeJSONBuffer(_ value: T) -> ByteBuffer { + let data = (try? JSONEncoder().encode(value)) ?? Data() + var buffer = ByteBufferAllocator().buffer(capacity: data.count) + buffer.writeBytes(data) + return buffer +} + +private func encodeSSEBuffer(_ value: T) -> ByteBuffer? { + guard let json = try? JSONEncoder().encode(value), + let jsonString = String(data: json, encoding: .utf8) else { + return nil + } + let line = "data: \(jsonString)\n\n" + var buffer = ByteBufferAllocator().buffer(capacity: line.utf8.count) + buffer.writeString(line) + return buffer +} diff --git a/src-tauri/plugins/tauri-plugin-foundation-models/swift-server/Sources/FoundationModelsServer/Types.swift b/src-tauri/plugins/tauri-plugin-foundation-models/swift-server/Sources/FoundationModelsServer/Types.swift new file mode 100644 index 000000000..86dc656f7 --- /dev/null +++ b/src-tauri/plugins/tauri-plugin-foundation-models/swift-server/Sources/FoundationModelsServer/Types.swift @@ -0,0 +1,98 @@ +import Foundation + +// MARK: - OpenAI Request Types + +struct ChatCompletionRequest: Codable, Sendable { + let model: String + let messages: [ChatMessage] + var temperature: Double? + var top_p: Double? + var max_tokens: Int? + var n_predict: Int? + var stream: Bool? + var stop: [String]? +} + +struct ChatMessage: Codable, Sendable { + let role: String + let content: String? +} + +// MARK: - OpenAI Response Types + +struct ChatCompletionResponse: Codable, Sendable { + let id: String + let object: String + let created: Int + let model: String + let choices: [ChatCompletionChoice] + let usage: UsageInfo +} + +struct ChatCompletionChoice: Codable, Sendable { + let index: Int + let message: ChatResponseMessage + let finish_reason: String +} + +struct ChatResponseMessage: Codable, Sendable { + let role: String + let content: String +} + +struct UsageInfo: Codable, Sendable { + let prompt_tokens: Int + let completion_tokens: Int + let total_tokens: Int +} + +// MARK: - Streaming Types + +struct ChatCompletionChunk: Codable, Sendable { + let id: String + let object: String + let created: Int + let model: String + let choices: [ChunkChoice] +} + +struct ChunkChoice: Codable, Sendable { + let index: Int + let delta: DeltaContent + let finish_reason: String? +} + +struct DeltaContent: Codable, Sendable { + let role: String? + let content: String? +} + +// MARK: - Model List Types + +struct ModelsListResponse: Codable, Sendable { + let object: String + let data: [ModelData] +} + +struct ModelData: Codable, Sendable { + let id: String + let object: String + let created: Int + let owned_by: String +} + +// MARK: - Health / Error Types + +struct HealthResponse: Codable, Sendable { + let status: String +} + +struct ErrorDetail: Codable, Sendable { + let message: String + let type: String + let code: String? +} + +struct ErrorResponse: Codable, Sendable { + let error: ErrorDetail +} diff --git a/web-app/src/services/providers/tauri.ts b/web-app/src/services/providers/tauri.ts index acd5e7844..5ecc1560f 100644 --- a/web-app/src/services/providers/tauri.ts +++ b/web-app/src/services/providers/tauri.ts @@ -45,8 +45,11 @@ export class TauriProvidersService extends DefaultProvidersService { } }).filter(Boolean) + // TODO: Re-enable foundation-models once migrated to apple-foundation-models crate + const hiddenProviders = new Set(['foundation-models']) const runtimeProviders: ModelProvider[] = [] for (const [providerName, value] of EngineManager.instance().engines) { + if (hiddenProviders.has(providerName)) continue const models = await value.list() ?? [] const provider: ModelProvider = { active: false, From 00978535ed30f0de565481977a4d0ec3fd172290 Mon Sep 17 00:00:00 2001 From: Vanalite Date: Fri, 20 Mar 2026 10:19:19 +0700 Subject: [PATCH 04/10] feat: remotely fetch Jan latest model during the onboarding --- web-app/src/constants/models.ts | 1 - web-app/src/containers/PromptJanModel.tsx | 24 ++++++++--------------- web-app/src/containers/SetupScreen.tsx | 22 +++++++-------------- web-app/src/services/models/default.ts | 19 ++++++++++++++++++ web-app/src/services/models/types.ts | 2 ++ web-app/src/types/global.d.ts | 1 + web-app/vite.config.ts | 3 +++ 7 files changed, 40 insertions(+), 32 deletions(-) diff --git a/web-app/src/constants/models.ts b/web-app/src/constants/models.ts index b43cdd9da..e087d1329 100644 --- a/web-app/src/constants/models.ts +++ b/web-app/src/constants/models.ts @@ -2,7 +2,6 @@ * Model-related constants */ -export const NEW_JAN_MODEL_HF_REPO = 'janhq/Jan-v3-4B-base-instruct-GGUF' export const JAN_CODE_HF_REPO = 'janhq/Jan-Code-4b-Gguf' export const DEFAULT_MODEL_QUANTIZATIONS = ['iq4_xs', 'q4_k_m'] diff --git a/web-app/src/containers/PromptJanModel.tsx b/web-app/src/containers/PromptJanModel.tsx index 8fb32d843..653091b10 100644 --- a/web-app/src/containers/PromptJanModel.tsx +++ b/web-app/src/containers/PromptJanModel.tsx @@ -5,10 +5,7 @@ import { useDownloadStore } from '@/hooks/useDownloadStore' import { useGeneralSetting } from '@/hooks/useGeneralSetting' import { useEffect, useState, useMemo, useCallback, useRef } from 'react' import type { CatalogModel } from '@/services/models/types' -import { - NEW_JAN_MODEL_HF_REPO, - SETUP_SCREEN_QUANTIZATIONS, -} from '@/constants/models' +import { SETUP_SCREEN_QUANTIZATIONS } from '@/constants/models' export function PromptJanModel() { @@ -27,22 +24,17 @@ export function PromptJanModel() { fetchAttempted.current = true try { - const repo = await serviceHub - .models() - .fetchHuggingFaceRepo(NEW_JAN_MODEL_HF_REPO, huggingfaceToken) + const model = await serviceHub.models().fetchLatestJanModel() - if (repo) { - const catalogModel = serviceHub - .models() - .convertHfRepoToCatalogModel(repo) - setJanNewModel(catalogModel) + if (model) { + setJanNewModel(model) } } catch (error) { - console.error('Error fetching Jan Model:', error) + console.error('Error fetching latest Jan model:', error) } finally { setIsLoading(false) } - }, [serviceHub, huggingfaceToken]) + }, [serviceHub]) useEffect(() => { fetchJanModel() @@ -98,7 +90,7 @@ export function PromptJanModel() {
Jan

- Jan v3 Model + {janNewModel?.display_name ?? janNewModel?.model_name ?? 'Jan Model'} {defaultVariant && ( {' '} @@ -108,7 +100,7 @@ export function PromptJanModel() {

- Get started with Jan v3, our recommended local AI model optimized for your device. + Get started with {janNewModel?.display_name ?? 'Jan'}, our recommended local AI model optimized for your device.