feat: add Rust RKLLM Server (Phase 5.1)

Implement a Rust replacement for the Python RKLLM Server, providing an OpenAI-compatible API for Qwen3-0.6B on the RK3588 NPU via librkllmrt.so. Structure: - rkllm-sys crate: FFI bindings with dynamic loading via libloading - backend.rs: async generate/stream with tokio mpsc callback bridge - chat.rs: ChatML formatting + /v1/chat/completions (streaming SSE) - config.rs: env-var configuration with model name auto-derivation - main.rs: axum server with clap CLI and graceful shutdown Key design decisions: - libloading for runtime .so loading (no compile-time dependency) - spawn_blocking for the blocking rkllm_run FFI call - usize-based pointer passing to satisfy Send bounds across threads - Box<Sender> as C callback userdata with proper lifecycle management 20 tests passing (ChatML formatting, config parsing, serialization). Release binary: 2.8MB stripped. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-08 05:58:32 +00:00
parent 0a698aca17
commit 81960f0be7
7 changed files with 1523 additions and 0 deletions
@@ -0,0 +1,37 @@
+[workspace]
+members = [
+    "crates/rkllm-sys",
+    ".",
+]
+resolver = "2"
+
+[workspace.package]
+version = "0.1.0"
+edition = "2021"
+license = "MIT"
+
+[package]
+name = "rkllm-server"
+version.workspace = true
+edition.workspace = true
+
+[dependencies]
+rkllm-sys = { path = "crates/rkllm-sys" }
+
+axum = "0.8"
+tokio = { version = "1", features = ["full"] }
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+clap = { version = "4", features = ["derive"] }
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+tower-http = { version = "0.6", features = ["cors", "trace"] }
+uuid = { version = "1", features = ["v4"] }
+tokio-stream = "0.1"
+futures = "0.3"
+anyhow = "1"
+
+[profile.release]
+opt-level = 3
+lto = "thin"
+strip = true
@@ -0,0 +1,7 @@
+[package]
+name = "rkllm-sys"
+version.workspace = true
+edition.workspace = true
+
+[dependencies]
+libloading = "0.8"
@@ -0,0 +1,243 @@
+//! FFI bindings for librkllmrt.so (RKLLM runtime).
+//!
+//! All types match the C ABI of the RKLLM SDK.
+//! The library is loaded dynamically at runtime via `libloading` — the .so
+//! need not be present at compile time.
+
+use std::ffi::{c_char, c_float, c_int, c_void};
+
+// ── Constants ────────────────────────────────────────────────────
+
+pub const RKLLM_RUN_NORMAL: c_int = 0;
+pub const RKLLM_RUN_WAITING: c_int = 1;
+pub const RKLLM_RUN_FINISH: c_int = 2;
+pub const RKLLM_RUN_ERROR: c_int = 3;
+
+pub const RKLLM_INPUT_PROMPT: c_int = 0;
+pub const RKLLM_INFER_GENERATE: c_int = 0;
+
+// ── Repr(C) structures ──────────────────────────────────────────
+
+#[repr(C)]
+pub struct RKLLMExtendParam {
+    pub base_domain_id: c_int,
+    pub embed_flash: i8,
+    pub enabled_cpus_num: i8,
+    pub enabled_cpus_mask: u32,
+    pub n_batch: u8,
+    pub use_cross_attn: i8,
+    pub reserved: [u8; 104],
+}
+
+impl Default for RKLLMExtendParam {
+    fn default() -> Self {
+        Self {
+            base_domain_id: 1,  // RK3588 A76
+            embed_flash: 1,
+            enabled_cpus_num: 4,
+            enabled_cpus_mask: 0xF0, // CPU 4-7 (A76 big cores)
+            n_batch: 1,
+            use_cross_attn: 0,
+            reserved: [0u8; 104],
+        }
+    }
+}
+
+#[repr(C)]
+pub struct RKLLMParam {
+    pub model_path: *const c_char,
+    pub max_context_len: c_int,
+    pub max_new_tokens: c_int,
+    pub top_k: c_float,
+    pub n_keep: c_int,
+    pub top_p: c_float,
+    pub temperature: c_float,
+    pub repeat_penalty: c_float,
+    pub frequency_penalty: c_float,
+    pub presence_penalty: c_float,
+    pub mirostat: c_int,
+    pub mirostat_tau: c_float,
+    pub mirostat_eta: c_float,
+    pub skip_special_token: bool,
+    pub is_async: bool,
+    pub img_start: *const c_char,
+    pub img_end: *const c_char,
+    pub img_content: *const c_char,
+    pub extend_param: RKLLMExtendParam,
+    pub use_gpu: bool,
+}
+
+#[repr(C)]
+#[derive(Debug, Clone, Copy, Default)]
+pub struct RKLLMPerfStat {
+    pub prefill_time_ms: c_float,
+    pub prefill_tokens: c_int,
+    pub generate_time_ms: c_float,
+    pub generate_tokens: c_int,
+    pub memory_usage_mb: c_float,
+}
+
+#[repr(C)]
+pub struct RKLLMResultLastHiddenLayer {
+    pub hidden_states: *const c_float,
+    pub embd_size: c_int,
+    pub num_tokens: c_int,
+}
+
+#[repr(C)]
+pub struct RKLLMResultLogits {
+    pub logits: *const c_float,
+    pub vocab_size: c_int,
+    pub num_tokens: c_int,
+}
+
+#[repr(C)]
+pub struct RKLLMResult {
+    pub text: *const c_char,
+    pub token_id: c_int,
+    pub last_hidden_layer: RKLLMResultLastHiddenLayer,
+    pub logits: RKLLMResultLogits,
+    pub perf: RKLLMPerfStat,
+}
+
+/// Union for different input types — we only use `prompt_input`.
+#[repr(C)]
+pub union RKLLMInputData {
+    pub prompt_input: *const c_char,
+    // embed_input, token_input, multimodal_input omitted — not needed
+    _pad: [u8; 64], // ensure union is large enough for any variant
+}
+
+#[repr(C)]
+pub struct RKLLMInput {
+    pub role: *const c_char,
+    pub enable_thinking: bool,
+    pub input_type: c_int,
+    pub input_data: RKLLMInputData,
+}
+
+#[repr(C)]
+pub struct RKLLMInferParam {
+    pub mode: c_int,
+    pub lora_params: *const c_void,
+    pub prompt_cache_params: *const c_void,
+    pub keep_history: c_int,
+}
+
+/// Callback type: void cb(RKLLMResult* result, void* userdata, int state)
+pub type RKLLMCallback = extern "C" fn(
+    result: *mut RKLLMResult,
+    userdata: *mut c_void,
+    state: c_int,
+);
+
+// ── Dynamic library wrapper ─────────────────────────────────────
+
+/// Dynamically loaded RKLLM runtime library.
+///
+/// Provides type-safe wrappers around the three core functions:
+/// - `rkllm_init`: load model + register callback
+/// - `rkllm_run`: run inference (blocking)
+/// - `rkllm_destroy`: release model resources
+pub struct RKLLMLibrary {
+    _lib: libloading::Library,
+    init_fn: unsafe extern "C" fn(*mut *mut c_void, *const RKLLMParam, RKLLMCallback) -> c_int,
+    run_fn: unsafe extern "C" fn(*mut c_void, *const RKLLMInput, *const RKLLMInferParam, *mut c_void) -> c_int,
+    destroy_fn: unsafe extern "C" fn(*mut c_void) -> c_int,
+}
+
+impl RKLLMLibrary {
+    /// Load librkllmrt.so from `path`.
+    ///
+    /// # Safety
+    /// The .so must be a valid RKLLM runtime library with the expected ABI.
+    pub unsafe fn load(path: &str) -> Result<Self, libloading::Error> {
+        let lib = unsafe { libloading::Library::new(path)? };
+
+        let init_fn = unsafe {
+            *lib.get::<unsafe extern "C" fn(*mut *mut c_void, *const RKLLMParam, RKLLMCallback) -> c_int>(
+                b"rkllm_init\0",
+            )?
+        };
+        let run_fn = unsafe {
+            *lib.get::<unsafe extern "C" fn(*mut c_void, *const RKLLMInput, *const RKLLMInferParam, *mut c_void) -> c_int>(
+                b"rkllm_run\0",
+            )?
+        };
+        let destroy_fn = unsafe {
+            *lib.get::<unsafe extern "C" fn(*mut c_void) -> c_int>(
+                b"rkllm_destroy\0",
+            )?
+        };
+
+        Ok(Self { _lib: lib, init_fn, run_fn, destroy_fn })
+    }
+
+    /// Call rkllm_init — load model and register callback.
+    ///
+    /// # Safety
+    /// `param` must point to a valid RKLLMParam with valid model_path.
+    pub unsafe fn init(
+        &self,
+        handle: *mut *mut c_void,
+        param: *const RKLLMParam,
+        callback: RKLLMCallback,
+    ) -> c_int {
+        unsafe { (self.init_fn)(handle, param, callback) }
+    }
+
+    /// Call rkllm_run — blocking inference.
+    ///
+    /// # Safety
+    /// `handle` must be a valid handle from `init`. `input` and `infer_param`
+    /// must point to valid structures. `userdata` is passed to the callback.
+    pub unsafe fn run(
+        &self,
+        handle: *mut c_void,
+        input: *const RKLLMInput,
+        infer_param: *const RKLLMInferParam,
+        userdata: *mut c_void,
+    ) -> c_int {
+        unsafe { (self.run_fn)(handle, input, infer_param, userdata) }
+    }
+
+    /// Call rkllm_destroy — release model.
+    ///
+    /// # Safety
+    /// `handle` must be a valid handle from `init`.
+    pub unsafe fn destroy(&self, handle: *mut c_void) -> c_int {
+        unsafe { (self.destroy_fn)(handle) }
+    }
+
+    /// Get the raw `rkllm_run` function pointer as a usize.
+    ///
+    /// Used to pass the function pointer across thread boundaries (usize is Send).
+    pub fn run_fn_ptr(&self) -> usize {
+        self.run_fn as usize
+    }
+}
+
+// The library handle is thread-safe (we protect access via Mutex in backend)
+unsafe impl Send for RKLLMLibrary {}
+unsafe impl Sync for RKLLMLibrary {}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::mem;
+
+    #[test]
+    fn extend_param_size() {
+        // RKLLMExtendParam should be: 4 + 1 + 1 + 4 + 1 + 1 + 104 = 116 bytes
+        // but with alignment padding it may differ. Just verify it's non-zero.
+        assert!(mem::size_of::<RKLLMExtendParam>() >= 116);
+    }
+
+    #[test]
+    fn constants() {
+        assert_eq!(RKLLM_RUN_NORMAL, 0);
+        assert_eq!(RKLLM_RUN_FINISH, 2);
+        assert_eq!(RKLLM_INPUT_PROMPT, 0);
+        assert_eq!(RKLLM_INFER_GENERATE, 0);
+    }
+}
@@ -0,0 +1,356 @@
+//! RKLLM NPU inference backend.
+//!
+//! Wraps the RKLLM C library (loaded dynamically via libloading) and provides
+//! async generate/stream methods that bridge the C callback to tokio channels.
+//!
+//! Key design:
+//! - `rkllm_run` blocks the calling thread => use `spawn_blocking`
+//! - Only one inference at a time => `Mutex` around handle
+//! - C callback fires on RKLLM internal thread => `mpsc::Sender` bridge to tokio
+
+use std::ffi::{c_int, c_void, CString};
+use std::ptr;
+use std::sync::Mutex;
+
+use rkllm_sys::*;
+use tokio::sync::mpsc;
+use tracing::{debug, error, info, warn};
+
+use crate::config::Config;
+
+// ── Callback event ──────────────────────────────────────────────
+
+/// Events sent from the C callback to the tokio async world.
+#[derive(Debug, Clone)]
+pub enum CallbackEvent {
+    /// A generated token fragment.
+    Token(String),
+    /// Inference finished — performance stats attached.
+    Finish {
+        prefill_ms: f32,
+        generate_ms: f32,
+        prompt_tokens: i32,
+        completion_tokens: i32,
+    },
+    /// Inference error.
+    Error,
+}
+
+/// Performance stats from the last inference run.
+#[derive(Debug, Clone, Default)]
+#[allow(dead_code)]
+pub struct PerfStats {
+    pub prefill_ms: f32,
+    pub generate_ms: f32,
+    pub prompt_tokens: i32,
+    pub completion_tokens: i32,
+}
+
+// ── Thread-safe handle wrapper ──────────────────────────────────
+
+/// A Send+Sync wrapper for the raw RKLLM handle and library function pointer.
+/// Safety: rkllm_run is only called under Mutex protection (one at a time).
+struct InferContext {
+    handle: usize,     // *mut c_void as usize
+    run_fn: usize,     // function pointer as usize
+    prompt: CString,
+    userdata: usize,   // *mut c_void as usize
+}
+
+// All fields are just integers + owned CString, so this is trivially Send.
+unsafe impl Send for InferContext {}
+
+// ── Backend ─────────────────────────────────────────────────────
+
+/// RKLLM inference backend.
+///
+/// Holds the loaded library, model handle, and an inference lock.
+/// Safe to share across tokio tasks via `Arc<RKLLMBackend>`.
+pub struct RKLLMBackend {
+    lib: RKLLMLibrary,
+    handle: Mutex<*mut c_void>,
+    /// Last inference perf stats (updated on Finish).
+    pub last_perf: Mutex<PerfStats>,
+}
+
+// We protect all handle access with Mutex; the C library is thread-safe
+// for separate calls (not concurrent rkllm_run).
+unsafe impl Send for RKLLMBackend {}
+unsafe impl Sync for RKLLMBackend {}
+
+impl RKLLMBackend {
+    /// Load the RKLLM library and initialize the model.
+    ///
+    /// This blocks for several seconds while the model loads onto the NPU.
+    pub fn init(config: &Config) -> anyhow::Result<Self> {
+        info!(lib = %config.lib_path, model = %config.model_path, "Loading RKLLM runtime");
+
+        let lib = unsafe { RKLLMLibrary::load(&config.lib_path) }
+            .map_err(|e| anyhow::anyhow!("Failed to load {}: {e}", config.lib_path))?;
+
+        // Build C strings that must outlive the init call
+        let model_path_c = CString::new(config.model_path.as_str())
+            .map_err(|e| anyhow::anyhow!("Invalid model path: {e}"))?;
+        let empty_c = CString::new("").unwrap();
+
+        let param = RKLLMParam {
+            model_path: model_path_c.as_ptr(),
+            max_context_len: config.max_context_len,
+            max_new_tokens: config.max_new_tokens,
+            top_k: config.top_k,
+            n_keep: 0,
+            top_p: config.top_p,
+            temperature: config.temperature,
+            repeat_penalty: 1.1,
+            frequency_penalty: 0.0,
+            presence_penalty: 0.0,
+            mirostat: 0,
+            mirostat_tau: 5.0,
+            mirostat_eta: 0.1,
+            skip_special_token: true,
+            is_async: false,
+            img_start: empty_c.as_ptr(),
+            img_end: empty_c.as_ptr(),
+            img_content: empty_c.as_ptr(),
+            extend_param: RKLLMExtendParam::default(),
+            use_gpu: true,
+        };
+
+        let callback: RKLLMCallback = rkllm_callback;
+
+        let mut handle: *mut c_void = ptr::null_mut();
+        let ret = unsafe { lib.init(&mut handle, &param, callback) };
+
+        if ret != 0 {
+            anyhow::bail!("rkllm_init failed (ret={ret}) path={}", config.model_path);
+        }
+
+        info!("RKLLM backend ready");
+
+        Ok(Self {
+            lib,
+            handle: Mutex::new(handle),
+            last_perf: Mutex::new(PerfStats::default()),
+        })
+    }
+
+    /// Non-streaming inference: returns the complete response text.
+    pub async fn generate(&self, prompt: String) -> anyhow::Result<String> {
+        let (tx, mut rx) = mpsc::channel::<CallbackEvent>(256);
+
+        self.run_inference(prompt, tx).await?;
+
+        let mut tokens = Vec::new();
+        while let Some(event) = rx.recv().await {
+            match event {
+                CallbackEvent::Token(t) => tokens.push(t),
+                CallbackEvent::Finish {
+                    prefill_ms,
+                    generate_ms,
+                    prompt_tokens,
+                    completion_tokens,
+                } => {
+                    let mut perf = self.last_perf.lock().unwrap();
+                    *perf = PerfStats {
+                        prefill_ms,
+                        generate_ms,
+                        prompt_tokens,
+                        completion_tokens,
+                    };
+                    break;
+                }
+                CallbackEvent::Error => {
+                    anyhow::bail!("RKLLM inference error");
+                }
+            }
+        }
+        Ok(tokens.join(""))
+    }
+
+    /// Streaming inference: returns a receiver that yields token events.
+    pub async fn stream(
+        &self,
+        prompt: String,
+    ) -> anyhow::Result<mpsc::Receiver<CallbackEvent>> {
+        let (tx, rx) = mpsc::channel::<CallbackEvent>(256);
+
+        self.run_inference(prompt, tx).await?;
+
+        Ok(rx)
+    }
+
+    /// Get the rkllm_run function pointer from the library.
+    fn get_run_fn(&self) -> usize {
+        self.lib.run_fn_ptr()
+    }
+
+    /// Spawn the blocking rkllm_run call on a dedicated thread.
+    async fn run_inference(
+        &self,
+        prompt: String,
+        tx: mpsc::Sender<CallbackEvent>,
+    ) -> anyhow::Result<()> {
+        // Box the sender so we can pass it as userdata through the C callback
+        let tx_box = Box::new(tx);
+        let userdata_ptr = Box::into_raw(tx_box) as *mut c_void;
+
+        let prompt_c = CString::new(prompt.as_str())
+            .map_err(|e| anyhow::anyhow!("Invalid prompt string: {e}"))?;
+
+        // Extract raw values as usize (integers are Send)
+        let handle_val = {
+            let guard = self.handle.lock().unwrap();
+            *guard as usize
+        };
+
+        let ctx = InferContext {
+            handle: handle_val,
+            run_fn: self.get_run_fn(),
+            prompt: prompt_c,
+            userdata: userdata_ptr as usize,
+        };
+
+        // spawn_blocking because rkllm_run blocks the calling thread
+        tokio::task::spawn_blocking(move || {
+            run_inference_blocking(ctx);
+        });
+
+        Ok(())
+    }
+
+    /// Update last perf stats (called from stream consumers).
+    pub fn update_perf(&self, stats: PerfStats) {
+        let mut perf = self.last_perf.lock().unwrap();
+        *perf = stats;
+    }
+
+    /// Get a copy of the last perf stats.
+    pub fn get_perf(&self) -> PerfStats {
+        self.last_perf.lock().unwrap().clone()
+    }
+}
+
+/// Execute rkllm_run on a blocking thread. All pointer values are
+/// passed as usize in `InferContext` to satisfy Send requirements.
+fn run_inference_blocking(ctx: InferContext) {
+    let input = RKLLMInput {
+        role: ptr::null(),
+        enable_thinking: false,
+        input_type: RKLLM_INPUT_PROMPT,
+        input_data: RKLLMInputData {
+            prompt_input: ctx.prompt.as_ptr(),
+        },
+    };
+
+    let mut infer_param: RKLLMInferParam = unsafe { std::mem::zeroed() };
+    infer_param.mode = RKLLM_INFER_GENERATE;
+    infer_param.keep_history = 0;
+
+    let handle = ctx.handle as *mut c_void;
+    let userdata = ctx.userdata as *mut c_void;
+
+    // Reconstruct the function pointer from the stored usize
+    let run_fn: unsafe extern "C" fn(
+        *mut c_void, *const RKLLMInput, *const RKLLMInferParam, *mut c_void
+    ) -> c_int = unsafe { std::mem::transmute(ctx.run_fn) };
+
+    let ret = unsafe { run_fn(handle, &input, &infer_param, userdata) };
+
+    if ret != 0 {
+        warn!(ret, "rkllm_run returned non-zero");
+        // Send error + clean up userdata if rkllm_run failed without calling callback
+        let tx = unsafe { Box::from_raw(userdata as *mut mpsc::Sender<CallbackEvent>) };
+        let _ = tx.try_send(CallbackEvent::Error);
+    }
+    // If ret == 0, the callback already consumed/sent events and we
+    // reconstruct the Box in the final Finish/Error callback to drop it.
+}
+
+impl Drop for RKLLMBackend {
+    fn drop(&mut self) {
+        let handle = self.handle.lock().unwrap();
+        if !handle.is_null() {
+            info!("Destroying RKLLM model");
+            let ret = unsafe { self.lib.destroy(*handle) };
+            if ret != 0 {
+                error!(ret, "rkllm_destroy failed");
+            }
+        }
+    }
+}
+
+// ── C callback ──────────────────────────────────────────────────
+
+/// The extern "C" callback invoked by the RKLLM runtime on its internal thread.
+///
+/// `userdata` is a `Box<mpsc::Sender<CallbackEvent>>`. On FINISH or ERROR,
+/// we reconstruct the Box to drop it (preventing memory leaks).
+extern "C" fn rkllm_callback(
+    result: *mut RKLLMResult,
+    userdata: *mut c_void,
+    state: c_int,
+) {
+    if userdata.is_null() {
+        return;
+    }
+
+    let tx = unsafe { &*(userdata as *const mpsc::Sender<CallbackEvent>) };
+
+    match state {
+        RKLLM_RUN_NORMAL => {
+            if result.is_null() {
+                return;
+            }
+            let text_ptr = unsafe { (*result).text };
+            if text_ptr.is_null() {
+                return;
+            }
+            let text = unsafe { std::ffi::CStr::from_ptr(text_ptr) };
+            if let Ok(s) = text.to_str() {
+                if !s.is_empty() {
+                    let _ = tx.try_send(CallbackEvent::Token(s.to_string()));
+                }
+            }
+        }
+        RKLLM_RUN_FINISH => {
+            if !result.is_null() {
+                let perf = unsafe { (*result).perf };
+                debug!(
+                    prefill_ms = perf.prefill_time_ms,
+                    prefill_tokens = perf.prefill_tokens,
+                    generate_ms = perf.generate_time_ms,
+                    generate_tokens = perf.generate_tokens,
+                    memory_mb = perf.memory_usage_mb,
+                    "RKLLM inference complete"
+                );
+                let _ = tx.try_send(CallbackEvent::Finish {
+                    prefill_ms: perf.prefill_time_ms,
+                    generate_ms: perf.generate_time_ms,
+                    prompt_tokens: perf.prefill_tokens,
+                    completion_tokens: perf.generate_tokens,
+                });
+            } else {
+                let _ = tx.try_send(CallbackEvent::Finish {
+                    prefill_ms: 0.0,
+                    generate_ms: 0.0,
+                    prompt_tokens: 0,
+                    completion_tokens: 0,
+                });
+            }
+            // Reconstruct and drop the Box to free memory
+            let _ = unsafe { Box::from_raw(userdata as *mut mpsc::Sender<CallbackEvent>) };
+        }
+        RKLLM_RUN_ERROR => {
+            error!("RKLLM inference error in callback");
+            let _ = tx.try_send(CallbackEvent::Error);
+            // Reconstruct and drop the Box
+            let _ = unsafe { Box::from_raw(userdata as *mut mpsc::Sender<CallbackEvent>) };
+        }
+        RKLLM_RUN_WAITING => {
+            // Prefill in progress, ignore
+        }
+        _ => {
+            warn!(state, "Unknown RKLLM callback state");
+        }
+    }
+}
@@ -0,0 +1,592 @@
+//! OpenAI-compatible chat completion endpoint.
+//!
+//! Provides `/v1/chat/completions` with both streaming (SSE) and non-streaming
+//! responses, plus `/v1/models` and `/health`.
+
+use std::sync::Arc;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use axum::extract::State;
+use axum::http::StatusCode;
+use axum::response::sse::{Event, KeepAlive, Sse};
+use axum::response::{IntoResponse, Json, Response};
+use futures::stream::Stream;
+use serde::{Deserialize, Serialize};
+use tokio_stream::wrappers::ReceiverStream;
+use tracing::error;
+
+use crate::backend::{CallbackEvent, PerfStats, RKLLMBackend};
+
+// ── App state ───────────────────────────────────────────────────
+
+/// Shared application state.
+#[derive(Clone)]
+#[allow(dead_code)]
+pub struct AppState {
+    pub backend: Arc<RKLLMBackend>,
+    pub model_name: String,
+    pub start_time: std::time::Instant,
+}
+
+// ── Request / Response types ────────────────────────────────────
+
+#[derive(Debug, Deserialize)]
+#[allow(dead_code)]
+pub struct ChatRequest {
+    #[serde(default)]
+    pub model: String,
+    #[serde(default)]
+    pub messages: Vec<Message>,
+    #[serde(default)]
+    pub max_tokens: Option<i32>,
+    #[serde(default)]
+    pub temperature: Option<f32>,
+    #[serde(default)]
+    pub top_p: Option<f32>,
+    #[serde(default)]
+    pub stream: bool,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct Message {
+    #[serde(default = "default_role")]
+    pub role: String,
+    #[serde(default)]
+    pub content: String,
+}
+
+fn default_role() -> String {
+    "user".to_string()
+}
+
+#[derive(Serialize)]
+pub struct ChatCompletion {
+    pub id: String,
+    pub object: &'static str,
+    pub created: u64,
+    pub model: String,
+    pub choices: Vec<CompletionChoice>,
+    pub usage: Usage,
+}
+
+#[derive(Serialize)]
+pub struct CompletionChoice {
+    pub index: u32,
+    pub message: Message,
+    pub finish_reason: &'static str,
+}
+
+#[derive(Serialize)]
+pub struct Usage {
+    pub prompt_tokens: i32,
+    pub completion_tokens: i32,
+    pub total_tokens: i32,
+}
+
+#[derive(Serialize)]
+pub struct ChatCompletionChunk {
+    pub id: String,
+    pub object: &'static str,
+    pub created: u64,
+    pub model: String,
+    pub choices: Vec<ChunkChoice>,
+}
+
+#[derive(Serialize)]
+pub struct ChunkChoice {
+    pub index: u32,
+    pub delta: Delta,
+    pub finish_reason: Option<&'static str>,
+}
+
+#[derive(Serialize)]
+pub struct Delta {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub role: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub content: Option<String>,
+}
+
+#[derive(Serialize)]
+pub struct ModelList {
+    pub object: &'static str,
+    pub data: Vec<ModelInfo>,
+}
+
+#[derive(Serialize)]
+pub struct ModelInfo {
+    pub id: String,
+    pub object: &'static str,
+    pub created: u64,
+    pub owned_by: &'static str,
+}
+
+#[derive(Serialize)]
+pub struct HealthResponse {
+    pub status: &'static str,
+}
+
+// ── ChatML formatting ───────────────────────────────────────────
+
+/// Format messages into Qwen3 ChatML template.
+///
+/// ```text
+/// <|im_start|>system
+/// You are a helpful assistant.<|im_end|>
+/// <|im_start|>user
+/// Hello<|im_end|>
+/// <|im_start|>assistant
+/// ```
+pub fn format_chatml(messages: &[Message]) -> String {
+    let mut parts = Vec::with_capacity(messages.len() + 1);
+    for msg in messages {
+        parts.push(format!(
+            "<|im_start|>{}\n{}<|im_end|>",
+            msg.role, msg.content
+        ));
+    }
+    parts.push("<|im_start|>assistant\n".to_string());
+    parts.join("\n")
+}
+
+// ── Helpers ─────────────────────────────────────────────────────
+
+fn unix_timestamp() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_secs()
+}
+
+fn completion_id() -> String {
+    format!("chatcmpl-{}", uuid::Uuid::new_v4().simple())
+}
+
+// ── Handlers ────────────────────────────────────────────────────
+
+/// GET /health
+pub async fn health(State(state): State<AppState>) -> Json<HealthResponse> {
+    let _ = state;
+    Json(HealthResponse { status: "ok" })
+}
+
+/// GET /v1/models
+pub async fn list_models(State(state): State<AppState>) -> Json<ModelList> {
+    Json(ModelList {
+        object: "list",
+        data: vec![ModelInfo {
+            id: state.model_name.clone(),
+            object: "model",
+            created: unix_timestamp(),
+            owned_by: "local",
+        }],
+    })
+}
+
+/// POST /v1/chat/completions
+pub async fn chat_completions(
+    State(state): State<AppState>,
+    Json(req): Json<ChatRequest>,
+) -> Response {
+    if req.messages.is_empty() {
+        return (
+            StatusCode::BAD_REQUEST,
+            Json(serde_json::json!({"error": {"message": "messages is required", "type": "invalid_request_error"}})),
+        )
+            .into_response();
+    }
+
+    let prompt = format_chatml(&req.messages);
+
+    if req.stream {
+        match handle_stream(state, prompt).await {
+            Ok(sse) => sse.into_response(),
+            Err(e) => {
+                error!("Stream error: {e}");
+                (
+                    StatusCode::INTERNAL_SERVER_ERROR,
+                    Json(serde_json::json!({"error": {"message": format!("Inference error: {e}"), "type": "server_error"}})),
+                )
+                    .into_response()
+            }
+        }
+    } else {
+        match handle_generate(state, prompt).await {
+            Ok(json) => Json(json).into_response(),
+            Err(e) => {
+                error!("Generate error: {e}");
+                let status = if e.to_string().contains("timed out") {
+                    StatusCode::GATEWAY_TIMEOUT
+                } else {
+                    StatusCode::INTERNAL_SERVER_ERROR
+                };
+                (
+                    status,
+                    Json(serde_json::json!({"error": {"message": format!("Inference error: {e}"), "type": "server_error"}})),
+                )
+                    .into_response()
+            }
+        }
+    }
+}
+
+/// Non-streaming: collect all tokens, return complete response.
+async fn handle_generate(
+    state: AppState,
+    prompt: String,
+) -> anyhow::Result<ChatCompletion> {
+    let content = state.backend.generate(prompt).await?;
+
+    let perf = state.backend.get_perf();
+    let id = completion_id();
+
+    Ok(ChatCompletion {
+        id,
+        object: "chat.completion",
+        created: unix_timestamp(),
+        model: state.model_name,
+        choices: vec![CompletionChoice {
+            index: 0,
+            message: Message {
+                role: "assistant".to_string(),
+                content,
+            },
+            finish_reason: "stop",
+        }],
+        usage: Usage {
+            prompt_tokens: perf.prompt_tokens,
+            completion_tokens: perf.completion_tokens,
+            total_tokens: perf.prompt_tokens + perf.completion_tokens,
+        },
+    })
+}
+
+/// Streaming: return SSE events as tokens arrive.
+async fn handle_stream(
+    state: AppState,
+    prompt: String,
+) -> anyhow::Result<Sse<impl Stream<Item = Result<Event, anyhow::Error>>>> {
+    let rx = state.backend.stream(prompt).await?;
+    let id = completion_id();
+    let created = unix_timestamp();
+    let model_name = state.model_name.clone();
+    let backend = state.backend.clone();
+
+    // Convert the mpsc receiver into an SSE stream
+    let (tx_sse, rx_sse) = tokio::sync::mpsc::channel::<Result<Event, anyhow::Error>>(256);
+
+    tokio::spawn(async move {
+        // First chunk: role
+        let role_chunk = ChatCompletionChunk {
+            id: id.clone(),
+            object: "chat.completion.chunk",
+            created,
+            model: model_name.clone(),
+            choices: vec![ChunkChoice {
+                index: 0,
+                delta: Delta {
+                    role: Some("assistant".to_string()),
+                    content: None,
+                },
+                finish_reason: None,
+            }],
+        };
+        let data = serde_json::to_string(&role_chunk).unwrap_or_default();
+        let _ = tx_sse.send(Ok(Event::default().data(data))).await;
+
+        // Token chunks from the backend
+        let mut rx = rx;
+        while let Some(event) = rx.recv().await {
+            match event {
+                CallbackEvent::Token(token) => {
+                    let chunk = ChatCompletionChunk {
+                        id: id.clone(),
+                        object: "chat.completion.chunk",
+                        created,
+                        model: model_name.clone(),
+                        choices: vec![ChunkChoice {
+                            index: 0,
+                            delta: Delta {
+                                role: None,
+                                content: Some(token),
+                            },
+                            finish_reason: None,
+                        }],
+                    };
+                    let data = serde_json::to_string(&chunk).unwrap_or_default();
+                    if tx_sse.send(Ok(Event::default().data(data))).await.is_err() {
+                        break; // client disconnected
+                    }
+                }
+                CallbackEvent::Finish {
+                    prefill_ms,
+                    generate_ms,
+                    prompt_tokens,
+                    completion_tokens,
+                } => {
+                    backend.update_perf(PerfStats {
+                        prefill_ms,
+                        generate_ms,
+                        prompt_tokens,
+                        completion_tokens,
+                    });
+
+                    // Final chunk with finish_reason
+                    let finish_chunk = ChatCompletionChunk {
+                        id: id.clone(),
+                        object: "chat.completion.chunk",
+                        created,
+                        model: model_name.clone(),
+                        choices: vec![ChunkChoice {
+                            index: 0,
+                            delta: Delta {
+                                role: None,
+                                content: None,
+                            },
+                            finish_reason: Some("stop"),
+                        }],
+                    };
+                    let data = serde_json::to_string(&finish_chunk).unwrap_or_default();
+                    let _ = tx_sse.send(Ok(Event::default().data(data))).await;
+
+                    // [DONE] sentinel
+                    let _ = tx_sse.send(Ok(Event::default().data("[DONE]"))).await;
+                    break;
+                }
+                CallbackEvent::Error => {
+                    let _ = tx_sse.send(Ok(Event::default().data("[DONE]"))).await;
+                    break;
+                }
+            }
+        }
+    });
+
+    let stream = ReceiverStream::new(rx_sse);
+    Ok(Sse::new(stream).keep_alive(KeepAlive::default()))
+}
+
+// ── Tests ───────────────────────────────────────────────────────
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_format_chatml_single_user() {
+        let messages = vec![Message {
+            role: "user".to_string(),
+            content: "Hello".to_string(),
+        }];
+        let result = format_chatml(&messages);
+        assert_eq!(
+            result,
+            "<|im_start|>user\nHello<|im_end|>\n<|im_start|>assistant\n"
+        );
+    }
+
+    #[test]
+    fn test_format_chatml_system_and_user() {
+        let messages = vec![
+            Message {
+                role: "system".to_string(),
+                content: "You are a helpful assistant.".to_string(),
+            },
+            Message {
+                role: "user".to_string(),
+                content: "Hi".to_string(),
+            },
+        ];
+        let result = format_chatml(&messages);
+        let expected = "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n\
+                        <|im_start|>user\nHi<|im_end|>\n\
+                        <|im_start|>assistant\n";
+        assert_eq!(result, expected);
+    }
+
+    #[test]
+    fn test_format_chatml_multi_turn() {
+        let messages = vec![
+            Message {
+                role: "user".to_string(),
+                content: "What is 2+2?".to_string(),
+            },
+            Message {
+                role: "assistant".to_string(),
+                content: "4".to_string(),
+            },
+            Message {
+                role: "user".to_string(),
+                content: "And 3+3?".to_string(),
+            },
+        ];
+        let result = format_chatml(&messages);
+        assert!(result.contains("<|im_start|>user\nWhat is 2+2?<|im_end|>"));
+        assert!(result.contains("<|im_start|>assistant\n4<|im_end|>"));
+        assert!(result.contains("<|im_start|>user\nAnd 3+3?<|im_end|>"));
+        assert!(result.ends_with("<|im_start|>assistant\n"));
+    }
+
+    #[test]
+    fn test_format_chatml_empty() {
+        let messages: Vec<Message> = vec![];
+        let result = format_chatml(&messages);
+        assert_eq!(result, "<|im_start|>assistant\n");
+    }
+
+    #[test]
+    fn test_completion_id_format() {
+        let id = completion_id();
+        assert!(id.starts_with("chatcmpl-"));
+        assert!(id.len() > 10);
+    }
+
+    #[test]
+    fn test_chat_completion_serialization() {
+        let completion = ChatCompletion {
+            id: "chatcmpl-test123".to_string(),
+            object: "chat.completion",
+            created: 1700000000,
+            model: "qwen3-0.6b".to_string(),
+            choices: vec![CompletionChoice {
+                index: 0,
+                message: Message {
+                    role: "assistant".to_string(),
+                    content: "Hello!".to_string(),
+                },
+                finish_reason: "stop",
+            }],
+            usage: Usage {
+                prompt_tokens: 10,
+                completion_tokens: 5,
+                total_tokens: 15,
+            },
+        };
+
+        let json = serde_json::to_value(&completion).unwrap();
+        assert_eq!(json["object"], "chat.completion");
+        assert_eq!(json["choices"][0]["message"]["content"], "Hello!");
+        assert_eq!(json["choices"][0]["finish_reason"], "stop");
+        assert_eq!(json["usage"]["total_tokens"], 15);
+    }
+
+    #[test]
+    fn test_chunk_serialization() {
+        let chunk = ChatCompletionChunk {
+            id: "chatcmpl-test".to_string(),
+            object: "chat.completion.chunk",
+            created: 1700000000,
+            model: "qwen3-0.6b".to_string(),
+            choices: vec![ChunkChoice {
+                index: 0,
+                delta: Delta {
+                    role: None,
+                    content: Some("Hello".to_string()),
+                },
+                finish_reason: None,
+            }],
+        };
+
+        let json = serde_json::to_value(&chunk).unwrap();
+        assert_eq!(json["object"], "chat.completion.chunk");
+        assert_eq!(json["choices"][0]["delta"]["content"], "Hello");
+        // role should not be present
+        assert!(json["choices"][0]["delta"].get("role").is_none());
+        assert!(json["choices"][0]["finish_reason"].is_null());
+    }
+
+    #[test]
+    fn test_role_chunk_serialization() {
+        let chunk = ChatCompletionChunk {
+            id: "chatcmpl-test".to_string(),
+            object: "chat.completion.chunk",
+            created: 1700000000,
+            model: "qwen3-0.6b".to_string(),
+            choices: vec![ChunkChoice {
+                index: 0,
+                delta: Delta {
+                    role: Some("assistant".to_string()),
+                    content: None,
+                },
+                finish_reason: None,
+            }],
+        };
+
+        let json = serde_json::to_value(&chunk).unwrap();
+        assert_eq!(json["choices"][0]["delta"]["role"], "assistant");
+        // content should not be present
+        assert!(json["choices"][0]["delta"].get("content").is_none());
+    }
+
+    #[test]
+    fn test_finish_chunk_serialization() {
+        let chunk = ChatCompletionChunk {
+            id: "chatcmpl-test".to_string(),
+            object: "chat.completion.chunk",
+            created: 1700000000,
+            model: "qwen3-0.6b".to_string(),
+            choices: vec![ChunkChoice {
+                index: 0,
+                delta: Delta {
+                    role: None,
+                    content: None,
+                },
+                finish_reason: Some("stop"),
+            }],
+        };
+
+        let json = serde_json::to_value(&chunk).unwrap();
+        assert_eq!(json["choices"][0]["finish_reason"], "stop");
+        // Both role and content should be absent
+        assert!(json["choices"][0]["delta"].get("role").is_none());
+        assert!(json["choices"][0]["delta"].get("content").is_none());
+    }
+
+    #[test]
+    fn test_model_list_serialization() {
+        let list = ModelList {
+            object: "list",
+            data: vec![ModelInfo {
+                id: "qwen3-0.6b".to_string(),
+                object: "model",
+                created: 1700000000,
+                owned_by: "local",
+            }],
+        };
+
+        let json = serde_json::to_value(&list).unwrap();
+        assert_eq!(json["object"], "list");
+        assert_eq!(json["data"][0]["id"], "qwen3-0.6b");
+        assert_eq!(json["data"][0]["owned_by"], "local");
+    }
+
+    #[test]
+    fn test_chat_request_deserialization() {
+        let json = serde_json::json!({
+            "model": "qwen3-0.6b",
+            "messages": [
+                {"role": "user", "content": "Hello"}
+            ],
+            "stream": true
+        });
+
+        let req: ChatRequest = serde_json::from_value(json).unwrap();
+        assert_eq!(req.model, "qwen3-0.6b");
+        assert_eq!(req.messages.len(), 1);
+        assert_eq!(req.messages[0].role, "user");
+        assert_eq!(req.messages[0].content, "Hello");
+        assert!(req.stream);
+        assert!(req.max_tokens.is_none());
+    }
+
+    #[test]
+    fn test_chat_request_defaults() {
+        let json = serde_json::json!({
+            "messages": [{"content": "Hi"}]
+        });
+
+        let req: ChatRequest = serde_json::from_value(json).unwrap();
+        assert_eq!(req.model, "");
+        assert!(!req.stream);
+        assert_eq!(req.messages[0].role, "user"); // default role
+    }
+}
@@ -0,0 +1,152 @@
+//! Environment-driven configuration for the RKLLM Server.
+//!
+//! All settings can be overridden via environment variables.
+//! Model name is auto-derived from the model file path if not set explicitly.
+
+use std::env;
+use std::path::Path;
+
+/// Server + model configuration, read from environment variables.
+#[derive(Debug, Clone)]
+pub struct Config {
+    /// Path to the .rkllm model file.
+    pub model_path: String,
+    /// Path to librkllmrt.so.
+    pub lib_path: String,
+    /// Listen port.
+    pub port: u16,
+    /// Listen host.
+    pub host: String,
+    /// Model name (auto-derived if empty).
+    pub model_name: String,
+    /// Maximum context length.
+    pub max_context_len: i32,
+    /// Maximum new tokens per inference.
+    pub max_new_tokens: i32,
+    /// Sampling temperature.
+    pub temperature: f32,
+    /// Top-p (nucleus) sampling.
+    pub top_p: f32,
+    /// Top-k sampling.
+    pub top_k: f32,
+}
+
+impl Config {
+    /// Load configuration from environment variables with sensible defaults.
+    pub fn from_env() -> Self {
+        Self {
+            model_path: env_or(
+                "RKLLM_MODEL",
+                "/var/lib/kvm-rkllm/models/Qwen3-0.6B-rk3588-w8a8-opt-1-hybrid-ratio-0.5.rkllm",
+            ),
+            lib_path: env_or("RKLLM_LIB", "/usr/lib/kvm-rkllm/librkllmrt.so"),
+            port: env_or("RKLLM_PORT", "8891").parse().unwrap_or(8891),
+            host: env_or("RKLLM_HOST", "0.0.0.0"),
+            model_name: env_or("RKLLM_MODEL_NAME", ""),
+            max_context_len: env_or("RKLLM_MAX_CTX", "2048").parse().unwrap_or(2048),
+            max_new_tokens: env_or("RKLLM_MAX_TOKENS", "2048").parse().unwrap_or(2048),
+            temperature: env_or("RKLLM_TEMPERATURE", "0.7").parse().unwrap_or(0.7),
+            top_p: env_or("RKLLM_TOP_P", "0.9").parse().unwrap_or(0.9),
+            top_k: env_or("RKLLM_TOP_K", "1").parse().unwrap_or(1.0),
+        }
+    }
+
+    /// Resolve the display model name.
+    ///
+    /// If `model_name` is explicitly set, return it.
+    /// Otherwise derive from the model file path:
+    /// `"Qwen3-0.6B-rk3588-w8a8-opt-1-hybrid-ratio-0.5.rkllm"` -> `"qwen3-0.6b"`
+    pub fn resolved_model_name(&self) -> String {
+        if !self.model_name.is_empty() {
+            return self.model_name.clone();
+        }
+        derive_model_name(&self.model_path)
+    }
+}
+
+/// Derive a short model name from a file path.
+///
+/// Strips the directory and `.rkllm` extension, lowercases, then truncates
+/// at the first occurrence of `-rk`, `-w8`, or `-w4`.
+pub fn derive_model_name(path: &str) -> String {
+    let basename = Path::new(path)
+        .file_name()
+        .and_then(|f| f.to_str())
+        .unwrap_or(path);
+
+    let lower = basename.to_lowercase();
+    let stem = lower.strip_suffix(".rkllm").unwrap_or(&lower);
+
+    for sep in ["-rk", "-w8", "-w4"] {
+        if let Some(idx) = stem.find(sep) {
+            if idx > 0 {
+                return stem[..idx].to_string();
+            }
+        }
+    }
+    stem.to_string()
+}
+
+fn env_or(key: &str, default: &str) -> String {
+    env::var(key).unwrap_or_else(|_| default.to_string())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn derive_qwen3() {
+        assert_eq!(
+            derive_model_name(
+                "/var/lib/kvm-rkllm/models/Qwen3-0.6B-rk3588-w8a8-opt-1-hybrid-ratio-0.5.rkllm"
+            ),
+            "qwen3-0.6b"
+        );
+    }
+
+    #[test]
+    fn derive_with_w8() {
+        assert_eq!(
+            derive_model_name("SomeModel-w8a8-quantized.rkllm"),
+            "somemodel"
+        );
+    }
+
+    #[test]
+    fn derive_no_markers() {
+        assert_eq!(derive_model_name("my-model.rkllm"), "my-model");
+    }
+
+    #[test]
+    fn derive_no_extension() {
+        assert_eq!(derive_model_name("qwen3-0.6b-rk3588"), "qwen3-0.6b");
+    }
+
+    #[test]
+    fn derive_explicit_name() {
+        let mut cfg = Config::from_env();
+        cfg.model_name = "custom-name".to_string();
+        assert_eq!(cfg.resolved_model_name(), "custom-name");
+    }
+
+    #[test]
+    fn default_config_values() {
+        // Clear env vars to test defaults (they might be set in CI)
+        let cfg = Config {
+            model_path: "/var/lib/kvm-rkllm/models/Qwen3-0.6B-rk3588-w8a8-opt-1-hybrid-ratio-0.5.rkllm".into(),
+            lib_path: "/usr/lib/kvm-rkllm/librkllmrt.so".into(),
+            port: 8891,
+            host: "0.0.0.0".into(),
+            model_name: String::new(),
+            max_context_len: 2048,
+            max_new_tokens: 2048,
+            temperature: 0.7,
+            top_p: 0.9,
+            top_k: 1.0,
+        };
+        assert_eq!(cfg.port, 8891);
+        assert_eq!(cfg.max_context_len, 2048);
+        assert!((cfg.temperature - 0.7).abs() < f32::EPSILON);
+    }
+}
@@ -0,0 +1,136 @@
+//! RKLLM Server — OpenAI-compatible API for Qwen3-0.6B on RK3588 NPU.
+//!
+//! Replaces the Python rkllm_server with a Rust implementation using:
+//! - axum for HTTP/SSE serving
+//! - libloading for dynamic RKLLM runtime loading
+//! - tokio for async I/O with spawn_blocking for FFI calls
+//!
+//! Device: NanoPC-T6 (RK3588, 6 TOPS NPU)
+//! Perf:   ~774 MB memory, ~214ms TTFT, ~32 tok/s
+
+mod backend;
+mod chat;
+mod config;
+
+use std::sync::Arc;
+use std::time::Instant;
+
+use axum::routing::{get, post};
+use axum::Router;
+use clap::Parser;
+use tower_http::cors::CorsLayer;
+use tower_http::trace::TraceLayer;
+use tracing::info;
+
+use backend::RKLLMBackend;
+use chat::AppState;
+use config::Config;
+
+#[derive(Parser)]
+#[command(
+    name = "rkllm-server",
+    about = "OpenAI-compatible LLM server for RK3588 NPU"
+)]
+struct Cli {
+    /// Override listen port (default: from RKLLM_PORT env or 8891).
+    #[arg(short, long)]
+    port: Option<u16>,
+
+    /// Override model path (default: from RKLLM_MODEL env).
+    #[arg(short, long)]
+    model: Option<String>,
+
+    /// Override librkllmrt.so path (default: from RKLLM_LIB env).
+    #[arg(short, long)]
+    lib: Option<String>,
+}
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    tracing_subscriber::fmt()
+        .with_env_filter(
+            tracing_subscriber::EnvFilter::try_from_default_env()
+                .unwrap_or_else(|_| "rkllm_server=info,tower_http=info".into()),
+        )
+        .init();
+
+    let cli = Cli::parse();
+    let mut cfg = Config::from_env();
+
+    // CLI overrides
+    if let Some(port) = cli.port {
+        cfg.port = port;
+    }
+    if let Some(model) = cli.model {
+        cfg.model_path = model;
+    }
+    if let Some(lib) = cli.lib {
+        cfg.lib_path = lib;
+    }
+
+    let model_name = cfg.resolved_model_name();
+    info!(
+        model = %model_name,
+        host = %cfg.host,
+        port = cfg.port,
+        "Starting RKLLM Server"
+    );
+
+    // Load model (blocks for several seconds)
+    let t0 = Instant::now();
+    let backend = Arc::new(RKLLMBackend::init(&cfg)?);
+    let load_time = t0.elapsed();
+    info!(elapsed_ms = load_time.as_millis(), "Model loaded");
+
+    let state = AppState {
+        backend,
+        model_name,
+        start_time: Instant::now(),
+    };
+
+    let app = Router::new()
+        // Chat endpoints
+        .route("/v1/chat/completions", post(chat::chat_completions))
+        .route("/v1/models", get(chat::list_models))
+        // Health
+        .route("/health", get(chat::health))
+        .with_state(state)
+        // Middleware
+        .layer(CorsLayer::permissive())
+        .layer(TraceLayer::new_for_http());
+
+    let addr = format!("{}:{}", cfg.host, cfg.port);
+    info!("RKLLM Server listening on {addr}");
+
+    let listener = tokio::net::TcpListener::bind(&addr).await?;
+    axum::serve(listener, app)
+        .with_graceful_shutdown(shutdown_signal())
+        .await?;
+
+    info!("RKLLM Server shutdown");
+    Ok(())
+}
+
+async fn shutdown_signal() {
+    let ctrl_c = async {
+        tokio::signal::ctrl_c()
+            .await
+            .expect("Failed to install CTRL+C handler");
+    };
+
+    #[cfg(unix)]
+    let terminate = async {
+        tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
+            .expect("Failed to install SIGTERM handler")
+            .recv()
+            .await;
+    };
+
+    #[cfg(not(unix))]
+    let terminate = std::future::pending::<()>();
+
+    tokio::select! {
+        _ = ctrl_c => info!("CTRL+C received"),
+        _ = terminate => info!("SIGTERM received"),
+    }
+}