Compare commits
100
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9bb799b26a | ||
|
|
0e2c765ea7 | ||
|
|
598c313e46 | ||
|
|
fa7e2cefa1 | ||
|
|
dbb8a93fa3 | ||
|
|
da8ecdce42 | ||
|
|
cd00c79262 | ||
|
|
48243bdf6b | ||
|
|
a10d29748a | ||
|
|
e069249512 | ||
|
|
ec464e3d9e | ||
|
|
e6115fbfd7 | ||
|
|
b0a3c6d099 | ||
|
|
81960f0be7 | ||
|
|
0a698aca17 | ||
|
|
40e286f26b | ||
|
|
9dececb78a | ||
|
|
a8fc01ee8b | ||
|
|
2a28466539 | ||
|
|
05435120a8 | ||
|
|
b15f1592fd | ||
|
|
5e6b7b1c5c | ||
|
|
c27aed5c16 | ||
|
|
3adb313fa1 | ||
|
|
8000170221 | ||
|
|
ff6ea73946 | ||
|
|
58ef04bfec | ||
|
|
04e387c7e9 | ||
|
|
7138af0b54 | ||
|
|
6bc7aef0f9 | ||
|
|
09a2212543 | ||
|
|
fa73fd0b6c | ||
|
|
df45cc81e7 | ||
|
|
d17031b9cc | ||
|
|
bb2e68fd3d | ||
|
|
0f05b3adf6 | ||
|
|
157d72f65d | ||
|
|
302c4aec28 | ||
|
|
465d9d38cc | ||
|
|
41aafca443 | ||
|
|
462bb9eec3 | ||
|
|
acf70034da | ||
|
|
18cdaa3251 | ||
|
|
07d77acc46 | ||
|
|
eb6e4329d4 | ||
|
|
6453bf1197 | ||
|
|
9b9121e362 | ||
|
|
58390e2e82 | ||
|
|
cc1ce4dae5 | ||
|
|
083ef77134 | ||
|
|
5ca5ffcd41 | ||
|
|
13ec3362aa | ||
|
|
502ae32c9f | ||
|
|
e34af87d96 | ||
|
|
2f73e25c4c | ||
|
|
619f30fcfe | ||
|
|
633e182a25 | ||
|
|
1c5cf01e4a | ||
|
|
346803ecd9 | ||
|
|
8ec50e141d | ||
|
|
6338f91f94 | ||
|
|
1bb44e0315 | ||
|
|
6e0e03b63b | ||
|
|
271b26e094 | ||
|
|
cfe7f9093a | ||
|
|
6a1684d693 | ||
|
|
b337f5dc06 | ||
|
|
07785330f1 | ||
|
|
baa900b46f | ||
|
|
9e884f68b3 | ||
|
|
03d16a63f9 | ||
|
|
a75c2696cc | ||
|
|
dc09306655 | ||
|
|
735d91cf26 | ||
|
|
477eb9f570 | ||
|
|
4e4f0798a8 | ||
|
|
9509889763 | ||
|
|
b2be690d14 | ||
|
|
3b1d2e3c70 | ||
|
|
2c39f8ec8f | ||
|
|
f7a7f3bb36 | ||
|
|
2b4bc9d997 | ||
|
|
7d84e36f5e | ||
|
|
88ce820fe9 | ||
|
|
522d9d8ec5 | ||
|
|
67c9c14564 | ||
|
|
ccd28566af | ||
|
|
e564db0b69 | ||
|
|
224d506f1b | ||
|
|
4030cf8504 | ||
|
|
5c56ba48e0 | ||
|
|
d3f838cc76 | ||
|
|
a8d29c1190 | ||
|
|
af2358c344 | ||
|
|
d567ad84c7 | ||
|
|
4a652bc38d | ||
|
|
703b67a313 | ||
|
|
ba19a68f59 | ||
|
|
d774f7a772 | ||
|
|
3510a1eebb |
+36
@@ -5,3 +5,39 @@ __pycache__/
|
||||
/deploy/systemd/*.service.override
|
||||
/services/privacy_gateway/__pycache__/
|
||||
.worktrees/
|
||||
build/deb/
|
||||
build/test-viz/
|
||||
dist/
|
||||
# Build artifacts in deb trees (populated by build scripts at build time)
|
||||
debian/kvm-mitm/usr/lib/
|
||||
debian/kvm-agent/usr/lib/
|
||||
debian/kvm-bridge/usr/lib/
|
||||
debian/kvm-rkllm/usr/lib/kvm-rkllm/rkllm_server/
|
||||
debian/kvm-rkllm/usr/lib/kvm-rkllm/librkllmrt.so
|
||||
debian/kvm-rkllm/lib/systemd/
|
||||
debian/kvm-npu/usr/local/bin/
|
||||
debian/kvm-npu/usr/share/kvm-npu/models/
|
||||
debian/kvm-privacy/usr/local/bin/
|
||||
|
||||
# Temporary test screenshots
|
||||
test-*.png
|
||||
e2e-verify-*.png
|
||||
|
||||
# Sensitive files
|
||||
api-key.txt
|
||||
|
||||
# Rust build artifacts
|
||||
target/
|
||||
|
||||
# Test output data
|
||||
testdata/results/
|
||||
|
||||
# IDE / tooling
|
||||
.playwright-mcp/
|
||||
firebase-debug.log
|
||||
|
||||
# Build artifacts — compiled binaries and model blobs (reproducible from source)
|
||||
debian/*/usr/sbin/
|
||||
debian/kvm-npu/usr/share/
|
||||
testdata/
|
||||
tools/workflow-dashboard/testdata/
|
||||
|
||||
@@ -6,17 +6,19 @@
|
||||
- Git commit 消息使用英文
|
||||
|
||||
## 项目概述
|
||||
KVM-Privacy 是一个基于 KVM-over-IP 的三层隐私保护系统,运行在 NanoPC-T6 (RK3588) 上。
|
||||
KVM-Privacy 是一个基于 KVM-over-IP 的两层隐私保护系统,运行在 NanoPC-T6 (RK3588) 上。所有防护完全在 KVM 设备端完成,无需在被控机器上部署任何软件。
|
||||
|
||||
### 核心服务
|
||||
|
||||
| 服务 | 端口 | 说明 |
|
||||
|------|------|------|
|
||||
| KVM Server (Go) | 8080 | KVM 控制 + React WebUI |
|
||||
| info-privacy-rs (Rust) | 8001 | RKNN PII 检测 |
|
||||
| mem-bridge router | 8002 | AI 路由服务 |
|
||||
| mem-bridge memory | 8003 | 会话存储 + FAISS 向量搜索 |
|
||||
| Privacy Gateway (Python) | 8888 | mitmproxy 网络拦截 |
|
||||
| 服务 | 端口 | DEB 包 | 说明 |
|
||||
|------|------|--------|------|
|
||||
| KVM Server (Go) | 8080 | kvm-server | KVM 控制 + React WebUI + WebRTC |
|
||||
| info-privacy-rs (Rust) | 8001 | kvm-privacy | RKNN PII 检测/脱敏 (NPU Bridge only, no Python workers) |
|
||||
| NPU Daemon (Rust) | 8004 | kvm-npu | 集中 RKNN 推理 (OCR/Face) |
|
||||
| embed-db (Rust) | 8003+8002 | kvm-bridge | USearch 向量搜索 + SQLite 会话存储 + AI 路由 |
|
||||
| Privacy Gateway (Rust) | 8888+8889 | kvm-mitm | hudsucker 透明代理 + axum API |
|
||||
| KVM Agent (Python) | 8890 | kvm-agent | AI Agent daemon (pending Rust migration) |
|
||||
| RKLLM Server (Rust) | 8891 | kvm-rkllm | 本地 LLM (Qwen3-0.6B NPU, libloading FFI) |
|
||||
|
||||
### 架构
|
||||
|
||||
@@ -36,19 +38,27 @@ KVM-Privacy 是一个基于 KVM-over-IP 的三层隐私保护系统,运行在
|
||||
|
||||
```
|
||||
services/
|
||||
kvm_agent/ # Python - KVM AI Agent
|
||||
privacy_gateway/ # Python - mitmproxy 隐私网关
|
||||
kvm_agent/ # Python - KVM AI Agent (pending Rust migration P3/P4)
|
||||
privacy-gateway-rs/ # Rust - hudsucker 隐私网关 (replaces Python mitmproxy)
|
||||
rkllm-server/ # Rust - RKLLM NPU LLM 服务 (replaces Python FastAPI)
|
||||
embed-db-rs/ # Rust - 向量搜索 + 会话存储 (replaces Python mem-bridge)
|
||||
npu_daemon/ # Rust - NPU 推理守护进程
|
||||
tools/
|
||||
smoke-test.sh # 服务健康检查脚本 (replaces workflow-dashboard)
|
||||
KVM/ # Git submodule - Go KVM 服务端
|
||||
deps/ # Git submodules - 依赖项目
|
||||
deploy/systemd/ # systemd 服务文件
|
||||
debian/ # DEB 包定义 (kvm-mitm, kvm-agent, kvm-bridge, kvm-npu, kvm-privacy, kvm-rkllm, kvm-meta)
|
||||
scripts/build-debs.sh # 全栈 DEB 构建脚本 (Rust + Python agent)
|
||||
```
|
||||
|
||||
## 编码规范
|
||||
- Python:asyncio + httpx,类型注解,dataclass 优先
|
||||
- 异步优先:所有 I/O 操作使用 async/await
|
||||
- Rust:tokio + axum + reqwest,serde 序列化,thiserror 错误处理
|
||||
- Python(仅 kvm_agent,待迁移):asyncio + httpx,类型注解,dataclass 优先
|
||||
- 异步优先:所有 I/O 操作使用 async/await (Rust: tokio, Python: asyncio)
|
||||
- 错误处理:finally 块中释放资源(HID 控制、隐私模式)
|
||||
- 测试:pytest + pytest-asyncio,mock 外部依赖
|
||||
- 配置:YAML 文件 + 环境变量,dataclass 承载
|
||||
- 测试:Rust #[tokio::test],Python pytest + pytest-asyncio,mock 外部依赖
|
||||
- 配置:YAML 文件 + 环境变量
|
||||
|
||||
## 测试方法论
|
||||
|
||||
@@ -80,6 +90,23 @@ action → sleep → screenshot → OCR → semantic assert
|
||||
- 仅用于判断"屏幕是否变化"
|
||||
- OCR 语义验证 > 像素对比
|
||||
|
||||
## 前端架构变更(2026-03)
|
||||
|
||||
### GatewayMode 类型
|
||||
- `privacyMode` 已从 `boolean` 升级为 `'off' | 'audit' | 'redact'`
|
||||
- 类型定义:`deps/KVM/web/src/types/privacy.ts`
|
||||
- PiiOverlay:`audit` = 红色边框,`redact` = 黑色填充,`off` = 清空
|
||||
|
||||
### 页面 Tab 结构
|
||||
- **PrivacyPage**:5 tabs — dashboard / network / screen / rules / settings
|
||||
- screen tab:`GET /api/v1/privacy/screen/detections`
|
||||
- rules tab:`GET/POST/PUT/DELETE /api/v1/privacy/patterns`
|
||||
- **AuditPage**:2 tabs — logs / recordings(OCR/patterns 已迁移到 Privacy)
|
||||
|
||||
### Agent 运行期间 HID 禁用
|
||||
- kvmStore 所有 HID 发送函数检查 `useAgentStore.getState().status === 'running'`
|
||||
- ConsolePage 粘贴/虚拟键盘按钮在 Agent 运行时禁用
|
||||
|
||||
## 架构决策:鼠标优先
|
||||
|
||||
### 背景
|
||||
@@ -106,32 +133,161 @@ escape, enter, tab, backspace, delete, space, up/down/left/right, shift, f1-f5,
|
||||
|
||||
```bash
|
||||
# 运行 KVM Agent 单次任务
|
||||
python -m kvm_agent --task "打开记事本" --kvm-url http://localhost:8080
|
||||
python3 -m kvm_agent --task "打开记事本" --kvm-url http://localhost:8080
|
||||
|
||||
# 运行测试
|
||||
cd services/kvm_agent && python -m pytest tests/ -v
|
||||
# 运行单元测试(不需要实体设备)
|
||||
cd services/kvm_agent && python3 -m pytest tests/ --ignore=tests/test_integration.py -v
|
||||
|
||||
# 运行集成测试(需要实体 KVM 设备)
|
||||
cd services/kvm_agent && python3 -m pytest tests/test_integration.py -v
|
||||
|
||||
# Go 后端编译(go.mod 在 deps/KVM/go/;系统 go 是 1.15 需用绝对路径)
|
||||
cd deps/KVM/go && /usr/local/go/bin/go build ./...
|
||||
|
||||
# kvm-server deb 打包(需要 Go 1.24 在 PATH 前面)
|
||||
cd deps/KVM && PATH="/usr/local/go/bin:$PATH" bash scripts/build-deb.sh
|
||||
|
||||
# 构建全部 DEB 包(Rust + Python agent + 闭源库)
|
||||
bash scripts/build-debs.sh
|
||||
# 跳过 Rust 编译(使用已有二进制)
|
||||
bash scripts/build-debs.sh --skip-rust
|
||||
|
||||
# 前端编译
|
||||
cd deps/KVM/web && npm run build
|
||||
|
||||
# 服务健康检查
|
||||
bash tools/smoke-test.sh
|
||||
|
||||
# 查看服务状态
|
||||
systemctl status kvm-agent mem-bridge-memory mem-bridge-router info-privacy privacy-gateway
|
||||
systemctl status kvm-agent embed-db info-privacy kvm-mitm npu-daemon rkllm-server
|
||||
|
||||
# 设备连接
|
||||
ssh pi@192.168.123.181
|
||||
```
|
||||
|
||||
## 部署规则(强制)
|
||||
|
||||
### 禁止直接拷贝二进制文件部署
|
||||
|
||||
- **绝对禁止** `cp`/`scp` 二进制文件到 `/usr/bin/`、`/usr/sbin/` 等系统路径
|
||||
- **所有服务部署必须通过 DEB 包**:`scripts/build-debs.sh` → `dpkg -i`
|
||||
- 原因:直接拷贝绕过 systemd 服务管理、conffile 保护、依赖检查、卸载清理
|
||||
- 唯一例外:开发时 `cargo run` / `python -m` 本地调试(不部署到系统路径)
|
||||
|
||||
### 标准部署流程
|
||||
|
||||
```bash
|
||||
# 1. 编译 + 打包(全栈)
|
||||
bash scripts/build-debs.sh
|
||||
|
||||
# 2. 安装指定包
|
||||
sudo dpkg -i debian/dist/kvm-privacy_*.deb
|
||||
|
||||
# 3. 验证服务状态
|
||||
systemctl is-active info-privacy && echo "PASS" || echo "FAIL"
|
||||
```
|
||||
|
||||
## Deb 包开发完整工作流
|
||||
|
||||
### 修改 → 构建 → 验证
|
||||
|
||||
每次修改 deb 包相关文件(debian/control, postinst, prerm, systemd service, Rust/Go 源码)后,按以下流程验证:
|
||||
|
||||
```bash
|
||||
# 1. 构建
|
||||
cd deps/KVM && PATH="/usr/local/go/bin:$PATH" bash scripts/build-deb.sh
|
||||
|
||||
# 2. 静态检查包内容(无需设备)
|
||||
DEB=$(ls deps/KVM/dist/kvm-server_*.deb | tail -1)
|
||||
dpkg -I "$DEB" | grep "Depends\|Recommends" # 检查依赖
|
||||
dpkg -c "$DEB" | grep "\.service\|\.timer\|usr/lib" # 检查安装文件
|
||||
dpkg -x "$DEB" /tmp/kvm-check && grep -n "systemctl" /tmp/kvm-check/DEBIAN/postinst
|
||||
dpkg -x "$DEB" /tmp/kvm-check && grep -n "systemctl" /tmp/kvm-check/DEBIAN/prerm
|
||||
grep "Requires\|Wants" /tmp/kvm-check/lib/systemd/system/kvm-server.service
|
||||
|
||||
# 3. 拷贝到目标设备
|
||||
scp "$DEB" pi@192.168.123.181:/tmp/
|
||||
```
|
||||
|
||||
### 标准安装/卸载/重装验证循环
|
||||
|
||||
**必须按此顺序在目标设备执行(ssh pi@192.168.123.181):**
|
||||
|
||||
```bash
|
||||
# [1] 首次安装
|
||||
sudo dpkg -i /tmp/kvm-server_*.deb
|
||||
sleep 5
|
||||
systemctl is-active kvm-server && echo "PASS" || echo "FAIL"
|
||||
curl -sf http://localhost:8080/api/v1/kvm/stream/stats
|
||||
|
||||
# [2] 卸载(保留 conffiles)
|
||||
sudo dpkg --remove kvm-server
|
||||
systemctl is-active kvm-server && echo "FAIL" || echo "PASS: stopped"
|
||||
ls /etc/kvm/config.json && echo "PASS: config preserved" || echo "FAIL: config lost"
|
||||
|
||||
# [3] 重装(验证 conffile 不被覆盖、服务恢复)
|
||||
sudo dpkg -i /tmp/kvm-server_*.deb
|
||||
sleep 5
|
||||
systemctl is-active kvm-server && echo "PASS" || echo "FAIL"
|
||||
curl -sf http://localhost:8080/api/v1/kvm/stream/stats
|
||||
|
||||
# [4] Purge(验证干净卸载)
|
||||
sudo dpkg --purge kvm-server
|
||||
ls /etc/kvm/config.json 2>/dev/null && echo "FAIL: conffile not purged" || echo "PASS: purged"
|
||||
|
||||
# [5] 最终 clean install
|
||||
sudo dpkg -i /tmp/kvm-server_*.deb
|
||||
```
|
||||
|
||||
### 关键验证点
|
||||
|
||||
| 验证项 | 命令 | 期望结果 |
|
||||
|--------|------|---------|
|
||||
| 安装不卡住 | `time dpkg -i *.deb` | < 60s |
|
||||
| 服务启动 | `systemctl is-active kvm-server` | `active` |
|
||||
| USB gadget 软依赖 | `systemctl is-active kvm-usb-gadget \|\| true` | 失败不影响 kvm-server |
|
||||
| OCR timer 已移除 | `systemctl list-units \| grep ocr-snapshot` | 空 |
|
||||
| API 可达 | `curl localhost:8080/api/v1/kvm/stream/stats` | JSON 响应 |
|
||||
| Conffile 保留 | `ls /etc/kvm/config.json` after remove | 文件存在 |
|
||||
| Conffile 清除 | `ls /etc/kvm/config.json` after purge | 文件不存在 |
|
||||
|
||||
### 常见 systemd 包问题(已修复)
|
||||
|
||||
| 问题 | 症状 | 修复 |
|
||||
|------|------|------|
|
||||
| 双重启动 | postinst 有手动 `systemctl restart` + dh_installsystemd 各触发一次 | 删除手动 systemctl 区域 |
|
||||
| 双重 stop | prerm 有手动 `systemctl stop` + dh 的 `deb-systemd-invoke stop` | 删除手动 systemctl stop |
|
||||
| USB gadget 硬依赖 | `Requires=kvm-usb-gadget` → gadget 失败导致 kvm-server 不启动 | 改为 `Wants=` |
|
||||
| setup-usb-gadget 崩溃 | `set -e` + UDC 绑定失败 → 整个脚本 exit 1 | 移除 `set -e`,容错化 |
|
||||
| 过时 timer 噪音 | kvm-ocr-snapshot.timer 每秒触发 oneshot,native 模式不需要 | 从包中移除 |
|
||||
|
||||
## 目标设备
|
||||
- 硬件:NanoPC-T6 (RK3588, 8核 ARM64, 6 TOPS NPU)
|
||||
- 硬件:NanoPC-T6 (RK3588, 8核 ARM64, 8GB RAM, 6 TOPS NPU)
|
||||
- 开发与运行同机:代码编辑、编译、服务运行均在本机完成,资源共享
|
||||
- 系统:Debian/Ubuntu ARM64
|
||||
- Python:3.12.3
|
||||
- Go:1.22+
|
||||
- NPU:rknn-toolkit-lite2 2.3.2
|
||||
- 资源约束:8GB RAM 需在多服务间分配,CPU 大核(A76)给 Go+HID,小核(A55)给 PII+mem-bridge
|
||||
|
||||
## 已知限制
|
||||
|
||||
| 类别 | 状态 | 说明 |
|
||||
|------|------|------|
|
||||
| 安全模型 | ✅ 良好 | 三层防护,白名单模式,Unicode NFKC 归一化 |
|
||||
| 安全模型 | ✅ 良好 | 两层防护(视频遮蔽+网络拦截),白名单模式,Unicode NFKC 归一化 |
|
||||
| 鼠标优先 | ✅ 完成 | mouse_ops + LLM 提示词 + agent 翻译层 |
|
||||
| 多 UI 状态 | ✅ 完成 | screen_state.py 检测 BIOS/锁屏/睡眠/桌面 |
|
||||
| 测试覆盖 | ✅ 良好 | safety/screen_state/mouse_ops 单元测试 + integration |
|
||||
| 隐私截图 | ❌ 未实现 | Go 端 privacy mode 不影响截图内容 |
|
||||
| 测试覆盖 | ✅ 良好 | 单元测试 462 个;integration 需实体设备(--ignore 跳过) |
|
||||
| 隐私遮蔽 | ✅ 完成 | C 视频管道实时 NV12 黑色填充 PII 区域 (WF6 Phase 6A-2) |
|
||||
| test_integration 一致性 | ⚠️ 部分 | 清理代码已迁移到 mouse_ops,测试目标仍用原始组合键 |
|
||||
| Privacy/Audit 重组 | ✅ 完成 | PrivacyPage 5 Tab,AuditPage 2 Tab,新增 5 条 API 路由 |
|
||||
| GatewayMode 类型 | ✅ 完成 | boolean→'off'\|'audit'\|'redact',PiiOverlay 视觉区分 |
|
||||
| Agent HID 保护 | ✅ 完成 | Agent 运行时前端键鼠/粘贴被阻断 |
|
||||
| OCR 内联 | ✅ 完成 | CGo libkvm_ocr.so 内联调用, 延迟 800ms→60ms (WF6) |
|
||||
| NPU 调度 | ✅ 完成 | OCR(Core0/1) + Embedding(Core2), 无争用 (WF6) |
|
||||
| CPU 亲和性 | ✅ 完成 | A76(Go+HID) / A55(PII+mem-bridge) (WF6) |
|
||||
| DMA-buf 零拷贝 | ✅ 完成 | V4L2→MPP 零拷贝编码 (WF6) |
|
||||
|
||||
## 架构文档
|
||||
|
||||
详细子系统文档见 [docs/architecture/](docs/architecture/overview.md)。
|
||||
|
||||
+134
-91
@@ -1,138 +1,181 @@
|
||||
# DEVELOP
|
||||
# 开发指南
|
||||
|
||||
## 环境要求
|
||||
|
||||
- 开发机:Linux x86_64,Python 3.10+,Go 1.21+
|
||||
- 目标设备:NanoPC-T6 (RK3588),192.168.123.181,user: pi
|
||||
- SSH 访问:`sshpass -p "pi" ssh pi@192.168.123.181`
|
||||
- 目标设备: NanoPC-T6 (RK3588), Debian/Ubuntu ARM64
|
||||
- Go: 1.22+ (`/usr/local/go/bin/go`)
|
||||
- Python: 3.12+
|
||||
- Node: 18+ (前端构建)
|
||||
- NPU: rknn-toolkit-lite2 2.3.2
|
||||
|
||||
## 目录结构
|
||||
|
||||
```
|
||||
KVM-privacy/
|
||||
├── deps/ # git submodule 依赖
|
||||
│ ├── KVM/ # KVM 核心(Go + React)
|
||||
│ ├── embedding/ # mem-bridge 记忆/路由服务
|
||||
│ └── info-privacy-rs/ # Rust PII 检测脱敏
|
||||
├── deps/
|
||||
│ ├── KVM/ # Go KVM 核心 + React + C 视频管道
|
||||
│ │ ├── go/ # Go 后端 (28 个 internal 包)
|
||||
│ │ ├── web/ # React 前端
|
||||
│ │ ├── src/video/ # C 视频管道 (V4L2/RGA/MPP)
|
||||
│ │ ├── src/audit/ # C OCR 模块 (RKNN)
|
||||
│ │ ├── debian/ # kvm-server deb 包定义
|
||||
│ │ └── scripts/ # 构建脚本
|
||||
│ ├── embedding/ # Python mem-bridge
|
||||
│ └── info-privacy-rs/ # Rust PII 检测
|
||||
├── services/
|
||||
│ └── privacy_gateway/ # mitmproxy 隐私网关(Python)
|
||||
├── deploy/
|
||||
│ └── systemd/ # systemd 服务文件(部署到设备)
|
||||
├── scripts/ # 部署脚本
|
||||
├── docs/plans/ # 设计文档(归档用,不做日常维护)
|
||||
└── Makefile
|
||||
│ ├── kvm_agent/ # Python AI Agent (26 模块)
|
||||
│ └── privacy_gateway/ # Python mitmproxy 网关
|
||||
├── debian/ # 5 个额外 deb 包定义
|
||||
├── deploy/systemd/ # 8 个 systemd 服务文件
|
||||
├── scripts/ # 构建/部署脚本
|
||||
└── docs/architecture/ # 子系统架构文档
|
||||
```
|
||||
|
||||
## 设备上的实际路径
|
||||
## 常用命令
|
||||
|
||||
| 本地开发路径 | 设备部署路径 |
|
||||
|------------|------------|
|
||||
| `deps/KVM/` | `/home/pi/Desktop/KVM/` |
|
||||
| `deps/embedding/` | `/home/pi/Desktop/embed-db/` |
|
||||
| `deps/info-privacy-rs/` | `/home/pi/Desktop/info-privacy-rs/` |
|
||||
| `services/privacy_gateway/` | `/data/project/KVM-privacy/services/privacy_gateway/` |
|
||||
|
||||
## Privacy Gateway 开发
|
||||
|
||||
### 本地测试
|
||||
### Go 后端
|
||||
|
||||
```bash
|
||||
cd services/privacy_gateway
|
||||
pip install mitmproxy httpx cryptography
|
||||
python -c "from audit_logger import AuditLogger; l=AuditLogger('/tmp/test.db'); print('OK')"
|
||||
# 编译 (go.mod 在 deps/KVM/go/)
|
||||
cd deps/KVM/go && /usr/local/go/bin/go build ./...
|
||||
|
||||
# 运行测试
|
||||
cd deps/KVM/go && /usr/local/go/bin/go test ./... -count=1
|
||||
|
||||
# 仅运行 api 包测试
|
||||
cd deps/KVM/go && /usr/local/go/bin/go test ./internal/api/... -v
|
||||
|
||||
# Go vet
|
||||
cd deps/KVM/go && /usr/local/go/bin/go vet ./...
|
||||
```
|
||||
|
||||
### 服务文件说明
|
||||
|
||||
| 文件 | 职责 |
|
||||
|------|------|
|
||||
| `addon.py` | mitmproxy 核心:AI 域名匹配 → 拦截解密 → 脱敏 → 放行 |
|
||||
| `interceptor.py` | 请求内容提取(Content-Type 检查、文件解析) |
|
||||
| `upload_scanner.py` | 调用 info-privacy-rs API 完成脱敏 |
|
||||
| `cert_manager.py` | 设备 CA 证书生成,存储于 `/etc/kvm-privacy/ca/` |
|
||||
| `audit_logger.py` | SQLite append-only 审计,不记录原文 |
|
||||
| `ai_domains.txt` | 拦截域名白名单 |
|
||||
|
||||
## 部署流程
|
||||
|
||||
### 初次部署(在本机执行)
|
||||
### deb 包构建
|
||||
|
||||
```bash
|
||||
# 克隆项目(含 submodule)
|
||||
git clone --recurse-submodules http://edge-stack.synology.me:3000/qiurui/KVM-privacy.git
|
||||
cd KVM-privacy
|
||||
# kvm-server (Go + C + React)
|
||||
cd deps/KVM && PATH="/usr/local/go/bin:$PATH" bash scripts/build-deb.sh
|
||||
|
||||
# 部署 Privacy Gateway(安装 mitmproxy + 上传服务文件 + 生成 CA 证书)
|
||||
make deploy-gateway
|
||||
# kvm-server + kvm-ocr
|
||||
cd deps/KVM && PATH="/usr/local/go/bin:$PATH" bash scripts/build-deb.sh --ocr
|
||||
|
||||
# 部署 mem-bridge 服务(上传 systemd 文件并启动)
|
||||
make deploy-memory
|
||||
# Python 包 (kvm-agent, kvm-mitm)
|
||||
bash scripts/build-python-debs.sh
|
||||
|
||||
# 其他包
|
||||
dpkg-deb --build debian/kvm-privacy build/deb/
|
||||
dpkg-deb --build debian/kvm-bridge build/deb/
|
||||
dpkg-deb --build debian/kvm-meta build/deb/
|
||||
```
|
||||
|
||||
### 更新 Privacy Gateway 服务文件
|
||||
|
||||
```bash
|
||||
sshpass -p pi scp services/privacy_gateway/*.py pi@192.168.123.181:/data/project/KVM-privacy/services/privacy_gateway/
|
||||
sshpass -p pi ssh pi@192.168.123.181 "sudo systemctl restart privacy-gateway"
|
||||
### kvm-server 构建顺序
|
||||
```
|
||||
1. npm build (前端)
|
||||
2. cmake (视频 C 库, 需要 rockchip_mpp)
|
||||
3. cmake (OCR 共享库, 需要 rknn_api) ← 必须在 Go 之前
|
||||
4. go build (自动检测 .so → CGo flags)
|
||||
5. dpkg-buildpackage
|
||||
```
|
||||
|
||||
### systemd 服务管理
|
||||
### 前端
|
||||
|
||||
```bash
|
||||
# 查看所有 KVM-Privacy 相关服务状态
|
||||
cd deps/KVM/web && npm install && npm run build
|
||||
```
|
||||
|
||||
### Python Agent
|
||||
|
||||
```bash
|
||||
# 运行单次任务
|
||||
python3 -m kvm_agent run --task "打开记事本" --kvm-url http://localhost:8080
|
||||
|
||||
# Daemon 模式
|
||||
python3 -m kvm_agent daemon
|
||||
|
||||
# 单元测试 (不需要设备)
|
||||
cd services/kvm_agent && python3 -m pytest tests/ --ignore=tests/test_integration.py -v
|
||||
|
||||
# 集成测试 (需要实体设备)
|
||||
cd services/kvm_agent && python3 -m pytest tests/test_integration.py -v
|
||||
```
|
||||
|
||||
### 服务管理
|
||||
|
||||
```bash
|
||||
# 查看状态
|
||||
systemctl status kvm-server kvm-agent info-privacy mem-bridge-memory mem-bridge-router
|
||||
|
||||
# 日志
|
||||
journalctl -u kvm-server -f
|
||||
journalctl -u kvm-agent -f
|
||||
|
||||
# Makefile 快捷命令
|
||||
make status
|
||||
|
||||
# 查看指定服务日志
|
||||
make logs SVC=info-privacy-rs
|
||||
make logs SVC=mem-bridge-memory
|
||||
make logs SVC=privacy-gateway
|
||||
make logs SVC=kvm-server
|
||||
```
|
||||
|
||||
## mem-bridge 开发(deps/embedding)
|
||||
## deb 包验证流程
|
||||
|
||||
开发路径:`/data/rockchip/embedding/`,设备路径:`/home/pi/Desktop/embed-db/`
|
||||
每次修改 deb 相关文件后,在目标设备执行完整验证循环:
|
||||
|
||||
```bash
|
||||
cd /data/rockchip/embedding
|
||||
python -m venv venv && source venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
# 1. 构建
|
||||
cd deps/KVM && PATH="/usr/local/go/bin:$PATH" bash scripts/build-deb.sh
|
||||
|
||||
# 运行 memory-service(端口 8003)
|
||||
python server.py --service memory
|
||||
# 2. 静态检查
|
||||
DEB=$(ls build/deb/kvm-server_*.deb | tail -1)
|
||||
dpkg -I "$DEB" | grep "Depends\|Recommends"
|
||||
dpkg -c "$DEB" | grep "\.service\|usr/lib"
|
||||
|
||||
# 运行 router-service(端口 8002)
|
||||
python server.py --service router
|
||||
# 3. 拷贝到设备
|
||||
scp "$DEB" pi@192.168.123.181:/tmp/
|
||||
|
||||
# 4. 安装/卸载/重装/purge/reinstall 验证
|
||||
ssh pi@192.168.123.181 << 'EOF'
|
||||
sudo dpkg -i /tmp/kvm-server_*.deb && sleep 5
|
||||
systemctl is-active kvm-server && echo "PASS" || echo "FAIL"
|
||||
sudo dpkg --remove kvm-server
|
||||
sudo dpkg -i /tmp/kvm-server_*.deb
|
||||
EOF
|
||||
```
|
||||
|
||||
## info-privacy-rs 开发(deps/info-privacy-rs)
|
||||
|
||||
开发路径:`/data/rockchip/info-privacy-rs/`,设备路径:`/home/pi/Desktop/info-privacy-rs/`
|
||||
## info-privacy-rs 开发
|
||||
|
||||
```bash
|
||||
# 交叉编译(在 x86 开发机,目标 aarch64)
|
||||
cd /data/rockchip/info-privacy-rs
|
||||
# 交叉编译 (x86 → aarch64)
|
||||
cd deps/info-privacy-rs
|
||||
cargo build --release --target aarch64-unknown-linux-gnu
|
||||
|
||||
# 本地编译 (ARM64 设备上)
|
||||
cargo build --release --features "rknn,rga"
|
||||
|
||||
# 上传到设备
|
||||
scp target/aarch64-unknown-linux-gnu/release/info-privacy-rs pi@192.168.123.181:/home/pi/Desktop/info-privacy-rs/target/release/
|
||||
scp target/release/info-privacy-rs pi@192.168.123.181:/home/pi/Desktop/info-privacy-rs/
|
||||
|
||||
# 验证
|
||||
curl http://192.168.123.181:8000/api/v1/health
|
||||
```
|
||||
|
||||
在设备上验证:
|
||||
## mem-bridge 开发
|
||||
|
||||
```bash
|
||||
curl http://192.168.123.181:8001/health
|
||||
# → {"status":"ok","rknn":true}
|
||||
cd deps/embedding
|
||||
|
||||
# Memory 服务
|
||||
python server.py --service memory # port 8001
|
||||
|
||||
# Router 服务
|
||||
python server.py --service router # port 8002
|
||||
```
|
||||
|
||||
## KVM 隐私 API
|
||||
## 关键 API 端点
|
||||
|
||||
KVM 服务在 8080 端口新增以下隐私相关端点(需 Bearer Token 鉴权):
|
||||
|
||||
```
|
||||
GET/POST /api/v1/privacy/masking 视频 PII 遮罩开关
|
||||
GET /api/v1/privacy/bboxes 当前活跃遮罩区域(调试)
|
||||
GET /api/v1/privacy/stats 今日拦截统计
|
||||
GET /api/v1/privacy/audit 审计日志(分页)
|
||||
GET /api/v1/privacy/cert 下载 CA 证书
|
||||
GET /api/v1/memory-admin/stats 记忆使用统计
|
||||
GET /api/v1/memory-admin/sessions 活跃会话列表
|
||||
```
|
||||
| 服务 | 端点 | 用途 |
|
||||
|------|------|------|
|
||||
| KVM | GET /api/health | 健康检查 |
|
||||
| KVM | GET /api/kvm/screenshot | 截图 (JPEG) |
|
||||
| KVM | GET/POST /api/v1/privacy/mode | 隐私模式 |
|
||||
| KVM | GET /api/v1/kvm/stream/stats | 视频流统计 |
|
||||
| Agent | POST /api/v1/agent/tasks | 提交任务 |
|
||||
| Agent | GET /api/v1/agent/status | Agent 状态 |
|
||||
| Privacy | POST /api/v1/analyze | PII 检测 |
|
||||
| Privacy | POST /api/v1/redact | PII 脱敏 |
|
||||
|
||||
@@ -1,66 +1,87 @@
|
||||
# KVM-Privacy Hub
|
||||
|
||||
在 KVM-over-IP 系统基础上叠加三层隐私保护能力,运行于 NanoPC-T6 (RK3588)。
|
||||
在 KVM-over-IP 系统基础上叠加两层隐私保护能力(视频遮蔽 + 网络拦截),运行于 NanoPC-T6 (RK3588)。所有防护完全在 KVM 设备端完成,无需在被控机器上部署任何软件。
|
||||
|
||||
## 架构概览
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ KVM 控制台 (8080) │
|
||||
│ Go 单体 + React Web UI + WebRTC 推流 │
|
||||
└──────────────┬──────────────────────┬───────────────────┘
|
||||
│ │
|
||||
┌──────────▼──────────┐ ┌────────▼────────────────┐
|
||||
│ Phase 1: 视频遮罩 │ │ Phase 3: AI 路由层 │
|
||||
│ OCR 检测 → YUV黑框 │ │ mem-bridge 8003/8002 │
|
||||
│ /api/v1/privacy/ │ │ Qwen2-1.5B(Ollama本地) │
|
||||
└──────────┬──────────┘ └─────────────────────────┘
|
||||
│
|
||||
┌──────────▼──────────┐
|
||||
│ Phase 2: 隐私网关 │
|
||||
│ mitmproxy (8888) │
|
||||
│ 拦截 AI 平台上传 │
|
||||
└──────────┬──────────┘
|
||||
│
|
||||
┌──────────▼──────────┐
|
||||
│ info-privacy-rs │
|
||||
│ RKNN PII 检测/脱敏 │
|
||||
│ Rust 加速 (8001) │
|
||||
└─────────────────────┘
|
||||
┌───────────────────────────────────────────────────────────┐
|
||||
│ KVM WebUI (React) │
|
||||
│ http://localhost:8080 │
|
||||
└────────────────────────┬──────────────────────────────────┘
|
||||
│
|
||||
┌────────────────────────▼──────────────────────────────────┐
|
||||
│ KVM Server (Go, port 8080) │
|
||||
│ 视频管道(MPP) | HID | WebRTC | RBAC | 审计 | 隐私遮蔽 │
|
||||
└──┬──────────┬──────────┬──────────────────────┬───────────┘
|
||||
│ │ │ │
|
||||
┌──▼────┐ ┌──▼──────┐ ┌─▼───────────┐ ┌───────▼──────────┐
|
||||
│隐私网关│ │KVM Agent│ │info-privacy │ │ mem-bridge │
|
||||
│mitmproxy│ │Python │ │Rust/RKNN │ │Memory:8001 │
|
||||
│ :8888 │ │ :8890 │ │ :8000 │ │Router:8002 │
|
||||
└────────┘ └─────────┘ └────────────┘ └──────────────────┘
|
||||
```
|
||||
|
||||
## 端口规划
|
||||
|
||||
| 服务 | 端口 | 说明 |
|
||||
|------|------|------|
|
||||
| kvm-server | 8080 | KVM 主服务(systemd 管理) |
|
||||
| info-privacy-rs | 8001 | Rust/RKNN PII 检测与脱敏 |
|
||||
| mem-bridge memory | 8003 | 会话记忆服务 |
|
||||
| mem-bridge router | 8002 | AI 路由服务 |
|
||||
| privacy-gateway | 8888 | mitmproxy 代理网关 |
|
||||
| 服务 | 端口 | 包名 | 说明 |
|
||||
|------|------|------|------|
|
||||
| KVM Server | 8080 | kvm-server | Go 主服务 + React WebUI + WebRTC |
|
||||
| info-privacy-rs | 8000 | kvm-privacy | Rust/RKNN PII 检测与脱敏 |
|
||||
| mem-bridge memory | 8001 | kvm-bridge | FAISS 向量搜索 + 会话存储 |
|
||||
| mem-bridge router | 8002 | kvm-bridge | AI 路由 (OpenAI/Gemini/本地) |
|
||||
| Privacy Gateway | 8888 | kvm-mitm | mitmproxy 代理网关 |
|
||||
| KVM Agent | 8890 | kvm-agent | Python AI Agent daemon |
|
||||
|
||||
## 依赖项目(submodule)
|
||||
## Debian 包
|
||||
|
||||
| 包名 | 架构 | 说明 |
|
||||
|------|------|------|
|
||||
| kvm-server | arm64 | Go KVM 核心 + C 视频管道 |
|
||||
| kvm-agent | all | Python AI Agent |
|
||||
| kvm-privacy | arm64 | Rust PII 检测 |
|
||||
| kvm-bridge | all | mem-bridge 记忆服务 |
|
||||
| kvm-mitm | all | mitmproxy 隐私网关 |
|
||||
| kvm-meta | all | 元包 (依赖全部) |
|
||||
|
||||
## 依赖项目 (submodule)
|
||||
|
||||
| 路径 | 说明 |
|
||||
|------|------|
|
||||
| `deps/KVM` | KVM-over-IP 核心服务(Go + React) |
|
||||
| `deps/embedding` | mem-bridge 记忆服务与 AI 路由 |
|
||||
| `deps/info-privacy-rs` | Rust/RKNN 加速的 PII 检测脱敏服务 |
|
||||
| `deps/KVM` | KVM-over-IP 核心 (Go + React + C 视频管道) |
|
||||
| `deps/embedding` | mem-bridge 记忆服务与 AI 路由 (Python) |
|
||||
| `deps/info-privacy-rs` | Rust/RKNN PII 检测脱敏引擎 |
|
||||
|
||||
## 快速部署
|
||||
|
||||
```bash
|
||||
# 克隆
|
||||
git clone --recurse-submodules http://edge-stack.synology.me:3000/qiurui/KVM-privacy.git
|
||||
cd KVM-privacy
|
||||
|
||||
# 部署隐私网关 + systemd 服务
|
||||
make deploy-gateway
|
||||
# 构建 kvm-server deb 包
|
||||
cd deps/KVM && PATH="/usr/local/go/bin:$PATH" bash scripts/build-deb.sh
|
||||
|
||||
# 部署 mem-bridge 记忆服务
|
||||
make deploy-memory
|
||||
# 构建 Python deb 包
|
||||
bash scripts/build-python-debs.sh
|
||||
|
||||
# 查看所有服务状态
|
||||
make status
|
||||
# 部署到目标设备
|
||||
scp build/deb/*.deb pi@192.168.123.181:/tmp/
|
||||
ssh pi@192.168.123.181 'sudo dpkg -i /tmp/kvm-*.deb && sudo apt-get install -f'
|
||||
|
||||
# 查看服务状态
|
||||
ssh pi@192.168.123.181 'systemctl status kvm-server kvm-agent info-privacy'
|
||||
```
|
||||
|
||||
详见 [DEVELOP.md](DEVELOP.md)。
|
||||
## 文档
|
||||
|
||||
- [系统架构总览](docs/architecture/overview.md)
|
||||
- [KVM Server (Go)](docs/architecture/kvm-server.md)
|
||||
- [原生视频管道](docs/architecture/video-pipeline.md)
|
||||
- [KVM Agent (Python)](docs/architecture/kvm-agent.md)
|
||||
- [隐私系统](docs/architecture/privacy-system.md)
|
||||
- [记忆桥接](docs/architecture/memory-bridge.md)
|
||||
- [RK3588 硬件优化](docs/architecture/hardware.md)
|
||||
- [部署打包](docs/architecture/deployment.md)
|
||||
- [开发指南](DEVELOP.md)
|
||||
- [版本历史](RELEASE.md)
|
||||
|
||||
+67
-28
@@ -1,34 +1,73 @@
|
||||
# RELEASE
|
||||
# 版本历史
|
||||
|
||||
## v1.0.0 (2026-03-05)
|
||||
|
||||
生产就绪版本。完成两层隐私保护(视频遮蔽 + 网络拦截) + AI 自主操控 + RK3588 硬件加速。
|
||||
|
||||
### 核心功能
|
||||
|
||||
**KVM Server (Go)**
|
||||
- 140+ REST API 端点, WebRTC 视频推流, USB HID 控制
|
||||
- RBAC 权限控制 (admin/auditor/operator) + JWT + TOTP 双因素认证
|
||||
- SM3 哈希链审计日志 + syslog 转发
|
||||
- HLS 视频录制 + SSH 终端会话 + 虚拟 USB 存储
|
||||
|
||||
**原生视频管道 (C/C++ + CGo)**
|
||||
- V4L2→RGA→MPP 直接编码 (绕过 GStreamer)
|
||||
- DMA-buf 零拷贝 (1080p@30fps 节省 ~90MB/s)
|
||||
- 隐私遮蔽: NV12 黑色填充 PII 区域 (编码前实时处理)
|
||||
- 自适应码率 AIMD (PLI 驱动)
|
||||
- ForceIDR 即时关键帧
|
||||
|
||||
**AI 自主操控 (Python)**
|
||||
- 鼠标优先架构: 20 键白名单, 组合键全部鼠标翻译
|
||||
- Vision LLM 规划 (gpt-4o / gemini-2.5-pro)
|
||||
- OCR 感知 + 屏幕状态检测 (BIOS/锁屏/睡眠/桌面)
|
||||
- 安全防护: Unicode NFKC 归一化 + 危险命令 regex
|
||||
- 任务队列 daemon + 模板回放
|
||||
|
||||
**隐私系统**
|
||||
- 三模式: off / audit (红色边框) / redact (黑色遮蔽)
|
||||
- mitmproxy 网络拦截: AI 域名匹配 → PII 检测/脱敏
|
||||
- info-privacy-rs: Rust/RKNN 加速 PII 检测 (docx/xlsx/pdf/img)
|
||||
- 完全设备端防护: 无需被控机器部署任何软件
|
||||
- Go 端实时视频遮蔽: OCR→regex→bbox→C pipeline
|
||||
|
||||
**记忆系统**
|
||||
- FAISS 向量搜索 + BM25 混合评分
|
||||
- RKNNLite 嵌入 (multilingual-e5-small, 384-dim)
|
||||
- AI 路由: OpenAI/Gemini/Deepseek/本地 Ollama
|
||||
|
||||
### RK3588 硬件优化 (WF6)
|
||||
- NPU 三核调度: OCR(Core 0/1) + Embedding(Core 2), 消除争用
|
||||
- CPU 亲和性: A76(Go+HID+编码) / A55(PII+mem-bridge)
|
||||
- OCR 内联化: CGo libkvm_ocr.so, 延迟 800ms→60ms
|
||||
- DMA-buf: V4L2→MPP 零拷贝编码
|
||||
|
||||
### 打包与部署
|
||||
- 6 个 Debian 包: kvm-server, kvm-agent, kvm-privacy, kvm-bridge, kvm-mitm, kvm-meta
|
||||
- 8 个 systemd 服务, CPU 亲和性配置
|
||||
- 完整安装/卸载/升级验证循环
|
||||
|
||||
### 前端
|
||||
- React WebUI: 5 Tab 隐私页面, 2 Tab 审计页面
|
||||
- GatewayMode 三态: off/audit/redact
|
||||
- PiiOverlay: audit=红色边框, redact=黑色填充
|
||||
- Agent 运行时 HID 前端禁用
|
||||
- 用户管理卡片网格布局
|
||||
|
||||
---
|
||||
|
||||
## v0.1.0 (2026-02-28)
|
||||
|
||||
初始版本,KVM-Privacy Hub 三层隐私能力落地。
|
||||
初始原型版本。
|
||||
|
||||
### 新功能
|
||||
|
||||
**Phase 1 — 视频 PII 遮罩**
|
||||
- OCR 检测屏幕 PII(身份证号、手机号等),在 WebRTC 推流前对帧叠加黑色矩形
|
||||
- Go API:`/api/v1/privacy/masking`(开关控制)、`/api/v1/privacy/bboxes`(遮罩坐标)
|
||||
- `frame_redactor.go`:纯 Go 实现 YUV420 矩形黑化,零外部依赖
|
||||
|
||||
**Phase 2 — AI 隐私网关**
|
||||
- mitmproxy 代理(端口 8888),拦截上传至 Claude/GPT 等 AI 平台的附件
|
||||
- 调用 info-privacy-rs 自动脱敏(PII → `████`),内存处理不落盘
|
||||
- SQLite append-only 审计日志,记录 PII 类型和文件哈希,不存原文
|
||||
- CA 证书管理 + KVM API 下载端点
|
||||
|
||||
**Phase 3 — AI 路由层**
|
||||
- mem-bridge 记忆服务(端口 8003):多维向量记忆 + 事实抽取
|
||||
- mem-bridge 路由服务(端口 8002):本地 Qwen2-1.5B / 云端 Claude 自动路由
|
||||
- KVM API 反向代理:`/api/v1/memory-admin/` 管理端点
|
||||
|
||||
### 依赖升级
|
||||
|
||||
- 替换 Python 版 info-privacy 为 Rust/RKNN 加速版 info-privacy-rs(端口 8001)
|
||||
- mem-bridge memory-service 端口 8001 → 8003(避免与 info-privacy-rs 冲突)
|
||||
### 功能
|
||||
- Phase 1: 视频 PII 遮罩 (OCR 检测→YUV 黑框)
|
||||
- Phase 2: mitmproxy 隐私网关 (AI 平台上传拦截)
|
||||
- Phase 3: mem-bridge 记忆服务 + AI 路由
|
||||
|
||||
### 已知限制
|
||||
|
||||
- eth1 网卡未接线,透明代理模式暂不可用,需手动配置系统代理指向 192.168.123.x:8888
|
||||
- Ollama/Qwen2-1.5B 需手动安装(`scripts/install_ollama.sh`),约占 1GB 存储
|
||||
- YUV 遮罩基于 OCR 检测坐标,OCR 触发间隔内新出现的 PII 不实时遮罩
|
||||
- 遮罩基于 OCR 间隔触发, 非实时
|
||||
- 透明代理需手动配置
|
||||
- Ollama 需手动安装
|
||||
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
"""Root conftest — shared fixtures for all Python test suites."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
# Ensure services/ packages are importable from project root
|
||||
_SERVICES = Path(__file__).parent / "services"
|
||||
if str(_SERVICES) not in sys.path:
|
||||
sys.path.insert(0, str(_SERVICES))
|
||||
|
||||
TESTDATA_DIR = Path(__file__).parent / "testdata"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def testdata_dir() -> Path:
|
||||
"""Return the project-wide testdata/ directory."""
|
||||
return TESTDATA_DIR
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def pii_samples(testdata_dir: Path) -> dict:
|
||||
"""Load testdata/pii/samples.json as a dict."""
|
||||
return json.loads((testdata_dir / "pii" / "samples.json").read_text())
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def pii_positive(pii_samples: dict) -> dict[str, list[dict]]:
|
||||
"""Shortcut: positive PII samples keyed by entity type."""
|
||||
return pii_samples["positive"]
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def pii_negative(pii_samples: dict) -> dict[str, list[dict]]:
|
||||
"""Shortcut: negative (false-positive trap) samples keyed by entity type."""
|
||||
return pii_samples["negative"]
|
||||
|
||||
|
||||
# ── Workflow fixtures (testdata/workflow/) ────────────────────────────────
|
||||
|
||||
def _load_json_dir(directory: Path) -> dict[str, dict]:
|
||||
"""Load all .json files from a directory into a {stem: data} dict."""
|
||||
result = {}
|
||||
if directory.is_dir():
|
||||
for f in sorted(directory.glob("*.json")):
|
||||
result[f.stem] = json.loads(f.read_text())
|
||||
return result
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def workflow_scenes(testdata_dir: Path) -> dict[str, dict]:
|
||||
"""Load testdata/workflow/scenes/*.json as {name: data}."""
|
||||
return _load_json_dir(testdata_dir / "workflow" / "scenes")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def llm_responses(testdata_dir: Path) -> dict[str, dict]:
|
||||
"""Load testdata/workflow/llm_responses/*.json as {name: data}."""
|
||||
return _load_json_dir(testdata_dir / "workflow" / "llm_responses")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def memory_responses(testdata_dir: Path) -> dict[str, dict]:
|
||||
"""Load testdata/workflow/memory/*.json as {name: data}."""
|
||||
return _load_json_dir(testdata_dir / "workflow" / "memory")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def operation_templates(testdata_dir: Path) -> dict[str, dict]:
|
||||
"""Load testdata/workflow/templates/*.json as {name: data}."""
|
||||
return _load_json_dir(testdata_dir / "workflow" / "templates")
|
||||
Vendored
+8
@@ -0,0 +1,8 @@
|
||||
Package: kvm-agent
|
||||
Version: 2.0.0-1
|
||||
Architecture: arm64
|
||||
Maintainer: KVM-Privacy <noreply@kvm-privacy.local>
|
||||
Depends: libc6 (>= 2.31), libssl3 | libssl1.1
|
||||
Description: KVM AI Agent v2 (Rust)
|
||||
Self-contained AI agent daemon for autonomous KVM control.
|
||||
Mouse-first architecture with NPU privacy redaction.
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
case "$1" in
|
||||
configure)
|
||||
mkdir -p /etc/kvm-agent
|
||||
if [ ! -f /etc/kvm-agent/secrets.env ]; then
|
||||
cat > /etc/kvm-agent/secrets.env <<'EOF'
|
||||
KVM_JWT_TOKEN=
|
||||
LLM_API_KEY=
|
||||
KVM_AGENT_DB_PASS=changeme
|
||||
EOF
|
||||
chmod 600 /etc/kvm-agent/secrets.env
|
||||
chown root:root /etc/kvm-agent/secrets.env
|
||||
echo "WARNING: Edit /etc/kvm-agent/secrets.env with actual credentials"
|
||||
fi
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable kvm-agent.service 2>/dev/null || true
|
||||
systemctl start kvm-agent.service 2>/dev/null || true
|
||||
;;
|
||||
esac
|
||||
exit 0
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
case "$1" in
|
||||
purge)
|
||||
rm -rf /etc/kvm-agent
|
||||
rm -rf /run/kvm-agent
|
||||
rm -rf /var/lib/kvm-agent
|
||||
;;
|
||||
esac
|
||||
exit 0
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
case "$1" in
|
||||
remove|purge)
|
||||
systemctl stop kvm-agent.service 2>/dev/null || true
|
||||
systemctl disable kvm-agent.service 2>/dev/null || true
|
||||
;;
|
||||
esac
|
||||
exit 0
|
||||
@@ -0,0 +1,22 @@
|
||||
[Unit]
|
||||
Description=KVM AI Agent v2
|
||||
After=network.target embed-db.service
|
||||
Wants=embed-db.service
|
||||
StartLimitInterval=300
|
||||
StartLimitBurst=5
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=pi
|
||||
EnvironmentFile=-/etc/kvm-agent/secrets.env
|
||||
RuntimeDirectory=kvm-agent
|
||||
RuntimeDirectoryMode=0750
|
||||
ExecStart=/usr/sbin/kvm-agent daemon
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
MemoryMax=128M
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
+1
@@ -0,0 +1 @@
|
||||
/usr/sbin/kvm-agent
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
/etc/kvm-bridge/bridge.env
|
||||
Vendored
+8
@@ -0,0 +1,8 @@
|
||||
Package: kvm-bridge
|
||||
Version: 2.0.0-1
|
||||
Architecture: arm64
|
||||
Maintainer: KVM-Privacy <noreply@kvm-privacy.local>
|
||||
Depends: libc6
|
||||
Description: KVM embed-db session memory service (Rust)
|
||||
USearch vector search + SQLite session storage for AI agent memory.
|
||||
Single binary serving memory API (8003) and router API (8002).
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
if [ "$1" = "configure" ]; then
|
||||
# Create config directory and default env file
|
||||
mkdir -p /etc/kvm-bridge
|
||||
if [ ! -f /etc/kvm-bridge/bridge.env ]; then
|
||||
cat > /etc/kvm-bridge/bridge.env <<'EOF'
|
||||
BRIDGE_DATA_DIR=/var/lib/kvm-bridge
|
||||
EMBED_MODEL_PATH=/usr/share/kvm-bridge/models/embedder.onnx
|
||||
EOF
|
||||
chmod 644 /etc/kvm-bridge/bridge.env
|
||||
fi
|
||||
|
||||
# Create runtime data directory
|
||||
mkdir -p /var/lib/kvm-bridge
|
||||
chown pi:pi /var/lib/kvm-bridge
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable embed-db.service || true
|
||||
systemctl start embed-db.service 2>/dev/null || true
|
||||
fi
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
case "$1" in
|
||||
purge)
|
||||
rm -rf /etc/kvm-bridge
|
||||
rm -rf /var/lib/kvm-bridge
|
||||
;;
|
||||
esac
|
||||
exit 0
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
case "$1" in
|
||||
remove|purge)
|
||||
systemctl stop embed-db.service 2>/dev/null || true
|
||||
systemctl disable embed-db.service 2>/dev/null || true
|
||||
;;
|
||||
esac
|
||||
exit 0
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
BRIDGE_DATA_DIR=/var/lib/kvm-bridge
|
||||
EMBED_MODEL_PATH=/usr/share/kvm-bridge/models/embedder.onnx
|
||||
@@ -0,0 +1,28 @@
|
||||
[Unit]
|
||||
Description=Embed-DB Memory + Router Service (Rust, ports 8003+8002)
|
||||
After=network.target
|
||||
Wants=network.target
|
||||
StartLimitInterval=300
|
||||
StartLimitBurst=5
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=pi
|
||||
Group=pi
|
||||
EnvironmentFile=-/etc/kvm-bridge/bridge.env
|
||||
Environment=MEMORY_PORT=8003
|
||||
Environment=ROUTER_PORT=8002
|
||||
ExecStart=/usr/sbin/embed-db
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
MemoryMax=1G
|
||||
CPUAffinity=4 5 6 7
|
||||
LimitNOFILE=4096
|
||||
StateDirectory=kvm-bridge
|
||||
RuntimeDirectory=kvm-bridge
|
||||
RuntimeDirectoryMode=0750
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Vendored
+12
@@ -0,0 +1,12 @@
|
||||
Package: kvm-meta
|
||||
Version: 2.0.0-1
|
||||
Architecture: all
|
||||
Maintainer: KVM-Privacy <noreply@kvm-privacy.local>
|
||||
Depends: kvm-server (>= 1.0.0), kvm-agent (>= 2.0.0), kvm-privacy (>= 1.0.0)
|
||||
Recommends: kvm-bridge, kvm-npu, kvm-rkllm, kvm-mitm
|
||||
Description: KVM-Privacy complete product (metapackage)
|
||||
Installs the full KVM-Privacy product stack:
|
||||
kvm-server (Go KVM core), kvm-agent (AI automation),
|
||||
kvm-privacy (PII detection), kvm-mitm (privacy gateway),
|
||||
kvm-npu (NPU inference), kvm-rkllm (local LLM),
|
||||
and optionally kvm-bridge (session memory).
|
||||
Vendored
+10
@@ -0,0 +1,10 @@
|
||||
Package: kvm-mitm
|
||||
Version: 2.0.0-1
|
||||
Architecture: arm64
|
||||
Maintainer: KVM-Privacy <noreply@kvm-privacy.local>
|
||||
Depends: libc6
|
||||
Recommends: dnsmasq, iptables
|
||||
Description: KVM Privacy Gateway (Rust)
|
||||
Privacy MITM interceptor using hudsucker transparent proxy.
|
||||
Scans uploads for PII using info-privacy-rs.
|
||||
Includes LAN gateway scripts for NAT + transparent proxy.
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
case "$1" in
|
||||
configure)
|
||||
mkdir -p /etc/kvm-mitm
|
||||
|
||||
# CA certificate directory
|
||||
mkdir -p /etc/kvm-privacy/ca
|
||||
chmod 700 /etc/kvm-privacy/ca
|
||||
|
||||
# State directory for privacy mode persistence
|
||||
mkdir -p /var/lib/kvm-privacy
|
||||
chown root:root /var/lib/kvm-privacy
|
||||
|
||||
# Install dnsmasq config if not present
|
||||
mkdir -p /etc/kvm
|
||||
if [ ! -f /etc/kvm/dnsmasq-lan.conf ]; then
|
||||
cp /usr/lib/kvm-mitm/network/dnsmasq-lan.conf /etc/kvm/dnsmasq-lan.conf 2>/dev/null || true
|
||||
fi
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable kvm-mitm.service 2>/dev/null || true
|
||||
systemctl start kvm-mitm.service 2>/dev/null || true
|
||||
;;
|
||||
esac
|
||||
exit 0
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
case "$1" in
|
||||
purge)
|
||||
rm -rf /usr/lib/kvm-mitm
|
||||
rm -rf /etc/kvm-mitm
|
||||
rm -f /etc/kvm/dnsmasq-lan.conf
|
||||
rm -rf /var/lib/kvm-privacy
|
||||
;;
|
||||
esac
|
||||
exit 0
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
case "$1" in
|
||||
remove|purge)
|
||||
systemctl stop kvm-mitm.service 2>/dev/null || true
|
||||
systemctl disable kvm-mitm.service 2>/dev/null || true
|
||||
systemctl stop kvm-gateway.service 2>/dev/null || true
|
||||
systemctl disable kvm-gateway.service 2>/dev/null || true
|
||||
;;
|
||||
esac
|
||||
exit 0
|
||||
@@ -0,0 +1,20 @@
|
||||
[Unit]
|
||||
Description=KVM LAN Gateway (NAT + Transparent Proxy)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
Before=kvm-mitm.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/lib/kvm-mitm/network/lan-gateway.sh
|
||||
ExecStop=/bin/bash -c "pkill -F /run/kvm-dnsmasq.pid 2>/dev/null || true"
|
||||
Environment=KVM_WAN_IF=eth0
|
||||
Environment=KVM_LAN_IF=eth1
|
||||
Environment=KVM_LAN_IP=10.42.0.1
|
||||
Environment=KVM_LAN_SUBNET=10.42.0.0/24
|
||||
Environment=KVM_PROXY_PORT=8888
|
||||
Environment=KVM_DNSMASQ_CONF=/etc/kvm/dnsmasq-lan.conf
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,29 @@
|
||||
[Unit]
|
||||
Description=KVM Privacy Gateway (Rust, hudsucker proxy + axum API)
|
||||
After=network.target info-privacy.service kvm-gateway.service
|
||||
Wants=info-privacy.service
|
||||
StartLimitInterval=300
|
||||
StartLimitBurst=5
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
EnvironmentFile=-/etc/kvm-mitm/secrets.env
|
||||
Environment=PROXY_PORT=8888
|
||||
Environment=API_PORT=8889
|
||||
Environment=INFO_PRIVACY_URL=http://localhost:8001
|
||||
Environment=RKLLM_URL=http://localhost:8891
|
||||
Environment=STATE_FILE=/var/lib/kvm-privacy/state.json
|
||||
Environment=CA_DIR=/etc/kvm-privacy/ca
|
||||
RuntimeDirectory=kvm-mitm
|
||||
RuntimeDirectoryMode=0750
|
||||
ExecStart=/usr/sbin/privacy-gateway-rs
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
MemoryMax=256M
|
||||
LimitNOFILE=8192
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
/etc/npu-daemon/config.yaml
|
||||
Vendored
+9
@@ -0,0 +1,9 @@
|
||||
Package: kvm-npu
|
||||
Version: 1.0.0-1
|
||||
Architecture: arm64
|
||||
Maintainer: KVM-Privacy <noreply@kvm-privacy.local>
|
||||
Depends: libc6 (>= 2.31)
|
||||
Description: NPU Daemon - Centralized RKNN inference (OCR, face, redaction)
|
||||
Manages RKNN NPU cores on RK3588 for OCR detection,
|
||||
recognition, face detection, and privacy redaction.
|
||||
Requires librknnrt.so at /usr/lib/ (from rknn-toolkit-lite2).
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
if [ "$1" = "configure" ]; then
|
||||
systemctl daemon-reload
|
||||
systemctl enable npu-daemon.service
|
||||
systemctl start npu-daemon.service || true
|
||||
fi
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
case "$1" in
|
||||
purge)
|
||||
rm -rf /etc/npu-daemon
|
||||
;;
|
||||
esac
|
||||
exit 0
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
if [ "$1" = "remove" ] || [ "$1" = "purge" ]; then
|
||||
systemctl stop npu-daemon.service || true
|
||||
systemctl disable npu-daemon.service || true
|
||||
fi
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
server:
|
||||
host: "0.0.0.0"
|
||||
port: 8004
|
||||
|
||||
models:
|
||||
# PP-OCRv4 detection model (text region detection)
|
||||
ocr_det: /data/project/KVM-privacy/deps/KVM/models/ocr/ppocrv4_det.rknn
|
||||
# PP-OCRv4 recognition model (Chinese)
|
||||
ocr_rec_ch: /data/project/KVM-privacy/deps/KVM/models/ocr/ch/ppocrv4_rec.rknn
|
||||
# PP-OCRv4 recognition model (English, optional)
|
||||
ocr_rec_en: /data/project/KVM-privacy/deps/KVM/models/ocr/en/ppocrv4_rec.rknn
|
||||
# Character dictionaries
|
||||
ocr_dict_ch: /data/project/KVM-privacy/deps/KVM/models/ocr/ch/ppocr_keys.txt
|
||||
ocr_dict_en: /data/project/KVM-privacy/deps/KVM/models/ocr/en/ppocr_keys.txt
|
||||
# MediaPipe face detection
|
||||
face_det: /data/project/KVM-privacy/deps/info-privacy-rs/deps/mediapipe-rknn/models/face_detection_short_range_rk3588.rknn
|
||||
|
||||
# NPU core assignment (RK3588 has 3 NPU cores)
|
||||
# OCR uses Core 0+1, Face uses Core 0 (serial with OCR det)
|
||||
# Embedding uses Core 2 (managed by mem-bridge, not this daemon)
|
||||
cores:
|
||||
ocr_det: core0
|
||||
ocr_rec: core1
|
||||
face_det: core0 # shares with det, runs serially via scheduler
|
||||
|
||||
scheduler:
|
||||
max_concurrent: 2 # max parallel NPU tasks
|
||||
queue_size: 32 # per-priority queue depth
|
||||
@@ -0,0 +1,38 @@
|
||||
[Unit]
|
||||
Description=NPU Daemon - Centralized RKNN inference service
|
||||
After=network.target
|
||||
Wants=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=pi
|
||||
Group=pi
|
||||
ExecStart=/usr/sbin/npu-daemon --config /etc/npu-daemon/config.yaml
|
||||
|
||||
# Environment
|
||||
Environment=RUST_LOG=npu_daemon=info
|
||||
Environment=RKNN_LIB_DIR=/usr/lib
|
||||
|
||||
# CPU affinity: A55 cores (4-7) for NPU management overhead
|
||||
# NPU cores handle actual inference, CPU just coordinates
|
||||
CPUAffinity=4 5 6 7
|
||||
|
||||
# Resource limits
|
||||
LimitNOFILE=65535
|
||||
MemoryMax=512M
|
||||
|
||||
# Restart policy
|
||||
Restart=on-failure
|
||||
RestartSec=3
|
||||
StartLimitBurst=5
|
||||
StartLimitIntervalSec=60
|
||||
|
||||
# Security hardening
|
||||
NoNewPrivileges=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=read-only
|
||||
ReadWritePaths=/tmp
|
||||
PrivateTmp=true
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Vendored
+2
@@ -0,0 +1,2 @@
|
||||
/etc/kvm-privacy/pii_rules.yaml
|
||||
/etc/kvm-privacy/surnames.txt
|
||||
Vendored
+9
@@ -0,0 +1,9 @@
|
||||
Package: kvm-privacy
|
||||
Version: 1.0.0-1
|
||||
Architecture: arm64
|
||||
Maintainer: KVM-Privacy <noreply@kvm-privacy.local>
|
||||
Depends: kvm-npu (>= 1.0.0)
|
||||
Description: KVM Privacy PII detection service (Rust/RKNN)
|
||||
Document privacy scanning and redaction service built with Rust.
|
||||
Uses NPU Daemon for OCR and face detection on RK3588.
|
||||
Provides REST API for analyze/redact operations.
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
if [ "$1" = "configure" ]; then
|
||||
# Create configs symlinks (info-privacy-rs reads relative configs/ path)
|
||||
mkdir -p /usr/lib/kvm-privacy/configs
|
||||
ln -sf /etc/kvm-privacy/pii_rules.yaml /usr/lib/kvm-privacy/configs/pii_rules.yaml
|
||||
ln -sf /etc/kvm-privacy/surnames.txt /usr/lib/kvm-privacy/configs/surnames.txt
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable info-privacy.service
|
||||
systemctl start info-privacy.service || true
|
||||
fi
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
case "$1" in
|
||||
purge)
|
||||
rm -rf /usr/lib/kvm-privacy
|
||||
rm -rf /etc/kvm-privacy
|
||||
;;
|
||||
esac
|
||||
exit 0
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
if [ "$1" = "remove" ] || [ "$1" = "purge" ]; then
|
||||
systemctl stop info-privacy.service || true
|
||||
systemctl disable info-privacy.service || true
|
||||
fi
|
||||
@@ -0,0 +1,76 @@
|
||||
# PII 正则规则
|
||||
regex:
|
||||
id_card:
|
||||
pattern: '(?<!\d)([1-9]\d{5})(19|20)\d{2}(0[1-9]|1[0-2])(0[1-9]|[12]\d|3[01])\d{3}[\dX](?!\d)'
|
||||
security_level: high
|
||||
phone:
|
||||
pattern: '(?<!\d)((\+?86[-\s]?)?1[3-9]\d{9})(?!\d)'
|
||||
security_level: high
|
||||
bank_card:
|
||||
pattern: '(?<!\d)([1-9]\d{15,18})(?!\d)'
|
||||
security_level: high
|
||||
email:
|
||||
pattern: '\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b'
|
||||
security_level: medium
|
||||
license_plate:
|
||||
pattern: '[京津沪渝冀豫云辽黑湘皖鲁新苏浙赣鄂桂甘晋蒙陕吉闽贵粤川青藏琼宁夏][A-Z][A-Z0-9]{5,6}'
|
||||
security_level: medium
|
||||
|
||||
# 保密关键词
|
||||
confidential_keywords:
|
||||
- 机密
|
||||
- 绝密
|
||||
- 保密
|
||||
- 内部文件
|
||||
- 内部资料
|
||||
- CONFIDENTIAL
|
||||
- SECRET
|
||||
- TOP SECRET
|
||||
- RESTRICTED
|
||||
- FOR INTERNAL USE ONLY
|
||||
|
||||
# OCR + 人脸检测配置(RK3588 RKNN 子进程)
|
||||
#
|
||||
# 子模块代码路径(git submodule update --init 后自动可用):
|
||||
# deps/ocr-rknn/ → PP-OCR RKNN Python 推理代码
|
||||
# deps/mediapipe-rknn → MediaPipe 人脸检测 RKNN 代码
|
||||
#
|
||||
# RKNN 模型文件需单独下载:
|
||||
# OCR: cd deps/ocr-rknn && python download_models.py --lang ch
|
||||
# 然后用 convert_to_rknn.py 转换,或直接复制预置 .rknn 文件
|
||||
# Face: cd deps/mediapipe-rknn && bash models/download_models.sh face
|
||||
# 然后用 convert.py 转换
|
||||
#
|
||||
# det_model / rec_model / face.model 为空时,对应功能自动禁用。
|
||||
rknn:
|
||||
npu_daemon_url: "http://localhost:8004"
|
||||
ocr:
|
||||
det_model: /data/project/KVM-privacy/deps/info-privacy-rs/deps/ocr-rknn/models/PP-OCRv4/det/ppocrv4_det_ch_int8.rknn
|
||||
rec_model: /data/project/KVM-privacy/deps/info-privacy-rs/deps/ocr-rknn/models/PP-OCRv4/rec/ppocrv4_rec_ch_fp16.rknn
|
||||
dict_path: deps/ocr-rknn/dicts/ppocr_keys_v1.txt
|
||||
target: rk3588
|
||||
ppocr_dir: ""
|
||||
ocr_rknn_dir: deps/ocr-rknn
|
||||
python_bin: python3
|
||||
face:
|
||||
model: /data/project/KVM-privacy/deps/info-privacy-rs/deps/mediapipe-rknn/models/face_detection_short_range_rk3588.rknn
|
||||
mediapipe_src: deps/mediapipe-rknn
|
||||
python_bin: python3
|
||||
|
||||
# 中文姓名词典规则
|
||||
ner:
|
||||
name:
|
||||
security_level: medium
|
||||
surnames_file: /etc/kvm-privacy/surnames.txt
|
||||
address:
|
||||
security_level: medium
|
||||
triggers:
|
||||
- 省
|
||||
- 市
|
||||
- 区
|
||||
- 县
|
||||
- 街道
|
||||
- 路
|
||||
- 号
|
||||
- 小区
|
||||
- 村
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
# 常见中文姓氏
|
||||
李
|
||||
王
|
||||
张
|
||||
刘
|
||||
陈
|
||||
杨
|
||||
黄
|
||||
赵
|
||||
吴
|
||||
周
|
||||
徐
|
||||
孙
|
||||
马
|
||||
朱
|
||||
胡
|
||||
郭
|
||||
何
|
||||
高
|
||||
林
|
||||
郑
|
||||
谢
|
||||
罗
|
||||
梁
|
||||
宋
|
||||
唐
|
||||
许
|
||||
邓
|
||||
韩
|
||||
冯
|
||||
曹
|
||||
彭
|
||||
曾
|
||||
萧
|
||||
田
|
||||
董
|
||||
潘
|
||||
袁
|
||||
蔡
|
||||
蒋
|
||||
余
|
||||
于
|
||||
杜
|
||||
叶
|
||||
程
|
||||
苏
|
||||
魏
|
||||
吕
|
||||
丁
|
||||
任
|
||||
卢
|
||||
姚
|
||||
沈
|
||||
钟
|
||||
姜
|
||||
崔
|
||||
谭
|
||||
陆
|
||||
汪
|
||||
范
|
||||
廖
|
||||
石
|
||||
金
|
||||
韦
|
||||
贾
|
||||
夏
|
||||
付
|
||||
方
|
||||
邹
|
||||
熊
|
||||
白
|
||||
孟
|
||||
秦
|
||||
邱
|
||||
侯
|
||||
江
|
||||
尹
|
||||
薛
|
||||
闫
|
||||
段
|
||||
雷
|
||||
龙
|
||||
黎
|
||||
史
|
||||
陶
|
||||
贺
|
||||
顾
|
||||
毛
|
||||
郝
|
||||
龚
|
||||
邵
|
||||
万
|
||||
钱
|
||||
严
|
||||
覃
|
||||
武
|
||||
戴
|
||||
莫
|
||||
孔
|
||||
向
|
||||
汤
|
||||
@@ -0,0 +1,27 @@
|
||||
[Unit]
|
||||
Description=Info-Privacy-RS PII Detection Service (Rust/RKNN)
|
||||
After=network.target npu-daemon.service
|
||||
Wants=network.target npu-daemon.service
|
||||
StartLimitInterval=300
|
||||
StartLimitBurst=5
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=pi
|
||||
Group=pi
|
||||
ExecStart=/usr/sbin/info-privacy-rs
|
||||
RuntimeDirectory=kvm-privacy
|
||||
RuntimeDirectoryMode=0750
|
||||
Environment=PORT=8001
|
||||
Environment=NPU_DAEMON_URL=http://localhost:8004
|
||||
Environment=PII_RULES=/etc/kvm-privacy/pii_rules.yaml
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
MemoryMax=512M
|
||||
CPUAffinity=0 1
|
||||
LimitNOFILE=4096
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Vendored
+9
@@ -0,0 +1,9 @@
|
||||
Package: kvm-rkllm
|
||||
Version: 2.0.0-1
|
||||
Architecture: arm64
|
||||
Maintainer: KVM-Privacy <noreply@kvm-privacy.local>
|
||||
Depends: libc6
|
||||
Description: RKLLM NPU inference server (Rust, Qwen on RK3588)
|
||||
OpenAI-compatible chat API backed by RKLLM NPU runtime.
|
||||
Includes librkllmrt.so for RK3588 NPU acceleration.
|
||||
LLM model not included - set RKLLM_MODEL env var.
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
if [ "$1" = "configure" ]; then
|
||||
# Ensure librkllmrt.so is discoverable by the dynamic linker
|
||||
ldconfig 2>/dev/null || true
|
||||
|
||||
# Create model directory and link known model locations
|
||||
mkdir -p /var/lib/kvm-rkllm/models
|
||||
chown pi:pi /var/lib/kvm-rkllm /var/lib/kvm-rkllm/models
|
||||
|
||||
# Auto-link model from known locations if not already present
|
||||
MODEL_NAME="Qwen3-0.6B-rk3588-w8a8-opt-1-hybrid-ratio-0.5.rkllm"
|
||||
if [ ! -e "/var/lib/kvm-rkllm/models/$MODEL_NAME" ]; then
|
||||
for dir in /opt/fileguard/models /home/pi/models; do
|
||||
if [ -f "$dir/$MODEL_NAME" ]; then
|
||||
ln -sf "$dir/$MODEL_NAME" "/var/lib/kvm-rkllm/models/$MODEL_NAME"
|
||||
echo "kvm-rkllm: linked model from $dir"
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable rkllm-server.service
|
||||
systemctl start rkllm-server.service 2>/dev/null || true
|
||||
fi
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
case "$1" in
|
||||
purge)
|
||||
rm -rf /var/lib/kvm-rkllm
|
||||
;;
|
||||
esac
|
||||
exit 0
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
if [ "$1" = "remove" ] || [ "$1" = "purge" ]; then
|
||||
systemctl stop rkllm-server.service || true
|
||||
systemctl disable rkllm-server.service || true
|
||||
fi
|
||||
@@ -0,0 +1,119 @@
|
||||
# KVM-Privacy TOML Configuration
|
||||
# Place at /etc/kvm/kvm.toml (takes priority over config.json)
|
||||
|
||||
[server]
|
||||
host = "0.0.0.0"
|
||||
port = 8080
|
||||
log_level = "info" # debug, info, warn, error
|
||||
dev_mode = false
|
||||
|
||||
[server.tls]
|
||||
enabled = false
|
||||
auto_cert = true # auto-generate self-signed cert
|
||||
cert_path = "/etc/kvm/tls/cert.pem"
|
||||
key_path = "/etc/kvm/tls/key.pem"
|
||||
|
||||
[database]
|
||||
host = "127.0.0.1"
|
||||
port = 3306
|
||||
user = "kvm"
|
||||
password = ""
|
||||
database = "kvm"
|
||||
|
||||
[audit]
|
||||
enabled = true
|
||||
retention_days = 30
|
||||
syslog_enabled = false
|
||||
# syslog_address = "localhost:514"
|
||||
# syslog_protocol = "udp"
|
||||
|
||||
[auth]
|
||||
jwt_expiry_hours = 24
|
||||
totp_issuer = "KVM"
|
||||
max_login_attempts = 5
|
||||
lockout_minutes = 15
|
||||
password_min_length = 12
|
||||
|
||||
[hid]
|
||||
keyboard_device = "/dev/hidg0"
|
||||
mouse_device = "/dev/hidg1"
|
||||
enabled = true
|
||||
|
||||
[gstreamer]
|
||||
video_device = "auto"
|
||||
width = 1920
|
||||
height = 1080
|
||||
fps = 30
|
||||
codec = "h264"
|
||||
bitrate = 2000000
|
||||
pipeline_backend = "native" # "gstreamer" or "native" (RKMPP+RGA)
|
||||
|
||||
[ocr]
|
||||
enabled = true
|
||||
binary_path = "/usr/bin/kvm-ocr"
|
||||
model_path = "/etc/kvm/models/ocr"
|
||||
language = "ch"
|
||||
sample_interval_sec = 2
|
||||
block_critical = true
|
||||
|
||||
[recording]
|
||||
enabled = false
|
||||
storage_path = "/var/lib/kvm/recordings"
|
||||
retention_days = 30
|
||||
|
||||
[terminal]
|
||||
enabled = true
|
||||
max_sessions = 10
|
||||
|
||||
[wireguard]
|
||||
enabled = false
|
||||
interface = "wg0"
|
||||
listen_port = 51820
|
||||
|
||||
[storage]
|
||||
enabled = false
|
||||
media_dir = "/var/lib/kvm/media"
|
||||
|
||||
# ── LAN Gateway (transparent proxy) ───────────────────
|
||||
# Requires NanoPC-T6 dual Ethernet: eth0=WAN, eth1=LAN→controlled PC
|
||||
|
||||
[network]
|
||||
gateway_enabled = false # enable dual-NIC transparent gateway
|
||||
wan_interface = "eth0"
|
||||
lan_interface = "eth1"
|
||||
lan_subnet = "10.42.0.0/24"
|
||||
lan_gateway = "10.42.0.1"
|
||||
dhcp_range = "10.42.0.100,10.42.0.200,12h"
|
||||
transparent_proxy = true
|
||||
|
||||
# ── Sub-service discovery ──────────────────────────────
|
||||
|
||||
[services.privacy]
|
||||
enabled = true
|
||||
address = "127.0.0.1:8000"
|
||||
|
||||
[services.agent]
|
||||
enabled = true
|
||||
address = "127.0.0.1:8890"
|
||||
|
||||
[services.gateway]
|
||||
enabled = true
|
||||
mode = "audit" # off, audit, redact
|
||||
|
||||
[services.bridge]
|
||||
enabled = false
|
||||
memory_address = "127.0.0.1:8001"
|
||||
router_address = "127.0.0.1:8002"
|
||||
|
||||
# ── DDNS Dynamic DNS ──────────────────────────────────
|
||||
|
||||
[ddns]
|
||||
enabled = false
|
||||
provider = "duckdns" # cloudflare | duckdns | noip | custom
|
||||
domain = ""
|
||||
update_interval_sec = 300
|
||||
api_token = "" # Cloudflare API Token
|
||||
duckdns_token = "" # DuckDNS Token
|
||||
noip_user = "" # No-IP username
|
||||
noip_pass = "" # No-IP password
|
||||
webhook_url = "" # Custom: supports {ip} {domain} templates
|
||||
@@ -0,0 +1,13 @@
|
||||
# systemd-networkd: WAN interface (DHCP client)
|
||||
[Match]
|
||||
Name=eth0
|
||||
|
||||
[Network]
|
||||
DHCP=yes
|
||||
DNS=8.8.8.8
|
||||
DNS=1.1.1.1
|
||||
|
||||
[DHCP]
|
||||
UseDNS=true
|
||||
UseNTP=true
|
||||
RouteMetric=100
|
||||
@@ -0,0 +1,8 @@
|
||||
# systemd-networkd: LAN interface (static IP, no DHCP)
|
||||
[Match]
|
||||
Name=eth1
|
||||
|
||||
[Network]
|
||||
Address=10.42.0.1/24
|
||||
DHCPServer=false
|
||||
IPForward=yes
|
||||
@@ -0,0 +1,32 @@
|
||||
# dnsmasq-lan.conf — DHCP + DNS for KVM controlled PC on LAN (eth1)
|
||||
#
|
||||
# Controlled PC gets: IP 10.42.0.100-200, GW 10.42.0.1, DNS 10.42.0.1
|
||||
# dnsmasq forwards DNS upstream via /etc/resolv.conf
|
||||
|
||||
# Only listen on LAN interface
|
||||
interface=eth1
|
||||
bind-interfaces
|
||||
|
||||
# DHCP range: 10.42.0.100 to 10.42.0.200, lease 12 hours
|
||||
dhcp-range=10.42.0.100,10.42.0.200,12h
|
||||
|
||||
# Gateway = this device
|
||||
dhcp-option=3,10.42.0.1
|
||||
|
||||
# DNS = this device (dnsmasq also serves DNS)
|
||||
dhcp-option=6,10.42.0.1
|
||||
|
||||
# Domain
|
||||
domain=kvm.local
|
||||
|
||||
# Logging
|
||||
log-dhcp
|
||||
|
||||
# Do not read /etc/hosts
|
||||
no-hosts
|
||||
|
||||
# PID file
|
||||
pid-file=/run/kvm-dnsmasq.pid
|
||||
|
||||
# Do not daemonize (systemd manages lifecycle)
|
||||
keep-in-foreground
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/bin/bash
|
||||
# lan-gateway.sh — Configure NanoPC-T6 as a transparent gateway for the controlled PC.
|
||||
#
|
||||
# Network topology:
|
||||
# Internet ←─eth0─ NanoPC-T6 ─eth1─→ Controlled PC (DHCP 10.42.0.x)
|
||||
# │
|
||||
# iptables PREROUTING
|
||||
# REDIRECT :80,:443 → :8888
|
||||
# │
|
||||
# mitmproxy:8888
|
||||
# (transparent mode)
|
||||
#
|
||||
# Environment variables (or defaults):
|
||||
# KVM_WAN_IF — WAN interface (default: eth0)
|
||||
# KVM_LAN_IF — LAN interface (default: eth1)
|
||||
# KVM_LAN_IP — LAN gateway IP (default: 10.42.0.1)
|
||||
# KVM_LAN_SUBNET — LAN subnet (default: 10.42.0.0/24)
|
||||
# KVM_PROXY_PORT — mitmproxy listen port (default: 8888)
|
||||
# KVM_DNSMASQ_CONF — dnsmasq config path (default: /etc/kvm/dnsmasq-lan.conf)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
WAN="${KVM_WAN_IF:-eth0}"
|
||||
LAN="${KVM_LAN_IF:-eth1}"
|
||||
LAN_IP="${KVM_LAN_IP:-10.42.0.1}"
|
||||
LAN_SUBNET="${KVM_LAN_SUBNET:-10.42.0.0/24}"
|
||||
PROXY_PORT="${KVM_PROXY_PORT:-8888}"
|
||||
DNSMASQ_CONF="${KVM_DNSMASQ_CONF:-/etc/kvm/dnsmasq-lan.conf}"
|
||||
|
||||
log() { echo "[kvm-gateway] $(date '+%H:%M:%S') $*"; }
|
||||
|
||||
# ── Cleanup function for idempotent re-runs ──
|
||||
cleanup_rules() {
|
||||
log "Flushing existing kvm-gateway iptables rules..."
|
||||
iptables -t nat -D POSTROUTING -o "$WAN" -s "$LAN_SUBNET" -j MASQUERADE 2>/dev/null || true
|
||||
iptables -D FORWARD -i "$LAN" -o "$WAN" -j ACCEPT 2>/dev/null || true
|
||||
iptables -D FORWARD -i "$WAN" -o "$LAN" -m state --state RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || true
|
||||
iptables -t nat -D PREROUTING -i "$LAN" -p tcp --dport 80 -j REDIRECT --to-port "$PROXY_PORT" 2>/dev/null || true
|
||||
iptables -t nat -D PREROUTING -i "$LAN" -p tcp --dport 443 -j REDIRECT --to-port "$PROXY_PORT" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# ── 1. Configure LAN interface ──
|
||||
log "Configuring $LAN with IP $LAN_IP/24"
|
||||
ip addr flush dev "$LAN" 2>/dev/null || true
|
||||
ip addr add "$LAN_IP/24" dev "$LAN" 2>/dev/null || true
|
||||
ip link set "$LAN" up
|
||||
|
||||
# ── 2. Enable IP forwarding ──
|
||||
log "Enabling IP forwarding"
|
||||
sysctl -w net.ipv4.ip_forward=1 >/dev/null
|
||||
|
||||
# ── 3. Clean up old rules then apply NAT ──
|
||||
cleanup_rules
|
||||
|
||||
log "Setting up NAT ($LAN_SUBNET → $WAN)"
|
||||
iptables -t nat -A POSTROUTING -o "$WAN" -s "$LAN_SUBNET" -j MASQUERADE
|
||||
iptables -A FORWARD -i "$LAN" -o "$WAN" -j ACCEPT
|
||||
iptables -A FORWARD -i "$WAN" -o "$LAN" -m state --state RELATED,ESTABLISHED -j ACCEPT
|
||||
|
||||
# ── 4. Transparent proxy redirect (HTTP/HTTPS → mitmproxy) ──
|
||||
log "Redirecting HTTP/HTTPS from $LAN to mitmproxy :$PROXY_PORT"
|
||||
iptables -t nat -A PREROUTING -i "$LAN" -p tcp --dport 80 -j REDIRECT --to-port "$PROXY_PORT"
|
||||
iptables -t nat -A PREROUTING -i "$LAN" -p tcp --dport 443 -j REDIRECT --to-port "$PROXY_PORT"
|
||||
|
||||
# ── 5. Start dnsmasq for DHCP + DNS on LAN ──
|
||||
if [ -f "$DNSMASQ_CONF" ]; then
|
||||
# Kill any existing dnsmasq on LAN interface
|
||||
pkill -f "dnsmasq.*$DNSMASQ_CONF" 2>/dev/null || true
|
||||
sleep 0.5
|
||||
log "Starting dnsmasq (DHCP + DNS) on $LAN"
|
||||
dnsmasq --conf-file="$DNSMASQ_CONF" --pid-file=/run/kvm-dnsmasq.pid
|
||||
else
|
||||
log "WARNING: dnsmasq config not found at $DNSMASQ_CONF — DHCP not started"
|
||||
fi
|
||||
|
||||
log "Gateway setup complete. Controlled PC should get IP via DHCP on $LAN."
|
||||
@@ -1,16 +1,25 @@
|
||||
[Unit]
|
||||
Description=Info-Privacy-RS PII Detection Service (Rust/RKNN)
|
||||
After=network.target
|
||||
Wants=network.target
|
||||
After=network.target npu-daemon.service
|
||||
Wants=network.target npu-daemon.service
|
||||
StartLimitInterval=300
|
||||
StartLimitBurst=5
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=pi
|
||||
Group=pi
|
||||
WorkingDirectory=/home/pi/Desktop/info-privacy-rs
|
||||
ExecStart=/home/pi/Desktop/info-privacy-rs/target/release/info-privacy-rs
|
||||
ExecStart=/usr/sbin/info-privacy-rs
|
||||
RuntimeDirectory=kvm-privacy
|
||||
RuntimeDirectoryMode=0750
|
||||
Environment=PORT=8001
|
||||
Environment=NPU_DAEMON_URL=http://localhost:8004
|
||||
Environment=PII_RULES=/etc/kvm-privacy/pii_rules.yaml
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
RestartSec=10
|
||||
MemoryMax=512M
|
||||
CPUAffinity=0 1
|
||||
LimitNOFILE=4096
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
# /etc/systemd/journald.conf.d/kvm.conf
|
||||
#
|
||||
# KVM-Privacy journald configuration for NanoPC-T6 (eMMC protection).
|
||||
# Volatile mode keeps logs in RAM only — protects eMMC from write wear.
|
||||
# Ring buffer capped at 128M, entries retained for 7 days max.
|
||||
#
|
||||
# Install: cp deploy/systemd/journald-kvm.conf /etc/systemd/journald.conf.d/kvm.conf
|
||||
# Reload: systemctl restart systemd-journald
|
||||
|
||||
[Journal]
|
||||
Storage=volatile
|
||||
RuntimeMaxUse=128M
|
||||
RuntimeKeepFree=64M
|
||||
RuntimeMaxFileSize=32M
|
||||
MaxRetentionSec=7day
|
||||
RateLimitIntervalSec=5s
|
||||
RateLimitBurst=1000
|
||||
Compress=yes
|
||||
@@ -0,0 +1,20 @@
|
||||
[Unit]
|
||||
Description=KVM AI Agent v2
|
||||
After=network.target mariadb.service
|
||||
Wants=mem-bridge-memory.service
|
||||
StartLimitInterval=300
|
||||
StartLimitBurst=5
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=pi
|
||||
EnvironmentFile=/etc/kvm-agent/secrets.env
|
||||
ExecStart=/usr/bin/kvm-agent daemon
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
MemoryMax=512M
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,20 @@
|
||||
[Unit]
|
||||
Description=KVM LAN Gateway (NAT + Transparent Proxy)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
Before=privacy-gateway.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/data/project/KVM-privacy/deploy/network/lan-gateway.sh
|
||||
ExecStop=/bin/bash -c "pkill -F /run/kvm-dnsmasq.pid 2>/dev/null || true"
|
||||
Environment=KVM_WAN_IF=eth0
|
||||
Environment=KVM_LAN_IF=eth1
|
||||
Environment=KVM_LAN_IP=10.42.0.1
|
||||
Environment=KVM_LAN_SUBNET=10.42.0.0/24
|
||||
Environment=KVM_PROXY_PORT=8888
|
||||
Environment=KVM_DNSMASQ_CONF=/etc/kvm/dnsmasq-lan.conf
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,21 @@
|
||||
[Unit]
|
||||
Description=KVM Privacy MITM Interceptor
|
||||
After=network.target mariadb.service info-privacy.service
|
||||
Wants=info-privacy.service
|
||||
StartLimitInterval=300
|
||||
StartLimitBurst=5
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
EnvironmentFile=/etc/kvm-privacy/secrets.env
|
||||
ExecStart=/usr/bin/kvm-mitm
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
MemoryMax=768M
|
||||
LimitNOFILE=8192
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,17 +1,26 @@
|
||||
[Unit]
|
||||
Description=Mem-Bridge Memory Service
|
||||
Description=Embed-DB Memory + Router Service (Rust, ports 8001+8002)
|
||||
After=network.target
|
||||
Wants=network.target
|
||||
StartLimitInterval=300
|
||||
StartLimitBurst=5
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=pi
|
||||
Group=pi
|
||||
WorkingDirectory=/home/pi/Desktop/embed-db
|
||||
EnvironmentFile=-/home/pi/Desktop/embed-db/.env
|
||||
ExecStart=/home/pi/Desktop/embed-db/venv/bin/python server.py --service memory
|
||||
EnvironmentFile=-/etc/kvm-bridge/bridge.env
|
||||
Environment=MEMORY_PORT=8003
|
||||
Environment=ROUTER_PORT=8002
|
||||
ExecStart=/usr/sbin/embed-db
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
RestartSec=10
|
||||
MemoryMax=1G
|
||||
CPUAffinity=2 3
|
||||
LimitNOFILE=4096
|
||||
StateDirectory=kvm-bridge
|
||||
RuntimeDirectory=kvm-bridge
|
||||
RuntimeDirectoryMode=0750
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
[Unit]
|
||||
Description=Mem-Bridge Router Service
|
||||
After=network.target mem-bridge-memory.service
|
||||
Wants=network.target
|
||||
Requires=mem-bridge-memory.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=pi
|
||||
Group=pi
|
||||
WorkingDirectory=/home/pi/Desktop/embed-db
|
||||
EnvironmentFile=-/home/pi/Desktop/embed-db/.env
|
||||
ExecStart=/home/pi/Desktop/embed-db/venv/bin/python server.py --service router
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,22 +0,0 @@
|
||||
[Unit]
|
||||
Description=KVM Privacy Gateway (mitmproxy)
|
||||
After=network.target info-privacy.service
|
||||
Wants=network.target info-privacy.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
WorkingDirectory=/data/project/KVM-privacy/services/privacy_gateway
|
||||
ExecStart=/usr/local/bin/mitmdump \
|
||||
--mode regular \
|
||||
--listen-host 0.0.0.0 \
|
||||
--listen-port 8888 \
|
||||
--ssl-insecure \
|
||||
-s /data/project/KVM-privacy/services/privacy_gateway/addon.py
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,23 @@
|
||||
[Unit]
|
||||
Description=RKLLM Server (Rust, Qwen3-0.6B on RK3588 NPU, port 8891)
|
||||
Documentation=https://github.com/airockchip/rknn-llm
|
||||
After=network.target
|
||||
Before=embed-db.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=pi
|
||||
Environment=RKLLM_MODEL=/var/lib/kvm-rkllm/models/Qwen3-0.6B-rk3588-w8a8-opt-1-hybrid-ratio-0.5.rkllm
|
||||
Environment=RKLLM_LIB=/usr/lib/kvm-rkllm/librkllmrt.so
|
||||
Environment=RKLLM_PORT=8891
|
||||
ExecStart=/usr/sbin/rkllm-server
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
CPUAffinity=0 1
|
||||
MemoryMax=2G
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rkllm-server
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Vendored
+1
-1
Submodule deps/KVM updated: 4da8090d0f...876a345c42
Vendored
+1
-1
Submodule deps/embedding updated: eb29ab36b4...d974f89337
Vendored
+1
-1
Submodule deps/info-privacy-rs updated: b1ed721f37...6b93cbcadd
@@ -0,0 +1,25 @@
|
||||
# Core services are deployed via DEB packages and systemd.
|
||||
# This file only runs the workflow dashboard for monitoring.
|
||||
services:
|
||||
workflow-dashboard:
|
||||
build:
|
||||
context: ./tools/workflow-dashboard
|
||||
ports:
|
||||
- "9099:9099"
|
||||
volumes:
|
||||
- ./tools/workflow-dashboard/testdata/results:/app/testdata/results
|
||||
environment:
|
||||
- RESULTS_DIR=testdata/results
|
||||
- KVM_URL=http://host.docker.internal:8080
|
||||
- KVM_JWT_TOKEN=${KVM_JWT_TOKEN:-}
|
||||
- PRIVACY_RS_URL=http://host.docker.internal:8001
|
||||
- AGENT_URL=http://host.docker.internal:8890
|
||||
- NPU_DAEMON_URL=http://host.docker.internal:8004
|
||||
- RKLLM_URL=http://host.docker.internal:8891
|
||||
- LLM_API_KEY=${LLM_API_KEY:-}
|
||||
- LLM_MODEL=${LLM_MODEL:-gpt-4o}
|
||||
- no_proxy=host.docker.internal,localhost,127.0.0.1
|
||||
- NO_PROXY=host.docker.internal,localhost,127.0.0.1
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
restart: unless-stopped
|
||||
@@ -0,0 +1,801 @@
|
||||
# KVM-Privacy AI Agent 架构评估:本地计算卸载与 Token 优化
|
||||
|
||||
> 评估日期:2026-03-06
|
||||
> 目标硬件:NanoPC-T6 (RK3588, 8 核 ARM64, 6 TOPS NPU)
|
||||
> 当前代码库:`services/kvm_agent/`
|
||||
|
||||
---
|
||||
|
||||
## 目录
|
||||
|
||||
1. [现有架构分析](#1-现有架构分析)
|
||||
2. [Playwright 工作流模式借鉴](#2-playwright-工作流模式借鉴)
|
||||
3. [最新 LLM 能力评估 (2025-2026)](#3-最新-llm-能力评估-2025-2026)
|
||||
4. [本地计算卸载策略](#4-本地计算卸载策略)
|
||||
5. [记忆系统 Token 优化](#5-记忆系统-token-优化)
|
||||
6. [Token 消耗定量分析](#6-token-消耗定量分析)
|
||||
7. [架构升级建议](#7-架构升级建议)
|
||||
8. [实施路线图](#8-实施路线图)
|
||||
|
||||
---
|
||||
|
||||
## 1. 现有架构分析
|
||||
|
||||
### 1.1 当前数据流
|
||||
|
||||
```
|
||||
截图 (JPEG ~50-150KB)
|
||||
↓
|
||||
OCR (本地 RKNN NPU, ~60ms) → SceneGraph (UIElement[])
|
||||
↓
|
||||
screen_state.py 检测 → PCState (SLEEP/BIOS/LOCK/DESKTOP/APP)
|
||||
↓
|
||||
hybrid_planner.py 三路分发:
|
||||
├── Path 1: TemplateStore (FAISS 语义匹配) → 本地重放 (0 token)
|
||||
├── Path 2: 本地 RKLLM (Qwen2.5-1.5B, text-only) → 简单任务
|
||||
└── Path 3: 云端 LLM (gemini-2.5-pro) → 复杂任务 (高 token)
|
||||
↓
|
||||
Action 执行 (HID: 鼠标/键盘)
|
||||
↓
|
||||
Validator 验证 (pixel_diff + OCR)
|
||||
↓
|
||||
Memory 记录 (mem-bridge: FAISS + SQLite)
|
||||
```
|
||||
|
||||
### 1.2 现有代码结构映射
|
||||
|
||||
| 模块 | 文件 | 职责 | 计算位置 |
|
||||
|------|------|------|----------|
|
||||
| 感知 | `perception.py` | OCR → SceneGraph → UIElement | 本地 NPU |
|
||||
| 状态检测 | `screen_state.py` | 截图哈希 + OCR 关键词 → PCState | 本地 CPU |
|
||||
| LLM 规划 | `llm_planner.py` | 截图+任务 → Action JSON | 云端 |
|
||||
| 混合规划 | `hybrid_planner.py` | 模板/本地/云端三路分发 | 混合 |
|
||||
| 验证 | `validator.py` | pixel_diff + OCR 验证 | 本地 CPU |
|
||||
| 模板 | `template_store.py` | FAISS 语义匹配 + 重放 | 本地 (RKNN embedding) |
|
||||
| 记忆 | `memory_client.py` | mem-bridge REST 客户端 | 本地 (FAISS) |
|
||||
| 执行 | `agent.py` | perceive-decide-act 循环 | 混合 |
|
||||
|
||||
### 1.3 现有架构优势
|
||||
|
||||
项目已具备良好的混合架构基础:
|
||||
|
||||
1. **三路分发机制** (`hybrid_planner.py`): 模板 → 本地 LLM → 云端 LLM 的降级链
|
||||
2. **OCR 优先坐标** (`agent.py:_click_with_retry`): 利用本地 OCR 精确定位,减少 LLM 坐标猜测
|
||||
3. **模板重放** (`template_store.py`): 已知任务零 token 执行
|
||||
4. **SceneGraph 结构化** (`perception.py:to_text_summary`): 区域分组、大小信息,减少 LLM 理解成本
|
||||
|
||||
### 1.4 已识别瓶颈
|
||||
|
||||
| 瓶颈 | 影响 | 量化 |
|
||||
|------|------|------|
|
||||
| 每步发送完整截图 | 高 token 消耗 | ~1290 tokens/image (1024x1024) |
|
||||
| system prompt 每次重发 | 固定 token 开销 | ~2000 tokens/turn |
|
||||
| scene_text 未压缩 | 冗余信息 | 30 个元素 ~1500 tokens |
|
||||
| 无状态 LLM 调用 | 历史仅保留最近 3 条 | 丢失上下文,重复规划 |
|
||||
| 验证-重试循环 | 失败时多次 LLM 调用 | 最坏 3x token 消耗 |
|
||||
|
||||
---
|
||||
|
||||
## 2. Playwright 工作流模式借鉴
|
||||
|
||||
### 2.1 Playwright 核心设计理念
|
||||
|
||||
Playwright 作为 2025-2026 年最主流的浏览器自动化框架,其设计理念对 KVM Agent 有重要参考价值:
|
||||
|
||||
**自动等待 (Auto-Waiting)**
|
||||
|
||||
Playwright 在执行每个操作前自动进行 actionability 检查:元素可见、可交互、稳定(无动画)、无遮挡。KVM Agent 当前仅使用 `asyncio.sleep()` 固定延迟,缺乏自适应等待。
|
||||
|
||||
借鉴:引入 **OCR 稳定性检测** — 连续两次 OCR 结果一致才认为屏幕稳定,替代固定延迟。
|
||||
|
||||
**重试机制 (Retry)**
|
||||
|
||||
Playwright 2026 的企业级框架推荐 `trace: 'on-first-retry'` 模式 — 首次重试时记录完整 trace 以便调试,而非每次都记录。
|
||||
|
||||
借鉴:KVM Agent 的 `_click_with_retry` 已实现 3 次重试,但缺少 **重试证据记录**。应在首次重试时保存截图+OCR 状态,用于后续分析和模板优化。
|
||||
|
||||
**Accessibility Tree vs 截图**
|
||||
|
||||
Playwright MCP (2026) 使用浏览器的 accessibility tree 而非截图进行元素定位,token 消耗降低 4x(114K → 27K tokens)。
|
||||
|
||||
关键差异:KVM Agent 无法访问 DOM/accessibility tree,只能通过 OCR 构建 SceneGraph。但 **SceneGraph 本质上就是 KVM 的 "accessibility tree"**,应进一步结构化以减少 LLM 理解成本。
|
||||
|
||||
### 2.2 ActionEngine 状态机模式 (2026 最新研究)
|
||||
|
||||
Georgia Tech 与 Microsoft Research 联合发表的 [ActionEngine](https://arxiv.org/abs/2602.20502) 提出了从 **响应式** (每步调用 LLM) 到 **编程式** (状态机驱动) 的范式转换:
|
||||
|
||||
- **Crawling Agent**: 离线探索 GUI,构建可更新的状态机记忆
|
||||
- **Execution Agent**: 利用状态机记忆合成完整 Python 程序执行任务
|
||||
- **效果**: WebArena Reddit 任务 95% 成功率,平均仅 **1 次 LLM 调用**,成本降低 **11.8x**
|
||||
|
||||
**对 KVM Agent 的启示**:
|
||||
|
||||
当前 `template_store.py` 的模板重放已实现类似思路,但模板是线性序列,缺乏分支和条件跳转。将模板升级为 **状态机** 可显著减少 LLM 调用:
|
||||
|
||||
```python
|
||||
# 当前: 线性模板
|
||||
steps = [click("搜索"), type("notepad"), click("记事本")]
|
||||
|
||||
# 升级: 状态机模板
|
||||
states = {
|
||||
"start": {"action": click("搜索"), "next": "search_open"},
|
||||
"search_open": {
|
||||
"verify": ocr_contains("输入搜索内容"),
|
||||
"action": type("notepad"),
|
||||
"next": "results_shown",
|
||||
"fallback": "start", # 搜索框未打开,重试
|
||||
},
|
||||
"results_shown": {
|
||||
"verify": ocr_contains("记事本"),
|
||||
"action": click("记事本"),
|
||||
"next": "done",
|
||||
"fallback": "search_open", # 结果未出现,重试搜索
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
### 2.3 可借鉴模式总结
|
||||
|
||||
| Playwright 模式 | KVM Agent 当前实现 | 建议改进 |
|
||||
|-----------------|-------------------|----------|
|
||||
| Auto-Wait | 固定 sleep | OCR 稳定性检测 |
|
||||
| Retry + Trace | 3 次重试,无 trace | 首次重试保存证据 |
|
||||
| Element Selector | OCR find_by_text (模糊匹配) | 增加位置+类型联合匹配 |
|
||||
| Assertion retry | validate_click 单次 | 异步轮询等待条件满足 |
|
||||
| 状态机 (ActionEngine) | 线性模板 | 带条件分支的状态机模板 |
|
||||
| Accessibility tree | SceneGraph | 进一步结构化 + 压缩 |
|
||||
|
||||
---
|
||||
|
||||
## 3. 最新 LLM 能力评估 (2025-2026)
|
||||
|
||||
### 3.1 主流模型对比 (2026-03)
|
||||
|
||||
| 模型 | 输入价格 (/M tokens) | 输出价格 (/M tokens) | Vision | Agent 能力 | 适用场景 |
|
||||
|------|---------------------|---------------------|--------|-----------|---------|
|
||||
| GPT-5.4 | $2.50 | $10.00 | 强 | 强 | 复杂推理+视觉 |
|
||||
| Claude Opus 4.6 | $15.00 | $75.00 | 强 | 极强 (Computer Use) | 代码+长推理 |
|
||||
| Gemini 2.5 Pro | $1.25 | $10.00 | 强 | 强 | **性价比首选** |
|
||||
| GPT-4o | $2.50 | $10.00 | 强 | 中 | 通用视觉任务 |
|
||||
| Claude Sonnet 4.5 | $3.00 | $15.00 | 强 | 强 | 平衡性价比 |
|
||||
|
||||
**当前配置 `gemini-2.5-pro` 是正确的性价比选择**,输入价格仅 $1.25/M,适合高频 agent 调用。
|
||||
|
||||
### 3.2 Computer Use 能力进展
|
||||
|
||||
2025-2026 年 LLM 的 Computer Use 能力取得重大进展:
|
||||
|
||||
- **Claude Computer Use**: Anthropic 在 2025 年推出 Chrome 扩展,允许 Claude 直接控制浏览器。Claude Opus 4.5/4.6 在 agentic tool use 方面表现最强
|
||||
- **GPT-5.x 推理突破**: GPT-5.2 突破 ARC-AGI-1 的 90% 门槛;GPT-5.4 在编程和专业工作流方面定位高端
|
||||
- **结构化输出**: 所有主流模型现在原生支持 JSON Schema 强制输出,KVM Agent 的 `_parse_action` JSON 解析可更可靠
|
||||
- **视觉理解提升**: 对 UI 元素、按钮、文本的识别精度显著提升,理论上可减少 OCR 依赖
|
||||
|
||||
### 3.3 对 KVM Agent 的影响
|
||||
|
||||
**短期可用改进**:
|
||||
1. **启用结构化输出** (Structured Output / JSON mode): `gemini-2.5-pro` 支持 `response_format: json_object`,可消除 JSON 解析失败
|
||||
2. **减少 system prompt**: 利用模型内在的 Computer Use 知识,system prompt 可精简 30-50%
|
||||
3. **图像分辨率控制**: 发送低分辨率截图 (512x384 而非 1920x1080) 可减少 ~75% 图像 token
|
||||
|
||||
**中期可探索方向**:
|
||||
1. **原生 Computer Use API**: 如果 Anthropic 开放 Computer Use API 的坐标输出,可直接替代 OCR + LLM 双调用
|
||||
2. **多模态本地模型**: Qwen2.5-VL-3B 已可在 RK3588 NPU 上运行,作为轻量级视觉理解备选
|
||||
|
||||
---
|
||||
|
||||
## 4. 本地计算卸载策略
|
||||
|
||||
### 4.1 RK3588 NPU 能力现状 (2025-2026)
|
||||
|
||||
根据最新 benchmark 数据和 RKNN-LLM 生态:
|
||||
|
||||
| 模型 | 参数量 | 量化 | 推理速度 | 内存占用 | 用途 |
|
||||
|------|--------|------|---------|---------|------|
|
||||
| TinyLlama 1.1B | 1.1B | W8A8 | 10-15 tok/s | ~1.5 GB | 简单文本任务 |
|
||||
| Qwen2.5-1.5B | 1.5B | W8A8 | 8-12 tok/s | ~2 GB | **当前本地 LLM** |
|
||||
| Qwen2.5-VL-3B | 3B | W8A8 | ~5-8 tok/s | ~4.6 GB | **潜力视觉模型** |
|
||||
| InternVL3.5-1B | 1B | W8A8 | ~24 tok/s | ~1.5 GB | 轻量视觉理解 |
|
||||
| InternVL3.5-2B | 2B | W8A8 | ~11.2 tok/s | ~3 GB | 中等视觉理解 |
|
||||
| ResNet18 (分类) | 11M | INT8 | 244 FPS | ~50 MB | 图像分类 |
|
||||
| multilingual-e5-small | 118M | INT8 | <10ms | ~200 MB | **当前 embedding** |
|
||||
|
||||
**关键限制**: RK3588 NPU 仅支持 W8A8 量化(RK3576 支持 W4A16),大模型精度受限。
|
||||
|
||||
### 4.2 本地计算资源分配
|
||||
|
||||
当前 NPU 核心分配已优化 (CLAUDE.md 记录):
|
||||
- **OCR**: Core 0/1 (高优先级,实时性要求)
|
||||
- **Embedding**: Core 2 (mem-bridge 语义搜索)
|
||||
|
||||
建议增加的本地计算任务:
|
||||
|
||||
#### 4.2.1 已在本地运行 (保持不变)
|
||||
|
||||
| 任务 | 实现 | 延迟 | 占用 |
|
||||
|------|------|------|------|
|
||||
| OCR (PaddleOCR RKNN) | CGo libkvm_ocr.so | ~60ms | NPU Core 0/1 |
|
||||
| Embedding (e5-small) | RKNN Lite | ~10ms | NPU Core 2 |
|
||||
| pixel_diff 验证 | Python 字节比较 | ~1ms | CPU |
|
||||
| 屏幕状态检测 | Python 哈希+关键词 | ~1ms | CPU |
|
||||
|
||||
#### 4.2.2 建议迁移到本地
|
||||
|
||||
| 任务 | 推荐模型 | 预估延迟 | 节省 | 优先级 |
|
||||
|------|---------|---------|------|--------|
|
||||
| **屏幕指纹匹配** | FAISS 向量 (复用 e5) | ~15ms | 跳过 LLM 调用 | **P0** |
|
||||
| **简单动作选择** | 规则引擎 + 模板 | ~5ms | 完全跳过 LLM | **P0** |
|
||||
| **对话框分类** | ResNet18 微调 | ~4ms | 跳过 LLM 规划 | **P1** |
|
||||
| **视觉验证** | InternVL3.5-1B | ~40ms | 替代 LLM 验证 | **P2** |
|
||||
| **本地 VLM 规划** | Qwen2.5-VL-3B | ~5s | 减少云端调用 | **P3** |
|
||||
|
||||
#### 4.2.3 必须保留在云端
|
||||
|
||||
| 任务 | 原因 |
|
||||
|------|------|
|
||||
| 复杂多步规划 | 需要强推理能力 (>7B 参数级别) |
|
||||
| 自然语言理解 | 任务描述解析需要大模型 |
|
||||
| 异常恢复决策 | 模板未覆盖的新场景 |
|
||||
| 跨应用工作流 | 多窗口切换需要全局理解 |
|
||||
|
||||
### 4.3 屏幕指纹系统 (P0 优先)
|
||||
|
||||
**核心思路**: 对已知屏幕状态计算向量指纹,下次遇到相同/相似屏幕时直接查表,完全跳过 LLM 调用。
|
||||
|
||||
```python
|
||||
# 概念设计
|
||||
class ScreenFingerprint:
|
||||
"""基于 OCR SceneGraph 的屏幕指纹。"""
|
||||
|
||||
def compute_fingerprint(self, scene: SceneGraph) -> np.ndarray:
|
||||
"""将 SceneGraph 转换为固定长度向量。
|
||||
|
||||
方法: 将排序后的 element texts 拼接,通过 e5-small 生成 embedding。
|
||||
优势: 复用已有 RKNN embedding 模型,无额外部署。
|
||||
"""
|
||||
text = " ".join(sorted(el.text for el in scene.elements[:20]))
|
||||
return self.embedding_model.encode(text) # ~10ms on NPU
|
||||
|
||||
async def lookup(self, fingerprint: np.ndarray) -> Optional[Action]:
|
||||
"""FAISS 最近邻搜索已知屏幕 → 预定义动作。"""
|
||||
scores, indices = self.index.search(fingerprint, k=1)
|
||||
if scores[0][0] > 0.92: # 高相似度阈值
|
||||
return self.action_table[indices[0][0]]
|
||||
return None # 未知屏幕,走 LLM 路径
|
||||
```
|
||||
|
||||
**预期效果**: 对重复任务(如"打开记事本"),第 2 次执行可实现 0 token、~25ms 延迟。
|
||||
|
||||
### 4.4 本地 VLM 评估 (Qwen2.5-VL-3B)
|
||||
|
||||
Qwen2.5-VL-3B 已有 RK3588 NPU 部署方案 ([Qengineering/Qwen2.5-VL-3B-NPU](https://github.com/Qengineering/Qwen2.5-VL-3B-NPU)):
|
||||
|
||||
**优势**:
|
||||
- 直接理解截图,无需 OCR → SceneGraph → text 转换
|
||||
- W8A8 量化后 ~4.6 GB 内存,RK3588 8GB 版本可承载
|
||||
- 支持中英文 UI 理解
|
||||
|
||||
**劣势**:
|
||||
- 推理速度 ~5-8 tok/s,单次规划需 3-5 秒
|
||||
- 与 OCR (NPU Core 0/1) 存在资源竞争
|
||||
- 复杂任务的推理质量远不如 Gemini/GPT
|
||||
|
||||
**建议**: 作为 **中间层** 使用 — 屏幕指纹未命中但任务简单时,先尝试本地 VLM,失败再回退云端。这需要 NPU 调度器升级以支持动态核心分配。
|
||||
|
||||
---
|
||||
|
||||
## 5. 记忆系统 Token 优化
|
||||
|
||||
### 5.1 当前 Token 消耗模型
|
||||
|
||||
分析 `llm_planner.py` 的 LLM 调用构成:
|
||||
|
||||
```
|
||||
每次 LLM 调用的 token 构成:
|
||||
┌─────────────────────────────────────────┐
|
||||
│ System Prompt (AGENT_SYSTEM_PROMPT) │ ~2,000 tokens (固定)
|
||||
│ Task + Step number │ ~30 tokens
|
||||
│ Historical Context (mem-bridge) │ ~500-2,000 tokens
|
||||
│ Scene Text (OCR elements, 30 items) │ ~800-1,500 tokens
|
||||
│ Recent Actions (history[-3:]) │ ~100-200 tokens
|
||||
│ Screenshot (JPEG base64, high detail) │ ~1,290 tokens (1024x1024)
|
||||
├─────────────────────────────────────────┤
|
||||
│ 输入总计 │ ~4,720-7,020 tokens
|
||||
│ 输出 (Action JSON) │ ~50-100 tokens
|
||||
│ max_tokens 设置 │ 512 tokens (上限)
|
||||
└─────────────────────────────────────────┘
|
||||
|
||||
一个 30 步任务的总 token 消耗:
|
||||
- 最佳情况 (模板命中): 0 tokens
|
||||
- 典型情况 (混合): ~30,000-50,000 tokens
|
||||
- 最坏情况 (全云端+重试): ~200,000+ tokens
|
||||
```
|
||||
|
||||
**成本估算** (Gemini 2.5 Pro 价格):
|
||||
- 典型任务: 40K input + 3K output = $0.05 + $0.03 = **$0.08/任务**
|
||||
- 高频使用 (100 任务/天): **$8/天 = $240/月**
|
||||
|
||||
### 5.2 优化策略矩阵
|
||||
|
||||
#### 策略 A: System Prompt 缓存 (预期节省 40%)
|
||||
|
||||
**现状**: `AGENT_SYSTEM_PROMPT` (~2000 tokens) 每次调用都完整发送。
|
||||
|
||||
**优化**: Gemini 2.5 Pro 和 GPT-4o 均支持 [system instruction caching](https://ai.google.dev/gemini-api/docs/caching),缓存后输入成本降低 ~90%,延迟降低 ~75%。
|
||||
|
||||
```python
|
||||
# llm_planner.py 改进
|
||||
class LLMPlanner:
|
||||
def __init__(self, ...):
|
||||
# 缓存 system prompt (首次调用创建,后续复用)
|
||||
self._cached_system = None
|
||||
|
||||
async def _ensure_cache(self):
|
||||
if self._cached_system is None:
|
||||
# Gemini: cachedContents API
|
||||
# OpenAI: 自动缓存长前缀
|
||||
self._cached_system = await self._create_cache(AGENT_SYSTEM_PROMPT)
|
||||
```
|
||||
|
||||
**预期效果**: 每步节省 ~2000 input tokens,30 步任务节省 ~60K tokens。
|
||||
|
||||
#### 策略 B: 渐进式上下文压缩 (预期节省 30%)
|
||||
|
||||
**现状**: `history` 仅保留最近 3 条 action 描述,更早的历史完全丢失。
|
||||
|
||||
**优化**: 实现三级上下文窗口:
|
||||
|
||||
```
|
||||
Level 1 (最近 3 步): 完整 action + scene_text
|
||||
Level 2 (3-10 步前): 压缩摘要 "步骤4-8: 打开搜索→输入notepad→点击结果"
|
||||
Level 3 (10 步前): 单句任务进度 "已完成: 打开记事本, 输入文本"
|
||||
```
|
||||
|
||||
```python
|
||||
# 上下文压缩器
|
||||
class ContextCompressor:
|
||||
def compress(self, history: list[str], step: int) -> str:
|
||||
recent = history[-3:] # Level 1: 完整
|
||||
middle = self._summarize(history[-10:-3]) # Level 2: 摘要
|
||||
old = self._progress_line(history[:-10]) # Level 3: 进度
|
||||
return f"{old}\n{middle}\n{'|'.join(recent)}"
|
||||
|
||||
def _summarize(self, actions: list[str]) -> str:
|
||||
# 本地规则压缩,不调用 LLM
|
||||
types = [a.split(":")[0].strip("[] 0123456789") for a in actions]
|
||||
return f"步骤概要: {', '.join(types)}"
|
||||
```
|
||||
|
||||
#### 策略 C: SceneGraph 智能裁剪 (预期节省 25%)
|
||||
|
||||
**现状**: `to_text_summary()` 发送最多 30 个元素,包含大量无关信息。
|
||||
|
||||
**优化**: 基于任务相关性和元素类型进行裁剪:
|
||||
|
||||
```python
|
||||
def to_text_summary_optimized(self, task_keywords: list[str], max_elements: int = 15) -> str:
|
||||
"""任务感知的 SceneGraph 压缩。"""
|
||||
scored = []
|
||||
for el in self.elements:
|
||||
score = 0
|
||||
# 任务相关性加权
|
||||
for kw in task_keywords:
|
||||
if kw.lower() in el.text.lower():
|
||||
score += 10
|
||||
# 可交互元素优先 (按钮大小)
|
||||
if el.width > 0.03 and el.height > 0.015:
|
||||
score += 5
|
||||
# 屏幕中心区域优先
|
||||
if 0.1 < el.center_y < 0.9:
|
||||
score += 3
|
||||
scored.append((score, el))
|
||||
|
||||
scored.sort(key=lambda x: -x[0])
|
||||
top_elements = [el for _, el in scored[:max_elements]]
|
||||
|
||||
# 紧凑格式: 去掉 conf 字段,合并坐标
|
||||
lines = [f"[{el.text}]({el.center_x:.2f},{el.center_y:.2f})"
|
||||
for el in top_elements]
|
||||
return " | ".join(lines)
|
||||
```
|
||||
|
||||
**效果**: 30 个元素的 ~1500 tokens → 15 个相关元素的 ~400 tokens。
|
||||
|
||||
#### 策略 D: 图像 Token 优化 (预期节省 50%)
|
||||
|
||||
**现状**: 截图以 `detail: "high"` 发送,消耗 ~1290 tokens。
|
||||
|
||||
**优化方案**:
|
||||
|
||||
1. **降低图像分辨率**: 发送 512x384 替代 1920x1080
|
||||
2. **使用 `detail: "low"`**: 固定消耗 85 tokens (但精度下降)
|
||||
3. **条件性发送图像**: 当 SceneGraph 已有足够信息时不发送截图
|
||||
|
||||
```python
|
||||
async def plan_action(self, screenshot_bytes, task, step, ...):
|
||||
# 如果 scene_text 包含足够元素且任务简单,跳过图像
|
||||
if scene_text and len(scene.elements) > 10 and self._is_simple_action(task):
|
||||
# Text-only 模式: 节省 ~1290 tokens
|
||||
user_content = [text_only_prompt]
|
||||
else:
|
||||
# Vision 模式: 包含截图
|
||||
user_content = [text_prompt, image_prompt]
|
||||
```
|
||||
|
||||
**实测建议**: 对 "点击已知按钮" 类操作使用 text-only 模式;对 "在屏幕中找到..." 类操作使用 vision 模式。
|
||||
|
||||
#### 策略 E: 情景记忆 (Episodic Memory)
|
||||
|
||||
**现状**: `memory_client.py` 存储对话轮次,但检索粒度粗 — `get_context()` 返回原始文本。
|
||||
|
||||
**参考**: Amazon Bedrock AgentCore 的情景记忆设计将记忆分为四层:
|
||||
- **Summarization**: 管理上下文长度
|
||||
- **Semantic**: 存储事实
|
||||
- **Preference**: 处理个性化
|
||||
- **Episodic**: 捕获经验
|
||||
|
||||
**KVM Agent 情景记忆设计**:
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class Episode:
|
||||
"""一次完整任务执行的压缩记录。"""
|
||||
task: str # "打开记事本"
|
||||
task_embedding: np.ndarray # FAISS 索引用
|
||||
success: bool
|
||||
total_steps: int
|
||||
key_actions: list[str] # 关键动作(非全部)
|
||||
obstacles: list[str] # 遇到的障碍
|
||||
screen_fingerprints: list[str] # 关键屏幕状态指纹
|
||||
duration_ms: int
|
||||
|
||||
class EpisodicMemory:
|
||||
async def recall(self, task: str) -> Optional[Episode]:
|
||||
"""语义搜索相似任务经验。"""
|
||||
# 复用 RKNN e5-small embedding
|
||||
embedding = self.encode(task)
|
||||
matches = self.faiss_index.search(embedding, k=3)
|
||||
return self._best_match(matches)
|
||||
|
||||
def to_context_hint(self, episode: Episode) -> str:
|
||||
"""将情景压缩为 LLM 提示 (~200 tokens)。"""
|
||||
return (
|
||||
f"Past experience: '{episode.task}' succeeded in {episode.total_steps} steps. "
|
||||
f"Key: {', '.join(episode.key_actions[:5])}. "
|
||||
f"Obstacles: {', '.join(episode.obstacles[:3])}."
|
||||
)
|
||||
```
|
||||
|
||||
**预期效果**: 上下文注入从 ~2000 tokens 降至 ~200 tokens,且信息更精准。
|
||||
|
||||
### 5.3 优化策略汇总
|
||||
|
||||
| 策略 | 实现难度 | Token 节省 | 延迟影响 | 优先级 |
|
||||
|------|---------|-----------|---------|--------|
|
||||
| A: System Prompt 缓存 | 低 | 40% input | -75% 首 token | **P0** |
|
||||
| B: 渐进式上下文压缩 | 低 | 30% context | 无 | **P0** |
|
||||
| C: SceneGraph 裁剪 | 中 | 25% scene | 无 | **P1** |
|
||||
| D: 图像 Token 优化 | 中 | 50% image | 需验证精度 | **P1** |
|
||||
| E: 情景记忆 | 中 | 75% context | ~15ms 本地查询 | **P1** |
|
||||
| F: 屏幕指纹跳过 | 高 | 100% (跳过 LLM) | ~25ms | **P0** |
|
||||
|
||||
**综合节省预估**: 对典型 30 步任务,从 ~180K tokens 降至 ~50K tokens (节省 ~72%)。
|
||||
|
||||
---
|
||||
|
||||
## 6. Token 消耗定量分析
|
||||
|
||||
### 6.1 当前消耗拆解
|
||||
|
||||
基于 `llm_planner.py` 代码分析,以一个 15 步 "打开记事本并输入文本" 任务为例:
|
||||
|
||||
```
|
||||
Step 1: 识别桌面状态
|
||||
system_prompt: 2,000 tokens
|
||||
task+step: 30 tokens
|
||||
scene_text: 1,200 tokens (桌面图标 25 个元素)
|
||||
screenshot: 1,290 tokens
|
||||
output: 60 tokens
|
||||
小计: 4,580 tokens
|
||||
|
||||
Step 2-5: 搜索+打开记事本
|
||||
每步类似: 4,200 tokens x 4 = 16,800 tokens
|
||||
|
||||
Step 6-15: 输入文本+验证
|
||||
每步 (text-only 潜力): 3,200 tokens x 10 = 32,000 tokens
|
||||
|
||||
总计: ~53,380 tokens
|
||||
成本: $0.067 input + $0.009 output = $0.076
|
||||
```
|
||||
|
||||
### 6.2 优化后消耗预估
|
||||
|
||||
```
|
||||
Step 1: 屏幕指纹命中 → 模板重放
|
||||
消耗: 0 tokens (本地 FAISS 查询 ~15ms)
|
||||
|
||||
Step 2-5: 模板重放 (已知流程)
|
||||
消耗: 0 tokens
|
||||
|
||||
Step 6: 模板未覆盖,本地 RKLLM 处理
|
||||
消耗: 0 cloud tokens (本地 ~2s)
|
||||
|
||||
Step 7-10: 简单输入,text-only 模式
|
||||
system (cached): 200 tokens (缓存折扣)
|
||||
compressed context: 150 tokens
|
||||
pruned scene: 400 tokens
|
||||
no screenshot: 0 tokens
|
||||
output: 60 tokens
|
||||
小计: 810 tokens x 4 = 3,240 tokens
|
||||
|
||||
Step 11-15: 验证+完成
|
||||
混合模式: ~1,500 tokens x 5 = 7,500 tokens
|
||||
|
||||
总计: ~10,740 tokens (节省 80%)
|
||||
成本: $0.013 input + $0.005 output = $0.018
|
||||
```
|
||||
|
||||
### 6.3 月度成本对比
|
||||
|
||||
| 场景 | 当前 | 优化后 | 节省 |
|
||||
|------|------|--------|------|
|
||||
| 轻度 (10 任务/天) | $24/月 | $5.4/月 | 78% |
|
||||
| 中度 (50 任务/天) | $120/月 | $27/月 | 78% |
|
||||
| 重度 (200 任务/天) | $480/月 | $108/月 | 78% |
|
||||
|
||||
---
|
||||
|
||||
## 7. 架构升级建议
|
||||
|
||||
### 7.1 目标架构:Memory-Augmented Programmatic Agent
|
||||
|
||||
```
|
||||
用户任务
|
||||
↓
|
||||
┌─────────────────────────────────────────────┐
|
||||
│ Task Router (本地, <5ms) │
|
||||
│ ├── 屏幕指纹匹配 → 状态机模板 (0 token) │
|
||||
│ ├── 情景记忆命中 → 经验引导规划 (少量 token) │
|
||||
│ └── 未知任务 → 完整 LLM 规划 │
|
||||
└─────────────────────────────────────────────┘
|
||||
↓
|
||||
┌─────────────────────────────────────────────┐
|
||||
│ 分层执行引擎 │
|
||||
│ │
|
||||
│ Layer 1: 确定性执行 (本地) │
|
||||
│ 状态机模板重放 + OCR 验证 │
|
||||
│ 延迟: ~100ms/步, 成本: 0 │
|
||||
│ │
|
||||
│ Layer 2: 本地智能 (NPU) │
|
||||
│ RKLLM (Qwen2.5-1.5B) text 规划 │
|
||||
│ InternVL3.5-1B 视觉验证 (可选) │
|
||||
│ 延迟: ~2s/步, 成本: 0 │
|
||||
│ │
|
||||
│ Layer 3: 云端智能 (API) │
|
||||
│ Gemini 2.5 Pro 视觉规划 │
|
||||
│ + 缓存 system prompt │
|
||||
│ + 压缩上下文 │
|
||||
│ + 自适应图像发送 │
|
||||
│ 延迟: ~1-3s/步, 成本: ~$0.005/步 │
|
||||
│ │
|
||||
│ Layer 4: 人工干预 │
|
||||
│ WebUI 提示用户协助 │
|
||||
│ 超时自动 abort │
|
||||
└─────────────────────────────────────────────┘
|
||||
↓
|
||||
┌─────────────────────────────────────────────┐
|
||||
│ 经验学习回路 │
|
||||
│ 成功 → 记录情景 + 生成/更新状态机模板 │
|
||||
│ 失败 → 记录障碍 + 降低模板置信度 │
|
||||
└─────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### 7.2 降级链路设计
|
||||
|
||||
```
|
||||
模板匹配 (相似度>0.92)
|
||||
├── 成功 → 完成 (0 token)
|
||||
└── 模板步骤验证失败
|
||||
↓
|
||||
本地 RKLLM (简单任务 + 记忆命中)
|
||||
├── 成功 → 完成 (0 cloud token)
|
||||
└── 本地 LLM 输出无效/解析失败
|
||||
↓
|
||||
云端 LLM (text-only 模式,若 SceneGraph 充分)
|
||||
├── 成功 → 完成 (~800 tokens)
|
||||
└── 需要视觉理解
|
||||
↓
|
||||
云端 LLM (vision 模式,压缩上下文)
|
||||
├── 成功 → 完成 (~3000 tokens)
|
||||
└── 连续 3 次失败
|
||||
↓
|
||||
用户干预请求
|
||||
```
|
||||
|
||||
### 7.3 关键组件改动
|
||||
|
||||
#### 7.3.1 `hybrid_planner.py` 增强
|
||||
|
||||
```python
|
||||
class HybridPlannerV3:
|
||||
"""四层降级 + 屏幕指纹 + 情景记忆。"""
|
||||
|
||||
async def plan(self, screenshot, task, step, history, max_steps):
|
||||
# 1. 屏幕指纹快速查表 (新增)
|
||||
scene = await self._perception.perceive(screenshot)
|
||||
fingerprint_action = await self._fingerprint.lookup(scene)
|
||||
if fingerprint_action:
|
||||
return PlanResult(action=fingerprint_action, source="fingerprint")
|
||||
|
||||
# 2. 模板匹配 (已有,增加状态机支持)
|
||||
if step == 0:
|
||||
self._active_template = await self._templates.find_template(task)
|
||||
if self._active_template:
|
||||
result = await self._replay_step(screenshot)
|
||||
if result:
|
||||
return result
|
||||
|
||||
# 3. 情景记忆引导 (新增)
|
||||
episode = await self._episodic.recall(task)
|
||||
context_hint = self._episodic.to_context_hint(episode) if episode else ""
|
||||
|
||||
# 4. 本地 RKLLM (已有,增加情景上下文)
|
||||
if await self._should_use_local(task, history):
|
||||
result = await self._local_llm_plan(screenshot, task, step, history, max_steps)
|
||||
if result:
|
||||
return result
|
||||
|
||||
# 5. 云端 LLM (优化: 自适应图像+压缩上下文)
|
||||
return await self._cloud_plan_optimized(
|
||||
screenshot, task, step, history, max_steps,
|
||||
scene=scene, context_hint=context_hint,
|
||||
)
|
||||
```
|
||||
|
||||
#### 7.3.2 `llm_planner.py` Token 优化
|
||||
|
||||
```python
|
||||
class LLMPlannerV3:
|
||||
"""Token 优化版 LLM 规划器。"""
|
||||
|
||||
async def plan_action(self, screenshot_bytes, task, step, history, **kwargs):
|
||||
scene = kwargs.get("scene")
|
||||
context_hint = kwargs.get("context_hint", "")
|
||||
|
||||
# 自适应图像发送
|
||||
include_image = self._should_send_image(scene, task, step)
|
||||
|
||||
# 压缩上下文
|
||||
compressed_context = self._compress_context(history, context_hint)
|
||||
|
||||
# 任务感知 SceneGraph 裁剪
|
||||
task_keywords = self._extract_keywords(task)
|
||||
pruned_scene = scene.to_text_summary_optimized(task_keywords, max_elements=15)
|
||||
|
||||
# 构建消息 (利用 system prompt 缓存)
|
||||
messages = self._build_messages(
|
||||
task, step, pruned_scene, compressed_context,
|
||||
screenshot_bytes if include_image else None,
|
||||
)
|
||||
|
||||
response = await self.client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=messages,
|
||||
max_tokens=128, # 降低:Action JSON 通常 <100 tokens
|
||||
temperature=self.temperature,
|
||||
response_format={"type": "json_object"}, # 结构化输出
|
||||
)
|
||||
...
|
||||
|
||||
def _should_send_image(self, scene, task, step) -> bool:
|
||||
"""决定是否发送截图。"""
|
||||
# 首步总是发送图像 (需要视觉理解)
|
||||
if step == 0:
|
||||
return True
|
||||
# 多元素场景 + 非导航任务 → text-only
|
||||
if scene and len(scene.elements) > 8:
|
||||
simple_verbs = ["点击", "输入", "click", "type", "关闭", "close"]
|
||||
if any(v in task.lower() for v in simple_verbs):
|
||||
return False
|
||||
return True
|
||||
```
|
||||
|
||||
### 7.4 NPU 资源调度升级
|
||||
|
||||
```
|
||||
当前 NPU 调度 (静态):
|
||||
Core 0: OCR (独占)
|
||||
Core 1: OCR (独占)
|
||||
Core 2: Embedding (独占)
|
||||
|
||||
建议 NPU 调度 (动态):
|
||||
Core 0-1: OCR (高优先级,按需)
|
||||
Core 2: Embedding / 分类模型 (共享)
|
||||
Core 0-2: 本地 VLM (低优先级,OCR 空闲时)
|
||||
|
||||
调度策略:
|
||||
- OCR 请求到达时中断 VLM 推理
|
||||
- VLM 推理在 OCR 间隙执行 (OCR ~60ms, 间隔 ~1000ms)
|
||||
- Embedding 查询 (~10ms) 与 OCR 无冲突
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 8. 实施路线图
|
||||
|
||||
### Phase 1: 低风险高收益 (1-2 周)
|
||||
|
||||
**目标**: 不改架构,通过配置和参数优化减少 40% token 消耗。
|
||||
|
||||
| 任务 | 改动文件 | 预期效果 |
|
||||
|------|---------|---------|
|
||||
| System prompt 缓存 | `llm_planner.py` | -40% input tokens |
|
||||
| `max_tokens` 降至 128 | `agent.yaml`, `llm_planner.py` | -75% 输出上限 |
|
||||
| `response_format: json_object` | `llm_planner.py` | 消除解析失败 |
|
||||
| `detail: "auto"` 图像 | `llm_planner.py` | -30% image tokens |
|
||||
| history 从 3 条扩到 5 条但压缩 | `llm_planner.py` | 更好上下文 |
|
||||
|
||||
### Phase 2: 屏幕指纹 + 情景记忆 (2-4 周)
|
||||
|
||||
**目标**: 对重复任务实现零 token 执行。
|
||||
|
||||
| 任务 | 新建/改动文件 | 预期效果 |
|
||||
|------|-------------|---------|
|
||||
| ScreenFingerprint 类 | 新建 `screen_fingerprint.py` | 已知屏幕跳过 LLM |
|
||||
| EpisodicMemory 类 | 新建 `episodic_memory.py` | 精简上下文注入 |
|
||||
| 状态机模板 | 改造 `template_store.py` | 条件分支+自动修复 |
|
||||
| HybridPlanner V3 | 改造 `hybrid_planner.py` | 四层降级链 |
|
||||
| SceneGraph 裁剪 | 改造 `perception.py` | 任务感知元素选择 |
|
||||
|
||||
### Phase 3: 本地 VLM 集成 (4-8 周)
|
||||
|
||||
**目标**: 简单视觉任务本地完成,进一步减少云端依赖。
|
||||
|
||||
| 任务 | 改动 | 预期效果 |
|
||||
|------|------|---------|
|
||||
| InternVL3.5-1B 部署 | 新建 `local_vlm.py` + RKNN 模型 | 本地视觉验证 |
|
||||
| NPU 动态调度器 | 改造 NPU 核心分配 | OCR/VLM 共存 |
|
||||
| 对话框分类器 (ResNet18) | 微调+部署 | 弹窗自动处理 |
|
||||
| Qwen2.5-VL-3B 评估 | 可选,视 Phase 2 效果 | 本地视觉规划 |
|
||||
|
||||
### Phase 4: 自学习闭环 (8-12 周)
|
||||
|
||||
**目标**: Agent 从执行经验中自动学习,持续优化。
|
||||
|
||||
| 任务 | 改动 | 预期效果 |
|
||||
|------|------|---------|
|
||||
| 自动模板生成 | 成功任务 → 状态机模板 | 自动积累知识 |
|
||||
| 模板退化检测 | 置信度衰减机制 | UI 变化时自动更新 |
|
||||
| A/B 路由优化 | 本地 vs 云端效果对比 | 自动调整路由阈值 |
|
||||
| 批量指纹预计算 | 离线 GUI 探索 | 冷启动优化 |
|
||||
|
||||
---
|
||||
|
||||
## 参考资料
|
||||
|
||||
### Playwright 与自动化模式
|
||||
- [Playwright Auto-Waiting](https://playwright.dev/docs/actionability)
|
||||
- [Playwright MCP: AI-Powered Test Automation 2026](https://www.testleaf.com/blog/playwright-mcp-ai-test-automation-2026/)
|
||||
- [Playwright and Playwright MCP: A Field Guide for Agentic Browser Automation](https://medium.com/@adnanmasood/playwright-and-playwright-mcp-a-field-guide-for-agentic-browser-automation-f11b9daa3627)
|
||||
|
||||
### LLM 与 Agent 架构
|
||||
- [ActionEngine: From Reactive to Programmatic GUI Agents via State Machine Memory](https://arxiv.org/abs/2602.20502)
|
||||
- [2025: The Year in LLMs](https://simonwillison.net/2025/Dec/31/the-year-in-llms/)
|
||||
- [GPT-5.4 Targets Anthropic's Claude With Premium Pricing](https://www.trendingtopics.eu/gpt-5-4-targets-anthropics-claude-with-premium-pricing-and-coding-muscle/)
|
||||
- [Best LLM for Coding 2026](https://smartscope.blog/en/generative-ai/chatgpt/llm-coding-benchmark-comparison-2026/)
|
||||
|
||||
### RK3588 NPU 与本地推理
|
||||
- [Rockchip RK3588 NPU Deep Dive](https://tinycomputers.io/posts/rockchip-rk3588-npu-benchmarks.html)
|
||||
- [RKNN-LLM (GitHub)](https://github.com/airockchip/rknn-llm)
|
||||
- [Qwen2.5-VL-3B on RK3588 NPU](https://github.com/Qengineering/Qwen2.5-VL-3B-NPU)
|
||||
- [Qwen3-VL-2B on RK3588 NPU](https://github.com/Qengineering/Qwen3-VL-2B-NPU)
|
||||
- [InternVL3.5-4B on RK3588 NPU](https://github.com/Qengineering/InternVL3.5-4B-NPU)
|
||||
- [Edge AI using the Rockchip NPU](https://tristanpenman.com/blog/posts/2025/07/20/edge-ai-using-the-rockchip-npu/)
|
||||
|
||||
### Token 优化与记忆架构
|
||||
- [LLM Token Optimization: Cut Costs & Latency in 2026](https://redis.io/blog/llm-token-optimization-speed-up-apps/)
|
||||
- [The Hidden Economics of AI Agents](https://online.stevens.edu/blog/hidden-economics-ai-agents-token-costs-latency/)
|
||||
- [LLM Cost Optimization: Complete Guide](https://ai.koombea.com/blog/llm-cost-optimization)
|
||||
- [Position: Episodic Memory is the Missing Piece for Long-Term LLM Agents](https://arxiv.org/pdf/2502.06975)
|
||||
- [Memory in the Age of AI Agents: A Survey](https://arxiv.org/abs/2512.13564)
|
||||
- [LLM Chat History Summarization Guide 2025](https://mem0.ai/blog/llm-chat-history-summarization-guide-2025)
|
||||
- [Amazon Bedrock AgentCore Episodic Memory](https://aws.amazon.com/blogs/machine-learning/build-agents-to-learn-from-experiences-using-amazon-bedrock-agentcore-episodic-memory/)
|
||||
@@ -0,0 +1,147 @@
|
||||
# 部署打包架构
|
||||
|
||||
> 5 个 Debian 包 + 8 个 systemd 服务
|
||||
|
||||
## 1. 包依赖关系
|
||||
|
||||
```
|
||||
kvm-meta (元包)
|
||||
├── Depends: kvm-server
|
||||
├── Depends: kvm-agent
|
||||
├── Depends: kvm-privacy
|
||||
└── Recommends: kvm-bridge
|
||||
│
|
||||
┌────┴─────────────────────────────────┐
|
||||
│ │
|
||||
▼ ▼
|
||||
kvm-server (Go) kvm-agent (Python)
|
||||
├─ /usr/bin/kvm-server ├─ /usr/bin/kvm-agent
|
||||
├─ /usr/lib/kvm/*.so ├─ /usr/lib/kvm-agent/
|
||||
├─ /etc/kvm/ ├─ /etc/kvm-agent/
|
||||
├─ kvm-server.service └─ kvm-agent.service
|
||||
└─ kvm-usb-gadget.service
|
||||
│
|
||||
┌────┴─────────────────────────────────┐
|
||||
│ │
|
||||
▼ ▼
|
||||
kvm-privacy (Rust) kvm-bridge (Python)
|
||||
├─ /usr/bin/info-privacy ├─ /opt/kvm-bridge/
|
||||
├─ /usr/lib/kvm-privacy/ ├─ mem-bridge-memory.service
|
||||
└─ info-privacy.service └─ mem-bridge-router.service
|
||||
|
||||
kvm-mitm (Python, 可选)
|
||||
├─ /usr/bin/kvm-mitm
|
||||
├─ /usr/lib/kvm-mitm/
|
||||
└─ kvm-mitm.service
|
||||
```
|
||||
|
||||
## 2. 包清单
|
||||
|
||||
| 包名 | 版本 | 架构 | 语言 | 说明 |
|
||||
|------|------|------|------|------|
|
||||
| kvm-server | 1.0.0-6 | arm64 | Go + C | KVM 核心服务 |
|
||||
| kvm-agent | 1.0.0-3 | all | Python | AI Agent daemon |
|
||||
| kvm-privacy | 1.0.0-1 | arm64 | Rust | PII 检测/脱敏 |
|
||||
| kvm-bridge | 1.0.0-1 | all | Python | 记忆 + AI 路由 |
|
||||
| kvm-mitm | 1.0.0-1 | all | Python | mitmproxy 网关 |
|
||||
| kvm-meta | 1.0.0-1 | all | — | 元包 (全部依赖) |
|
||||
|
||||
## 3. 构建命令
|
||||
|
||||
### kvm-server
|
||||
```bash
|
||||
cd deps/KVM && PATH="/usr/local/go/bin:$PATH" bash scripts/build-deb.sh
|
||||
# 输出: build/deb/kvm-server_1.0.0-6_arm64.deb
|
||||
```
|
||||
|
||||
### Python 包 (kvm-agent, kvm-mitm)
|
||||
```bash
|
||||
bash scripts/build-python-debs.sh
|
||||
# 输出: build/deb/kvm-agent_1.0.0-3_all.deb
|
||||
# build/deb/kvm-mitm_1.0.0-1_all.deb
|
||||
```
|
||||
|
||||
### 其他包
|
||||
```bash
|
||||
# kvm-privacy, kvm-bridge, kvm-meta 使用 dpkg-deb 直接打包
|
||||
dpkg-deb --build debian/kvm-privacy build/deb/
|
||||
dpkg-deb --build debian/kvm-bridge build/deb/
|
||||
dpkg-deb --build debian/kvm-meta build/deb/
|
||||
```
|
||||
|
||||
## 4. systemd 服务
|
||||
|
||||
| 服务 | 包 | 端口 | CPU 亲和性 |
|
||||
|------|-----|------|-----------|
|
||||
| kvm-server | kvm-server | 8080 | 4 5 6 7 (A76) |
|
||||
| kvm-usb-gadget | kvm-server | — | — |
|
||||
| kvm-agent | kvm-agent | 8890 | — |
|
||||
| info-privacy | kvm-privacy | 8000 | 0 1 (A55) |
|
||||
| mem-bridge-memory | kvm-bridge | 8001 | 2 3 (A55) |
|
||||
| mem-bridge-router | kvm-bridge | 8002 | 2 3 (A55) |
|
||||
| kvm-mitm | kvm-mitm | 8888 | — |
|
||||
| privacy-gateway | kvm-mitm | 8888 | — |
|
||||
|
||||
## 5. 安装验证清单
|
||||
|
||||
### 标准验证流程
|
||||
```bash
|
||||
# [1] 首次安装
|
||||
sudo dpkg -i kvm-server_*.deb
|
||||
sleep 5
|
||||
systemctl is-active kvm-server # → active
|
||||
|
||||
# [2] API 可达
|
||||
curl -sf http://localhost:8080/api/v1/kvm/stream/stats # → JSON
|
||||
|
||||
# [3] 卸载 (保留 conffiles)
|
||||
sudo dpkg --remove kvm-server
|
||||
systemctl is-active kvm-server # → inactive
|
||||
ls /etc/kvm/config.json # → 存在
|
||||
|
||||
# [4] 重装
|
||||
sudo dpkg -i kvm-server_*.deb
|
||||
systemctl is-active kvm-server # → active
|
||||
|
||||
# [5] Purge
|
||||
sudo dpkg --purge kvm-server
|
||||
ls /etc/kvm/config.json 2>/dev/null # → 不存在
|
||||
|
||||
# [6] Clean install
|
||||
sudo dpkg -i kvm-server_*.deb
|
||||
```
|
||||
|
||||
### 关键检查项
|
||||
| 检查 | 命令 | 期望 |
|
||||
|------|------|------|
|
||||
| 安装 < 60s | `time dpkg -i` | < 60s |
|
||||
| 服务启动 | `systemctl is-active` | active |
|
||||
| USB gadget 软依赖 | gadget 失败不影响 kvm-server | Wants= |
|
||||
| API 可达 | `curl :8080/api/health` | 200 |
|
||||
| conffile 保留 | remove 后 config 存在 | 存在 |
|
||||
| conffile 清除 | purge 后 config 不存在 | 不存在 |
|
||||
|
||||
## 6. 远程部署
|
||||
|
||||
```bash
|
||||
# 拷贝到目标设备
|
||||
scp build/deb/*.deb pi@192.168.123.181:/tmp/
|
||||
|
||||
# 安装
|
||||
ssh pi@192.168.123.181 'sudo dpkg -i /tmp/kvm-*.deb && sudo apt-get install -f'
|
||||
|
||||
# 验证
|
||||
ssh pi@192.168.123.181 'systemctl status kvm-server kvm-agent info-privacy'
|
||||
```
|
||||
|
||||
## 7. 配置文件位置
|
||||
|
||||
| 文件 | 包 | 用途 |
|
||||
|------|-----|------|
|
||||
| /etc/kvm/config.json | kvm-server | KVM 主配置 |
|
||||
| /etc/kvm/kvm-default.json | kvm-server | 默认配置模板 |
|
||||
| /etc/kvm/turnserver.conf | kvm-server | TURN 服务配置 |
|
||||
| /etc/kvm/sensitive_commands.json | kvm-server | 敏感命令规则 |
|
||||
| /etc/kvm-agent/secrets.env | kvm-agent | API 密钥 |
|
||||
| /etc/kvm-privacy/secrets.env | kvm-mitm | MITM 数据库凭证 |
|
||||
| /var/lib/kvm-privacy/state.json | kvm-server | 隐私模式状态 |
|
||||
@@ -0,0 +1,144 @@
|
||||
# RK3588 硬件优化架构
|
||||
|
||||
> 平台: NanoPC-T6 (RK3588) | 8核 ARM64 | 6 TOPS NPU | RGA 2D
|
||||
|
||||
## 1. 硬件资源
|
||||
|
||||
### CPU 布局
|
||||
```
|
||||
Core 0-3: ARM Cortex-A55 (低功耗, 1.8 GHz)
|
||||
Core 4-7: ARM Cortex-A76 (高性能, 2.4 GHz)
|
||||
```
|
||||
|
||||
### 核心分配方案
|
||||
| 核心 | 类型 | 分配 | 服务 |
|
||||
|------|------|------|------|
|
||||
| 0-1 | A55 | info-privacy-rs + OCR 后处理 | PII 检测 |
|
||||
| 2-3 | A55 | mem-bridge | 记忆 + 路由 |
|
||||
| 4 | A76 | HID Dispatcher (SCHED_RR) | 实时 HID |
|
||||
| 5-6 | A76 | Go runtime (GOMAXPROCS=3) | kvm-server |
|
||||
| 7 | A76 | 视频编码回调线程 | V4L2→MPP |
|
||||
|
||||
### systemd 亲和性配置
|
||||
```ini
|
||||
# kvm-server.service
|
||||
CPUAffinity=4 5 6 7
|
||||
|
||||
# info-privacy.service
|
||||
CPUAffinity=0 1
|
||||
|
||||
# mem-bridge-memory.service / mem-bridge-router.service
|
||||
CPUAffinity=2 3
|
||||
```
|
||||
|
||||
### Go 进程级别
|
||||
```go
|
||||
// main.go
|
||||
func pinToA76BigCores() {
|
||||
var mask [16]byte // 128 cores max
|
||||
mask[0] = 0xF0 // bits 4,5,6,7
|
||||
syscall.RawSyscall(SYS_SCHED_SETAFFINITY, 0, 16, uintptr(unsafe.Pointer(&mask[0])))
|
||||
runtime.GOMAXPROCS(3)
|
||||
}
|
||||
```
|
||||
|
||||
## 2. NPU (Neural Processing Unit)
|
||||
|
||||
### 三核调度
|
||||
```
|
||||
NPU Core 0: OCR text detection (RKNN_NPU_CORE_0)
|
||||
NPU Core 1: OCR text recognition (RKNN_NPU_CORE_1)
|
||||
NPU Core 2: Embedding vectorization (RKNN_NPU_CORE_2)
|
||||
```
|
||||
|
||||
### 之前的问题
|
||||
embedding 使用 `NPU_CORE_ALL`(全部 3 核),与 OCR 的 CORE_0/CORE_1 争用,导致推理延迟抖动。
|
||||
|
||||
### 修复
|
||||
```python
|
||||
# deps/embedding/src/embed_db/embedder.py
|
||||
ret = rknn.init_runtime(core_mask=RKNNLite.NPU_CORE_2) # 独占 Core 2
|
||||
```
|
||||
|
||||
### 验证
|
||||
```bash
|
||||
cat /sys/kernel/debug/rknpu/load
|
||||
# 应显示三核独立负载,无争用
|
||||
```
|
||||
|
||||
## 3. RGA (2D 图形加速器)
|
||||
|
||||
### 用途
|
||||
- **色彩转换**: BGR24 → NV12 (im2d.h: imcvtcolor)
|
||||
- **缩放**: 任意分辨率 → 编码目标分辨率 (imresize)
|
||||
- **隐私遮蔽**: NV12 区域黑色填充 (memset, RGA imfill 未验证)
|
||||
|
||||
### 色彩转换路径
|
||||
```
|
||||
V4L2 BGR24 帧
|
||||
→ wrapbuffer_virtualaddr(src, w, h, RK_FORMAT_BGR_888)
|
||||
→ wrapbuffer_virtualaddr(dst, w, h, RK_FORMAT_YCbCr_420_SP)
|
||||
→ imcvtcolor(src, dst, src_format, dst_format)
|
||||
→ NV12 数据
|
||||
```
|
||||
|
||||
## 4. MPP (Media Process Platform)
|
||||
|
||||
### H.264 编码
|
||||
- 编码器类型: MPP_VIDEO_CodingAVC
|
||||
- 帧率: 1-60 fps (动态可调)
|
||||
- 码率: 100Kbps - 20Mbps (动态可调)
|
||||
- GOP: 与帧率相同 (每秒一个关键帧)
|
||||
|
||||
### 关键操作
|
||||
```c
|
||||
mpp_encoder_force_idr() // 立即产生 IDR 关键帧
|
||||
mpp_encoder_update_rc() // 动态更新码率+帧率
|
||||
```
|
||||
|
||||
### DMA-buf 零拷贝
|
||||
```
|
||||
V4L2 buffer → VIDIOC_EXPBUF → DMA-buf fd
|
||||
→ mpp_buffer_import_with_tag(fd) // MPP 直接使用 V4L2 buffer
|
||||
→ 编码 (零 CPU memcpy)
|
||||
```
|
||||
|
||||
启用条件:
|
||||
1. V4L2 输出 NV12 格式
|
||||
2. 无缩放 (capture == encode 分辨率)
|
||||
3. 无隐私遮蔽 (遮蔽会修改 buffer)
|
||||
4. DMA-buf 导出成功
|
||||
|
||||
节省: 1080p@30fps 约 90MB/s 内存带宽
|
||||
|
||||
## 5. OCR 内联化
|
||||
|
||||
### 之前: exec 模式
|
||||
```
|
||||
Go → exec.Command("kvm-ocr", args...) → fork+exec
|
||||
延迟: 800-2000ms (含进程启动 + RKNN 模型加载)
|
||||
```
|
||||
|
||||
### 之后: CGo 内联
|
||||
```
|
||||
Go → CGo → libkvm_ocr.so → RKNN (常驻, 无 fork)
|
||||
延迟: 60-100ms
|
||||
```
|
||||
|
||||
### 实现
|
||||
- **C 层**: `src/audit/ocr_analyzer.cpp` 新增 `extern "C"` API
|
||||
- **共享库**: CMakeLists.txt 新增 `libkvm_ocr SHARED` 目标
|
||||
- **Go 桥接**: `ocr_bridge_cgo.go` (build tag: `cgo && rknn`)
|
||||
- **接口**: `OCRAnalyzer` 统一 exec/CGo 两种实现
|
||||
|
||||
## 6. 构建集成
|
||||
|
||||
```bash
|
||||
# 构建顺序 (build-deb.sh):
|
||||
1. 前端 (npm build)
|
||||
2. 视频 C 库 (cmake, 需要 rockchip_mpp)
|
||||
3. OCR 共享库 (cmake, 需要 rknn_api) ← 必须在 Go 之前!
|
||||
4. Go 后端 (自动检测 .so → 设置 CGo flags + build tags)
|
||||
5. OCR 独立二进制 (可选, --ocr flag)
|
||||
6. dpkg-buildpackage
|
||||
```
|
||||
@@ -0,0 +1,179 @@
|
||||
# KVM Agent (Python) 架构
|
||||
|
||||
> 子系统: services/kvm_agent/ | 端口: 8890 | 包名: kvm-agent
|
||||
|
||||
## 1. 概述
|
||||
|
||||
KVM Agent 是基于 Vision LLM 的 AI 自主操控系统。通过 OCR 感知屏幕内容、LLM 规划操作序列、鼠标/键盘执行动作、截图验证结果,实现对目标主机的自动化操作。
|
||||
|
||||
## 2. 核心循环
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────┐
|
||||
│ KVMAgent.run_task() │
|
||||
│ │
|
||||
│ 1. 获取 HID 控制权 │
|
||||
│ 2. 设置隐私模式 (audit) │
|
||||
│ │
|
||||
│ ┌─ while step < max_steps: ───────────┐ │
|
||||
│ │ │ │
|
||||
│ │ ┌─ perceive ──────────────────┐ │ │
|
||||
│ │ │ screenshot() → JPEG │ │ │
|
||||
│ │ │ screen_state.detect() → │ │ │
|
||||
│ │ │ SLEEP? → wake_from_sleep │ │ │
|
||||
│ │ │ perception.perceive() → │ │ │
|
||||
│ │ │ SceneGraph + raw_text │ │ │
|
||||
│ │ └─────────────────────────────┘ │ │
|
||||
│ │ │ │
|
||||
│ │ ┌─ plan ──────────────────────┐ │ │
|
||||
│ │ │ LLMPlanner.plan_action( │ │ │
|
||||
│ │ │ screenshot, task, │ │ │
|
||||
│ │ │ scene_text, history, │ │ │
|
||||
│ │ │ memory_context) │ │ │
|
||||
│ │ │ → Action{type, x, y, text} │ │ │
|
||||
│ │ └─────────────────────────────┘ │ │
|
||||
│ │ │ │
|
||||
│ │ ┌─ validate + execute ────────┐ │ │
|
||||
│ │ │ safety.validate(action) │ │ │
|
||||
│ │ │ _execute_action(action) │ │ │
|
||||
│ │ │ validator.verify() │ │ │
|
||||
│ │ │ type=done? → return success │ │ │
|
||||
│ │ └─────────────────────────────┘ │ │
|
||||
│ └─────────────────────────────────────┘ │
|
||||
│ │
|
||||
│ 3. 释放 HID 控制权 │
|
||||
│ 4. 恢复隐私模式 │
|
||||
└──────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## 3. 模块清单
|
||||
|
||||
| 模块 | 文件 | 职责 |
|
||||
|------|------|------|
|
||||
| Agent 主循环 | agent.py | perceive→plan→execute→verify 循环 |
|
||||
| 感知引擎 | perception.py | OCR 结果归一化 + SceneGraph 构建 |
|
||||
| LLM 规划器 | llm_planner.py | Vision LLM 调用 + 动作解析 |
|
||||
| 混合规划器 | hybrid_planner.py | LLM + 本地模型混合决策 |
|
||||
| 鼠标操作 | mouse_ops.py | click_element, close_window, drag 等 |
|
||||
| 窗口管理 | window_manager.py | 全屏化、贴靠、最小化 |
|
||||
| 屏幕状态 | screen_state.py | BIOS/锁屏/睡眠/桌面 检测 |
|
||||
| 验证器 | validator.py | 截图对比 + OCR 语义验证 |
|
||||
| 安全层 | safety.py | 白名单 + Unicode 归一化 + 危险命令拦截 |
|
||||
| KVM 客户端 | kvm_client.py | HTTP REST 封装 (截图/HID/OCR) |
|
||||
| 任务执行器 | runner.py | 持久化队列 + daemon 进程 |
|
||||
| 记忆客户端 | memory_client.py | mem-bridge 会话存储 |
|
||||
| 模板系统 | template_store.py / template_recorder.py | 成功任务转模板 |
|
||||
| 配置 | config.py | YAML + ENV + dataclass |
|
||||
| 动作定义 | actions.py | Action 类型枚举 |
|
||||
| API 服务器 | api_server.py | aiohttp REST (port 8890) |
|
||||
| CLI | \_\_main\_\_.py | run/daemon/queue 子命令 |
|
||||
|
||||
## 4. 鼠标优先架构
|
||||
|
||||
### 设计原因
|
||||
实测暴露组合键系统性风险:
|
||||
- Alt+F4 → Windows 关机对话框
|
||||
- Win+D → 二次触发恢复窗口
|
||||
- IME 拦截 Enter/Space
|
||||
|
||||
### 安全键白名单 (20 个)
|
||||
```
|
||||
escape, enter, tab, backspace, delete, space,
|
||||
up, down, left, right, shift,
|
||||
f1, f2, f3, f4, f5, f11, f12
|
||||
```
|
||||
|
||||
所有其他键和所有组合键被 safety.py 拒绝。
|
||||
|
||||
### 快捷键翻译表
|
||||
| 快捷键 | 鼠标替代 | 函数 |
|
||||
|--------|---------|------|
|
||||
| Alt+F4 | 点击标题栏 X | close_window() |
|
||||
| Win+D | 点击 (0.999, 0.999) | show_desktop() |
|
||||
| Win+R | 搜索→输入→点击 | launch_from_taskbar() |
|
||||
| Ctrl+S | OCR 找"保存"→点击 | click_element("保存") |
|
||||
|
||||
### mouse_ops.py 核心函数
|
||||
```python
|
||||
click_element(kvm, text, perception) # OCR 找文字→点击中心
|
||||
close_window() # 标题栏 X 按钮
|
||||
show_desktop() # 任务栏右下角
|
||||
launch_from_taskbar(app) # 搜索框启动应用
|
||||
safe_cleanup() # 状态机清理 (关机→取消等)
|
||||
drag(x1, y1, x2, y2, steps=10) # 平滑拖拽
|
||||
```
|
||||
|
||||
## 5. 屏幕状态检测
|
||||
|
||||
```python
|
||||
class PCState(Enum):
|
||||
UNKNOWN # 未知
|
||||
SLEEP # 休眠 (HDMI 冻结)
|
||||
BIOS # BIOS 界面
|
||||
BOOT # 启动中
|
||||
LOCK_SCREEN # 锁屏
|
||||
DESKTOP # 桌面
|
||||
APP_WINDOW # 应用窗口
|
||||
```
|
||||
|
||||
| 状态 | 检测方法 | 信心度 |
|
||||
|------|---------|--------|
|
||||
| SLEEP | 连续 3+ 张截图 MD5 相同 | 0.95 |
|
||||
| BIOS | ≥2 个 BIOS 关键词 | 0.85 |
|
||||
| LOCK_SCREEN | lock 词 + 无任务栏 | 0.80 |
|
||||
| DESKTOP | 桌面指示词 + 无应用指示词 | 0.75 |
|
||||
| APP_WINDOW | 应用指示词存在 | 0.70 |
|
||||
|
||||
## 6. 安全验证
|
||||
|
||||
### validate_type_action(text) 三步检查
|
||||
1. **长度限制**: ≤ 10000 字符
|
||||
2. **Unicode NFKC 归一化**: 全角→ASCII (rm→rm)
|
||||
3. **逐行 regex 匹配**: rm -rf, shutdown, dd, chmod 777, curl|sh 等
|
||||
|
||||
### BLOCKED_PATTERNS
|
||||
```python
|
||||
r'rm\s+(-[rfvdi]*\s+)*/', # rm -rf /
|
||||
r'shutdown|reboot|halt', # 系统关机
|
||||
r'dd\s+if=', # 磁盘写入
|
||||
r'chmod\s+777', # 权限打开
|
||||
r'curl.*\|\s*(ba)?sh', # RCE
|
||||
r':\(\)\s*\{\s*:\s*\|', # Fork bomb
|
||||
```
|
||||
|
||||
## 7. API (port 8890)
|
||||
|
||||
| 方法 | 路径 | 说明 |
|
||||
|------|------|------|
|
||||
| GET | /api/v1/agent/status | 运行状态 + 队列深度 |
|
||||
| GET | /api/v1/agent/tasks | 任务列表 |
|
||||
| POST | /api/v1/agent/tasks | 提交新任务 |
|
||||
| GET | /api/v1/agent/tasks/{id} | 任务详情 |
|
||||
| DELETE | /api/v1/agent/tasks/{id} | 取消任务 |
|
||||
| GET | /api/v1/agent/config | 获取配置 |
|
||||
| PATCH | /api/v1/agent/config | 修改配置 |
|
||||
|
||||
## 8. 配置
|
||||
|
||||
```yaml
|
||||
# configs/agent.yaml
|
||||
kvm_url: "http://localhost:8080"
|
||||
kvm_jwt_token: "${KVM_JWT_TOKEN}"
|
||||
llm_model: "gpt-4o"
|
||||
max_steps: 30
|
||||
step_delay: 1.0
|
||||
click_verify_enabled: true
|
||||
memory_enabled: true
|
||||
template_enabled: true
|
||||
```
|
||||
|
||||
## 9. 测试
|
||||
|
||||
```bash
|
||||
# 单元测试 (19 个文件, 不需要设备)
|
||||
cd services/kvm_agent
|
||||
python3 -m pytest tests/ --ignore=tests/test_integration.py -v
|
||||
|
||||
# 集成测试 (需要实体 KVM 设备)
|
||||
python3 -m pytest tests/test_integration.py -v
|
||||
```
|
||||
@@ -0,0 +1,192 @@
|
||||
# KVM Server (Go) 架构
|
||||
|
||||
> 子系统: deps/KVM/go/ | 端口: 8080 | 包名: kvm-server
|
||||
|
||||
## 1. 概述
|
||||
|
||||
KVM Server 是系统核心,Go 单体服务提供 140+ REST API、WebRTC 视频推流、USB HID 控制、RBAC 鉴权、审计日志和 Web UI。
|
||||
|
||||
## 2. 包结构
|
||||
|
||||
```
|
||||
go/
|
||||
├── cmd/kvm-server/main.go # 入口 (18 阶段初始化)
|
||||
└── internal/
|
||||
├── api/ # HTTP 路由 + 处理器 (16 files)
|
||||
│ ├── router.go # 140+ 路由注册
|
||||
│ ├── middleware.go # CORS/JWT/RBAC/审计/gzip
|
||||
│ ├── kvm_handler.go # 视频/HID/录制/OCR
|
||||
│ ├── privacy_handler.go # 隐私模式 + PII 检测 + 遮蔽回调
|
||||
│ ├── auth_handler.go # 登录/TOTP/JWT
|
||||
│ ├── user_handler.go # RBAC 用户管理
|
||||
│ ├── audit_handler.go # 审计日志/录制/时间线
|
||||
│ ├── system_handler.go # 健康/指标/系统信息
|
||||
│ ├── terminal_handler.go # SSH 终端会话
|
||||
│ ├── storage_handler.go # 虚拟 USB 存储
|
||||
│ ├── config_handler.go # 服务发现
|
||||
│ └── proxy_handler.go # Agent 反代 (→:8890)
|
||||
├── videopipe/ # 原生视频管道 (CGo)
|
||||
├── gstreamer/ # GStreamer 备选管道
|
||||
├── webrtc/ # pion WebRTC 推流
|
||||
├── hid/ # USB HID 控制
|
||||
│ ├── manager.go # 键鼠统一接口
|
||||
│ ├── dispatcher.go # RT 优先级 HID 线程
|
||||
│ ├── keyboard.go # USB 键盘协议
|
||||
│ ├── mouse.go # USB 鼠标协议
|
||||
│ └── charmap.go # 字符→HID 键码映射
|
||||
├── ai/ # OCR + 模式匹配
|
||||
│ ├── ocr_bridge.go # exec 模式 OCR
|
||||
│ ├── ocr_bridge_cgo.go # CGo 内联 OCR (build: cgo && rknn)
|
||||
│ ├── ocr_monitor.go # 事件驱动 OCR 触发
|
||||
│ └── pattern_evaluator.go # 敏感命令匹配
|
||||
├── auth/ # JWT + TOTP
|
||||
├── rbac/ # Casbin ACL
|
||||
├── audit/ # 审计日志 + syslog
|
||||
├── store/ # GORM MySQL 存储
|
||||
├── config/ # TOML/JSON 配置
|
||||
├── control/ # HID 互斥控制
|
||||
├── recorder/ # HLS 录制
|
||||
├── signaling/ # WebSocket 信令
|
||||
├── terminal/ # SSH/PTY 终端
|
||||
├── storage/ # USB 虚拟媒体
|
||||
├── gateway/ # 隐私网关集成
|
||||
├── crypto/ # SM3 国密
|
||||
├── logging/ # slog JSON
|
||||
├── metrics/ # Prometheus 指标
|
||||
├── timeline/ # 事件时间线
|
||||
├── turn/ # STUN/TURN 凭证
|
||||
├── web/ # 嵌入 React UI
|
||||
└── wireguard/ # WireGuard (stub)
|
||||
```
|
||||
|
||||
## 3. 启动序列 (main.go)
|
||||
|
||||
```
|
||||
1. pinToA76BigCores() CPU 亲和性: cores 4-7
|
||||
2. config.Load() /etc/kvm/config.json
|
||||
3. logging.Setup() slog JSON
|
||||
4. store.OpenDB() MySQL + auto-migrate
|
||||
5. auth.Init() JWT secret + TOTP
|
||||
6. rbac.Init() Casbin ACL 策略
|
||||
7. audit.Init() 审计日志 + syslog (可选)
|
||||
8. hid.Init() /dev/hidg0 + /dev/hidg1
|
||||
9. videoPipeline.Start() Native MPP 或 GStreamer
|
||||
10. webrtc.Init() pion PeerConnection 工厂
|
||||
11. recorder.Init() HLS 分段录制
|
||||
12. signaling.Init() WebSocket 信令
|
||||
13. ocr.Init() OCR Monitor (事件驱动)
|
||||
14. privacy.Init() 隐私处理器 + 遮蔽回调
|
||||
15. terminal.Init() SSH 终端管理
|
||||
16. router.Build() 140+ 路由 + 中间件
|
||||
17. http.ListenAndServe() :8080
|
||||
18. signal.Notify() SIGINT/SIGTERM 优雅关闭
|
||||
```
|
||||
|
||||
## 4. 核心 API 端点
|
||||
|
||||
### 认证
|
||||
| 方法 | 路径 | 说明 |
|
||||
|------|------|------|
|
||||
| POST | /api/auth/login | 用户名+密码登录 |
|
||||
| POST | /api/auth/login/totp | TOTP 二次验证 |
|
||||
| POST | /api/auth/logout | 登出 |
|
||||
| GET | /api/auth/me | 当前用户信息 |
|
||||
|
||||
### KVM 控制
|
||||
| 方法 | 路径 | 说明 |
|
||||
|------|------|------|
|
||||
| GET | /api/kvm/screenshot | JPEG 截图 |
|
||||
| POST | /api/kvm/mouse/move | 鼠标移动 (归一化坐标) |
|
||||
| POST | /api/kvm/mouse/click | 鼠标点击 |
|
||||
| POST | /api/kvm/keyboard/type | 文本输入 |
|
||||
| POST | /api/kvm/keyboard/key | 单键按下 |
|
||||
| POST | /api/kvm/video/keyframe | 强制 IDR |
|
||||
| PUT | /api/kvm/video/settings | 更新码率/帧率 |
|
||||
|
||||
### 隐私
|
||||
| 方法 | 路径 | 说明 |
|
||||
|------|------|------|
|
||||
| GET/POST | /api/v1/privacy/mode | 隐私模式 (off/audit/redact) |
|
||||
| GET | /api/v1/privacy/bboxes | 当前 PII 区域坐标 |
|
||||
| GET | /api/v1/privacy/screen/detections | OCR 检测结果 |
|
||||
| CRUD | /api/v1/privacy/patterns | 自定义检测规则 |
|
||||
|
||||
### 审计
|
||||
| 方法 | 路径 | 说明 |
|
||||
|------|------|------|
|
||||
| GET | /api/audit/logs | 审计日志 (分页) |
|
||||
| POST | /api/audit/logs/verify | SM3 哈希链验证 |
|
||||
| GET | /api/audit/recordings | 录制列表 |
|
||||
| GET | /api/audit/timeline | 事件时间线 |
|
||||
| GET | /api/audit/ai | OCR/模式匹配日志 |
|
||||
|
||||
完整路由清单见 `go/internal/api/router.go`。
|
||||
|
||||
## 5. HID 控制子系统
|
||||
|
||||
### 线程模型
|
||||
```
|
||||
WebSocket 消息 → signaling.OnBinaryHID()
|
||||
→ control.TryAcquire(clientID) [互斥, 5min 超时]
|
||||
→ HIDDispatcher.Dispatch(event) [非阻塞, ch 缓冲 1024]
|
||||
→ Worker 线程 (A76 core 4, SCHED_RR 实时优先级)
|
||||
├─ 鼠标移动合并 (高频消息优化)
|
||||
├─ write(/dev/hidg0) 或 write(/dev/hidg1)
|
||||
├─ onKeyPress() → OCR L1 触发
|
||||
└─ onMouseClick() → OCR L2 触发
|
||||
```
|
||||
|
||||
### HID 协议
|
||||
- 键盘: USB HID Report (8 bytes: modifiers + keycode)
|
||||
- 鼠标: 绝对坐标 (x:u16 y:u16 buttons:u8)
|
||||
|
||||
## 6. OCR 监控
|
||||
|
||||
### 触发机制 (事件驱动, 非轮询)
|
||||
| 级别 | 触发源 | 冷却时间 |
|
||||
|------|--------|---------|
|
||||
| L1 | 控制键 (Enter, Ctrl+C) | 100ms |
|
||||
| L2 | 鼠标点击 | 300ms |
|
||||
| L3 | 普通按键 | 500ms |
|
||||
|
||||
### 双模式实现
|
||||
- **exec 模式**: fork `/usr/bin/kvm-ocr` (800ms, 通用)
|
||||
- **CGo 模式**: libkvm_ocr.so 内联调用 (60ms, 需 RKNN)
|
||||
- Build tag: `//go:build cgo && rknn`
|
||||
|
||||
## 7. 配置
|
||||
|
||||
配置文件: `/etc/kvm/config.json` (JSON) 或 `.toml`
|
||||
|
||||
关键配置节:
|
||||
```toml
|
||||
[server]
|
||||
port = 8080
|
||||
|
||||
[gstreamer]
|
||||
pipeline_backend = "native" # "native" 或 "gstreamer"
|
||||
|
||||
[hid]
|
||||
keyboard_device = "/dev/hidg0"
|
||||
mouse_device = "/dev/hidg1"
|
||||
|
||||
[ocr]
|
||||
enabled = true
|
||||
binary_path = "/usr/bin/kvm-ocr"
|
||||
sample_interval_sec = 2
|
||||
|
||||
[services.privacy]
|
||||
enabled = true
|
||||
address = "127.0.0.1:8000"
|
||||
```
|
||||
|
||||
## 8. 数据库模型
|
||||
|
||||
| 表 | 说明 |
|
||||
|-----|------|
|
||||
| users | 用户 + 密码哈希 + 角色 |
|
||||
| sessions | 登录会话 |
|
||||
| audit_log | SM3 哈希链审计 |
|
||||
| recordings | HLS 录制元数据 |
|
||||
| ai_logs | OCR 检测 + 模式匹配 |
|
||||
| request_logs | HTTP 请求审计 |
|
||||
@@ -0,0 +1,110 @@
|
||||
# 记忆桥接 (mem-bridge) 架构
|
||||
|
||||
> 子系统: deps/embedding/ | 端口: 8001 (memory) + 8002 (router) | 包名: kvm-bridge
|
||||
|
||||
## 1. 概述
|
||||
|
||||
mem-bridge 提供两个 FastAPI 服务:Memory Service 负责向量搜索和会话存储,Router Service 负责 LLM 路由和上下文注入。
|
||||
|
||||
## 2. 双服务架构
|
||||
|
||||
```
|
||||
Agent/前端 → Router Service (port 8002)
|
||||
├─ 复杂度评分
|
||||
├─ Query 改写
|
||||
├─ Memory 检索 (→ port 8001)
|
||||
├─ 上下文注入
|
||||
└─ 后端路由 (OpenAI/Gemini/Deepseek/本地)
|
||||
│
|
||||
▼
|
||||
Memory Service (port 8001)
|
||||
├─ Turn Store (对话轮次, SQLite)
|
||||
├─ Fact Store (事实提取)
|
||||
├─ Compressor (上下文压缩)
|
||||
└─ Vector Index (FAISS + RKNNLite)
|
||||
├─ multilingual-e5-small (384-dim)
|
||||
├─ IVF-PQ 索引 (>5000 向量)
|
||||
└─ FlatIP 备选 (<5000 向量)
|
||||
```
|
||||
|
||||
## 3. Memory Service API
|
||||
|
||||
| 方法 | 路径 | 说明 |
|
||||
|------|------|------|
|
||||
| POST | /v1/add_turn | 保存对话轮次 |
|
||||
| POST | /v1/search | 语义搜索 (混合 BM25 + embedding) |
|
||||
| POST | /v1/compress | 上下文压缩 (token 限制) |
|
||||
| POST | /v1/facts | 事实提取和存储 |
|
||||
|
||||
### 混合搜索评分
|
||||
```
|
||||
final = semantic_score * (1 - bm25_weight) + bm25_score * bm25_weight
|
||||
```
|
||||
|
||||
## 4. Router Service API
|
||||
|
||||
| 方法 | 路径 | 说明 |
|
||||
|------|------|------|
|
||||
| POST | /v1/chat/completions | OpenAI 兼容聊天 API |
|
||||
|
||||
### 路由策略
|
||||
1. 复杂度评分 (ComplexityScorer)
|
||||
2. Memory 检索上下文
|
||||
3. Query 改写 (QueryRewriter)
|
||||
4. 后端选择:
|
||||
- gpt-4o → OpenAI API
|
||||
- gemini-2.5-pro → Google API
|
||||
- deepseek → Deepseek API
|
||||
- local → Ollama (Qwen2-1.5B)
|
||||
|
||||
## 5. 嵌入模型
|
||||
|
||||
| 参数 | 值 |
|
||||
|------|-----|
|
||||
| 模型 | multilingual-e5-small |
|
||||
| 维度 | 384 |
|
||||
| 量化 | RKNN INT8 (NPU Core 2) |
|
||||
| FAISS 索引 | IVF-PQ (nlist=100, m=48) |
|
||||
| 压缩比 | 32:1 (每向量 48 bytes) |
|
||||
| 最大存储 | 3GB |
|
||||
|
||||
## 6. NPU 核心分配
|
||||
|
||||
```
|
||||
Core 0: OCR detection (RKNN_NPU_CORE_0)
|
||||
Core 1: OCR recognition (RKNN_NPU_CORE_1)
|
||||
Core 2: Embedding (RKNN_NPU_CORE_2) ← 独占, 不与 OCR 争用
|
||||
```
|
||||
|
||||
## 7. 配置
|
||||
|
||||
```yaml
|
||||
db_dir: "~/.embed_db/"
|
||||
memory_host: "0.0.0.0"
|
||||
memory_port: 8001
|
||||
router_host: "0.0.0.0"
|
||||
router_port: 8002
|
||||
|
||||
backends:
|
||||
openai:
|
||||
api_key: "${OPENAI_API_KEY}"
|
||||
model: "gpt-4o"
|
||||
google:
|
||||
api_key: "${GOOGLE_API_KEY}"
|
||||
model: "gemini-2.5-pro"
|
||||
|
||||
default_backend: "openai"
|
||||
```
|
||||
|
||||
## 8. systemd 服务
|
||||
|
||||
```ini
|
||||
# mem-bridge-memory.service
|
||||
ExecStart=python server.py --service memory
|
||||
CPUAffinity=2 3 # A55 小核
|
||||
|
||||
# mem-bridge-router.service
|
||||
Requires=mem-bridge-memory.service
|
||||
ExecStart=python server.py --service router
|
||||
CPUAffinity=2 3 # A55 小核
|
||||
```
|
||||
@@ -0,0 +1,185 @@
|
||||
# KVM-Privacy 系统架构总览
|
||||
|
||||
> 最后更新: 2026-03-05 | 版本: 1.0.0
|
||||
|
||||
## 1. 系统定位
|
||||
|
||||
KVM-Privacy 是基于 KVM-over-IP 的两层隐私保护系统,运行于 NanoPC-T6 (RK3588)。通过硬件加速(NPU + RGA + MPP)实现实时 PII 检测、视频流遮蔽、网络上传拦截和 AI 自主操控。所有防护完全在 KVM 设备端完成,无需在被控机器上部署任何软件。
|
||||
|
||||
## 2. 服务架构
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ KVM WebUI (React) │
|
||||
│ http://localhost:8080 │
|
||||
└──────────────┬──────────────────────────────────────────────┘
|
||||
│ HTTP + WebSocket + WebRTC
|
||||
┌──────────────▼──────────────────────────────────────────────┐
|
||||
│ KVM Server (Go, port 8080) │
|
||||
│ ┌──────────┐ ┌───────┐ ┌────────┐ ┌──────┐ ┌───────────┐ │
|
||||
│ │ 视频管道 │ │ HID │ │ WebRTC │ │ RBAC │ │ 隐私处理 │ │
|
||||
│ │ V4L2→MPP │ │hidg0/1│ │ pion │ │Casbin│ │ OCR+遮蔽 │ │
|
||||
│ └──────────┘ └───────┘ └────────┘ └──────┘ └───────────┘ │
|
||||
│ ┌──────────┐ ┌───────┐ ┌────────┐ ┌──────┐ ┌───────────┐ │
|
||||
│ │ 审计日志 │ │ 录制 │ │ 终端 │ │ 存储 │ │ Agent代理 │ │
|
||||
│ │ SM3链+DB │ │ HLS │ │SSH/WS │ │虚拟U盘│ │ →:8890 │ │
|
||||
│ └──────────┘ └───────┘ └────────┘ └──────┘ └───────────┘ │
|
||||
└──┬───────┬───────────┬──────────────────────┬──────────────┘
|
||||
│ │ │ │
|
||||
│ ┌────▼────┐ ┌────▼──────────┐ ┌───────▼──────────┐
|
||||
│ │隐私网关 │ │ KVM Agent │ │ info-privacy-rs │
|
||||
│ │mitmproxy│ │ Python 8890 │ │ Rust/RKNN 8000 │
|
||||
│ │port 8888│ │ LLM+OCR+HID │ │ PII 检测/脱敏 │
|
||||
│ └────┬────┘ └───────┬───────┘ └──────────────────┘
|
||||
│ │ │
|
||||
│ ┌────▼────┐ ┌─────▼──────────────────┐
|
||||
│ │ 审计DB │ │ mem-bridge (Python) │
|
||||
│ │MariaDB/ │ │ Memory :8001 (FAISS) │
|
||||
│ │SQLite │ │ Router :8002 (LLM路由) │
|
||||
│ └─────────┘ └────────────────────────┘
|
||||
│
|
||||
┌──▼──────────────────────┐
|
||||
│ 目标主机 (HDMI + USB) │
|
||||
│ V4L2 采集 + HID gadget │
|
||||
└─────────────────────────┘
|
||||
```
|
||||
|
||||
## 3. 服务端口规划
|
||||
|
||||
| 服务 | 端口 | 协议 | 包名 | 说明 |
|
||||
|------|------|------|------|------|
|
||||
| KVM Server | 8080 | HTTP/WS/WebRTC | kvm-server | Go 主服务 |
|
||||
| info-privacy-rs | 8000 | HTTP | kvm-privacy | Rust PII 检测/脱敏 |
|
||||
| mem-bridge memory | 8001 | HTTP | kvm-bridge | FAISS 向量搜索 + 会话存储 |
|
||||
| mem-bridge router | 8002 | HTTP | kvm-bridge | AI 路由 (OpenAI/Gemini/本地) |
|
||||
| Privacy Gateway | 8888 | HTTP Proxy | kvm-mitm | mitmproxy 网络拦截 |
|
||||
| KVM Agent | 8890 | HTTP | kvm-agent | Python AI Agent daemon |
|
||||
| Privacy API | 8889 | HTTP | kvm-mitm | 隐私网关 REST API |
|
||||
|
||||
## 4. 子系统划分
|
||||
|
||||
| 子系统 | 语言 | 文档 | 核心职责 |
|
||||
|--------|------|------|---------|
|
||||
| KVM Server | Go 1.24 | [kvm-server.md](kvm-server.md) | 视频流、HID、WebRTC、RBAC、审计 |
|
||||
| 原生视频管道 | C/C++ + CGo | [video-pipeline.md](video-pipeline.md) | V4L2→RGA→MPP 编码、隐私遮蔽、DMA-buf |
|
||||
| KVM Agent | Python 3.12 | [kvm-agent.md](kvm-agent.md) | AI 自主操控(感知→规划→执行→验证) |
|
||||
| 隐私系统 | Go+Python+Rust | [privacy-system.md](privacy-system.md) | PII 检测、视频遮蔽、网络拦截、Chrome 扩展 |
|
||||
| 记忆桥接 | Python 3.12 | [memory-bridge.md](memory-bridge.md) | FAISS 向量搜索、LLM 路由、上下文压缩 |
|
||||
| 硬件优化 | C/C++ | [hardware.md](hardware.md) | RK3588 NPU/RGA/DMA-buf/CPU 亲和性 |
|
||||
| 部署打包 | Shell/Debian | [deployment.md](deployment.md) | 5 个 deb 包、systemd 服务、CI 流程 |
|
||||
|
||||
## 5. 数据流概览
|
||||
|
||||
### 5.1 视频流路径
|
||||
```
|
||||
HDMI 输入 → V4L2 (/dev/video0) → [BGR/NV12]
|
||||
→ RGA 色彩转换/缩放 → NV12 buffer
|
||||
→ 隐私遮蔽 (redact_nv12_regions, 如果 mode=redact)
|
||||
→ MPP H.264 编码 (或 DMA-buf 零拷贝)
|
||||
→ RTP 打包 (pion/rtp)
|
||||
→ WebRTC DTLS-SRTP → 浏览器 <video>
|
||||
```
|
||||
|
||||
### 5.2 OCR 感知路径
|
||||
```
|
||||
V4L2 BGR 帧 → JPEG 编码 → /tmp/kvm-ocr/frame.jpg
|
||||
→ HID 事件触发 OCR (L1/L2/L3 优先级)
|
||||
→ kvm-ocr 分析 (exec 或 CGo 内联)
|
||||
→ 文本 + 区域坐标 → 模式匹配 (sensitive_commands.json)
|
||||
→ 危险命令? → HID 阻断
|
||||
→ mode=redact? → detectPIIBboxes → SetRedactRegions → 视频管道
|
||||
```
|
||||
|
||||
### 5.3 Agent 自主操控
|
||||
```
|
||||
用户提交任务 → POST /api/v1/agent/tasks
|
||||
→ LLM 规划 (Vision: screenshot + OCR + 历史)
|
||||
→ 执行动作 (鼠标优先, 安全白名单)
|
||||
→ 截图验证 (OCR 语义检查)
|
||||
→ 成功? → 保存模板 | 失败? → 重试/放弃
|
||||
```
|
||||
|
||||
### 5.4 隐私拦截
|
||||
```
|
||||
浏览器流量 → mitmproxy (8888)
|
||||
→ AI 域名匹配 → 文件上传检测
|
||||
→ info-privacy-rs API 分析 PII
|
||||
→ audit: 记录日志 | redact: 脱敏后放行 | off: 直通
|
||||
```
|
||||
|
||||
## 6. 技术栈
|
||||
|
||||
| 层 | 技术 | 版本 |
|
||||
|----|------|------|
|
||||
| 前端 | React + TypeScript | 18+ / 5.3+ |
|
||||
| Go 后端 | Go + gorm + pion WebRTC | 1.24.0 |
|
||||
| Python 服务 | asyncio + aiohttp + httpx | 3.12 |
|
||||
| Rust 加速 | Axum + RKNN + RGA | 2024 edition |
|
||||
| 视频编码 | Rockchip MPP + RGA (native) / GStreamer (fallback) | — |
|
||||
| 向量搜索 | FAISS + RKNNLite (embedding) | — |
|
||||
| 数据库 | MariaDB + SQLite | 10.4+ |
|
||||
| 认证 | JWT (RS256) + TOTP + Casbin RBAC | — |
|
||||
| 国密 | SM3 哈希 (审计链) | emmansun/gmsm |
|
||||
|
||||
## 7. 安全架构
|
||||
|
||||
### 纵深防御(设备端完成,不依赖被控机器)
|
||||
```
|
||||
Layer 1: Agent Safety (Python)
|
||||
├─ 20 键白名单, 组合键全禁
|
||||
├─ Unicode NFKC 归一化 (防全角绕过)
|
||||
├─ 危险命令 regex 拦截
|
||||
└─ 文本长度 ≤10000
|
||||
|
||||
Layer 2: KVM Server (Go)
|
||||
├─ OCR 模式匹配 (sensitive_commands.json)
|
||||
├─ HID 实时阻断 (检测到危险命令)
|
||||
├─ RBAC 权限控制 (admin/auditor/operator)
|
||||
├─ JWT + TOTP 双因素认证
|
||||
└─ SM3 哈希链审计日志
|
||||
|
||||
Layer 3: Privacy Gateway (Python + Rust)
|
||||
├─ MITM SSL 拦截 AI 平台上传
|
||||
├─ info-privacy-rs PII 检测/脱敏
|
||||
└─ append-only 审计日志
|
||||
```
|
||||
|
||||
## 8. 目录结构
|
||||
|
||||
```
|
||||
KVM-privacy/
|
||||
├── CLAUDE.md # 项目编码规范
|
||||
├── README.md # 项目概览
|
||||
├── DEVELOP.md # 开发指南
|
||||
├── RELEASE.md # 版本历史
|
||||
├── Makefile # 部署命令
|
||||
├── debian/ # 5 个 deb 包定义
|
||||
│ ├── kvm-agent/ # Python AI Agent
|
||||
│ ├── kvm-bridge/ # mem-bridge 记忆服务
|
||||
│ ├── kvm-meta/ # 元包 (依赖全部)
|
||||
│ ├── kvm-mitm/ # mitmproxy 网关
|
||||
│ └── kvm-privacy/ # Rust PII 检测
|
||||
├── deploy/
|
||||
│ ├── kvm.toml.example # KVM 配置模板
|
||||
│ └── systemd/ # 8 个 systemd 服务文件
|
||||
├── deps/ # Git submodules
|
||||
│ ├── KVM/ # Go KVM 核心 + React WebUI
|
||||
│ │ ├── go/ # Go 后端 (28 个 internal 包)
|
||||
│ │ ├── web/ # React 前端
|
||||
│ │ ├── src/video/ # C 视频管道 (V4L2/RGA/MPP)
|
||||
│ │ ├── src/audit/ # C OCR 模块 (RKNN)
|
||||
│ │ ├── systemd/ # KVM systemd 服务
|
||||
│ │ ├── debian/ # kvm-server deb 定义
|
||||
│ │ └── scripts/ # 构建脚本
|
||||
│ ├── embedding/ # Python mem-bridge
|
||||
│ └── info-privacy-rs/ # Rust PII 检测
|
||||
├── services/
|
||||
│ ├── kvm_agent/ # Python AI Agent (26 个模块)
|
||||
│ └── privacy_gateway/ # Python mitmproxy 网关
|
||||
├── scripts/ # 部署/测试脚本
|
||||
├── docs/
|
||||
│ ├── architecture/ # 子系统架构文档 (本目录)
|
||||
│ └── plans/ # 历史设计文档 (14 个)
|
||||
└── web/
|
||||
└── src/ # TypeScript 组件库
|
||||
```
|
||||
@@ -0,0 +1,135 @@
|
||||
# 隐私系统架构
|
||||
|
||||
> 跨子系统: Go privacy_handler + Python gateway + Rust info-privacy-rs
|
||||
|
||||
## 1. 概述
|
||||
|
||||
隐私系统提供两层 PII 防护:视频流实时遮蔽、网络上传拦截脱敏。所有防护完全在 KVM 设备端完成,无需在被控机器上部署任何软件。
|
||||
|
||||
## 2. 两层防护
|
||||
|
||||
```
|
||||
Layer 1: 视频流遮蔽 (Go + C)
|
||||
├─ OCR 检测屏幕 PII
|
||||
├─ 归一化坐标 → C 视频管道
|
||||
└─ NV12 黑色填充 → MPP 编码输出
|
||||
|
||||
Layer 2: 网络拦截 (Python + Rust)
|
||||
├─ mitmproxy SSL 中间人 (port 8888)
|
||||
├─ AI 域名匹配 → 文件上传检测
|
||||
├─ info-privacy-rs 分析/脱敏
|
||||
└─ append-only 审计日志
|
||||
```
|
||||
|
||||
## 3. 隐私模式
|
||||
|
||||
| 模式 | 视频流 | 网络 | 前端 |
|
||||
|------|--------|------|------|
|
||||
| off | 无处理 | 直通 | 无 overlay |
|
||||
| audit | 红色边框 overlay | 记录日志,放行 | 红色边框 |
|
||||
| redact | 黑色填充遮蔽 | 脱敏后放行 | 黑色填充 |
|
||||
|
||||
## 4. 视频遮蔽 (Go + C)
|
||||
|
||||
### 数据流
|
||||
```
|
||||
OCRMonitor.processFrame()
|
||||
→ OCRResult{text, regions}
|
||||
→ onAfterScan callback
|
||||
→ PrivacyHandler.ScanAndUpdateRedaction()
|
||||
→ detectPIIBboxes() [Go regex: phone/id_card/bank_card/email]
|
||||
→ onRedactUpdate(bboxes [][4]float32)
|
||||
→ NativePipeline.SetRedactRegions()
|
||||
→ bridgeSetRedactBboxes() [CGo]
|
||||
→ video_pipeline_set_redact_bboxes() [C, mutex]
|
||||
→ 每帧编码前: redact_nv12_regions()
|
||||
```
|
||||
|
||||
### PII 检测正则 (Go, privacy_handler.go)
|
||||
```go
|
||||
phone: 1[3-9]\d{9}
|
||||
id_card: \d{17}[\dXx]
|
||||
bank_card: \d{16,19}
|
||||
email: [\w.+-]+@[\w.-]+\.\w{2,}
|
||||
```
|
||||
|
||||
### NV12 黑色填充 (C, color_convert.cpp)
|
||||
- Y plane = 16 (BT.601 黑色, 非 0)
|
||||
- UV plane = 128 (中性色度)
|
||||
- 坐标对齐到 2 像素 (色度子采样)
|
||||
|
||||
## 5. Privacy Gateway (Python)
|
||||
|
||||
### 组件
|
||||
| 文件 | 职责 |
|
||||
|------|------|
|
||||
| addon.py | mitmproxy 插件: 域名匹配→拦截→脱敏 |
|
||||
| interceptor.py | 请求内容提取 (multipart/JSON) |
|
||||
| upload_scanner.py | 调用 info-privacy-rs 分析 |
|
||||
| cert_manager.py | CA 证书生成 (/etc/kvm-privacy/ca/) |
|
||||
| audit_logger.py | 审计日志 (MariaDB/SQLite) |
|
||||
| privacy_api.py | REST API (port 8889) |
|
||||
|
||||
### AI 域名白名单
|
||||
```
|
||||
api.openai.com
|
||||
api.anthropic.com
|
||||
generativelanguage.googleapis.com
|
||||
api.deepseek.com
|
||||
...
|
||||
```
|
||||
|
||||
### 拦截流程
|
||||
```
|
||||
HTTP Request → addon.py:request()
|
||||
→ should_intercept(url)?
|
||||
→ 文件上传? → upload_scanner.scan()
|
||||
→ POST info-privacy-rs /api/v1/analyze
|
||||
→ classification:
|
||||
normal → 放行 (toast)
|
||||
sensitive_partial → 记录 + 用户决定
|
||||
classified → 拦截 (阻止)
|
||||
→ mode=redact? → POST /api/v1/redact → 替换内容
|
||||
```
|
||||
|
||||
## 6. info-privacy-rs (Rust)
|
||||
|
||||
### API
|
||||
| 方法 | 路径 | 说明 |
|
||||
|------|------|------|
|
||||
| POST | /api/v1/analyze | 检测 PII (返回 DetectionReport) |
|
||||
| POST | /api/v1/redact | 脱敏 (需传 redact_types 配置!) |
|
||||
| GET | /api/v1/health | 健康检查 |
|
||||
| GET | /api/v1/types | 支持的实体类型列表 |
|
||||
|
||||
### 支持格式
|
||||
docx, xlsx, pdf, jpg, jpeg, png, bmp (**不支持 txt**)
|
||||
|
||||
### 实体类型
|
||||
```
|
||||
id_card, phone, bank_card, email,
|
||||
license_plate, name, address, face
|
||||
```
|
||||
|
||||
### 分类标准
|
||||
- **normal**: 无 PII
|
||||
- **sensitive_partial**: < 5 个高风险实体
|
||||
- **classified**: ≥ 5 个高风险实体 (id_card/phone/bank_card)
|
||||
|
||||
### 重要: redact 必须传 config
|
||||
```bash
|
||||
# 正确 (返回脱敏文件)
|
||||
curl -F "file=@doc.pdf" -F 'config={"redact_types":["id_card","phone"]}' \
|
||||
http://localhost:8000/api/v1/redact
|
||||
|
||||
# 错误 (返回原文件!)
|
||||
curl -F "file=@doc.pdf" http://localhost:8000/api/v1/redact
|
||||
```
|
||||
|
||||
## 7. 状态存储
|
||||
|
||||
| 文件 | 内容 |
|
||||
|------|------|
|
||||
| /var/lib/kvm-privacy/state.json | `{"mode":"off\|audit\|redact"}` |
|
||||
| /var/lib/kvm-privacy/audit.db | SQLite 审计日志 |
|
||||
| /etc/kvm-privacy/ca/ca.crt | MITM CA 证书 |
|
||||
@@ -0,0 +1,183 @@
|
||||
# 原生视频管道架构
|
||||
|
||||
> 子系统: deps/KVM/src/video/ + go/internal/videopipe/
|
||||
|
||||
## 1. 概述
|
||||
|
||||
原生视频管道绕过 GStreamer,直接使用 RK3588 硬件加速器链:V4L2 采集 → RGA 色彩转换/缩放 → MPP H.264 编码。通过 CGo 桥接到 Go 层,最终由 pion/rtp 打包推送到 WebRTC 客户端。
|
||||
|
||||
## 2. 管道架构
|
||||
|
||||
```
|
||||
HDMI-RX (/dev/video0)
|
||||
│
|
||||
▼ V4L2 MMAP (4 buffers, BGR24 或 NV12)
|
||||
│
|
||||
┌───▼──────────────────────────────────────┐
|
||||
│ v4l2_capture.cpp │
|
||||
│ ├─ pthread capture_thread (select loop) │
|
||||
│ ├─ DMA-buf EXPBUF (零拷贝, 可选) │
|
||||
│ └─ 回调: data + size + pts + buffer_idx │
|
||||
└───┬──────────────────────────────────────┘
|
||||
│
|
||||
┌───▼──────────────────────────────────────┐
|
||||
│ video_pipeline.cpp (internal_v4l2_callback)│
|
||||
│ │
|
||||
│ 1. BGR 快照回调 (snapshot_fps 控制频率) │
|
||||
│ └─ NV12→BGR 转换 (如果源是 NV12) │
|
||||
│ │
|
||||
│ 2. 色彩转换 + 缩放 │
|
||||
│ ├─ BGR→NV12: convert_bgr_to_nv12() │
|
||||
│ ├─ BGR→NV12+resize: _resize() │
|
||||
│ ├─ NV12→NV12 resize: resize_nv12() │
|
||||
│ └─ NV12 直拷: memcpy │
|
||||
│ │
|
||||
│ 3. 隐私遮蔽 (mutex 保护) │
|
||||
│ └─ redact_nv12_regions(nv12, bboxes) │
|
||||
│ │
|
||||
│ 4. MPP 编码 │
|
||||
│ ├─ DMA-buf: mpp_encoder_encode_dmabuf │
|
||||
│ └─ memcpy: mpp_encoder_encode_nv12 │
|
||||
└───┬──────────────────────────────────────┘
|
||||
│
|
||||
┌───▼──────────────────────────────────────┐
|
||||
│ mpp_encoder.cpp │
|
||||
│ ├─ MppApi::encode_put_frame │
|
||||
│ ├─ MppApi::encode_get_packet │
|
||||
│ ├─ IDR 强制 (MPP_ENC_SET_IDR_FRAME) │
|
||||
│ └─ 动态码率 (MPP_ENC_SET_RC_CFG) │
|
||||
└───┬──────────────────────────────────────┘
|
||||
│ H.264 NAL units
|
||||
┌───▼──────────────────────────────────────┐
|
||||
│ Go: cgo_bridge.go → pipeline.go │
|
||||
│ ├─ onH264(data, pts, keyframe) │
|
||||
│ ├─ RTPPacketizer (pion/rtp H.264) │
|
||||
│ └─ AdaptiveController (PLI→码率调整) │
|
||||
└───┬──────────────────────────────────────┘
|
||||
│ RTP packets
|
||||
▼ WebRTC PeerConnection → 浏览器
|
||||
```
|
||||
|
||||
## 3. 源文件清单
|
||||
|
||||
### C/C++ 层 (deps/KVM/src/video/)
|
||||
|
||||
| 文件 | 行数 | 职责 |
|
||||
|------|------|------|
|
||||
| video_pipeline.h | 55 | 公共 C API (create/start/stop/redact) |
|
||||
| video_pipeline.cpp | 364 | 管道编排: 采集→转换→遮蔽→编码 |
|
||||
| v4l2_capture.h | 55 | V4L2 采集 API (含 DMA-buf) |
|
||||
| v4l2_capture.cpp | 354 | V4L2 MMAP + EXPBUF + capture thread |
|
||||
| mpp_encoder.h | 30 | MPP 编码器 API (含 DMA-buf encode) |
|
||||
| mpp_encoder.cpp | ~300 | Rockchip MPP H.264 编码 |
|
||||
| color_convert.h | 20 | 色彩转换 API |
|
||||
| color_convert.cpp | ~400 | BGR↔NV12 + RGA resize + NV12 redact |
|
||||
| CMakeLists.txt | 45 | 构建 libmpp_video.so |
|
||||
|
||||
### CGo 桥接层 (go/internal/videopipe/)
|
||||
|
||||
| 文件 | Build Tag | 职责 |
|
||||
|------|-----------|------|
|
||||
| cgo_bridge.go | cgo | Go→C 函数封装 + RedactBbox 类型 |
|
||||
| cgo_exports.go | cgo | //export Go 回调 (H.264 + BGR snapshot) |
|
||||
| cgo_shim.c | — | C 回调 shim (路由到 Go) |
|
||||
| pipeline.go | — | NativePipeline 公共接口 |
|
||||
| adaptive.go | — | AIMD 自适应码率控制 |
|
||||
| rtp_packetizer.go | — | H.264 NAL→RTP 转换 |
|
||||
|
||||
## 4. 关键数据结构
|
||||
|
||||
### VideoPipeline (C++)
|
||||
```c
|
||||
struct VideoPipeline {
|
||||
MppEncoder* encoder;
|
||||
V4L2Capture* capture;
|
||||
H264PacketCallback h264_callback;
|
||||
BGRFrameCallback bgr_callback;
|
||||
|
||||
int width, height; // 编码目标分辨率
|
||||
int cap_width, cap_height; // V4L2 实际采集分辨率
|
||||
uint32_t cap_format; // BGR24 或 NV12
|
||||
bool needs_resize; // 采集≠编码分辨率时启用
|
||||
|
||||
uint8_t* nv12_buffer; // 可复用转换缓冲区
|
||||
uint8_t* bgr_buffer; // NV12→BGR 快照缓冲区
|
||||
|
||||
int snapshot_fps; // BGR 快照帧率 (0=每帧)
|
||||
int snapshot_interval; // = capture_fps / snapshot_fps
|
||||
|
||||
float* redact_bboxes; // [x0,y0,w0,h0, x1,y1,w1,h1, ...]
|
||||
int redact_count;
|
||||
std::mutex redact_mutex; // 保护 bbox 更新
|
||||
};
|
||||
```
|
||||
|
||||
### NativePipeline (Go)
|
||||
```go
|
||||
type NativePipeline struct {
|
||||
cfg Config // Device, Width, Height, FPS, Bitrate
|
||||
onH264 func([]byte, int64, bool) // H.264 输出
|
||||
onSnapshot func([]byte) // JPEG 快照
|
||||
pipelineID int // C 回调路由 ID
|
||||
}
|
||||
```
|
||||
|
||||
## 5. DMA-buf 零拷贝路径
|
||||
|
||||
当满足以下条件时,跳过 CPU memcpy:
|
||||
1. V4L2 输出为 NV12(非 BGR)
|
||||
2. 无缩放(cap_width == width && cap_height == height)
|
||||
3. 无隐私遮蔽(redact_count == 0)
|
||||
4. DMA-buf 导出成功(VIDIOC_EXPBUF)
|
||||
|
||||
```
|
||||
V4L2 DMA-buf fd → MPP mpp_buffer_import_with_tag → 零拷贝编码
|
||||
```
|
||||
|
||||
节省 1080p@30fps 约 90MB/s 内存带宽。
|
||||
|
||||
## 6. 隐私遮蔽机制
|
||||
|
||||
### 调用链
|
||||
```
|
||||
Go: PrivacyHandler.ScanAndUpdateRedaction()
|
||||
→ detectPIIBboxes() (regex: phone/id_card/bank_card/email)
|
||||
→ onRedactUpdate(bboxes) callback
|
||||
→ NativePipeline.SetRedactRegions(regions)
|
||||
→ bridgeSetRedactBboxes() [CGo]
|
||||
→ video_pipeline_set_redact_bboxes() [C, mutex 保护]
|
||||
```
|
||||
|
||||
### NV12 遮蔽算法 (color_convert.cpp)
|
||||
```
|
||||
对每个 bbox (归一化 0.0-1.0):
|
||||
1. 转换为像素坐标 (对齐到 2 像素, NV12 色度采样)
|
||||
2. Y 平面: memset 为 16 (BT.601 黑色)
|
||||
3. UV 平面: memset 为 128 (中性色度)
|
||||
```
|
||||
|
||||
## 7. 自适应码率
|
||||
|
||||
AIMD (Additive Increase Multiplicative Decrease) 算法:
|
||||
- PLI (Picture Loss Indication) 作为拥塞信号
|
||||
- 拥塞时码率减半 (multiplicative decrease ÷2)
|
||||
- 稳定期线性增加 (additive increase)
|
||||
- 码率范围: 100Kbps ~ 20Mbps
|
||||
- FPS 范围: 1 ~ 60
|
||||
|
||||
## 8. 构建
|
||||
|
||||
```bash
|
||||
# 构建 C 库 (需要 rockchip_mpp, rga 开发库)
|
||||
cd deps/KVM
|
||||
cmake -B build/video -S src/video -DCMAKE_BUILD_TYPE=Release
|
||||
cmake --build build/video -j$(nproc)
|
||||
|
||||
# Go 构建 (链接 libmpp_video.so)
|
||||
cd go
|
||||
CGO_CFLAGS="-I../src/video" \
|
||||
CGO_LDFLAGS="-L../build/video -lmpp_video -Wl,-rpath,/usr/lib/kvm" \
|
||||
CGO_ENABLED=1 go build ./cmd/kvm-server/
|
||||
```
|
||||
|
||||
安装位置: `/usr/lib/kvm/libmpp_video.so`
|
||||
@@ -0,0 +1,218 @@
|
||||
# AI Agent Command Bar + Privacy Gateway Three-Mode Design
|
||||
|
||||
**Date:** 2026-03-03
|
||||
**Branch:** feature/enhance-testing-security-states
|
||||
**Status:** Approved
|
||||
|
||||
## Overview
|
||||
|
||||
Add two missing frontend entry points:
|
||||
1. **AI Agent Command Bar** — bottom of ConsolePage, submit tasks to KVM Agent
|
||||
2. **Privacy Gateway Mode Selector** — extend /privacy page CertificateTab with OFF/AUDIT/REDACT modes
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Browser (port 8080)
|
||||
├── /api/v1/agent/* → Go reverse proxy → KVM Agent (port 8890)
|
||||
└── /api/v1/privacy/* → Go reverse proxy → Privacy Gateway (port 8889)
|
||||
```
|
||||
|
||||
Go adds `proxy_handler.go` using `httputil.ReverseProxy`, one instance per target.
|
||||
|
||||
## Backend Changes
|
||||
|
||||
### Go: proxy_handler.go (new)
|
||||
|
||||
- `AgentProxy`: `/api/v1/agent/*` → `http://localhost:8890`
|
||||
- `PrivacyProxy`: `/api/v1/privacy/*` → `http://localhost:8889`
|
||||
- Register in `router.go`
|
||||
- Strip prefix before forwarding
|
||||
- Log proxy errors
|
||||
|
||||
### privacy_api.py (redesign)
|
||||
|
||||
**Remove:** `privacy_enabled: bool`, old boolean format
|
||||
|
||||
**New state.json format:**
|
||||
```json
|
||||
{"mode": "off"}
|
||||
```
|
||||
|
||||
**Endpoints:**
|
||||
```
|
||||
GET /api/v1/privacy/mode → {mode: "off"|"audit"|"redact"}
|
||||
POST /api/v1/privacy/mode body: {mode: "off"|"audit"|"redact"} → {mode: "..."}
|
||||
GET /api/v1/privacy/stats → {requests, files, pii_by_type, actions:{allow,audit,redact}, top_domains}
|
||||
GET /api/v1/privacy/audit ?limit=50 → [{...log entries}]
|
||||
GET /api/v1/privacy/cert → (unchanged)
|
||||
POST /api/v1/privacy/masking → (unchanged)
|
||||
```
|
||||
|
||||
### addon.py (update)
|
||||
|
||||
Read `mode` from state.json, three-branch logic:
|
||||
```python
|
||||
if mode == "off": pass_through() # no logging
|
||||
elif mode == "audit": log_request(); allow() # log only
|
||||
elif mode == "redact": log_request(); redact() # log + redact
|
||||
```
|
||||
|
||||
### audit_logger.py (extend)
|
||||
|
||||
Log schema per intercepted request:
|
||||
```json
|
||||
{
|
||||
"timestamp": "ISO8601",
|
||||
"mode": "audit|redact",
|
||||
"domain": "api.openai.com",
|
||||
"method": "POST",
|
||||
"path": "/v1/chat/completions",
|
||||
"action": "allow|redact",
|
||||
"files_detected": 1,
|
||||
"pii_types": ["id_card", "phone"],
|
||||
"source_ip": "192.168.1.x",
|
||||
"request_size_bytes": 12345
|
||||
}
|
||||
```
|
||||
|
||||
Storage: SQLite table `audit_log`, supports `GET /api/v1/privacy/audit?limit=50`.
|
||||
|
||||
### runner.py (task cancellation)
|
||||
|
||||
- Add `asyncio.Event cancel_event` per running task
|
||||
- `DELETE /api/v1/agent/tasks/{id}` sets the event
|
||||
- runner checks event between action steps → sets status `CANCELLED`
|
||||
- KVM HID cleanup via `safe_cleanup()` on cancel
|
||||
|
||||
### kvm_agent/api_server.py
|
||||
|
||||
No changes needed — `DELETE /api/v1/agent/tasks/{id}` already exists.
|
||||
|
||||
## Frontend Changes
|
||||
|
||||
### New: `AgentCommandBar.tsx`
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ [任务描述输入框.............................] [执行] [终止] │
|
||||
│ 状态: 空闲 | 运行中 (task_id: abc123) │
|
||||
│ 最近: ✅ 打开记事本 ❌ 找不到文件 │
|
||||
└─────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
- Placed below `.video-container` in ConsolePage
|
||||
- [终止] button visible only when task is running
|
||||
- Task history: last 5 entries, success/fail icons
|
||||
|
||||
### New: `agentStore.ts`
|
||||
|
||||
```typescript
|
||||
interface AgentStore {
|
||||
status: "idle" | "running" | "error"
|
||||
currentTask: Task | null
|
||||
taskHistory: Task[]
|
||||
submitTask(description: string): Promise<void>
|
||||
cancelTask(id: string): Promise<void>
|
||||
pollStatus(id: string): void // 2s interval, stops on terminal state
|
||||
}
|
||||
```
|
||||
|
||||
### Modified: `PrivacyPage.tsx` — CertificateTab
|
||||
|
||||
Replace binary ToggleCard with three-segment mode selector:
|
||||
|
||||
```
|
||||
隐私网关模式:
|
||||
┌──────────┬──────────┬──────────┐
|
||||
│ 关闭 │ 审计 │ 脱敏 │
|
||||
└──────────┴──────────┴──────────┘
|
||||
描述: 不拦截任何请求 / 记录但不拦截 / 记录并脱敏文件
|
||||
```
|
||||
|
||||
- Calls `POST /api/v1/privacy/mode {mode: "..."}`
|
||||
- Loads current mode on mount via `GET /api/v1/privacy/mode`
|
||||
|
||||
### i18n keys to add
|
||||
|
||||
```json
|
||||
"agent.command_placeholder": "描述任务,例如:打开记事本",
|
||||
"agent.submit": "执行",
|
||||
"agent.cancel": "终止",
|
||||
"agent.status_idle": "空闲",
|
||||
"agent.status_running": "运行中",
|
||||
"agent.task_success": "成功",
|
||||
"agent.task_failed": "失败",
|
||||
"agent.task_cancelled": "已终止",
|
||||
"privacy.mode_off": "关闭",
|
||||
"privacy.mode_audit": "审计",
|
||||
"privacy.mode_redact": "脱敏",
|
||||
"privacy.mode_off_desc": "不拦截任何请求",
|
||||
"privacy.mode_audit_desc": "记录 PII 但不拦截",
|
||||
"privacy.mode_redact_desc": "记录并自动脱敏文件"
|
||||
```
|
||||
|
||||
## Data Flow
|
||||
|
||||
### Agent Task Submission
|
||||
|
||||
```
|
||||
User input → agentStore.submitTask()
|
||||
→ POST /api/v1/agent/tasks {task_type: "general", description: "..."}
|
||||
→ Go proxy → api_server.py
|
||||
→ TaskQueue (SQLite) → runner.py executes
|
||||
→ Poll GET /api/v1/agent/tasks/{id} every 2s
|
||||
→ On COMPLETED/FAILED/CANCELLED → stop polling, update history
|
||||
```
|
||||
|
||||
### Agent Task Cancellation
|
||||
|
||||
```
|
||||
User clicks [终止] → agentStore.cancelTask(id)
|
||||
→ DELETE /api/v1/agent/tasks/{id}
|
||||
→ Go proxy → api_server.py
|
||||
→ Sets cancel_event on runner
|
||||
→ runner finishes current action step → stops
|
||||
→ safe_cleanup() → status = CANCELLED
|
||||
→ Frontend poll detects CANCELLED → shows "已终止"
|
||||
```
|
||||
|
||||
### Privacy Mode Switch
|
||||
|
||||
```
|
||||
User clicks mode → POST /api/v1/privacy/mode {mode: "audit"}
|
||||
→ Go proxy → privacy_api.py
|
||||
→ Writes state.json {mode: "audit"}
|
||||
→ addon.py reads mode on next intercept
|
||||
```
|
||||
|
||||
### Audit Log Query (PrivacyPage AuditTab)
|
||||
|
||||
```
|
||||
AuditTab mount → GET /api/v1/privacy/audit?limit=50
|
||||
→ Go proxy → privacy_api.py
|
||||
→ SQLite query audit_log → return entries
|
||||
→ Render table with timestamp/domain/action/pii columns
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
| Layer | What to test |
|
||||
|-------|-------------|
|
||||
| Go proxy | `httptest` mock downstream, verify routing + error forwarding |
|
||||
| privacy_api | pytest: mode switch, stats format, audit log insert/query |
|
||||
| addon.py | mock three-mode branches (off/audit/redact) |
|
||||
| runner.py | cancel_event propagation, CANCELLED status |
|
||||
| AgentCommandBar | vitest: submit → poll → complete flow, cancel button visibility |
|
||||
| PrivacyPage | vitest: mode selector loads current mode, calls correct endpoint |
|
||||
|
||||
## Implementation Order
|
||||
|
||||
1. **Go proxy** (unblocks all frontend API calls)
|
||||
2. **privacy_api.py redesign** (mode enum + audit log)
|
||||
3. **addon.py update** (three-mode branch)
|
||||
4. **runner.py cancellation**
|
||||
5. **agentStore.ts + AgentCommandBar.tsx**
|
||||
6. **PrivacyPage CertificateTab mode selector**
|
||||
7. **i18n + wiring**
|
||||
8. **Tests**
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,278 @@
|
||||
# Frontend Enhancement Design
|
||||
|
||||
**Date:** 2026-03-03
|
||||
**Branch:** feature/enhance-testing-security-states
|
||||
**Scope:** KVM WebUI 隐私功能集成 + Chrome 扩展 Popup 仪表板化
|
||||
|
||||
---
|
||||
|
||||
## 背景与目标
|
||||
|
||||
当前前端状态:
|
||||
- KVM WebUI (React 18 + Vite) 功能完整,但缺少隐私功能入口
|
||||
- `web/src/PrivacyPage.tsx` 已实现三 tab 布局,但未集成到 KVM WebUI
|
||||
- Chrome 扩展 popup 为简单状态页(320px),缺乏统计可视化
|
||||
- 视频流隐私遮罩 API 已有(`/api/v1/privacy/bboxes`),前端无叠层实现
|
||||
|
||||
目标:
|
||||
1. Phase A:将隐私功能集成进 KVM WebUI,含视频 PII Canvas 叠层
|
||||
2. Phase B:Chrome 扩展 popup 仪表板化,含统计卡片、PII 分布图、近期扫描列表
|
||||
|
||||
---
|
||||
|
||||
## Phase A:KVM WebUI 隐私功能集成
|
||||
|
||||
### A1. 路由与导航
|
||||
|
||||
**文件:`deps/KVM/web/src/App.tsx`**
|
||||
|
||||
添加路由:
|
||||
```tsx
|
||||
<Route path="/privacy" element={<PrivacyPage />} />
|
||||
```
|
||||
|
||||
**文件:`deps/KVM/web/src/components/layout/Navbar.tsx`**
|
||||
|
||||
导航项列表末尾添加:
|
||||
```tsx
|
||||
{ path: '/privacy', icon: ShieldCheckIcon, labelKey: 'nav.privacy' }
|
||||
```
|
||||
|
||||
i18n key 添加到 `public/locales/zh/translation.json` 和 `en/translation.json`:
|
||||
```json
|
||||
{ "nav": { "privacy": "隐私" } }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### A2. PrivacyPage(自包含,写入 submodule)
|
||||
|
||||
**文件:`deps/KVM/web/src/pages/PrivacyPage.tsx`(新增)**
|
||||
|
||||
完整独立实现,不跨目录引用 `web/src/`。包含三个 tab:
|
||||
|
||||
#### DashboardTab
|
||||
- 统计卡片行:拦截次数 / 已扫描文件 / 检出 PII 数 / 涉及域名数
|
||||
- PII 类型分布表(id_card、phone、bank_card、email、license_plate、name、address、face)
|
||||
- 今日域名 Top 5(按请求量排序)
|
||||
- 数据来源:`GET /api/v1/privacy/stats` → `PrivacyStats`
|
||||
- 刷新:30 秒自动刷新 + 手动刷新按钮
|
||||
|
||||
#### AuditTab
|
||||
- 分页表格(page_size=20)
|
||||
- 列:时间 / 域名 / 动作(allow/block/redact)/ PII 类型 / 文件名 / 文件大小 / 客户端 IP
|
||||
- 过滤:域名输入框(防抖 500ms)
|
||||
- 动作徽章颜色:allow=绿 / block=红 / redact=黄
|
||||
- 数据来源:`GET /api/v1/privacy/audit?page=N&domain=X`
|
||||
|
||||
#### CertificateTab
|
||||
- 隐私模式开关(大号 toggle):`GET/POST /api/v1/privacy/mode`
|
||||
- 开启时同步写入 `ocrStore.privacyMode = true`
|
||||
- 视频遮罩开关:`GET/POST /api/v1/privacy/masking`
|
||||
- CA 证书下载按钮:链接至 `GET /api/v1/privacy/cert`
|
||||
- 代理配置说明(手动设置系统代理 127.0.0.1:8888 的步骤)
|
||||
|
||||
**TypeScript 接口**(在同文件或单独的 `types/privacy.ts`):
|
||||
```typescript
|
||||
interface PrivacyStats {
|
||||
requests: number;
|
||||
files: number;
|
||||
pii_by_type: Record<string, number>;
|
||||
actions: { allow: number; block: number; redact: number };
|
||||
top_domains: Array<{ domain: string; count: number }>;
|
||||
}
|
||||
|
||||
interface PrivacyAuditEntry {
|
||||
id: number;
|
||||
ts: string;
|
||||
domain: string;
|
||||
pii_types: Record<string, number>;
|
||||
action: 'allow' | 'block' | 'redact';
|
||||
filename: string;
|
||||
file_size: number;
|
||||
client_ip: string;
|
||||
}
|
||||
```
|
||||
|
||||
**样式:** 复用 KVM WebUI 已有 Tailwind 类 + CSS 变量,不引入新的 CSS 文件。
|
||||
|
||||
---
|
||||
|
||||
### A3. PiiOverlay Canvas 叠层
|
||||
|
||||
**文件:`deps/KVM/web/src/components/console/PiiOverlay.tsx`(新增)**
|
||||
|
||||
```
|
||||
Props:
|
||||
enabled: boolean // 隐私模式是否开启
|
||||
videoRef: RefObject<HTMLVideoElement>
|
||||
|
||||
行为:
|
||||
- enabled=false → canvas display:none,不轮询
|
||||
- enabled=true → 每 2000ms 轮询 GET /api/v1/privacy/bboxes
|
||||
- 响应格式:{ bboxes: [{x, y, w, h, label}] }(归一化 0-1)
|
||||
- 绘制:ctx.fillStyle = 'rgba(0,0,0,0.85)',fillRect(x*W, y*H, w*W, h*H)
|
||||
- canvas 大小跟随 videoRef 实际渲染尺寸(ResizeObserver)
|
||||
- unmount 时 clearInterval + observer.disconnect()
|
||||
```
|
||||
|
||||
**挂载点(`ConsolePage.tsx` 修改):**
|
||||
```tsx
|
||||
// VideoPlayer 外层加 relative 容器
|
||||
<div style={{ position: 'relative' }}>
|
||||
<VideoPlayer ref={videoRef} ... />
|
||||
<PiiOverlay enabled={privacyMode} videoRef={videoRef} />
|
||||
</div>
|
||||
```
|
||||
|
||||
`privacyMode` 从 `kvmStore` 读取(需在 store 中添加该字段,初始值通过 `GET /api/v1/privacy/mode` 加载)。
|
||||
|
||||
---
|
||||
|
||||
### A4. `/api/v1/privacy/bboxes` 端点约定
|
||||
|
||||
Go 后端需提供(已有 `/api/v1/privacy/masking`,类似扩展):
|
||||
```json
|
||||
GET /api/v1/privacy/bboxes
|
||||
Response: {
|
||||
"bboxes": [
|
||||
{ "x": 0.1, "y": 0.2, "w": 0.3, "h": 0.05, "label": "id_card" }
|
||||
]
|
||||
}
|
||||
```
|
||||
若无活跃遮罩,返回 `{ "bboxes": [] }`。
|
||||
|
||||
---
|
||||
|
||||
## Phase B:Chrome 扩展 Popup 仪表板化
|
||||
|
||||
### B1. 布局变更
|
||||
|
||||
- `popup/popup.html`:width 320px → 400px,height 从固定 → min-height 480px
|
||||
- 结构改为三区域:
|
||||
|
||||
```
|
||||
┌────────────────────────────────────────┐ 400px
|
||||
│ Header: Logo + 状态指示 + [⚙] 按钮 │ 52px
|
||||
├────────────────────────────────────────┤
|
||||
│ Stats Bar: 扫描 / PII / 拦截 / 脱敏 │ 72px
|
||||
├──────────────────┬─────────────────────┤
|
||||
│ PII 分布标题 │ │
|
||||
│ 手机号 ██████ 2 │ │ ~120px
|
||||
│ 身份证 ███ 1 │ │
|
||||
│ 邮箱 ██ 1 │ │
|
||||
├────────────────────────────────────────┤
|
||||
│ Recent Scans 标题 │
|
||||
│ ✅ report.pdf normal 1分前 │ ~150px
|
||||
│ 🚫 id_scan.jpg sensitive 5分前 │
|
||||
│ ✅ contract.docx normal 12分前 │
|
||||
└────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
设置视图(点击 ⚙ 切换,不跳转新页面):
|
||||
```
|
||||
┌────────────────────────────────────────┐
|
||||
│ ← 返回 设置 │
|
||||
├────────────────────────────────────────┤
|
||||
│ 服务地址 [http://localhost:8001 ] │
|
||||
│ 自动拦截高风险文件 [●———] │
|
||||
│ 净文件扫描通知 [●———] │
|
||||
│ [保存] │
|
||||
└────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### B2. Stats Bar 数据
|
||||
|
||||
来源:`chrome.runtime.sendMessage({ type: 'getStats' })`
|
||||
|
||||
Service Worker 已有 `getStats()` 返回:
|
||||
```javascript
|
||||
{ todayScans, todayPII, todayBlocked, totalScans }
|
||||
```
|
||||
|
||||
需**扩展**返回 `todayRedacted`(脱敏次数)。
|
||||
在 `service-worker.js` 的 `recordScan()` 中增加计数。
|
||||
|
||||
---
|
||||
|
||||
### B3. PII 分布图(纯 CSS 条形图)
|
||||
|
||||
不引入第三方库。使用 HTML + CSS:
|
||||
|
||||
```html
|
||||
<div class="pii-bar-row">
|
||||
<span class="pii-label">手机号</span>
|
||||
<div class="pii-bar-track">
|
||||
<div class="pii-bar-fill" style="width: 66%"></div>
|
||||
</div>
|
||||
<span class="pii-count">2</span>
|
||||
</div>
|
||||
```
|
||||
|
||||
宽度百分比 = `(count / maxCount) * 100`,颜色对应 PII 类型(与 `ui.js` 已有颜色一致)。
|
||||
|
||||
数据来源:`scanHistory` 条目中的 `piiTypes`(现有 `reportScan` 消息里有 `pii_types`)。
|
||||
|
||||
---
|
||||
|
||||
### B4. Recent Scans 列表
|
||||
|
||||
从 `chrome.storage.local` 读 `scanHistory`,取最新 5 条:
|
||||
|
||||
| 图标 | 条件 |
|
||||
|------|------|
|
||||
| ✅ | result = 'normal' |
|
||||
| ⚠️ | result = 'sensitive_partial' |
|
||||
| 🚫 | result = 'classified' 或 action = 'block' |
|
||||
| ✂️ | action = 'redact' |
|
||||
|
||||
时间显示:相对时间(`1分前`、`5分前`、`1小时前`)。
|
||||
|
||||
---
|
||||
|
||||
### B5. 涉及文件清单
|
||||
|
||||
| 文件 | 变更类型 | 说明 |
|
||||
|------|--------|------|
|
||||
| `web/chrome-extension/popup/popup.html` | 修改 | 宽度 400px,新增结构区块 |
|
||||
| `web/chrome-extension/popup/popup.js` | 修改 | 渲染 stats/pii/history,设置视图切换 |
|
||||
| `web/chrome-extension/popup/popup.css` | 修改 | stats bar、pii bar、recent list 样式 |
|
||||
| `web/chrome-extension/background/service-worker.js` | 修改 | getStats 增加 todayRedacted |
|
||||
| `web/chrome-extension/_locales/zh_CN/messages.json` | 修改 | 新 i18n key |
|
||||
| `web/chrome-extension/_locales/en/messages.json` | 修改 | 新 i18n key |
|
||||
|
||||
---
|
||||
|
||||
## 涉及文件汇总(Phase A)
|
||||
|
||||
| 文件 | 变更类型 | 说明 |
|
||||
|------|--------|------|
|
||||
| `deps/KVM/web/src/App.tsx` | 修改 | 添加 `/privacy` 路由 |
|
||||
| `deps/KVM/web/src/components/layout/Navbar.tsx` | 修改 | 添加隐私导航项 |
|
||||
| `deps/KVM/web/src/pages/PrivacyPage.tsx` | 新增 | 完整隐私页面(三 tab)|
|
||||
| `deps/KVM/web/src/components/console/PiiOverlay.tsx` | 新增 | Canvas PII 叠层 |
|
||||
| `deps/KVM/web/src/pages/ConsolePage.tsx` | 修改 | 挂载 PiiOverlay |
|
||||
| `deps/KVM/web/src/stores/kvmStore.ts` | 修改 | 添加 privacyMode 字段 |
|
||||
| `deps/KVM/web/public/locales/zh/translation.json` | 修改 | nav.privacy 等 key |
|
||||
| `deps/KVM/web/public/locales/en/translation.json` | 修改 | nav.privacy 等 key |
|
||||
|
||||
---
|
||||
|
||||
## 与现有计划的对应
|
||||
|
||||
| 原计划 | 本设计 |
|
||||
|--------|--------|
|
||||
| Phase 1: Web UI 隐私仪表盘 + 审计日志 | ✅ Phase A(PrivacyPage 三 tab)|
|
||||
| Phase 3: KVM 视频 PII 遮罩叠层 | ✅ Phase A(PiiOverlay Canvas)|
|
||||
| Chrome 扩展 UI 改进(未列入原计划)| ✅ Phase B(popup 仪表板化)|
|
||||
|
||||
---
|
||||
|
||||
## 不在本次范围内
|
||||
|
||||
- Go 后端 `frame_redactor.go` 修复(视频截图遮罩 bug)
|
||||
- native 视频管道切换(config.json 改动)
|
||||
- 审计日志轮换策略
|
||||
- 透明代理 iptables 配置
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,353 @@
|
||||
# KVM-Privacy Deb Packaging & Architecture Redesign
|
||||
|
||||
**Date:** 2026-03-04
|
||||
**Branch:** feature/enhance-testing-security-states
|
||||
**Status:** Approved — pending implementation plan
|
||||
|
||||
---
|
||||
|
||||
## 1. Goals
|
||||
|
||||
1. Produce proper Debian packages for all KVM-Privacy services
|
||||
2. Eliminate Python packaging dependency hell via PyInstaller binaries
|
||||
3. Replace SQLite with MariaDB (already a system dependency) to fix multi-process write contention
|
||||
4. Maximise service independence: each service owns exactly the data it writes
|
||||
5. Address P0/P1 robustness issues identified in architecture review
|
||||
|
||||
---
|
||||
|
||||
## 2. Package Structure
|
||||
|
||||
| Package | Version | Contents | Build tool |
|
||||
|---------|---------|----------|-----------|
|
||||
| `kvm-server` | 1.0.0-3 | Go binary + embedded React frontend + Privacy Go handlers | Go + npm + dpkg-buildpackage |
|
||||
| `kvm-mitm` | 1.0.0-1 | PyInstaller self-contained binary (mitmproxy + addon) | PyInstaller + dpkg-deb |
|
||||
| `kvm-agent` | 1.0.0-1 | PyInstaller self-contained binary (AI agent daemon) | PyInstaller + dpkg-deb |
|
||||
|
||||
---
|
||||
|
||||
## 3. Database Architecture
|
||||
|
||||
### 3.1 MariaDB replaces SQLite everywhere
|
||||
|
||||
MariaDB is **already installed** on the target machine (`kvm-server` depends on `mariadb-server`).
|
||||
|
||||
**New tables** in the existing `kvm` database:
|
||||
|
||||
```sql
|
||||
-- Privacy gateway audit log (replaces /var/lib/kvm-privacy/audit.db)
|
||||
CREATE TABLE kvm.privacy_audit_log (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
ts DATETIME(3) NOT NULL DEFAULT NOW(3),
|
||||
domain VARCHAR(255) NOT NULL,
|
||||
pii_types JSON,
|
||||
action VARCHAR(20) NOT NULL, -- allow | redact | scan_failed
|
||||
doc_hash VARCHAR(64),
|
||||
file_count INT DEFAULT 0,
|
||||
client_ip VARCHAR(45) DEFAULT '',
|
||||
request_url VARCHAR(2048) DEFAULT '',
|
||||
filename VARCHAR(255) DEFAULT '',
|
||||
file_size INT DEFAULT 0,
|
||||
redacted_pii_count INT DEFAULT 0,
|
||||
INDEX idx_ts (ts),
|
||||
INDEX idx_domain (domain),
|
||||
INDEX idx_action (action)
|
||||
);
|
||||
|
||||
-- Agent task queue (replaces task_queue.db)
|
||||
CREATE TABLE kvm.agent_tasks (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
task_type VARCHAR(50) NOT NULL DEFAULT 'task',
|
||||
description TEXT NOT NULL,
|
||||
variables JSON NOT NULL DEFAULT ('{}'),
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'pending',
|
||||
created_at DOUBLE NOT NULL,
|
||||
started_at DOUBLE,
|
||||
finished_at DOUBLE,
|
||||
result TEXT DEFAULT '',
|
||||
success TINYINT(1) DEFAULT 0,
|
||||
INDEX idx_status (status),
|
||||
INDEX idx_created (created_at)
|
||||
);
|
||||
```
|
||||
|
||||
**Existing tables unchanged:** `audit_logs`, `request_logs`, `ai_interaction_logs`, `users`, `sessions`, `recordings`, etc.
|
||||
|
||||
### 3.2 Per-service MariaDB users (principle of least privilege)
|
||||
|
||||
```sql
|
||||
CREATE USER 'kvm_mitm'@'localhost' IDENTIFIED BY '...';
|
||||
CREATE USER 'kvm_agent'@'localhost' IDENTIFIED BY '...';
|
||||
CREATE USER 'kvm_server'@'localhost' IDENTIFIED BY '...'; -- already exists; extend grants
|
||||
|
||||
GRANT INSERT, SELECT ON kvm.privacy_audit_log TO 'kvm_mitm'@'localhost';
|
||||
GRANT ALL PRIVILEGES ON kvm.agent_tasks TO 'kvm_agent'@'localhost';
|
||||
GRANT SELECT ON kvm.privacy_audit_log TO 'kvm_server'@'localhost';
|
||||
GRANT SELECT ON kvm.agent_tasks TO 'kvm_server'@'localhost';
|
||||
```
|
||||
|
||||
### 3.3 Test database
|
||||
|
||||
```sql
|
||||
CREATE DATABASE kvm_test;
|
||||
-- Grant same users full access to kvm_test for CI
|
||||
GRANT ALL PRIVILEGES ON kvm_test.* TO 'kvm_mitm'@'localhost';
|
||||
GRANT ALL PRIVILEGES ON kvm_test.* TO 'kvm_agent'@'localhost';
|
||||
GRANT ALL PRIVILEGES ON kvm_test.* TO 'kvm_server'@'localhost';
|
||||
```
|
||||
|
||||
Tests `setUp` creates tables; `tearDown` truncates them.
|
||||
Python tests use `PyMySQL`; Go tests use GORM with `kvm_test` DSN.
|
||||
|
||||
### 3.4 Python connector
|
||||
|
||||
**PyMySQL** (pure Python, zero C extensions → PyInstaller compatible).
|
||||
Replace all `import sqlite3` with `import pymysql`.
|
||||
|
||||
---
|
||||
|
||||
## 4. Service Ownership Boundaries
|
||||
|
||||
Each service has **exclusive write access** to its own data; all other services read-only.
|
||||
|
||||
| Service | Writes | Reads (read-only) | External deps |
|
||||
|---------|--------|-------------------|--------------|
|
||||
| `kvm-server` | `state.json` (atomic rename) | `kvm.privacy_audit_log`, `kvm.agent_tasks` | MariaDB |
|
||||
| `kvm-mitm` | `kvm.privacy_audit_log`, `/etc/kvm-privacy/ca/` | `state.json` | MariaDB, info-privacy :8001 |
|
||||
| `kvm-agent` | `kvm.agent_tasks` | KVM API :8080, LLM API | MariaDB, mem-bridge :8003 (soft) |
|
||||
|
||||
`state.json` stays as a **file** (hot path: mitmproxy reads it per-request; file page-cache read is ~10× faster than a MariaDB round-trip). Go writes it with atomic `rename`.
|
||||
|
||||
---
|
||||
|
||||
## 5. Package B — privacy_api Moved into Go kvm-server
|
||||
|
||||
Remove the `/api/v1/privacy/*` reverse-proxy route.
|
||||
Add `go/internal/api/privacy_handler.go` using the existing GORM `*gorm.DB`:
|
||||
|
||||
| Endpoint | Method | Action |
|
||||
|----------|--------|--------|
|
||||
| `/api/v1/privacy/mode` | GET | Read `state.json` |
|
||||
| `/api/v1/privacy/mode` | POST/PUT | Validate mode, atomic-write `state.json` |
|
||||
| `/api/v1/privacy/stats` | GET | Aggregate query on `kvm.privacy_audit_log` |
|
||||
| `/api/v1/privacy/audit` | GET | Paginated query on `kvm.privacy_audit_log` |
|
||||
| `/api/v1/privacy/cert` | GET | Read `/etc/kvm-privacy/ca/ca.crt` (503 if not yet generated) |
|
||||
|
||||
`/api/v1/agent/*` proxy to :8890 **unchanged**.
|
||||
|
||||
---
|
||||
|
||||
## 6. Package C — PyInstaller Binaries
|
||||
|
||||
### 6.1 kvm-mitm
|
||||
|
||||
**Key change:** replace `-s addon.py` file-load with programmatic API so all Python source is compiled into the binary:
|
||||
|
||||
```python
|
||||
# services/privacy_gateway/mitm_launcher.py (new entry point)
|
||||
from mitmproxy.tools.dump import DumpMaster
|
||||
from mitmproxy import options
|
||||
from privacy_gateway.addon import PrivacyGatewayAddon # compiled in
|
||||
import asyncio, signal, sys
|
||||
|
||||
async def main():
|
||||
opts = options.Options(
|
||||
listen_host="0.0.0.0", listen_port=8888,
|
||||
mode="regular", ssl_insecure=True,
|
||||
)
|
||||
master = DumpMaster(opts, with_termlog=False, with_dumper=False)
|
||||
master.addons.add(PrivacyGatewayAddon())
|
||||
loop = asyncio.get_event_loop()
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
loop.add_signal_handler(sig, master.shutdown)
|
||||
await master.run()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
```
|
||||
|
||||
Build command:
|
||||
```bash
|
||||
pyinstaller --onefile \
|
||||
--collect-all mitmproxy \
|
||||
--hidden-import pymysql \
|
||||
--name kvm-mitm \
|
||||
services/privacy_gateway/mitm_launcher.py
|
||||
# → dist/kvm-mitm (~120 MB raw, ~40 MB xz in deb)
|
||||
```
|
||||
|
||||
Installed to: `/usr/bin/kvm-mitm`
|
||||
|
||||
### 6.2 kvm-agent
|
||||
|
||||
`kvm_agent/__main__.py` is already a clean CLI entry point (`run` / `daemon` / `queue` subcommands).
|
||||
|
||||
Build command:
|
||||
```bash
|
||||
pyinstaller --onefile \
|
||||
--hidden-import kvm_agent.actions \
|
||||
--hidden-import kvm_agent.workflows \
|
||||
--hidden-import pymysql \
|
||||
--name kvm-agent \
|
||||
services/kvm_agent/__main__.py
|
||||
# → dist/kvm-agent (~80 MB raw, ~25 MB xz in deb)
|
||||
```
|
||||
|
||||
Installed to: `/usr/bin/kvm-agent`
|
||||
|
||||
---
|
||||
|
||||
## 7. Credential Externalisation
|
||||
|
||||
Sensitive values removed from `.service` files.
|
||||
Each service reads an `EnvironmentFile`:
|
||||
|
||||
| File | Owner | Permissions | Contents |
|
||||
|------|-------|-------------|----------|
|
||||
| `/etc/kvm-privacy/secrets.env` | root | 600 | `KVM_MITM_DB_PASS=...` |
|
||||
| `/etc/kvm-agent/secrets.env` | root | 600 | `KVM_JWT_TOKEN=...`, `LLM_API_KEY=...`, `KVM_AGENT_DB_PASS=...` |
|
||||
|
||||
`deb postinst` creates empty template files with correct permissions if absent.
|
||||
|
||||
---
|
||||
|
||||
## 8. Robustness Improvements
|
||||
|
||||
| Issue | Fix | Where |
|
||||
|-------|-----|-------|
|
||||
| SQLite multi-process write contention | Replaced by MariaDB | audit_logger.py, runner.py |
|
||||
| state.json TOCTOU | Go atomic `rename` write | privacy_handler.go |
|
||||
| Credentials in service files | EnvironmentFile (600) | all .service files |
|
||||
| Unlimited restart loop | `StartLimitInterval=300; StartLimitBurst=5` | all .service files |
|
||||
| kvm-agent: memory unavailable at start | Exponential-backoff reconnect in `memory_client.py` | memory_client.py |
|
||||
| CA cert generation blocks startup | Background generation; `/cert` returns 503 until ready | cert_manager.py |
|
||||
| info-privacy down → silent pass-through | Explicit degradation: audit→log `scan_failed`, redact→block upload | addon.py |
|
||||
|
||||
---
|
||||
|
||||
## 9. systemd Service Files (final)
|
||||
|
||||
### kvm-mitm.service
|
||||
```ini
|
||||
[Unit]
|
||||
Description=KVM Privacy MITM Interceptor
|
||||
After=network.target mariadb.service info-privacy.service
|
||||
Wants=info-privacy.service
|
||||
StartLimitInterval=300
|
||||
StartLimitBurst=5
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
EnvironmentFile=/etc/kvm-privacy/secrets.env
|
||||
ExecStart=/usr/bin/kvm-mitm
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
MemoryMax=768M
|
||||
LimitNOFILE=8192
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
### kvm-agent.service
|
||||
```ini
|
||||
[Unit]
|
||||
Description=KVM AI Agent v2
|
||||
After=network.target mariadb.service
|
||||
Wants=mem-bridge-memory.service
|
||||
StartLimitInterval=300
|
||||
StartLimitBurst=5
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=pi
|
||||
EnvironmentFile=/etc/kvm-agent/secrets.env
|
||||
ExecStart=/usr/bin/kvm-agent daemon
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
MemoryMax=512M
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 10. Build Script
|
||||
|
||||
`scripts/build-python-debs.sh` (new):
|
||||
|
||||
```
|
||||
1. Pre-flight: verify pyinstaller, pymysql installed; verify MariaDB reachable
|
||||
2. Build kvm-mitm binary (pyinstaller)
|
||||
3. Build kvm-agent binary (pyinstaller)
|
||||
4. Assemble kvm-mitm deb:
|
||||
- DEBIAN/{control, postinst, prerm, postrm, conffiles}
|
||||
- usr/bin/kvm-mitm
|
||||
- lib/systemd/system/kvm-mitm.service
|
||||
- etc/kvm-privacy/ (directory, postinst sets perms)
|
||||
5. Assemble kvm-agent deb:
|
||||
- DEBIAN/{control, postinst, prerm, postrm, conffiles}
|
||||
- usr/bin/kvm-agent
|
||||
- lib/systemd/system/kvm-agent.service
|
||||
- etc/kvm-agent/agent.yaml (conffile)
|
||||
6. dpkg-deb --build for each
|
||||
7. Output → build/deb/
|
||||
```
|
||||
|
||||
`kvm-server` rebuild uses existing `scripts/build-deb.sh` with `--version 1.0.0-3`.
|
||||
|
||||
---
|
||||
|
||||
## 11. Installation Sequence
|
||||
|
||||
```bash
|
||||
# 1. Install/upgrade kvm-server (creates MariaDB users + tables in postinst)
|
||||
sudo dpkg -i kvm-server_1.0.0-3_arm64.deb
|
||||
|
||||
# 2. Install kvm-mitm
|
||||
sudo dpkg -i kvm-mitm_1.0.0-1_arm64.deb
|
||||
|
||||
# 3. Install kvm-agent
|
||||
sudo dpkg -i kvm-agent_1.0.0-1_arm64.deb
|
||||
|
||||
# 4. Populate secrets files (one-time, per deployment)
|
||||
sudo editor /etc/kvm-privacy/secrets.env
|
||||
sudo editor /etc/kvm-agent/secrets.env
|
||||
|
||||
# 5. Start new services
|
||||
sudo systemctl start kvm-mitm kvm-agent
|
||||
|
||||
# 6. Verify
|
||||
systemctl status kvm-server kvm-mitm kvm-agent
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 12. Files Changed / Created
|
||||
|
||||
### New files
|
||||
- `go/internal/api/privacy_handler.go`
|
||||
- `go/internal/api/privacy_handler_test.go`
|
||||
- `services/privacy_gateway/mitm_launcher.py`
|
||||
- `scripts/build-python-debs.sh`
|
||||
- `debian/kvm-mitm/` (DEBIAN control tree)
|
||||
- `debian/kvm-agent/` (DEBIAN control tree)
|
||||
|
||||
### Modified files
|
||||
- `go/internal/api/router.go` — remove privacy/* proxy, add privacy handler
|
||||
- `services/privacy_gateway/audit_logger.py` — sqlite3 → PyMySQL
|
||||
- `services/privacy_gateway/addon.py` — explicit scan_failed degradation
|
||||
- `services/privacy_gateway/cert_manager.py` — non-blocking background generation
|
||||
- `services/kvm_agent/runner.py` — sqlite3 → PyMySQL
|
||||
- `services/kvm_agent/config.py` — `queue_db_path` → MariaDB DSN fields
|
||||
- `services/kvm_agent/configs/agent.yaml` — update DB config
|
||||
- `services/kvm_agent/memory_client.py` — exponential-backoff reconnect
|
||||
- `services/kvm_agent/tests/test_runner*.py` — SQLite tmp → MariaDB kvm_test
|
||||
- `deploy/systemd/kvm-agent.service` — EnvironmentFile, StartLimit
|
||||
- `deps/KVM/debian/kvm-server/DEBIAN/postinst` — add privacy_audit_log, agent_tasks DDL + user grants
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,73 @@
|
||||
# 导航权限收紧 + 用户管理卡片布局设计
|
||||
|
||||
Date: 2026-03-04
|
||||
|
||||
## 背景
|
||||
|
||||
1. 导航栏 Privacy 和 AI Agent 链接对 operator/viewer 角色无条件可见,与后端 RBAC 策略不一致。
|
||||
2. 用户管理 Tab(SettingsPage → Users)使用传统表格布局,在 800px 容器内横向溢出,操作按钮文字冗长。
|
||||
|
||||
## 问题一:导航权限控制
|
||||
|
||||
### 现状
|
||||
```
|
||||
审计中心 → auditor | security_admin | system_admin ✅(已控制)
|
||||
隐私保护 → 所有人可见 ❌
|
||||
AI Agent → 所有人可见 ❌
|
||||
```
|
||||
|
||||
### 方案:扩展现有角色判断
|
||||
|
||||
在 `Layout.tsx` 增加 `isPrivileged` 变量(auditor/security_admin/system_admin),复用于三个受控链接:
|
||||
|
||||
```tsx
|
||||
const isPrivileged = user?.role_name === 'auditor'
|
||||
|| user?.role_name === 'security_admin'
|
||||
|| user?.role_name === 'system_admin'
|
||||
|
||||
// 审计、隐私保护、AI Agent 均用 isPrivileged 控制
|
||||
{isPrivileged && <Link to="/audit">...</Link>}
|
||||
{isPrivileged && <Link to="/privacy">...</Link>}
|
||||
{isPrivileged && <Link to="/agent">...</Link>}
|
||||
```
|
||||
|
||||
**改动范围:** `Layout.tsx` 仅重构现有条件判断,不增加新逻辑。
|
||||
|
||||
## 问题二:用户管理卡片布局(方案 A)
|
||||
|
||||
### 目标
|
||||
- 替换宽表格为响应式卡片网格
|
||||
- 每张卡片信息完整、操作直观
|
||||
- 与 AuditPage 录像卡片风格一致
|
||||
|
||||
### 卡片设计
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────┐
|
||||
│ [AB] admin [自己] │ ← 头像(角色色)+ 用户名 + 自己标记
|
||||
│ system_admin 角色徽章 │ ← 角色
|
||||
│ ─────────────────────────────── │
|
||||
│ 2FA: 已启用 状态: 正常 │ ← 状态行
|
||||
│ 最后登录: 2026-03-04 09:11 │ ← 时间
|
||||
│ ─────────────────────────────── │
|
||||
│ [禁用] [解锁] [删除] │ ← 操作(非自身才显示)
|
||||
└─────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### 布局规则
|
||||
- 容器:`display: grid; grid-template-columns: repeat(auto-fill, minmax(280px, 1fr))`
|
||||
- 头像:40×40px 圆形,背景色取角色色(蓝/橙/绿/灰),文字为用户名首字母大写
|
||||
- 操作按钮:图标(SVG inline)+ 短文字,hover 显色
|
||||
|
||||
### 文件改动
|
||||
| 文件 | 改动 |
|
||||
|------|------|
|
||||
| `components/settings/UserManagementTab.tsx` | JSX 重构为卡片网格,state/函数不变 |
|
||||
| `pages/UsersPage.css` | 新增卡片 CSS(`.user-card`, `.user-avatar`, `.user-card-grid`) |
|
||||
| `components/Layout.tsx` | Privacy/Agent 链接添加 isPrivileged 条件 |
|
||||
|
||||
## 验证
|
||||
1. `npm run build` 无 TypeScript 错误
|
||||
2. operator 登录 → 导航栏无 Privacy/Audit/Agent 链接
|
||||
3. system_admin 登录 → Settings > Users Tab 显示卡片布局
|
||||
4. 卡片操作功能:启用/禁用/解锁/删除正常工作
|
||||
@@ -0,0 +1,502 @@
|
||||
# 导航权限收紧 + 用户管理卡片布局实现计划
|
||||
|
||||
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
|
||||
|
||||
**Goal:** operator/viewer 角色不可见 Privacy/Audit/Agent 导航链接;用户管理 Tab 表格替换为响应式卡片网格。
|
||||
|
||||
**Architecture:** 纯前端改动,不涉及后端路由或 RBAC。Layout.tsx 提取 `isPrivileged` 复用于三个受控链接;UserManagementTab.tsx 仅替换 JSX render 层(state/函数完全不变);新增卡片 CSS 到 UsersPage.css。
|
||||
|
||||
**Tech Stack:** React 18 + TypeScript + CSS variables(项目既有 `--primary`, `--warning`, `--success`, `--danger`, `--border`, `--bg-gray`, `--bg-dark`, `--text-primary`, `--text-secondary`)
|
||||
|
||||
---
|
||||
|
||||
### Task 1:Layout.tsx 导航权限收紧
|
||||
|
||||
**Files:**
|
||||
- Modify: `deps/KVM/web/src/components/Layout.tsx:33-37`
|
||||
|
||||
**Step 1:修改 Layout.tsx**
|
||||
|
||||
将原来三处条件替换为共享 `isPrivileged` 常量:
|
||||
|
||||
```tsx
|
||||
// 在 const { user, logout } = useAuthStore() 之后添加
|
||||
const isPrivileged = user?.role_name === 'auditor'
|
||||
|| user?.role_name === 'security_admin'
|
||||
|| user?.role_name === 'system_admin'
|
||||
```
|
||||
|
||||
然后将导航菜单改为:
|
||||
|
||||
```tsx
|
||||
<div className="navbar-menu">
|
||||
<Link to="/" className="nav-link">{t('nav.console')}</Link>
|
||||
<Link to="/terminal" className="nav-link">{t('nav.terminal')}</Link>
|
||||
<Link to="/settings" className="nav-link">{t('nav.settings')}</Link>
|
||||
{isPrivileged && (
|
||||
<Link to="/audit" className="nav-link">{t('nav.audit')}</Link>
|
||||
)}
|
||||
{isPrivileged && (
|
||||
<Link to="/privacy" className="nav-link">{t('nav.privacy')}</Link>
|
||||
)}
|
||||
{isPrivileged && (
|
||||
<Link to="/agent" className="nav-link">{t('nav.agent')}</Link>
|
||||
)}
|
||||
</div>
|
||||
```
|
||||
|
||||
**Step 2:编译验证**
|
||||
|
||||
```bash
|
||||
cd deps/KVM/web && npm run build 2>&1 | tail -5
|
||||
```
|
||||
|
||||
期望:`✓ built in` 无 TypeScript 错误
|
||||
|
||||
**Step 3:提交**
|
||||
|
||||
```bash
|
||||
git add deps/KVM/web/src/components/Layout.tsx
|
||||
git commit -m "feat: restrict privacy/audit/agent nav links to privileged roles"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2:UsersPage.css 新增卡片样式
|
||||
|
||||
**Files:**
|
||||
- Modify: `deps/KVM/web/src/pages/UsersPage.css`(在文件末尾追加)
|
||||
|
||||
**Step 1:在 UsersPage.css 末尾追加以下 CSS**
|
||||
|
||||
```css
|
||||
/* ── User Card Grid ── */
|
||||
.user-card-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(260px, 1fr));
|
||||
gap: 12px;
|
||||
margin-top: 4px;
|
||||
}
|
||||
|
||||
.user-card {
|
||||
background: var(--bg-dark);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
padding: 16px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 12px;
|
||||
transition: border-color 0.15s;
|
||||
}
|
||||
|
||||
.user-card:hover {
|
||||
border-color: var(--primary);
|
||||
}
|
||||
|
||||
/* Header: avatar + name */
|
||||
.user-card-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.user-avatar {
|
||||
width: 40px;
|
||||
height: 40px;
|
||||
border-radius: 50%;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-size: 16px;
|
||||
font-weight: 700;
|
||||
flex-shrink: 0;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.user-avatar.role-system_admin { background: rgba(59,130,246,0.2); color: var(--primary); }
|
||||
.user-avatar.role-security_admin { background: rgba(245,158,11,0.2); color: var(--warning); }
|
||||
.user-avatar.role-auditor { background: rgba(16,185,129,0.2); color: var(--success); }
|
||||
.user-avatar.role-operator { background: rgba(156,163,175,0.2); color: var(--text-secondary); }
|
||||
|
||||
.user-card-name {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 2px;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.user-card-username {
|
||||
font-weight: 600;
|
||||
font-size: 14px;
|
||||
color: var(--text-primary);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
/* Role row */
|
||||
.user-card-role {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.user-card-role select {
|
||||
background: var(--bg-dark);
|
||||
border: 1px solid var(--border);
|
||||
color: var(--text-primary);
|
||||
padding: 2px 6px;
|
||||
border-radius: 4px;
|
||||
font-size: 12px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
/* Meta: 2FA + status + last login */
|
||||
.user-card-meta {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
font-size: 12px;
|
||||
color: var(--text-secondary);
|
||||
border-top: 1px solid var(--border);
|
||||
padding-top: 10px;
|
||||
}
|
||||
|
||||
.user-card-meta-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.user-card-meta-row .dot {
|
||||
width: 6px;
|
||||
height: 6px;
|
||||
border-radius: 50%;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.user-card-meta-row .dot.enabled { background: var(--success); }
|
||||
.user-card-meta-row .dot.disabled { background: var(--danger); }
|
||||
|
||||
/* Actions */
|
||||
.user-card-actions {
|
||||
display: flex;
|
||||
gap: 6px;
|
||||
flex-wrap: wrap;
|
||||
border-top: 1px solid var(--border);
|
||||
padding-top: 10px;
|
||||
}
|
||||
|
||||
.user-card-actions .btn-sm {
|
||||
flex: 1;
|
||||
min-width: 60px;
|
||||
text-align: center;
|
||||
padding: 5px 8px;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
/* Empty state for card grid */
|
||||
.user-card-empty {
|
||||
grid-column: 1 / -1;
|
||||
text-align: center;
|
||||
color: var(--text-secondary);
|
||||
padding: 40px;
|
||||
border: 1px dashed var(--border);
|
||||
border-radius: 8px;
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2:编译验证(CSS 无 TS 类型检查,只需确认无语法破坏)**
|
||||
|
||||
```bash
|
||||
cd deps/KVM/web && npm run build 2>&1 | tail -5
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3:UserManagementTab.tsx 卡片 JSX 重构
|
||||
|
||||
**Files:**
|
||||
- Modify: `deps/KVM/web/src/components/settings/UserManagementTab.tsx`
|
||||
|
||||
只替换 `return (...)` 中的表格部分,**所有 state、函数、import 完全不变**。
|
||||
|
||||
**Step 1:将 UserManagementTab.tsx 的 return 块替换为以下内容**
|
||||
|
||||
```tsx
|
||||
return (
|
||||
<div className="users-container">
|
||||
<div className="users-header">
|
||||
<h2>{t('users.title')}</h2>
|
||||
<button className="btn-primary" onClick={() => setShowCreate(true)}>
|
||||
{t('users.create')}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="alert alert-error">
|
||||
{error}
|
||||
<button onClick={() => setError('')} className="alert-close">×</button>
|
||||
</div>
|
||||
)}
|
||||
{message && (
|
||||
<div className="alert alert-success">
|
||||
{message}
|
||||
<button onClick={() => setMessage('')} className="alert-close">×</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Create User Form — overlay 不变 */}
|
||||
{showCreate && (
|
||||
<div className="create-form-overlay">
|
||||
<div className="create-form">
|
||||
<h3>{t('users.create_title')}</h3>
|
||||
<form onSubmit={handleCreate}>
|
||||
<div className="form-row">
|
||||
<div className="form-group">
|
||||
<label>{t('users.username')} *</label>
|
||||
<input
|
||||
type="text"
|
||||
value={newUser.username}
|
||||
onChange={(e) => setNewUser({ ...newUser, username: e.target.value })}
|
||||
placeholder={t('users.username_placeholder')}
|
||||
required
|
||||
autoFocus
|
||||
/>
|
||||
</div>
|
||||
<div className="form-group">
|
||||
<label>{t('users.password')} * ({t('users.password_hint')})</label>
|
||||
<input
|
||||
type="password"
|
||||
value={newUser.password}
|
||||
onChange={(e) => setNewUser({ ...newUser, password: e.target.value })}
|
||||
placeholder={t('users.password_placeholder')}
|
||||
required
|
||||
minLength={12}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="form-row">
|
||||
<div className="form-group">
|
||||
<label>{t('users.display_name')}</label>
|
||||
<input
|
||||
type="text"
|
||||
value={newUser.display_name}
|
||||
onChange={(e) => setNewUser({ ...newUser, display_name: e.target.value })}
|
||||
placeholder={t('users.display_name_placeholder')}
|
||||
/>
|
||||
</div>
|
||||
<div className="form-group">
|
||||
<label>{t('users.email')}</label>
|
||||
<input
|
||||
type="email"
|
||||
value={newUser.email}
|
||||
onChange={(e) => setNewUser({ ...newUser, email: e.target.value })}
|
||||
placeholder={t('users.email_placeholder')}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="form-group">
|
||||
<label>{t('users.role')}</label>
|
||||
<select
|
||||
value={newUser.role_name}
|
||||
onChange={(e) => setNewUser({ ...newUser, role_name: e.target.value })}
|
||||
>
|
||||
{ROLE_OPTIONS.map((role) => (
|
||||
<option key={role} value={role}>{ROLE_LABELS[role]}</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
<div className="form-actions">
|
||||
<button type="button" className="btn-secondary" onClick={() => setShowCreate(false)}>
|
||||
{t('users.cancel')}
|
||||
</button>
|
||||
<button type="submit" className="btn-primary">
|
||||
{t('users.submit_create')}
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* User Card Grid */}
|
||||
{loading ? (
|
||||
<div className="loading">{t('users.loading')}</div>
|
||||
) : (
|
||||
<div className="user-card-grid">
|
||||
{users.length === 0 ? (
|
||||
<div className="user-card-empty">{t('users.no_data')}</div>
|
||||
) : (
|
||||
users.map((user) => (
|
||||
<div key={user.id} className="user-card">
|
||||
|
||||
{/* Header: avatar + username + self badge */}
|
||||
<div className="user-card-header">
|
||||
<div className={`user-avatar role-${user.role_name}`}>
|
||||
{user.username.charAt(0)}
|
||||
</div>
|
||||
<div className="user-card-name">
|
||||
<span className="user-card-username">
|
||||
{user.username}
|
||||
{user.id === currentUser?.id && (
|
||||
<span className="badge-self">{t('users.current_user')}</span>
|
||||
)}
|
||||
</span>
|
||||
{user.display_name && (
|
||||
<span style={{ fontSize: '12px', color: 'var(--text-secondary)' }}>
|
||||
{user.display_name}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Role — click to edit (non-self) */}
|
||||
<div className="user-card-role">
|
||||
{editingRole === user.id ? (
|
||||
<select
|
||||
value={user.role_name}
|
||||
onChange={(e) => handleSetRole(user.id, e.target.value)}
|
||||
onBlur={() => setEditingRole(null)}
|
||||
autoFocus
|
||||
>
|
||||
{ROLE_OPTIONS.map((role) => (
|
||||
<option key={role} value={role}>{ROLE_LABELS[role]}</option>
|
||||
))}
|
||||
</select>
|
||||
) : (
|
||||
<span
|
||||
className={`role-badge role-${user.role_name}`}
|
||||
onClick={() => user.id !== currentUser?.id && setEditingRole(user.id)}
|
||||
title={user.id !== currentUser?.id ? t('users.click_edit_role') : ''}
|
||||
style={{ cursor: user.id !== currentUser?.id ? 'pointer' : 'default' }}
|
||||
>
|
||||
{ROLE_LABELS[user.role_name] || user.role_name}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Meta: 2FA / status / last login */}
|
||||
<div className="user-card-meta">
|
||||
<div className="user-card-meta-row">
|
||||
<span className={`dot ${user.totp_enabled ? 'enabled' : 'disabled'}`} />
|
||||
<span>2FA: {user.totp_enabled ? t('users.totp_enabled') : t('users.totp_disabled')}</span>
|
||||
</div>
|
||||
<div className="user-card-meta-row">
|
||||
<span className={`dot ${user.enabled ? 'enabled' : 'disabled'}`} />
|
||||
<span>{user.enabled ? t('users.status_active') : t('users.status_disabled')}</span>
|
||||
</div>
|
||||
<div className="user-card-meta-row">
|
||||
<span style={{ color: 'var(--text-secondary)', fontSize: '11px' }}>
|
||||
{user.last_login_at
|
||||
? new Date(user.last_login_at).toLocaleString()
|
||||
: t('users.never_logged_in')}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Actions (non-self only) */}
|
||||
{user.id !== currentUser?.id && (
|
||||
<div className="user-card-actions">
|
||||
<button
|
||||
className={`btn-sm ${user.enabled ? 'btn-toggle' : 'btn-unlock'}`}
|
||||
onClick={() => handleToggleEnabled(user)}
|
||||
title={user.enabled ? t('users.action_disable') : t('users.action_enable')}
|
||||
>
|
||||
{user.enabled ? t('users.action_disable') : t('users.action_enable')}
|
||||
</button>
|
||||
<button
|
||||
className="btn-sm btn-unlock"
|
||||
onClick={() => handleUnlock(user.id)}
|
||||
title={t('users.action_unlock')}
|
||||
>
|
||||
{t('users.action_unlock')}
|
||||
</button>
|
||||
<button
|
||||
className="btn-sm btn-delete"
|
||||
onClick={() => handleDelete(user)}
|
||||
title={t('users.action_delete')}
|
||||
>
|
||||
{t('users.action_delete')}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
</div>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
```
|
||||
|
||||
**Step 2:编译验证**
|
||||
|
||||
```bash
|
||||
cd deps/KVM/web && npm run build 2>&1 | tail -8
|
||||
```
|
||||
|
||||
期望:`✓ built in` 无 TypeScript 错误
|
||||
|
||||
**Step 3:提交**
|
||||
|
||||
```bash
|
||||
git add deps/KVM/web/src/components/settings/UserManagementTab.tsx \
|
||||
deps/KVM/web/src/pages/UsersPage.css
|
||||
git commit -m "feat: user management card grid layout in settings tab"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4:全量构建并安装 deb
|
||||
|
||||
**Step 1:递增 changelog 版本到 1.0.0-6**
|
||||
|
||||
在 `deps/KVM/debian/changelog` 顶部插入:
|
||||
|
||||
```
|
||||
kvm-server (1.0.0-6) unstable; urgency=medium
|
||||
|
||||
* Restrict privacy/audit/agent nav to privileged roles (auditor+)
|
||||
* User management: replace table with responsive card grid
|
||||
|
||||
-- KVM Project <kvm@example.com> Tue, 04 Mar 2026 12:00:00 +0000
|
||||
```
|
||||
|
||||
**Step 2:构建 deb**
|
||||
|
||||
```bash
|
||||
cd /data/project/KVM-privacy/deps/KVM && PATH="/usr/local/go/bin:$PATH" bash scripts/build-deb.sh 2>&1 | tail -20
|
||||
```
|
||||
|
||||
期望:`kvm-server_1.0.0-6_arm64.deb` 生成
|
||||
|
||||
**Step 3:安装**
|
||||
|
||||
```bash
|
||||
sudo dpkg -i /data/project/KVM-privacy/deps/KVM/build/deb/kvm-server_1.0.0-6_arm64.deb
|
||||
```
|
||||
|
||||
**Step 4:验证服务**
|
||||
|
||||
```bash
|
||||
systemctl is-active kvm-server && curl -sf http://localhost:8080/api/health
|
||||
```
|
||||
|
||||
期望:`active` + `{"status":"ok"}`
|
||||
|
||||
---
|
||||
|
||||
### 验证清单
|
||||
|
||||
1. `npm run build` TypeScript 无错误
|
||||
2. operator 登录 → 导航栏只显示:Console / Terminal / Settings
|
||||
3. system_admin 登录 → 导航栏显示全部链接;Settings > Users Tab 为卡片布局
|
||||
4. 卡片头像字母色 = 对应角色色(蓝/橙/绿/灰)
|
||||
5. 点击角色徽章 → 出现下拉切换(非自身)
|
||||
6. 禁用/解锁/删除按钮功能正常
|
||||
7. 创建用户 overlay 正常打开/关闭
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,139 @@
|
||||
# 设计文档:WebRTC 稳定性增强 + 虚拟媒体存储
|
||||
|
||||
日期:2026-03-04
|
||||
分支:feature/enhance-testing-security-states
|
||||
|
||||
---
|
||||
|
||||
## 一、WebRTC 稳定性(方案 B)
|
||||
|
||||
### 根因
|
||||
|
||||
| 问题 | 文件 | 当前值 | 目标值 |
|
||||
|------|------|--------|--------|
|
||||
| GStreamer PLI 是空函数 | gstreamer/pipeline.go:1007 | no-op | 切 native 管道 |
|
||||
| WS 心跳间隔过长 | signaling/websocket.go | 27s ping / 30s pong | 10s / 15s |
|
||||
| stale 检测阈值过长 | useWebRTC.ts:182 | 8 次×2s=16s | 4 次×2s=8s |
|
||||
| disconnected 超时过长 | useWebRTC.ts:248 | 3000ms | 1000ms |
|
||||
| track.onmute/onended 无监听 | useWebRTC.ts | 无 | 即时触发重连 |
|
||||
| framesDecoded 无冻结检测 | useWebRTC.ts | 无 | bytes↑但 frames 停滞→重连 |
|
||||
| WriteRTP 错误无处理 | webrtc/server.go | 忽略 | 日志+移除死 peer |
|
||||
|
||||
### 方案
|
||||
|
||||
#### 1. 切换 native 视频管道(解决 PLI 无响应)
|
||||
- `deps/KVM/config/kvm-default.json`:`pipeline_backend` 改为 `"native"`
|
||||
- native 管道已实现,`ForceIDR()` 通过 MPP 即时触发,PLI→IDR 延迟 <1ms
|
||||
|
||||
#### 2. 调优心跳参数(Go)
|
||||
- `signaling/websocket.go`:
|
||||
- `pongWait` 30s → 15s
|
||||
- `pingPeriod` 27s → 10s(= pongWait × 2/3)
|
||||
- `writeWait` 10s 不变
|
||||
|
||||
#### 3. 前端稳定性增强(TypeScript)
|
||||
- `useWebRTC.ts`:
|
||||
- stale 阈值:`>= 8` → `>= 4`(8s)
|
||||
- disconnected timeout:3000 → 1000ms
|
||||
- `ontrack` 回调中监听 `track.onmute` 和 `track.onended` → 立即调用 `scheduleReconnect`
|
||||
- stats 循环中增加 framesDecoded 冻结检测:当 bytesDelta > 0 但 framesDelta === 0 连续 4 次,触发重连
|
||||
|
||||
#### 4. WriteRTP 错误处理(Go)
|
||||
- `webrtc/server.go` `WriteRTP` 方法:错误时 log + 收集死 peer ID,写完后统一移除
|
||||
|
||||
---
|
||||
|
||||
## 二、虚拟媒体存储(USB Mass Storage Gadget)
|
||||
|
||||
### 架构
|
||||
|
||||
```
|
||||
用户浏览器
|
||||
↕ 上传/管理 ISO 文件(multipart)
|
||||
Go API (/api/kvm/storage/*)
|
||||
↕ 读写 configfs
|
||||
Linux USB Gadget (mass_storage.0)
|
||||
↕ USB OTG (fe800000.usb)
|
||||
被控机(Target PC 看到 USB CD-ROM/磁盘)
|
||||
```
|
||||
|
||||
### 核心设计决策
|
||||
|
||||
**预置 mass_storage 功能(空文件状态)**:gadget 初始化时就创建 `functions/mass_storage.0`,`lun.0/file` 为空(被控机看到"无介质")。挂载时只写文件路径,**无需重绑 UDC,HID 不中断**。
|
||||
|
||||
**媒体目录**:`/var/lib/kvm/media/`(persistent,随 deb 创建)
|
||||
|
||||
**文件类型**:
|
||||
- `.iso` → 自动 cdrom=true, ro=true
|
||||
- `.img` → cdrom=false, ro=false(可写磁盘)
|
||||
|
||||
### 组件
|
||||
|
||||
#### A. 脚本层
|
||||
`setup-usb-gadget.sh` 追加:
|
||||
```bash
|
||||
modprobe usb_f_mass_storage 2>/dev/null || true
|
||||
mkdir -p functions/mass_storage.0
|
||||
echo 1 > functions/mass_storage.0/stall
|
||||
echo 1 > functions/mass_storage.0/lun.0/removable
|
||||
echo "" > functions/mass_storage.0/lun.0/file # 初始无介质
|
||||
ln -s functions/mass_storage.0 configs/c.1/
|
||||
```
|
||||
|
||||
#### B. Go 层
|
||||
新包 `go/internal/storage/manager.go`:
|
||||
```
|
||||
StorageManager
|
||||
├── Mount(filename, cdrom) error // 写 lun.0/file
|
||||
├── Unmount() error // 写空字符串
|
||||
├── Status() StorageStatus // 读 lun.0/file
|
||||
└── ListImages() []ImageInfo // 扫描 /var/lib/kvm/media/
|
||||
```
|
||||
|
||||
新文件 `go/internal/api/storage_handler.go`:
|
||||
```
|
||||
GET /api/kvm/storage/status → 当前挂载状态
|
||||
POST /api/kvm/storage/mount → { filename, cdrom }
|
||||
POST /api/kvm/storage/unmount → 弹出
|
||||
GET /api/kvm/storage/images → 镜像列表
|
||||
POST /api/kvm/storage/upload → multipart 上传(限 8GB)
|
||||
DELETE /api/kvm/storage/images/{name} → 删除镜像
|
||||
```
|
||||
|
||||
`router.go` 注册 6 条路由(withAuth 保护)。
|
||||
|
||||
#### C. 前端层
|
||||
- `web/src/services/api/kvm.ts`:新增 `storageAPI` 方法组
|
||||
- `web/src/stores/storageStore.ts`:zustand store,管理 status + images 列表
|
||||
- `web/src/components/VirtualMediaModal.tsx`:
|
||||
- 镜像列表(文件名、大小、类型、挂载按钮)
|
||||
- 拖拽/点击上传区(进度条)
|
||||
- 当前挂载状态 badge
|
||||
- 弹出按钮
|
||||
- `web/src/components/QuickActions.tsx`:
|
||||
- 移除 `disabled`,onClick 打开 VirtualMediaModal
|
||||
- "挂载 ISO" / "挂载 CD-ROM" / "弹出媒体" 按钮连接 store
|
||||
|
||||
### 安全考虑
|
||||
|
||||
- 上传文件名: 仅允许 `[a-zA-Z0-9_\-.]`,拒绝路径穿越(`../`)
|
||||
- 文件大小: 上限 8GB(multipart limit)
|
||||
- 仅允许 `.iso` 和 `.img` 扩展名
|
||||
- RBAC: 存储操作要求 `operator` 以上角色
|
||||
|
||||
---
|
||||
|
||||
## 三、实现顺序(单次部署)
|
||||
|
||||
1. native pipeline 配置切换(1个文件)
|
||||
2. Go 心跳参数(1个文件)
|
||||
3. Go WriteRTP 错误处理(1个文件)
|
||||
4. 前端 useWebRTC 4项修复(1个文件)
|
||||
5. setup-usb-gadget.sh 追加 mass_storage
|
||||
6. Go StorageManager(新包)
|
||||
7. Go StorageHandler(新文件)
|
||||
8. router.go 注册路由
|
||||
9. main.go 初始化 StorageManager
|
||||
10. 前端 kvmAPI + storageStore + VirtualMediaModal
|
||||
11. QuickActions 激活按钮
|
||||
12. deb 打包安装(changelog → 1.0.0-7)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,338 @@
|
||||
# HDMI-TX DRM 本地输出 + AI Agent 优化实施计划
|
||||
|
||||
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
|
||||
|
||||
**Goal:** 实现 HDMI-TX 本地遮蔽输出 + OSD 隐私警告,并优化 AI Agent token 消耗 72%。
|
||||
|
||||
**Architecture:** HDMI DRM 模块作为 `deps/KVM` 视频管道的扩展(C/CGo),Agent 优化基于 `services/kvm_agent` 和 `deps/embedding`。所有模型已同步到项目目录。
|
||||
|
||||
**Tech Stack:** C (libdrm/DRM-KMS), CGo, Go, Python asyncio, RKNN NPU, FAISS
|
||||
|
||||
---
|
||||
|
||||
## 模块一: HDMI-TX DRM 本地输出 (deps/KVM)
|
||||
|
||||
### Task 1: DRM 基础输出模块
|
||||
|
||||
**Files:**
|
||||
- Create: `deps/KVM/src/video/drm_output.h`
|
||||
- Create: `deps/KVM/src/video/drm_output.c`
|
||||
- Modify: `deps/KVM/src/video/CMakeLists.txt`
|
||||
|
||||
**内容:**
|
||||
1. 实现 `drm_output_create()`: 打开 `/dev/dri/card0`, 查找 HDMI-A-2 connector, 获取 CRTC + encoder
|
||||
2. 实现 `drm_output_present()`: memcpy NV12 数据到 DRM dumb buffer → `drmModeAtomicCommit()` 页翻转
|
||||
3. 实现 `drm_output_destroy()`: 释放资源
|
||||
4. CMakeLists.txt 添加 `pkg_check_modules(DRM REQUIRED libdrm)` 依赖
|
||||
|
||||
**验证:**
|
||||
```bash
|
||||
cd deps/KVM && cmake -B build/video -S src/video && cmake --build build/video
|
||||
# 编写独立测试: 输出纯色帧到 HDMI-A-2, 接上显示器确认有画面
|
||||
```
|
||||
|
||||
### Task 2: V4L2 → DRM 实时 Loopback
|
||||
|
||||
**Files:**
|
||||
- Modify: `deps/KVM/src/video/video_pipeline.cpp` (internal_v4l2_callback 分叉)
|
||||
- Modify: `deps/KVM/src/video/video_pipeline.h` (添加 drm_output_ctx 字段)
|
||||
|
||||
**内容:**
|
||||
1. `VideoPipeline` 结构体添加 `drm_output_ctx* drm_ctx` 和 `bool drm_output_enabled`
|
||||
2. `internal_v4l2_callback()` 中遮蔽后、MPP 编码前,调用 `drm_output_present()`
|
||||
3. `video_pipeline_create()` 读取配置决定是否启用 DRM 输出
|
||||
4. `video_pipeline_destroy()` 中销毁 DRM 上下文
|
||||
|
||||
**关键约束:**
|
||||
- DRM present 不能阻塞 MPP 编码路径 — 使用非阻塞 page flip + 丢帧策略
|
||||
- 若 HDMI-A-2 未连接,静默跳过(不报错)
|
||||
|
||||
**验证:**
|
||||
```bash
|
||||
# 构建后安装到设备
|
||||
PATH="/usr/local/go/bin:$PATH" bash deps/KVM/scripts/build-deb.sh
|
||||
# 设备上: 接上 HDMI-A-2 显示器, 开启 KVM, 确认本地显示器有遮蔽后的画面
|
||||
```
|
||||
|
||||
### Task 3: DMA-buf 零拷贝优化
|
||||
|
||||
**Files:**
|
||||
- Modify: `deps/KVM/src/video/drm_output.c` (添加 dmabuf import 路径)
|
||||
|
||||
**内容:**
|
||||
1. 实现 `drm_output_present_dmabuf()`: `drmPrimeFDToHandle()` → `drmModeAddFB2()` → atomic commit
|
||||
2. 在 `video_pipeline.cpp` 中,当 V4L2 使用 DMA-buf 模式时,优先使用零拷贝路径
|
||||
3. 隐私遮蔽场景下退回 memcpy(遮蔽修改了 buffer 内容,不能零拷贝原始帧)
|
||||
|
||||
**验证:** 对比有无 DMA-buf 时的 CPU 占用率(预期节省 ~2ms/帧)
|
||||
|
||||
### Task 4: OSD 覆盖层 (Esmart Plane)
|
||||
|
||||
**Files:**
|
||||
- Modify: `deps/KVM/src/video/drm_output.h` (OSD API)
|
||||
- Modify: `deps/KVM/src/video/drm_output.c` (Esmart plane + ARGB framebuffer)
|
||||
|
||||
**内容:**
|
||||
1. 初始化时查找 Esmart plane (支持 ARGB8888),绑定到同一 CRTC
|
||||
2. 分配 ARGB8888 OSD framebuffer (例如 1920x120 状态栏)
|
||||
3. 实现 `drm_output_show_osd()`: CPU 渲染简单矩形+文字到 ARGB buffer → atomic commit
|
||||
4. 实现 `drm_output_clear_osd()`: 清空 ARGB buffer
|
||||
5. OSD 渲染: 简单 bitmap font(无依赖)或 FreeType(如需中文)
|
||||
|
||||
**验证:** 调用 `drm_output_show_osd("TEST", ...)` 确认显示器上出现半透明覆盖文字
|
||||
|
||||
### Task 5: CGo 桥接 + Go API
|
||||
|
||||
**Files:**
|
||||
- Create: `deps/KVM/go/internal/videopipe/drm_output_cgo.go`
|
||||
- Modify: `deps/KVM/go/internal/api/router.go` (添加 OSD API 路由)
|
||||
- Create: `deps/KVM/go/internal/api/display_handler.go`
|
||||
|
||||
**内容:**
|
||||
1. CGo 封装: `ShowOSD(text string, level int, durationMs int)` 调用 C `drm_output_show_osd()`
|
||||
2. `EnableLocalDisplay()` / `DisableLocalDisplay()` 控制 DRM 输出开关
|
||||
3. REST API:
|
||||
- `POST /api/v1/display/osd` — 显示 OSD 消息
|
||||
- `GET /api/v1/display/status` — 获取 HDMI-TX 状态
|
||||
- `PUT /api/v1/display/config` — 配置本地显示参数
|
||||
4. RBAC: 添加 `/api/v1/display/*` 到 `defaultPolicy`
|
||||
|
||||
**验证:**
|
||||
```bash
|
||||
curl -X POST localhost:8080/api/v1/display/osd \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"text":"文件上传已拦截","level":"warning","duration_ms":5000}'
|
||||
# 确认本地显示器出现 OSD
|
||||
```
|
||||
|
||||
### Task 6: Privacy Gateway → OSD 事件集成
|
||||
|
||||
**Files:**
|
||||
- Modify: `services/privacy_gateway/privacy_api.py` (上传拦截时发送 OSD)
|
||||
- Modify: `deps/KVM/go/internal/privacy/handler.go` (隐私模式切换时发送 OSD)
|
||||
|
||||
**内容:**
|
||||
1. Privacy Gateway 拦截文件上传后,POST 到 `/api/v1/display/osd` 发送警告
|
||||
2. 隐私模式 off/audit/redact 切换时,更新状态栏 OSD
|
||||
3. OCR PII 检测结果更新到状态栏(检测到几处 PII)
|
||||
4. Agent 任务开始/结束时更新 OSD
|
||||
|
||||
---
|
||||
|
||||
## 模块二: AI Agent Token 优化 (services/kvm_agent)
|
||||
|
||||
### Task 7: System Prompt 缓存 + 结构化输出
|
||||
|
||||
**Files:**
|
||||
- Modify: `services/kvm_agent/llm_planner.py`
|
||||
- Modify: `services/kvm_agent/configs/agent.yaml`
|
||||
|
||||
**内容:**
|
||||
1. 为 Gemini API 启用 `cachedContents` 缓存 system prompt (TTL 1 小时)
|
||||
2. 为 OpenAI API 确保 system prompt 在每次请求中位置一致(自动触发前缀缓存)
|
||||
3. `response_format: {"type": "json_object"}` 启用结构化输出
|
||||
4. `max_tokens` 从 512 降至 128(Action JSON 通常 <100 tokens)
|
||||
5. agent.yaml 新增 `llm.cache_system_prompt: true` 配置项
|
||||
|
||||
**验证:**
|
||||
```bash
|
||||
cd services/kvm_agent && python3 -m pytest tests/test_hybrid_planner.py -v
|
||||
```
|
||||
|
||||
### Task 8: SceneGraph 智能裁剪
|
||||
|
||||
**Files:**
|
||||
- Modify: `services/kvm_agent/perception.py` (添加 `to_text_summary_optimized`)
|
||||
|
||||
**内容:**
|
||||
1. 新方法 `SceneGraph.to_text_summary_optimized(task_keywords, max_elements=15)`
|
||||
2. 基于任务关键词相关性 + 元素大小 + 位置进行评分排序
|
||||
3. 紧凑输出格式: `[text](x,y)` 替代完整元素描述
|
||||
4. `hybrid_planner.py` 调用时传入任务关键词
|
||||
|
||||
**验证:**
|
||||
```bash
|
||||
cd services/kvm_agent && python3 -m pytest tests/test_perception.py -v
|
||||
```
|
||||
|
||||
### Task 9: 渐进式上下文压缩
|
||||
|
||||
**Files:**
|
||||
- Create: `services/kvm_agent/context_compressor.py`
|
||||
- Modify: `services/kvm_agent/llm_planner.py` (使用压缩上下文)
|
||||
|
||||
**内容:**
|
||||
1. `ContextCompressor` 类: 三级压缩
|
||||
- Level 1 (最近 3 步): 完整 action + scene 描述
|
||||
- Level 2 (3-10 步前): 压缩为动作类型序列
|
||||
- Level 3 (10+ 步前): 单句进度摘要
|
||||
2. `llm_planner.py` 的 `_prepare_user_content()` 使用压缩后的上下文
|
||||
3. history 保留从 3 条扩展到 10 条(但总 token 更少)
|
||||
|
||||
**验证:**
|
||||
```bash
|
||||
cd services/kvm_agent && python3 -m pytest tests/ -v -k "context"
|
||||
```
|
||||
|
||||
### Task 10: 自适应图像发送
|
||||
|
||||
**Files:**
|
||||
- Modify: `services/kvm_agent/llm_planner.py` (添加 `_should_send_image`)
|
||||
|
||||
**内容:**
|
||||
1. `_should_send_image(scene, task, step)` 决策逻辑:
|
||||
- step=0: 总是发送(初始视觉理解)
|
||||
- 简单动作(点击/输入)+ 充足 SceneGraph (>8 元素): text-only
|
||||
- 验证步骤: 发送(需要确认视觉变化)
|
||||
2. Text-only 模式不发送截图,节省 ~1290 tokens/步
|
||||
3. 可通过 `agent.yaml` 配置 `llm.adaptive_image: true` 开关
|
||||
|
||||
**验证:** Mock 测试确认不同场景下 `_should_send_image` 返回正确决策
|
||||
|
||||
### Task 11: 屏幕指纹匹配系统
|
||||
|
||||
**Files:**
|
||||
- Create: `services/kvm_agent/screen_fingerprint.py`
|
||||
- Modify: `services/kvm_agent/hybrid_planner.py` (集成屏幕指纹)
|
||||
|
||||
**内容:**
|
||||
1. `ScreenFingerprint` 类:
|
||||
- `compute_fingerprint(scene)`: 排序 OCR 文本 → e5-small embedding → 384-dim 向量
|
||||
- `lookup(fingerprint)`: FAISS 最近邻搜索 → 返回预定义 Action
|
||||
- `register(fingerprint, action)`: 注册新的屏幕→动作映射
|
||||
2. 指纹库存储在 `~/.kvm-agent/fingerprints.faiss` + `fingerprints.json`
|
||||
3. `hybrid_planner.py` 在模板匹配前先查指纹(更快、更通用)
|
||||
4. 利用 `deps/embedding` 的 e5-small 模型进行编码
|
||||
|
||||
**依赖:** `deps/embedding/models/embedder.onnx` (已同步到项目目录)
|
||||
|
||||
**验证:**
|
||||
```bash
|
||||
cd services/kvm_agent && python3 -m pytest tests/ -v -k "fingerprint"
|
||||
```
|
||||
|
||||
### Task 12: 情景记忆系统
|
||||
|
||||
**Files:**
|
||||
- Create: `services/kvm_agent/episodic_memory.py`
|
||||
- Modify: `services/kvm_agent/agent.py` (任务完成后记录情景)
|
||||
- Modify: `services/kvm_agent/hybrid_planner.py` (使用情景记忆)
|
||||
|
||||
**内容:**
|
||||
1. `Episode` dataclass: task, success, total_steps, key_actions, obstacles, screen_fingerprints
|
||||
2. `EpisodicMemory` 类:
|
||||
- `record(episode)`: 将情景向量化存入 FAISS
|
||||
- `recall(task)`: 语义搜索相似任务经验
|
||||
- `to_context_hint(episode)`: 压缩为 ~200 tokens 的 LLM 提示
|
||||
3. `agent.py` 的 `run_task()` 完成后调用 `episodic.record()`
|
||||
4. `hybrid_planner.py` 在 LLM 调用前注入情景上下文
|
||||
|
||||
**依赖:** `deps/embedding` 的 FAISS + e5-small
|
||||
|
||||
**验证:**
|
||||
```bash
|
||||
cd services/kvm_agent && python3 -m pytest tests/ -v -k "episodic"
|
||||
```
|
||||
|
||||
### Task 13: 状态机模板升级
|
||||
|
||||
**Files:**
|
||||
- Modify: `services/kvm_agent/template_store.py` (支持条件分支)
|
||||
|
||||
**内容:**
|
||||
1. 模板格式从线性步骤列表升级为状态机:
|
||||
```python
|
||||
states = {
|
||||
"start": {"action": ..., "verify": ..., "next": ..., "fallback": ...},
|
||||
}
|
||||
```
|
||||
2. `replay_step()` 执行当前状态的 action,OCR 验证 `verify` 条件
|
||||
3. 验证通过 → 转到 `next`;失败 → 转到 `fallback`(或重试)
|
||||
4. 成功任务自动生成状态机模板(从 episode 提取)
|
||||
|
||||
**验证:**
|
||||
```bash
|
||||
cd services/kvm_agent && python3 -m pytest tests/ -v -k "template"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 模块三: 隐私处理模型集成
|
||||
|
||||
### Task 14: OCR 模型路径统一
|
||||
|
||||
**Files:**
|
||||
- Modify: `deps/KVM/go/internal/config/config.go` (默认模型路径)
|
||||
- Modify: `deps/KVM/scripts/build-deb.sh` (打包时包含模型)
|
||||
- Modify: `debian/kvm-server/DEBIAN/postinst` (安装时复制模型)
|
||||
|
||||
**内容:**
|
||||
1. OCR 模型已同步到 `deps/KVM/models/ocr/` (ppocrv4_det.rknn + ch/en 子目录)
|
||||
2. build-deb.sh 打包时将 `models/ocr/` 复制到 deb 包的 `/etc/kvm/models/ocr/`
|
||||
3. config.go 默认路径指向 `/etc/kvm/models/ocr`(已是当前配置)
|
||||
4. 确保 deb 包安装后 OCR 模型自动就位
|
||||
|
||||
**验证:**
|
||||
```bash
|
||||
PATH="/usr/local/go/bin:$PATH" bash deps/KVM/scripts/build-deb.sh
|
||||
dpkg -c dist/kvm-server_*.deb | grep models/ocr
|
||||
```
|
||||
|
||||
### Task 15: Embedding 模型部署路径
|
||||
|
||||
**Files:**
|
||||
- Modify: `deps/embedding/src/embedder.py` (模型搜索路径)
|
||||
|
||||
**内容:**
|
||||
1. 模型已同步到 `deps/embedding/models/` (embedder.onnx + tokenizer.json)
|
||||
2. embedder.py 搜索顺序: `./models/` → `~/.embed_db/models/` → `/etc/kvm/models/embedding/`
|
||||
3. kvm-bridge deb 包打包时包含 embedding 模型
|
||||
|
||||
**验证:**
|
||||
```bash
|
||||
cd deps/embedding && python3 -c "from src.embedder import Embedder; e = Embedder(); print(e.model_path)"
|
||||
```
|
||||
|
||||
### Task 16: Face Detection 模型部署
|
||||
|
||||
**Files:**
|
||||
- Modify: `deps/info-privacy-rs/src/face_detect.rs` (模型路径配置)
|
||||
|
||||
**内容:**
|
||||
1. 模型已同步到 `deps/info-privacy-rs/deps/mediapipe-rknn/models/`
|
||||
2. face_detect.rs 默认路径指向 `/etc/kvm/models/face/` (安装时复制)
|
||||
3. kvm-privacy deb 包打包时包含 face detection 模型
|
||||
|
||||
---
|
||||
|
||||
## 实施优先级
|
||||
|
||||
| 优先级 | 模块 | Task | 预期效果 |
|
||||
|--------|------|------|---------|
|
||||
| **P0** | Agent | T7: System Prompt 缓存 | -40% input tokens |
|
||||
| **P0** | Agent | T8: SceneGraph 裁剪 | -25% scene tokens |
|
||||
| **P0** | Agent | T9: 上下文压缩 | -30% context tokens |
|
||||
| **P0** | Agent | T10: 自适应图像 | -50% image tokens |
|
||||
| **P1** | Agent | T11: 屏幕指纹 | 重复任务 0 token |
|
||||
| **P1** | Agent | T12: 情景记忆 | -75% context tokens |
|
||||
| **P1** | Agent | T13: 状态机模板 | 已知流程 0 token |
|
||||
| **P1** | HDMI | T1: DRM 基础输出 | 本地显示可用 |
|
||||
| **P1** | HDMI | T2: V4L2→DRM 连通 | 实时 loopback |
|
||||
| **P2** | HDMI | T3: DMA-buf 零拷贝 | -2ms/帧 |
|
||||
| **P2** | HDMI | T4: OSD 覆盖层 | 隐私警告显示 |
|
||||
| **P2** | HDMI | T5: Go API | REST 控制 |
|
||||
| **P2** | HDMI | T6: 事件集成 | 自动 OSD |
|
||||
| **P2** | Model | T14-16: 模型路径统一 | deb 包自带模型 |
|
||||
|
||||
## 验证
|
||||
|
||||
```bash
|
||||
# Agent 测试
|
||||
cd services/kvm_agent && python3 -m pytest tests/ -v
|
||||
|
||||
# KVM Server 编译
|
||||
cd deps/KVM && PATH="/usr/local/go/bin:$PATH" bash scripts/build-deb.sh
|
||||
|
||||
# C 视频管道编译 (含 DRM)
|
||||
cd deps/KVM && cmake -B build/video -S src/video && cmake --build build/video
|
||||
```
|
||||
@@ -0,0 +1,293 @@
|
||||
# HDMI-TX DRM 本地输出 + OSD 设计
|
||||
|
||||
> 日期: 2026-03-06
|
||||
> 目标: 通过 NanoPC-T6 HDMI-TX 输出遮蔽后画面 + OSD 隐私警告,本地用户无需浏览器
|
||||
|
||||
## 1. 问题背景
|
||||
|
||||
当前视频管道:
|
||||
```
|
||||
被控 PC → HDMI-RX → V4L2 → RGA → MPP H.264 → WebRTC → 浏览器
|
||||
```
|
||||
|
||||
本地用户必须通过浏览器访问 WebRTC 流(150-350ms 延迟)。NanoPC-T6 有 2 个 HDMI-TX 输出口,应利用其中一路直接输出遮蔽后的视频 + OSD 隐私警告。
|
||||
|
||||
**设计目标**:
|
||||
- 本地显示遮蔽后的画面(PII 已黑化)
|
||||
- OSD 覆盖层显示隐私警告(如"文件上传已拦截")
|
||||
- 端到端延迟 ≤ 2 帧(~35ms @60fps)
|
||||
- 零 CPU 拷贝(DMA-buf 直通 VOP2)
|
||||
|
||||
## 2. RK3588 HDMI-TX 硬件
|
||||
|
||||
### 接口状态
|
||||
|
||||
| 接口 | DRM Connector | 当前状态 | VOP2 VP |
|
||||
|------|--------------|---------|---------|
|
||||
| HDMI-A-1 (主) | connector-157 | connected (桌面) | video_port0 |
|
||||
| HDMI-A-2 (副) | connector-163 | disconnected (可用) | video_port1 (free) |
|
||||
|
||||
### VOP2 平面资源
|
||||
|
||||
RK3588 VOP2 有 24 个硬件平面:
|
||||
- **4 Cluster 平面**: 支持 NV12/NV16/NV24 + AFBC,适合视频层
|
||||
- **4 Esmart 平面**: 支持 ARGB8888/XRGB8888,适合 OSD 覆盖层
|
||||
- **每个 VP 可绑定**: 1 Cluster + 1 Esmart(至少)
|
||||
|
||||
### 多平面 OSD 架构
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────┐
|
||||
│ Cluster Plane (z=0): NV12 视频 │ ← V4L2 DMA-buf 零拷贝
|
||||
│ ┌─────────────────────────────┐ │
|
||||
│ │ 遮蔽后的 HDMI-RX 画面 │ │
|
||||
│ │ (PII 区域已黑化) │ │
|
||||
│ └─────────────────────────────┘ │
|
||||
│ │
|
||||
│ Esmart Plane (z=1): ARGB OSD │ ← CPU 渲染 ARGB8888
|
||||
│ ┌─────────────────────────────┐ │
|
||||
│ │ ⚠ 文件上传已拦截 │ │ 半透明背景 + 文字
|
||||
│ │ 🔒 隐私模式: redact │ │ 状态栏 / 弹窗通知
|
||||
│ └─────────────────────────────┘ │
|
||||
│ │
|
||||
│ VOP2 硬件混合 → HDMI-A-2 输出 │ ← 无 CPU 混合开销
|
||||
└─────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## 3. 数据流设计
|
||||
|
||||
### 视频路径(零拷贝)
|
||||
|
||||
```
|
||||
V4L2 /dev/video0
|
||||
→ DMA-buf export (VIDIOC_EXPBUF, 已有)
|
||||
→ RGA NV12 处理 (已有)
|
||||
→ 隐私遮蔽 redact_nv12_regions() (已有)
|
||||
→ 分叉 ─┬─→ MPP H.264 编码 → WebRTC (不变)
|
||||
└─→ DRM plane import (DMA-buf fd) → HDMI-TX scanout (新增)
|
||||
```
|
||||
|
||||
**关键**: V4L2 capture 已实现 DMA-buf 导出(`v4l2_capture.cpp:260-277`),VOP2 Cluster 平面原生支持 NV12 DMA-buf import。
|
||||
|
||||
### OSD 路径(CPU 渲染 + DRM 平面)
|
||||
|
||||
```
|
||||
OSD 事件 (Go/Python → C callback)
|
||||
→ CPU 渲染 ARGB8888 framebuffer (简单矩形+文字)
|
||||
→ DRM Esmart plane atomic commit
|
||||
→ VOP2 硬件 alpha blending
|
||||
→ HDMI-TX 混合输出
|
||||
```
|
||||
|
||||
OSD 更新频率低(事件驱动,非每帧),CPU 开销可忽略。
|
||||
|
||||
### 集成点
|
||||
|
||||
`video_pipeline.cpp` 中的 `internal_v4l2_callback()`:
|
||||
|
||||
```cpp
|
||||
// 现有: 隐私遮蔽之后
|
||||
redact_nv12_regions(pipeline->nv12_buffer, ...);
|
||||
|
||||
// <<<< 新增: DRM 输出 >>>>
|
||||
if (pipeline->drm_output_enabled) {
|
||||
drm_output_present(pipeline->drm_ctx, pipeline->nv12_buffer,
|
||||
pipeline->width, pipeline->height);
|
||||
}
|
||||
|
||||
// 现有: MPP 编码
|
||||
mpp_encoder_encode(pipeline->encoder, pipeline->nv12_buffer, ...);
|
||||
```
|
||||
|
||||
## 4. C 模块设计
|
||||
|
||||
### 新文件: `src/video/drm_output.h`
|
||||
|
||||
```c
|
||||
typedef struct drm_output_ctx drm_output_ctx;
|
||||
|
||||
// 初始化 DRM 输出 (指定 HDMI-TX connector)
|
||||
drm_output_ctx* drm_output_create(const char* connector_name);
|
||||
|
||||
// 提交 NV12 帧到 DRM plane (DMA-buf 或 memcpy fallback)
|
||||
int drm_output_present(drm_output_ctx* ctx,
|
||||
const uint8_t* nv12_data,
|
||||
int width, int height);
|
||||
|
||||
// 设置 DMA-buf fd 直接 scanout (零拷贝路径)
|
||||
int drm_output_present_dmabuf(drm_output_ctx* ctx,
|
||||
int dmabuf_fd,
|
||||
int width, int height,
|
||||
uint32_t fourcc); // DRM_FORMAT_NV12
|
||||
|
||||
// OSD 覆盖层
|
||||
int drm_output_show_osd(drm_output_ctx* ctx,
|
||||
const char* text,
|
||||
int x, int y,
|
||||
uint32_t color, // ARGB
|
||||
int duration_ms); // 0 = permanent
|
||||
|
||||
int drm_output_clear_osd(drm_output_ctx* ctx);
|
||||
|
||||
void drm_output_destroy(drm_output_ctx* ctx);
|
||||
```
|
||||
|
||||
### DRM 初始化流程
|
||||
|
||||
```
|
||||
open("/dev/dri/card0")
|
||||
→ drmModeGetResources()
|
||||
→ 查找 HDMI-A-2 connector (connector-163)
|
||||
→ 获取对应 CRTC + encoder
|
||||
→ 查找 Cluster plane (支持 NV12) → 绑定视频
|
||||
→ 查找 Esmart plane (支持 ARGB8888) → 绑定 OSD
|
||||
→ drmModeAtomicCommit() 设置初始模式
|
||||
```
|
||||
|
||||
### 关键 DRM API
|
||||
|
||||
| 操作 | API | 说明 |
|
||||
|------|-----|------|
|
||||
| 打开设备 | `open("/dev/dri/card0")` | 主 DRM 设备 |
|
||||
| 查找 connector | `drmModeGetConnector()` | HDMI-A-2 |
|
||||
| 设置模式 | `drmModeAtomicCommit()` | Atomic modesetting |
|
||||
| 创建 framebuffer | `drmModeAddFB2()` | NV12 双平面 |
|
||||
| DMA-buf import | `drmPrimeFDToHandle()` | V4L2 fd → GEM handle |
|
||||
| 页翻转 | `DRM_MODE_PAGE_FLIP_EVENT` | 垂直同步 |
|
||||
|
||||
### 依赖
|
||||
|
||||
```cmake
|
||||
# CMakeLists.txt 新增
|
||||
find_package(PkgConfig REQUIRED)
|
||||
pkg_check_modules(DRM REQUIRED libdrm)
|
||||
target_link_libraries(drm_output ${DRM_LIBRARIES})
|
||||
target_include_directories(drm_output PRIVATE ${DRM_INCLUDE_DIRS})
|
||||
```
|
||||
|
||||
## 5. OSD 内容设计
|
||||
|
||||
### 状态栏(常驻底部)
|
||||
|
||||
```
|
||||
┌────────────────────────────────────────┐
|
||||
│ │
|
||||
│ (遮蔽后的视频画面) │
|
||||
│ │
|
||||
│ │
|
||||
├────────────────────────────────────────┤
|
||||
│ 🔒 隐私模式: redact | PII: 3处已遮蔽 │ ← 半透明黑底
|
||||
└────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### 弹窗通知(事件触发,自动消失)
|
||||
|
||||
```
|
||||
┌────────────────────────────────────────┐
|
||||
│ ┌─────────────────────────┐ │
|
||||
│ │ ⚠ 文件上传已拦截 │ │
|
||||
│ │ 目标: chat.openai.com │ │
|
||||
│ │ 包含: 身份证号 x2 │ │
|
||||
│ └─────────────────────────┘ │
|
||||
│ │
|
||||
│ (遮蔽后的视频画面) │
|
||||
│ │
|
||||
└────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### OSD 事件来源
|
||||
|
||||
| 事件 | 来源 | 触发 |
|
||||
|------|------|------|
|
||||
| 隐私模式切换 | Go KVM Server | WebUI 操作 |
|
||||
| PII 检测结果 | OCR → regex | 每次 OCR 扫描 |
|
||||
| 文件上传拦截 | Privacy Gateway | mitmproxy 检测 |
|
||||
| Agent 任务状态 | KVM Agent | 任务开始/结束 |
|
||||
| 网络异常 | Privacy Gateway | 连接断开/恢复 |
|
||||
|
||||
### OSD 通信 API
|
||||
|
||||
Go → C 回调(CGo):
|
||||
|
||||
```go
|
||||
// go/internal/videopipe/drm_output.go
|
||||
func ShowOSD(text string, level int, durationMs int) {
|
||||
cText := C.CString(text)
|
||||
defer C.free(unsafe.Pointer(cText))
|
||||
C.drm_output_show_osd(pipeline.drmCtx, cText, 10, 10,
|
||||
C.uint32_t(levelToColor(level)),
|
||||
C.int(durationMs))
|
||||
}
|
||||
```
|
||||
|
||||
Python → Go HTTP → C:
|
||||
|
||||
```
|
||||
POST /api/v1/display/osd
|
||||
{
|
||||
"text": "文件上传已拦截: chat.openai.com",
|
||||
"level": "warning", // info|warning|error
|
||||
"duration_ms": 5000 // 0 = permanent
|
||||
}
|
||||
```
|
||||
|
||||
## 6. 延迟分析
|
||||
|
||||
| 阶段 | 延迟 | 说明 |
|
||||
|------|------|------|
|
||||
| V4L2 采集 | ~16ms | 1 帧 @60fps |
|
||||
| RGA + 遮蔽 | ~2ms | 已有,无额外开销 |
|
||||
| DRM page flip | ~16ms | 1 帧 vsync 等待 |
|
||||
| **总计** | **~35ms** | **约 2 帧** |
|
||||
|
||||
对比:
|
||||
- WebRTC: 150-350ms (10-20x 延迟)
|
||||
- HDMI 分路器: ~0ms (但无遮蔽/无 OSD)
|
||||
- **DRM 输出: ~35ms (遮蔽 + OSD + 极低延迟)**
|
||||
|
||||
## 7. 配置设计
|
||||
|
||||
```json
|
||||
// config.json
|
||||
{
|
||||
"hdmi_output": {
|
||||
"enabled": true,
|
||||
"connector": "HDMI-A-2",
|
||||
"osd": {
|
||||
"status_bar": true,
|
||||
"notifications": true,
|
||||
"font_size": 24,
|
||||
"opacity": 0.8
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 8. 实现分期
|
||||
|
||||
| 阶段 | 内容 | 产出 |
|
||||
|------|------|------|
|
||||
| Phase 1 | DRM 基础输出 | `drm_output.c`: 打开 HDMI-A-2, 输出 NV12 静态帧 |
|
||||
| Phase 2 | V4L2 → DRM 连通 | `video_pipeline.cpp` 分叉, 实时 loopback |
|
||||
| Phase 3 | DMA-buf 零拷贝 | `drmPrimeFDToHandle()` 替代 memcpy |
|
||||
| Phase 4 | OSD Esmart 平面 | ARGB8888 覆盖层, 状态栏 + 弹窗 |
|
||||
| Phase 5 | Go/Python API | CGo 桥接 + REST API + 事件驱动 OSD |
|
||||
| Phase 6 | 热插拔 + 配置 | HDMI-TX 连接检测, config.json 配置化 |
|
||||
|
||||
## 9. 风险与缓解
|
||||
|
||||
| 风险 | 影响 | 缓解 |
|
||||
|------|------|------|
|
||||
| VOP2 DMA-buf import 不支持 | 退回 memcpy (~2ms 额外) | Phase 1 先用 memcpy 验证 |
|
||||
| HDMI-A-2 未被 BSP 内核启用 | 无法输出 | 检查 DTS 配置, 必要时编译自定义 DT overlay |
|
||||
| NV24 格式不被 VOP2 支持 | 某些 HDMI-RX 模式不兼容 | 已有 RGA NV24→NV12 转换 |
|
||||
| OSD 中文渲染 | 需要字体支持 | 使用 FreeType + Noto Sans CJK |
|
||||
| DRM master 冲突 | 其他进程占用 DRM | 检查是否有 Weston/X11 运行 |
|
||||
|
||||
## 10. 参考
|
||||
|
||||
- [Rockchip VOP2 DRM 驱动 (内核源码)](https://github.com/torvalds/linux/tree/master/drivers/gpu/drm/rockchip)
|
||||
- [RK3588 HDMI-TX 内核补丁 (Collabora)](https://lwn.net/Articles/992163/)
|
||||
- [libdrm API 文档](https://gitlab.freedesktop.org/mesa/drm)
|
||||
- [DRM Atomic Modesetting HOWTO](https://docs.kernel.org/gpu/drm-kms.html)
|
||||
- [V4L2 DMA-buf → DRM 示例](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/tools/testing/selftests/dmabuf-heaps)
|
||||
@@ -0,0 +1,970 @@
|
||||
# Hotplug Stability + Video Pipeline Process Isolation — Implementation Plan
|
||||
|
||||
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
|
||||
|
||||
**Goal:** Isolate the C video pipeline (V4L2→RGA→MPP) into a separate process communicating via shared memory, add HID hot-reconnect, DeviceWatchdog, health API, and fix udev rules.
|
||||
|
||||
**Architecture:** The C pipeline becomes a standalone binary (`kvm-video`) managed as a child process by `kvm-server`. IPC uses POSIX shared memory ring buffer (H.264), double buffer (BGR), Unix socket (control), and eventfd (notifications). HID gains automatic device recovery. A DeviceWatchdog monitors all subsystems with systemd sd_notify integration.
|
||||
|
||||
**Tech Stack:** C11 (video process), Go 1.22+ (kvm-server), POSIX shm/mmap, eventfd, Unix sockets, CMake, systemd
|
||||
|
||||
**Design doc:** `docs/plans/2026-03-06-hotplug-stability-process-isolation.md`
|
||||
|
||||
---
|
||||
|
||||
## Phase 1: C Shared Memory Primitives
|
||||
|
||||
### Task 1: SPSC Ring Buffer — C Header + Implementation
|
||||
|
||||
**Files:**
|
||||
- Create: `deps/KVM/src/video/shm_ring.h`
|
||||
- Create: `deps/KVM/src/video/shm_ring.c`
|
||||
|
||||
**Step 1: Write shm_ring.h**
|
||||
|
||||
```c
|
||||
// shm_ring.h — SPSC lock-free ring buffer over POSIX shared memory
|
||||
#ifndef SHM_RING_H
|
||||
#define SHM_RING_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdatomic.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
// Ring buffer header — lives at offset 0 of shared memory
|
||||
typedef struct {
|
||||
_Atomic uint64_t write_pos; // Producer write offset (bytes)
|
||||
_Atomic uint64_t read_pos; // Consumer read offset (bytes)
|
||||
uint64_t ring_size; // Data region size (fixed at creation)
|
||||
_Atomic uint64_t heartbeat; // Producer increments per frame
|
||||
_Atomic uint64_t error_code; // 0 = normal, >0 = error
|
||||
_Atomic uint64_t frame_count; // Total frames (including dropped)
|
||||
uint64_t _reserved[2]; // Pad to 64 bytes
|
||||
} ShmRingHeader;
|
||||
|
||||
// Entry header — prepended to each data entry
|
||||
typedef struct {
|
||||
uint32_t entry_size; // Payload size (not including this header)
|
||||
uint8_t type; // 0=H264, 1=JPEG_SNAPSHOT
|
||||
uint8_t flags; // bit0=keyframe
|
||||
uint16_t _pad;
|
||||
int64_t pts; // Timestamp (microseconds)
|
||||
} __attribute__((packed)) ShmRingEntry;
|
||||
|
||||
#define SHM_RING_ENTRY_HEADER_SIZE 16
|
||||
#define SHM_RING_DATA_OFFSET 64 // sizeof(ShmRingHeader) padded
|
||||
#define SHM_RING_TYPE_H264 0
|
||||
#define SHM_RING_TYPE_JPEG 1
|
||||
#define SHM_RING_FLAG_KEYFRAME 0x01
|
||||
|
||||
// Create or open shared memory ring. Returns pointer to mmap'd region, or NULL.
|
||||
// If create=true, initializes header; if false, opens existing.
|
||||
// shm_size = total size including header (e.g., 4*1024*1024).
|
||||
void* shm_ring_create(const char* name, size_t shm_size, bool create);
|
||||
|
||||
// Destroy shared memory ring (unmaps + unlinks if owner).
|
||||
void shm_ring_destroy(void* ring, size_t shm_size, bool owner);
|
||||
|
||||
// Producer: write an entry to the ring. Returns 0 on success, -1 if full (frame dropped).
|
||||
int shm_ring_write(void* ring, uint8_t type, uint8_t flags, int64_t pts,
|
||||
const uint8_t* data, uint32_t data_size);
|
||||
|
||||
// Consumer: read next entry from ring. Returns pointer to entry in ring memory,
|
||||
// or NULL if ring is empty. Caller must process entry before calling again.
|
||||
// Sets *out_size to total entry size (header + payload).
|
||||
const ShmRingEntry* shm_ring_read(void* ring, uint32_t* out_size);
|
||||
|
||||
// Consumer: advance read position after processing.
|
||||
void shm_ring_read_advance(void* ring, uint32_t consumed);
|
||||
|
||||
// Get header pointer for direct atomic access (heartbeat, error_code, etc.)
|
||||
static inline ShmRingHeader* shm_ring_header(void* ring) {
|
||||
return (ShmRingHeader*)ring;
|
||||
}
|
||||
|
||||
// Get available bytes for consumer to read.
|
||||
static inline uint64_t shm_ring_available(void* ring) {
|
||||
ShmRingHeader* h = (ShmRingHeader*)ring;
|
||||
return atomic_load_explicit(&h->write_pos, memory_order_acquire) -
|
||||
atomic_load_explicit(&h->read_pos, memory_order_relaxed);
|
||||
}
|
||||
|
||||
#endif // SHM_RING_H
|
||||
```
|
||||
|
||||
**Step 2: Write shm_ring.c**
|
||||
|
||||
```c
|
||||
// shm_ring.c — SPSC lock-free ring buffer implementation
|
||||
#include "shm_ring.h"
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/stat.h>
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#include <errno.h>
|
||||
|
||||
void* shm_ring_create(const char* name, size_t shm_size, bool create) {
|
||||
int flags = create ? (O_CREAT | O_RDWR) : O_RDWR;
|
||||
int fd = shm_open(name, flags, 0600);
|
||||
if (fd < 0) {
|
||||
perror("shm_open");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (create) {
|
||||
if (ftruncate(fd, shm_size) < 0) {
|
||||
perror("ftruncate");
|
||||
close(fd);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
void* ptr = mmap(NULL, shm_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
|
||||
close(fd); // fd not needed after mmap
|
||||
|
||||
if (ptr == MAP_FAILED) {
|
||||
perror("mmap");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (create) {
|
||||
memset(ptr, 0, SHM_RING_DATA_OFFSET);
|
||||
ShmRingHeader* h = (ShmRingHeader*)ptr;
|
||||
h->ring_size = shm_size - SHM_RING_DATA_OFFSET;
|
||||
}
|
||||
|
||||
return ptr;
|
||||
}
|
||||
|
||||
void shm_ring_destroy(void* ring, size_t shm_size, bool owner) {
|
||||
if (!ring) return;
|
||||
munmap(ring, shm_size);
|
||||
// Caller responsible for shm_unlink if owner
|
||||
}
|
||||
|
||||
static inline uint8_t* ring_data(void* ring) {
|
||||
return (uint8_t*)ring + SHM_RING_DATA_OFFSET;
|
||||
}
|
||||
|
||||
static inline uint64_t align8(uint64_t v) {
|
||||
return (v + 7) & ~(uint64_t)7;
|
||||
}
|
||||
|
||||
int shm_ring_write(void* ring, uint8_t type, uint8_t flags, int64_t pts,
|
||||
const uint8_t* data, uint32_t data_size) {
|
||||
ShmRingHeader* h = (ShmRingHeader*)ring;
|
||||
uint64_t entry_total = align8(SHM_RING_ENTRY_HEADER_SIZE + data_size);
|
||||
|
||||
uint64_t wp = atomic_load_explicit(&h->write_pos, memory_order_relaxed);
|
||||
uint64_t rp = atomic_load_explicit(&h->read_pos, memory_order_acquire);
|
||||
|
||||
// Check space: used = wp - rp; free = ring_size - used
|
||||
if ((wp - rp) + entry_total > h->ring_size) {
|
||||
return -1; // Full — caller should drop frame
|
||||
}
|
||||
|
||||
uint8_t* base = ring_data(ring);
|
||||
uint64_t offset = wp % h->ring_size;
|
||||
|
||||
// Build entry header
|
||||
ShmRingEntry entry = {
|
||||
.entry_size = data_size,
|
||||
.type = type,
|
||||
.flags = flags,
|
||||
._pad = 0,
|
||||
.pts = pts,
|
||||
};
|
||||
|
||||
// Write header (may wrap around ring boundary)
|
||||
uint64_t first_chunk = h->ring_size - offset;
|
||||
if (first_chunk >= entry_total) {
|
||||
// No wrap
|
||||
memcpy(base + offset, &entry, SHM_RING_ENTRY_HEADER_SIZE);
|
||||
memcpy(base + offset + SHM_RING_ENTRY_HEADER_SIZE, data, data_size);
|
||||
} else {
|
||||
// Wrap around — copy in two parts
|
||||
uint8_t tmp[entry_total];
|
||||
memcpy(tmp, &entry, SHM_RING_ENTRY_HEADER_SIZE);
|
||||
memcpy(tmp + SHM_RING_ENTRY_HEADER_SIZE, data, data_size);
|
||||
memset(tmp + SHM_RING_ENTRY_HEADER_SIZE + data_size, 0,
|
||||
entry_total - SHM_RING_ENTRY_HEADER_SIZE - data_size);
|
||||
|
||||
memcpy(base + offset, tmp, first_chunk);
|
||||
memcpy(base, tmp + first_chunk, entry_total - first_chunk);
|
||||
}
|
||||
|
||||
// Publish: advance write_pos with release semantics
|
||||
atomic_store_explicit(&h->write_pos, wp + entry_total, memory_order_release);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const ShmRingEntry* shm_ring_read(void* ring, uint32_t* out_size) {
|
||||
ShmRingHeader* h = (ShmRingHeader*)ring;
|
||||
uint64_t rp = atomic_load_explicit(&h->read_pos, memory_order_relaxed);
|
||||
uint64_t wp = atomic_load_explicit(&h->write_pos, memory_order_acquire);
|
||||
|
||||
if (rp >= wp) {
|
||||
return NULL; // Empty
|
||||
}
|
||||
|
||||
uint8_t* base = ring_data(ring);
|
||||
uint64_t offset = rp % h->ring_size;
|
||||
|
||||
const ShmRingEntry* entry = (const ShmRingEntry*)(base + offset);
|
||||
uint64_t entry_total = align8(SHM_RING_ENTRY_HEADER_SIZE + entry->entry_size);
|
||||
|
||||
if (out_size) *out_size = (uint32_t)entry_total;
|
||||
return entry;
|
||||
}
|
||||
|
||||
void shm_ring_read_advance(void* ring, uint32_t consumed) {
|
||||
ShmRingHeader* h = (ShmRingHeader*)ring;
|
||||
uint64_t rp = atomic_load_explicit(&h->read_pos, memory_order_relaxed);
|
||||
atomic_store_explicit(&h->read_pos, rp + consumed, memory_order_release);
|
||||
}
|
||||
```
|
||||
|
||||
**Step 3: Compile and verify**
|
||||
|
||||
Run: `cd /data/project/KVM-privacy/deps/KVM && gcc -c -std=c11 -I src/video src/video/shm_ring.c -o /tmp/shm_ring.o && echo "OK"`
|
||||
Expected: `OK` (no errors or warnings)
|
||||
|
||||
**Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add src/video/shm_ring.h src/video/shm_ring.c
|
||||
git commit -m "feat(video): add SPSC lock-free ring buffer for process isolation IPC"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: BGR Double Buffer — C Header + Implementation
|
||||
|
||||
**Files:**
|
||||
- Create: `deps/KVM/src/video/shm_bgr.h`
|
||||
- Create: `deps/KVM/src/video/shm_bgr.c`
|
||||
|
||||
**Step 1: Write shm_bgr.h**
|
||||
|
||||
```c
|
||||
// shm_bgr.h — Double-buffered BGR24 frame in shared memory (for OCR snapshots)
|
||||
#ifndef SHM_BGR_H
|
||||
#define SHM_BGR_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdatomic.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
typedef struct {
|
||||
_Atomic uint32_t active_idx; // Buffer producer is writing to (0 or 1)
|
||||
_Atomic uint32_t ready_idx; // Buffer consumer can read (0 or 1)
|
||||
uint32_t width;
|
||||
uint32_t height;
|
||||
_Atomic int64_t pts; // Timestamp of ready frame
|
||||
_Atomic uint64_t seq; // Sequence number (consumer detects new frames)
|
||||
uint64_t _reserved[2]; // Pad to 64 bytes
|
||||
} ShmBGRHeader;
|
||||
|
||||
#define SHM_BGR_HEADER_SIZE 64
|
||||
|
||||
// Create or open BGR double buffer. Returns mmap'd pointer or NULL.
|
||||
// total_size = SHM_BGR_HEADER_SIZE + 2 * width * height * 3
|
||||
void* shm_bgr_create(const char* name, uint32_t width, uint32_t height, bool create);
|
||||
|
||||
// Compute total shared memory size for given resolution.
|
||||
size_t shm_bgr_total_size(uint32_t width, uint32_t height);
|
||||
|
||||
// Destroy (unmap). Caller responsible for shm_unlink if owner.
|
||||
void shm_bgr_destroy(void* bgr, uint32_t width, uint32_t height, bool owner);
|
||||
|
||||
// Producer: get pointer to active write buffer.
|
||||
uint8_t* shm_bgr_write_ptr(void* bgr);
|
||||
|
||||
// Producer: publish written buffer (swaps active/ready, increments seq).
|
||||
void shm_bgr_publish(void* bgr, int64_t pts);
|
||||
|
||||
// Consumer: get pointer to ready buffer for reading (NULL if no frame yet).
|
||||
const uint8_t* shm_bgr_read_ptr(void* bgr);
|
||||
|
||||
// Consumer: get sequence number (poll this to detect new frames).
|
||||
static inline uint64_t shm_bgr_seq(void* bgr) {
|
||||
return atomic_load_explicit(&((ShmBGRHeader*)bgr)->seq, memory_order_acquire);
|
||||
}
|
||||
|
||||
static inline ShmBGRHeader* shm_bgr_header(void* bgr) {
|
||||
return (ShmBGRHeader*)bgr;
|
||||
}
|
||||
|
||||
#endif // SHM_BGR_H
|
||||
```
|
||||
|
||||
**Step 2: Write shm_bgr.c**
|
||||
|
||||
```c
|
||||
// shm_bgr.c — Double-buffered BGR24 shared memory
|
||||
#include "shm_bgr.h"
|
||||
#include <string.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/stat.h>
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#include <stdio.h>
|
||||
|
||||
size_t shm_bgr_total_size(uint32_t width, uint32_t height) {
|
||||
return SHM_BGR_HEADER_SIZE + 2 * (size_t)width * height * 3;
|
||||
}
|
||||
|
||||
void* shm_bgr_create(const char* name, uint32_t width, uint32_t height, bool create) {
|
||||
size_t total = shm_bgr_total_size(width, height);
|
||||
int flags = create ? (O_CREAT | O_RDWR) : O_RDWR;
|
||||
int fd = shm_open(name, flags, 0600);
|
||||
if (fd < 0) { perror("shm_open bgr"); return NULL; }
|
||||
|
||||
if (create && ftruncate(fd, total) < 0) {
|
||||
perror("ftruncate bgr"); close(fd); return NULL;
|
||||
}
|
||||
|
||||
void* ptr = mmap(NULL, total, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
|
||||
close(fd);
|
||||
if (ptr == MAP_FAILED) { perror("mmap bgr"); return NULL; }
|
||||
|
||||
if (create) {
|
||||
memset(ptr, 0, SHM_BGR_HEADER_SIZE);
|
||||
ShmBGRHeader* h = (ShmBGRHeader*)ptr;
|
||||
h->width = width;
|
||||
h->height = height;
|
||||
}
|
||||
return ptr;
|
||||
}
|
||||
|
||||
void shm_bgr_destroy(void* bgr, uint32_t width, uint32_t height, bool owner) {
|
||||
if (!bgr) return;
|
||||
munmap(bgr, shm_bgr_total_size(width, height));
|
||||
}
|
||||
|
||||
static inline uint8_t* buffer_ptr(void* bgr, uint32_t idx) {
|
||||
ShmBGRHeader* h = (ShmBGRHeader*)bgr;
|
||||
size_t frame_size = (size_t)h->width * h->height * 3;
|
||||
return (uint8_t*)bgr + SHM_BGR_HEADER_SIZE + idx * frame_size;
|
||||
}
|
||||
|
||||
uint8_t* shm_bgr_write_ptr(void* bgr) {
|
||||
uint32_t idx = atomic_load_explicit(&((ShmBGRHeader*)bgr)->active_idx,
|
||||
memory_order_relaxed);
|
||||
return buffer_ptr(bgr, idx);
|
||||
}
|
||||
|
||||
void shm_bgr_publish(void* bgr, int64_t pts) {
|
||||
ShmBGRHeader* h = (ShmBGRHeader*)bgr;
|
||||
uint32_t idx = atomic_load_explicit(&h->active_idx, memory_order_relaxed);
|
||||
|
||||
// Publish: make this buffer readable
|
||||
atomic_store_explicit(&h->ready_idx, idx, memory_order_release);
|
||||
atomic_store_explicit(&h->pts, pts, memory_order_release);
|
||||
|
||||
// Swap: next write goes to other buffer
|
||||
atomic_store_explicit(&h->active_idx, idx ^ 1, memory_order_relaxed);
|
||||
|
||||
// Increment sequence for consumer polling
|
||||
atomic_fetch_add_explicit(&h->seq, 1, memory_order_release);
|
||||
}
|
||||
|
||||
const uint8_t* shm_bgr_read_ptr(void* bgr) {
|
||||
ShmBGRHeader* h = (ShmBGRHeader*)bgr;
|
||||
if (atomic_load_explicit(&h->seq, memory_order_acquire) == 0)
|
||||
return NULL; // No frame published yet
|
||||
uint32_t idx = atomic_load_explicit(&h->ready_idx, memory_order_acquire);
|
||||
return buffer_ptr(bgr, idx);
|
||||
}
|
||||
```
|
||||
|
||||
**Step 3: Compile and verify**
|
||||
|
||||
Run: `cd /data/project/KVM-privacy/deps/KVM && gcc -c -std=c11 -I src/video src/video/shm_bgr.c -o /tmp/shm_bgr.o && echo "OK"`
|
||||
Expected: `OK`
|
||||
|
||||
**Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add src/video/shm_bgr.h src/video/shm_bgr.c
|
||||
git commit -m "feat(video): add BGR double-buffer shared memory for snapshot IPC"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Unix Socket Control Protocol — C Header + Implementation
|
||||
|
||||
**Files:**
|
||||
- Create: `deps/KVM/src/video/ctrl_socket.h`
|
||||
- Create: `deps/KVM/src/video/ctrl_socket.c`
|
||||
|
||||
**Step 1: Write ctrl_socket.h**
|
||||
|
||||
```c
|
||||
// ctrl_socket.h — Unix socket control channel between kvm-server (Go) and kvm-video (C)
|
||||
#ifndef CTRL_SOCKET_H
|
||||
#define CTRL_SOCKET_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
// Message types: Go->C commands
|
||||
#define CTRL_CMD_START 0x01
|
||||
#define CTRL_CMD_STOP 0x02
|
||||
#define CTRL_CMD_FORCE_IDR 0x03
|
||||
#define CTRL_CMD_UPDATE_RC 0x04
|
||||
#define CTRL_CMD_UPDATE_PROFILE 0x05
|
||||
#define CTRL_CMD_SET_REDACT 0x06
|
||||
#define CTRL_CMD_GET_STATUS 0x07
|
||||
#define CTRL_CMD_SET_SNAPSHOT_FPS 0x08
|
||||
|
||||
// Message types: C->Go responses/events
|
||||
#define CTRL_RSP_OK 0x80
|
||||
#define CTRL_RSP_ERROR 0x81
|
||||
#define CTRL_RSP_STATUS 0x82
|
||||
#define CTRL_EVT_DEVICE 0x83
|
||||
|
||||
// Device event subtypes
|
||||
#define DEV_EVT_V4L2_LOST 1
|
||||
#define DEV_EVT_V4L2_RECOVERED 2
|
||||
#define DEV_EVT_HDMI_DISCONNECTED 3
|
||||
#define DEV_EVT_HDMI_CONNECTED 4
|
||||
#define DEV_EVT_SIGNAL_CHANGED 5
|
||||
|
||||
// Wire format: [type:1][_pad:1][length:2][payload:length]
|
||||
#define CTRL_MAX_PAYLOAD 4096
|
||||
|
||||
typedef struct {
|
||||
uint8_t type;
|
||||
uint8_t _pad;
|
||||
uint16_t length; // Payload length (network byte order)
|
||||
} __attribute__((packed)) CtrlMsgHeader;
|
||||
|
||||
#define CTRL_MSG_HEADER_SIZE 4
|
||||
|
||||
// Start command payload
|
||||
typedef struct {
|
||||
char device[64];
|
||||
int32_t width;
|
||||
int32_t height;
|
||||
int32_t fps;
|
||||
int32_t bitrate;
|
||||
int32_t profile;
|
||||
int32_t snapshot_fps;
|
||||
} __attribute__((packed)) CtrlStartPayload;
|
||||
|
||||
// Rate control update payload
|
||||
typedef struct {
|
||||
int32_t bitrate;
|
||||
int32_t fps;
|
||||
} __attribute__((packed)) CtrlRCPayload;
|
||||
|
||||
// Redact bbox (matches RedactBbox in video_pipeline.h)
|
||||
typedef struct {
|
||||
float x, y, w, h;
|
||||
} __attribute__((packed)) CtrlRedactBbox;
|
||||
|
||||
// Status response payload
|
||||
typedef struct {
|
||||
uint8_t running;
|
||||
uint8_t _pad[3];
|
||||
int32_t fps;
|
||||
int32_t bitrate;
|
||||
uint64_t frame_count;
|
||||
uint32_t capture_errors;
|
||||
uint32_t encode_errors;
|
||||
} __attribute__((packed)) CtrlStatusPayload;
|
||||
|
||||
// Device event payload
|
||||
typedef struct {
|
||||
uint8_t event_type;
|
||||
uint8_t _pad[3];
|
||||
char detail[128];
|
||||
} __attribute__((packed)) CtrlDeviceEventPayload;
|
||||
|
||||
// Send a complete message (header + payload). Returns 0 on success, -1 on error.
|
||||
int ctrl_send(int fd, uint8_t type, const void* payload, uint16_t length);
|
||||
|
||||
// Receive a complete message. Caller provides buf of at least CTRL_MAX_PAYLOAD bytes.
|
||||
// Returns message type on success, -1 on error/disconnect.
|
||||
int ctrl_recv(int fd, void* buf, uint16_t* out_length);
|
||||
|
||||
// Convenience: send OK response for a command type.
|
||||
int ctrl_send_ok(int fd, uint8_t cmd_type);
|
||||
|
||||
// Convenience: send error response.
|
||||
int ctrl_send_error(int fd, uint8_t cmd_type, int32_t code, const char* msg);
|
||||
|
||||
#endif // CTRL_SOCKET_H
|
||||
```
|
||||
|
||||
**Step 2: Write ctrl_socket.c**
|
||||
|
||||
```c
|
||||
// ctrl_socket.c — Unix socket control message send/recv
|
||||
#include "ctrl_socket.h"
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
#include <errno.h>
|
||||
#include <arpa/inet.h>
|
||||
#include <stdio.h>
|
||||
|
||||
static int write_all(int fd, const void* buf, size_t n) {
|
||||
const uint8_t* p = (const uint8_t*)buf;
|
||||
size_t remaining = n;
|
||||
while (remaining > 0) {
|
||||
ssize_t w = write(fd, p, remaining);
|
||||
if (w < 0) {
|
||||
if (errno == EINTR) continue;
|
||||
return -1;
|
||||
}
|
||||
p += w;
|
||||
remaining -= w;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int read_all(int fd, void* buf, size_t n) {
|
||||
uint8_t* p = (uint8_t*)buf;
|
||||
size_t remaining = n;
|
||||
while (remaining > 0) {
|
||||
ssize_t r = read(fd, p, remaining);
|
||||
if (r <= 0) {
|
||||
if (r < 0 && errno == EINTR) continue;
|
||||
return -1;
|
||||
}
|
||||
p += r;
|
||||
remaining -= r;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int ctrl_send(int fd, uint8_t type, const void* payload, uint16_t length) {
|
||||
CtrlMsgHeader hdr = {
|
||||
.type = type,
|
||||
._pad = 0,
|
||||
.length = htons(length),
|
||||
};
|
||||
if (write_all(fd, &hdr, CTRL_MSG_HEADER_SIZE) < 0) return -1;
|
||||
if (length > 0 && payload) {
|
||||
if (write_all(fd, payload, length) < 0) return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int ctrl_recv(int fd, void* buf, uint16_t* out_length) {
|
||||
CtrlMsgHeader hdr;
|
||||
if (read_all(fd, &hdr, CTRL_MSG_HEADER_SIZE) < 0) return -1;
|
||||
|
||||
uint16_t len = ntohs(hdr.length);
|
||||
if (len > CTRL_MAX_PAYLOAD) return -1;
|
||||
|
||||
if (len > 0) {
|
||||
if (read_all(fd, buf, len) < 0) return -1;
|
||||
}
|
||||
if (out_length) *out_length = len;
|
||||
return hdr.type;
|
||||
}
|
||||
|
||||
int ctrl_send_ok(int fd, uint8_t cmd_type) {
|
||||
uint8_t payload = cmd_type;
|
||||
return ctrl_send(fd, CTRL_RSP_OK, &payload, 1);
|
||||
}
|
||||
|
||||
int ctrl_send_error(int fd, uint8_t cmd_type, int32_t code, const char* msg) {
|
||||
uint8_t buf[CTRL_MAX_PAYLOAD];
|
||||
buf[0] = cmd_type;
|
||||
int32_t nc = htonl(code);
|
||||
memcpy(buf + 1, &nc, 4);
|
||||
size_t msg_len = msg ? strlen(msg) : 0;
|
||||
if (msg_len > CTRL_MAX_PAYLOAD - 5) msg_len = CTRL_MAX_PAYLOAD - 5;
|
||||
if (msg_len > 0) memcpy(buf + 5, msg, msg_len);
|
||||
return ctrl_send(fd, CTRL_RSP_ERROR, buf, 5 + msg_len);
|
||||
}
|
||||
```
|
||||
|
||||
**Step 3: Compile and verify**
|
||||
|
||||
Run: `cd /data/project/KVM-privacy/deps/KVM && gcc -c -std=c11 -I src/video src/video/ctrl_socket.c -o /tmp/ctrl_socket.o && echo "OK"`
|
||||
Expected: `OK`
|
||||
|
||||
**Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add src/video/ctrl_socket.h src/video/ctrl_socket.c
|
||||
git commit -m "feat(video): add Unix socket control protocol for video process IPC"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Phase 2: C Video Process
|
||||
|
||||
### Task 4: kvm-video Entry Point
|
||||
|
||||
**Files:**
|
||||
- Create: `deps/KVM/src/video/kvm_video_main.c`
|
||||
- Modify: `deps/KVM/src/video/CMakeLists.txt`
|
||||
|
||||
**Step 1: Write kvm_video_main.c**
|
||||
|
||||
Standalone C video pipeline process. Receives fd 3 (control socket), fd 4 (eventfd H.264), fd 5 (eventfd BGR) from parent. Command line: `kvm-video --shm-ring <name> --shm-bgr <name>`.
|
||||
|
||||
Full implementation: see design doc section "4. C 独立进程:kvm-video" for the complete source. Key features:
|
||||
- `on_h264()` callback writes to shm ring + signals eventfd
|
||||
- `on_bgr()` callback writes to BGR double buffer + signals eventfd
|
||||
- `handle_command()` dispatches control messages (START/STOP/FORCE_IDR/UPDATE_RC/SET_REDACT/etc.)
|
||||
- Main loop: `poll()` on control socket, exits when socket disconnects (parent died)
|
||||
- Signal handling: SIGTERM/SIGINT for graceful shutdown
|
||||
- Cleanup: unlinks shared memory on exit
|
||||
|
||||
**Step 2: Update CMakeLists.txt**
|
||||
|
||||
Add after existing `add_library(mpp_video ...)` (after line 47):
|
||||
|
||||
```cmake
|
||||
# Standalone video process (for process isolation)
|
||||
add_executable(kvm-video
|
||||
kvm_video_main.c
|
||||
shm_ring.c
|
||||
shm_bgr.c
|
||||
ctrl_socket.c
|
||||
)
|
||||
target_link_libraries(kvm-video mpp_video pthread rt)
|
||||
|
||||
install(TARGETS kvm-video
|
||||
RUNTIME DESTINATION /usr/lib/kvm
|
||||
)
|
||||
```
|
||||
|
||||
**Step 3: Build and verify**
|
||||
|
||||
Run: `cd /data/project/KVM-privacy/deps/KVM && cmake -B build/video -S src/video -DCMAKE_BUILD_TYPE=Release && cmake --build build/video -j$(nproc) 2>&1 | tail -5`
|
||||
Expected: Both `libmpp_video.so` and `kvm-video` built.
|
||||
|
||||
Run: `./build/video/kvm-video --help`
|
||||
Expected: Usage message printed.
|
||||
|
||||
**Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add src/video/kvm_video_main.c src/video/CMakeLists.txt
|
||||
git commit -m "feat(video): add kvm-video standalone process entry point"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: V4L2 Capture Thread Error Recovery
|
||||
|
||||
**Files:**
|
||||
- Modify: `deps/KVM/src/video/v4l2_capture.cpp` — the `capture_thread()` function
|
||||
|
||||
**Step 1: Read current capture_thread to understand exact line ranges**
|
||||
|
||||
Read `deps/KVM/src/video/v4l2_capture.cpp`, find the `while (cap->running)` loop.
|
||||
|
||||
**Step 2: Add error recovery**
|
||||
|
||||
Current behavior: `select()` error -> `break` (thread exits). New behavior:
|
||||
- EINTR -> `continue`
|
||||
- EAGAIN/ENOMEM -> sleep 100ms, increment error counter, `continue`
|
||||
- ENODEV/EBADF -> log error, `break` (device truly lost, parent restarts process)
|
||||
- EIO -> sleep 50ms, increment error counter, `continue` (common during HDMI signal change)
|
||||
|
||||
Add `uint32_t capture_errors` to V4L2Capture struct for health reporting.
|
||||
|
||||
**Step 3: Build and verify**
|
||||
|
||||
Run: `cmake --build build/video -j$(nproc) 2>&1 | tail -3`
|
||||
Expected: Build succeeds.
|
||||
|
||||
**Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add src/video/v4l2_capture.cpp
|
||||
git commit -m "fix(video): add error recovery to V4L2 capture thread
|
||||
|
||||
EINTR and EAGAIN/ENOMEM now retry instead of exiting the capture
|
||||
thread. EIO retries with 50ms backoff. Only true device loss causes exit."
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Phase 3: Go SharedPipeline
|
||||
|
||||
### Task 6: Go Shared Memory Ring Buffer Reader
|
||||
|
||||
**Files:**
|
||||
- Create: `deps/KVM/go/internal/videopipe/shm_ring.go`
|
||||
- Create: `deps/KVM/go/internal/videopipe/shm_ring_test.go`
|
||||
|
||||
**Step 1: Write shm_ring_test.go (failing test)**
|
||||
|
||||
Test that `ShmRingHeader` is exactly 64 bytes and `ShmRingEntryHeader` is exactly 16 bytes (must match C layout).
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Expected: FAIL — types not defined
|
||||
|
||||
**Step 3: Write shm_ring.go**
|
||||
|
||||
Go reader for the SPSC ring buffer. Uses `unix.Mmap` to map shared memory, `atomic.LoadUint64` for lock-free reads. `ReadNext()` returns a `RingEntry` with copied data (safe for async use). Build tag: `//go:build !cgo_video`.
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Expected: PASS
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add go/internal/videopipe/shm_ring.go go/internal/videopipe/shm_ring_test.go
|
||||
git commit -m "feat(videopipe): add Go shared memory ring buffer reader"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 7: Go BGR Double Buffer Reader
|
||||
|
||||
**Files:**
|
||||
- Create: `deps/KVM/go/internal/videopipe/shm_bgr.go`
|
||||
|
||||
Go reader for BGR double buffer. `HasNewFrame()` checks sequence number. `ReadFrame()` copies ready buffer. Build tag: `//go:build !cgo_video`.
|
||||
|
||||
**Commit:**
|
||||
```bash
|
||||
git add go/internal/videopipe/shm_bgr.go
|
||||
git commit -m "feat(videopipe): add Go BGR double-buffer shared memory reader"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 8: Go Control Socket Client
|
||||
|
||||
**Files:**
|
||||
- Create: `deps/KVM/go/internal/videopipe/ctrl_client.go`
|
||||
|
||||
Mirrors the C `ctrl_socket.h` protocol. Methods: `SendStart(cfg)`, `SendStop()`, `SendForceIDR()`, `SendUpdateRC(bitrate, fps)`, `SendUpdateProfile(profile)`, `SendSetRedact(regions)`. Uses `io.ReadFull` for reliable reads and mutex-serialized commands. Build tag: `//go:build !cgo_video`.
|
||||
|
||||
**Commit:**
|
||||
```bash
|
||||
git add go/internal/videopipe/ctrl_client.go
|
||||
git commit -m "feat(videopipe): add Go control socket client for video process IPC"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 9: SharedPipeline — Go Process Manager
|
||||
|
||||
**Files:**
|
||||
- Create: `deps/KVM/go/internal/videopipe/shared_pipeline.go`
|
||||
|
||||
Main Go type replacing NativePipeline. Implements the same method set. Key internals:
|
||||
- `spawnChild()` — creates eventfds, socketpair, shared memory; starts `kvm-video` child process; sends START command
|
||||
- `readLoop()` — epoll on eventfd, reads ring entries, dispatches to `onH264` callback; heartbeat watchdog (5s stall -> restart child)
|
||||
- `bgrLoop()` — polls BGR seq at 10fps, JPEG-encodes snapshots, dispatches to `onSnapshot`
|
||||
- `waitChild()` — waits for child exit, auto-restarts if `running`
|
||||
- `restartChild()` — kills old child, spawns new one
|
||||
|
||||
Build tag: `//go:build !cgo_video`.
|
||||
|
||||
**Commit:**
|
||||
```bash
|
||||
git add go/internal/videopipe/shared_pipeline.go
|
||||
git commit -m "feat(videopipe): add SharedPipeline process-isolated video manager"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 10: Pipeline Interface + Build Tag Routing
|
||||
|
||||
**Files:**
|
||||
- Create: `deps/KVM/go/internal/videopipe/pipeline_iface.go` — no build tag
|
||||
- Modify: `deps/KVM/go/internal/videopipe/pipeline.go` — add `//go:build cgo_video`
|
||||
- Modify: `deps/KVM/go/internal/videopipe/cgo_bridge.go` — add `//go:build cgo_video`
|
||||
- Modify: `deps/KVM/go/internal/videopipe/cgo_exports.go` — add `//go:build cgo_video`
|
||||
- Modify: `deps/KVM/go/cmd/kvm-server/main.go` — use `videopipe.Pipeline` interface
|
||||
|
||||
**Step 1: Create pipeline_iface.go with Pipeline interface + shared types (Config, RedactBbox)**
|
||||
|
||||
Move `Config` and `RedactBbox` from `pipeline.go` to `pipeline_iface.go` (no build tag). Define `Pipeline` interface with all methods both implementations share.
|
||||
|
||||
**Step 2: Add `//go:build cgo_video` to CGo files**
|
||||
|
||||
**Step 3: Update main.go to use `videopipe.Pipeline` interface**
|
||||
|
||||
Change `var nativePipe *videopipe.NativePipeline` to `var nativePipe videopipe.Pipeline`.
|
||||
|
||||
**Step 4: Build both modes**
|
||||
|
||||
- Default (no CGo): `CGO_ENABLED=0 go build ./cmd/kvm-server/`
|
||||
- CGo: `CGO_ENABLED=1 go build -tags cgo_video ./cmd/kvm-server/`
|
||||
|
||||
**Commit:**
|
||||
```bash
|
||||
git add go/internal/videopipe/pipeline_iface.go go/internal/videopipe/pipeline.go \
|
||||
go/internal/videopipe/cgo_bridge.go go/internal/videopipe/cgo_exports.go \
|
||||
go/cmd/kvm-server/main.go
|
||||
git commit -m "refactor(videopipe): extract Pipeline interface for build-tag switching"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Phase 4: HID Hot-Reconnect
|
||||
|
||||
### Task 11: DeviceLostError + writeWithRetry Improvement
|
||||
|
||||
**Files:**
|
||||
- Create: `deps/KVM/go/internal/hid/errors.go`
|
||||
- Modify: `deps/KVM/go/internal/hid/mouse.go:130-154`
|
||||
- Modify: `deps/KVM/go/internal/hid/keyboard.go:110-143`
|
||||
|
||||
Add `DeviceLostError` type. Update `writeWithRetry` to return it on EBADF/EIO/ENXIO/ENODEV. Update `ledReadLoop` to exit cleanly on device loss.
|
||||
|
||||
**Commit:**
|
||||
```bash
|
||||
git add go/internal/hid/errors.go go/internal/hid/mouse.go go/internal/hid/keyboard.go
|
||||
git commit -m "feat(hid): add DeviceLostError detection for USB hot-unplug"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 12: HIDManager Hot-Reconnect Loop
|
||||
|
||||
**Files:**
|
||||
- Modify: `deps/KVM/go/internal/hid/manager.go`
|
||||
- Modify: `deps/KVM/go/internal/hid/keyboard.go` — add `Reopen()`, `LostCh()`, `Path()`
|
||||
- Modify: `deps/KVM/go/internal/hid/mouse.go` — add `Reopen()`, `LostCh()`, `Path()`
|
||||
|
||||
Add `reopenLoop` goroutine to HIDManager. Each device gets `lostCh` (signaled on DeviceLostError), `Reopen()` (close old fd, open new), `markLost()` (send to lostCh once). Retry with exponential backoff 500ms-5s.
|
||||
|
||||
**Commit:**
|
||||
```bash
|
||||
git add go/internal/hid/manager.go go/internal/hid/keyboard.go go/internal/hid/mouse.go
|
||||
git commit -m "feat(hid): add automatic reconnect loop for USB hot-unplug"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Phase 5: DeviceWatchdog + Health API
|
||||
|
||||
### Task 13: DeviceWatchdog with sd_notify
|
||||
|
||||
**Files:**
|
||||
- Create: `deps/KVM/go/internal/watchdog/watchdog.go`
|
||||
|
||||
Monitors pipeline health and HID status every 5s. Sends `WATCHDOG=1` to systemd via `unixgram` socket. Emits `DeviceEvent` for WebSocket push.
|
||||
|
||||
**Commit:**
|
||||
```bash
|
||||
git add go/internal/watchdog/watchdog.go
|
||||
git commit -m "feat: add DeviceWatchdog with systemd sd_notify integration"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 14: Health API Endpoint
|
||||
|
||||
**Files:**
|
||||
- Create: `deps/KVM/go/internal/api/health_handler.go`
|
||||
- Modify: `deps/KVM/go/internal/api/router.go`
|
||||
|
||||
`GET /api/health` (no auth) returns JSON: `{status, uptime_seconds, components: {video, keyboard, mouse}}`. Each component: `{status: "healthy"|"stopped"|"disconnected"}`.
|
||||
|
||||
**Commit:**
|
||||
```bash
|
||||
git add go/internal/api/health_handler.go go/internal/api/router.go
|
||||
git commit -m "feat(api): add /api/health endpoint for device status monitoring"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Phase 6: Deployment
|
||||
|
||||
### Task 15: systemd Service + udev Rules
|
||||
|
||||
**Files:**
|
||||
- Modify: `deps/KVM/systemd/kvm-server.service` — add `WatchdogSec=30`
|
||||
- Modify: `deps/KVM/config/udev/99-kvm-video.rules` — fix service name if needed
|
||||
- Create: `deps/KVM/scripts/hid-hotplug.sh`
|
||||
|
||||
**Commit:**
|
||||
```bash
|
||||
git add systemd/kvm-server.service config/udev/99-kvm-video.rules scripts/hid-hotplug.sh
|
||||
git commit -m "deploy: add WatchdogSec, fix udev rules, add HID hotplug script"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 16: Build System
|
||||
|
||||
**Files:**
|
||||
- Modify: `deps/KVM/scripts/build-deb.sh` — install `kvm-video` binary + `hid-hotplug.sh`
|
||||
|
||||
**Commit:**
|
||||
```bash
|
||||
git add scripts/build-deb.sh
|
||||
git commit -m "build: include kvm-video binary and hid-hotplug.sh in deb package"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Phase 7: Integration
|
||||
|
||||
### Task 17: Wire Everything in main.go
|
||||
|
||||
**Files:**
|
||||
- Modify: `deps/KVM/go/cmd/kvm-server/main.go`
|
||||
|
||||
Use `videopipe.NewShared(cfg)` when backend is "native". Initialize `watchdog.New()`, `go wd.Run(ctx)`. Register `HealthHandler`. Add SIGUSR1 handler for HID reopen.
|
||||
|
||||
**Commit:**
|
||||
```bash
|
||||
git add go/cmd/kvm-server/main.go
|
||||
git commit -m "feat: integrate SharedPipeline, DeviceWatchdog, and health API"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 18: HIDManager Interface Methods
|
||||
|
||||
**Files:**
|
||||
- Modify: `deps/KVM/go/internal/hid/manager.go` — add `IsKeyboardOpen()`, `IsMouseOpen()`
|
||||
- Modify: `deps/KVM/go/internal/hid/keyboard.go` — add `IsOpen()`
|
||||
- Modify: `deps/KVM/go/internal/hid/mouse.go` — add `IsOpen()`
|
||||
|
||||
**Commit:**
|
||||
```bash
|
||||
git add go/internal/hid/manager.go go/internal/hid/keyboard.go go/internal/hid/mouse.go
|
||||
git commit -m "feat(hid): add IsKeyboardOpen/IsMouseOpen for watchdog"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 19: Final Build + Integration Verification
|
||||
|
||||
1. Full Go build (no CGo): `CGO_ENABLED=0 go build ./cmd/kvm-server/`
|
||||
2. Full Go build (CGo fallback): `CGO_ENABLED=1 go build -tags cgo_video ./cmd/kvm-server/`
|
||||
3. C build: `cmake --build build/video` — verify both `libmpp_video.so` and `kvm-video`
|
||||
4. All Go tests: `go test ./...`
|
||||
5. Verify: `grep WatchdogSec systemd/kvm-server.service`
|
||||
6. Verify: `grep '/api/health' go/internal/api/router.go`
|
||||
|
||||
**Commit:**
|
||||
```bash
|
||||
git commit --allow-empty -m "chore: all builds and tests verified"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
| Phase | Tasks | Key Deliverables |
|
||||
|-------|-------|------------------|
|
||||
| 1: C IPC | 1-3 | shm_ring, shm_bgr, ctrl_socket (C) |
|
||||
| 2: C Process | 4-5 | kvm-video binary, V4L2 error recovery |
|
||||
| 3: Go Pipeline | 6-10 | SharedPipeline, Pipeline interface, build tags |
|
||||
| 4: HID | 11-12 | DeviceLostError, hot-reconnect loop |
|
||||
| 5: Watchdog | 13-14 | DeviceWatchdog, /api/health |
|
||||
| 6: Deploy | 15-16 | systemd, udev, build-deb.sh |
|
||||
| 7: Integration | 17-19 | main.go wiring, final verification |
|
||||
@@ -0,0 +1,741 @@
|
||||
# 硬件热插拔稳定性 + 视频管道进程隔离
|
||||
|
||||
## 背景
|
||||
|
||||
### 触发事件
|
||||
kvm-server 原生 C 视频管道(V4L2→RGA→MPP)的采集线程静默退出后,Go 主进程存活但无法产出视频帧。
|
||||
WebRTC 客户端陷入 3.5s 连接→断开循环,需手动重启服务。
|
||||
|
||||
### 根因分析
|
||||
1. **C 采集线程脆弱**:`v4l2_capture.cpp` 中 `select()` 或 `VIDIOC_DQBUF` 出错后直接 `break` 退出线程,无重试
|
||||
2. **CGo 共享地址空间**:C 线程崩溃(segfault)→ 整个 Go 进程被内核 SIGKILL,无法捕获
|
||||
3. **Go 侧无活性检测**:`pipeline.running` 标志在 C 线程退出后仍为 true
|
||||
4. **USB HID 无热重连**:设备拔出后 write 返回 EBADF,永不重试 open
|
||||
5. **udev 规则指向错误服务名**(kvm-all 而非 kvm-server)
|
||||
|
||||
---
|
||||
|
||||
## 架构设计
|
||||
|
||||
### 进程模型
|
||||
|
||||
```
|
||||
┌─────────────────────────────┐ ┌─────────────────────────┐
|
||||
│ kvm-server (Go) │ │ kvm-video (C 独立进程) │
|
||||
│ │ │ │
|
||||
│ WebRTC Server │ │ V4L2 capture │
|
||||
│ HID Manager │ │ ↓ DMA-buf (零拷贝) │
|
||||
│ REST API │ │ RGA color convert │
|
||||
│ Privacy Handler │ │ ↓ DMA-buf (零拷贝) │
|
||||
│ OCR Monitor │ │ MPP H.264 encode │
|
||||
│ Signaling Server │ │ ↓ DMA-buf (零拷贝) │
|
||||
│ DeviceWatchdog │ │ Privacy redaction │
|
||||
│ │ │ │
|
||||
│ ShmRingReader ◄──── /dev/shm/kvm-video-ring ────► ShmRingWriter │
|
||||
│ (mmap读取H.264) eventfd │ │ (mmap写入H.264) │
|
||||
│ │ │ │
|
||||
│ ShmBGRReader ◄──── /dev/shm/kvm-video-bgr ─────► ShmBGRWriter │
|
||||
│ (mmap读取BGR) eventfd │ │ (mmap写入BGR快照) │
|
||||
│ │ │ │
|
||||
│ CtrlSocket ◄──── /run/kvm/video.sock ───────────► CtrlSocket │
|
||||
│ (命令/状态) │ │ (接收命令/报告错误) │
|
||||
│ │ │ │
|
||||
│ waitpid() 监控子进程 │ │ 检测 socket 断开→退出 │
|
||||
└─────────────────────────────┘ └─────────────────────────┘
|
||||
```
|
||||
|
||||
### DMA-buf 路径不变
|
||||
|
||||
```
|
||||
V4L2 /dev/video40
|
||||
│ VIDIOC_DQBUF → DMA-buf fd
|
||||
↓
|
||||
RGA (rk_rga)
|
||||
│ DMA-buf import → BGR→NV12 转换
|
||||
↓
|
||||
MPP (mpp_enc)
|
||||
│ DMA-buf import → H.264 编码
|
||||
↓
|
||||
H.264 NAL → memcpy → 共享内存环形缓冲区
|
||||
↓
|
||||
Go mmap 读取 → RTP 打包 → WebRTC
|
||||
```
|
||||
|
||||
关键:DMA-buf 零拷贝链在 C 进程内闭合,与之前完全一致。
|
||||
C→Go 的 H.264 传输从 `C.GoBytes()` (CGo memcpy) 变为 shm memcpy,成本等价。
|
||||
|
||||
---
|
||||
|
||||
## 组件详细设计
|
||||
|
||||
### 1. 共享内存环形缓冲区(H.264 通道)
|
||||
|
||||
路径:`/dev/shm/kvm-video-ring`
|
||||
大小:4 MB(可容纳 ~10 个 IDR 帧或 ~400 个 P 帧)
|
||||
|
||||
#### 内存布局
|
||||
|
||||
```
|
||||
Offset Size Field Description
|
||||
─────────────────────────────────────────────────────
|
||||
0x0000 8 write_pos (atomic) 生产者写位置(字节偏移)
|
||||
0x0008 8 read_pos (atomic) 消费者读位置(字节偏移)
|
||||
0x0010 8 ring_size 数据区总大小
|
||||
0x0018 8 heartbeat (atomic) C 每编码一帧递增
|
||||
0x0020 8 error_code (atomic) C 侧错误码(0=正常)
|
||||
0x0028 8 frame_count (atomic) 总帧计数(含丢弃帧)
|
||||
0x0030 16 _reserved 对齐到 64 字节
|
||||
0x0040 ring_size data[] 环形数据区
|
||||
```
|
||||
|
||||
#### 数据条目格式
|
||||
|
||||
```
|
||||
Offset Size Field
|
||||
──────────────────────────
|
||||
0 4 entry_size (不含此 header)
|
||||
4 1 type 0=H264, 1=JPEG_SNAPSHOT
|
||||
5 1 flags bit0=keyframe
|
||||
6 2 _pad
|
||||
8 8 pts 时间戳(微秒)
|
||||
16 N data H.264 NAL / JPEG 数据
|
||||
```
|
||||
|
||||
#### 无锁 SPSC 协议
|
||||
|
||||
```
|
||||
生产者(C 进程):
|
||||
1. 计算 entry_total = header(16) + data_size,对齐到 8 字节
|
||||
2. 检查 ring 可用空间:(write_pos - read_pos) < ring_size - entry_total
|
||||
3. 如果空间不足:丢弃帧,递增 drop_count(降级策略)
|
||||
4. memcpy entry 到 data[write_pos % ring_size](处理环绕)
|
||||
5. memory_order_release: write_pos += entry_total
|
||||
6. write(eventfd, 1) // 通知消费者
|
||||
|
||||
消费者(Go 进程):
|
||||
1. epoll_wait(eventfd) / select 阻塞
|
||||
2. memory_order_acquire: 读取 write_pos
|
||||
3. 循环读取 [read_pos, write_pos) 中的所有条目
|
||||
4. 处理每个条目(RTP 打包 / JPEG 保存)
|
||||
5. memory_order_release: read_pos += consumed_bytes
|
||||
```
|
||||
|
||||
### 2. BGR 快照双缓冲
|
||||
|
||||
路径:`/dev/shm/kvm-video-bgr`
|
||||
大小:`64 + 2 × (1920×1080×3)` ≈ 12.4 MB
|
||||
|
||||
#### 内存布局
|
||||
|
||||
```
|
||||
Offset Size Field
|
||||
──────────────────────────────
|
||||
0x0000 8 active_idx (atomic) 当前写入的 buffer 索引 (0/1)
|
||||
0x0008 8 ready_idx (atomic) 可供读取的 buffer 索引 (0/1)
|
||||
0x0010 8 width
|
||||
0x0018 8 height
|
||||
0x0020 8 pts (atomic) 快照时间戳
|
||||
0x0028 8 seq (atomic) 序列号(消费者检测新帧)
|
||||
0x0030 16 _reserved
|
||||
0x0040 W×H×3 buffer[0] BGR24 帧数据
|
||||
0x0040+B W×H×3 buffer[1] BGR24 帧数据
|
||||
```
|
||||
|
||||
#### 双缓冲协议
|
||||
|
||||
```
|
||||
生产者(C):
|
||||
1. 写入 buffer[active_idx]
|
||||
2. ready_idx = active_idx(release)
|
||||
3. active_idx ^= 1(切换)
|
||||
4. seq++(通知新帧)
|
||||
5. write(eventfd_bgr, 1)
|
||||
|
||||
消费者(Go):
|
||||
1. 定期检查 seq 或 epoll(eventfd_bgr)
|
||||
2. 读取 buffer[ready_idx](与写入无竞争)
|
||||
3. encodeBGRToJPEG()(仍在 Go 侧,保持现有逻辑)
|
||||
```
|
||||
|
||||
### 3. Unix Socket 控制通道
|
||||
|
||||
路径:`/run/kvm/video.sock`(SOCK_STREAM)
|
||||
|
||||
#### 消息协议
|
||||
|
||||
```
|
||||
Header: [type:1][_pad:1][length:2][payload:length]
|
||||
|
||||
Go→C 命令:
|
||||
0x01 START {device, width, height, fps, bitrate, profile, snapshot_fps}
|
||||
0x02 STOP {}
|
||||
0x03 FORCE_IDR {}
|
||||
0x04 UPDATE_RC {bitrate:i32, fps:i32}
|
||||
0x05 UPDATE_PROFILE {profile:i32}
|
||||
0x06 SET_REDACT {count:i32, bboxes:[{x,y,w,h}×N]} (每个 16 字节)
|
||||
0x07 GET_STATUS {}
|
||||
|
||||
C→Go 响应/事件:
|
||||
0x80 OK {original_type:u8}
|
||||
0x81 ERROR {original_type:u8, code:i32, msg:string}
|
||||
0x82 STATUS {running:bool, fps:i32, bitrate:i32, frame_count:u64,
|
||||
capture_errors:u32, encode_errors:u32}
|
||||
0x83 DEVICE_EVENT {event_type:u8, detail:string}
|
||||
event_type: 1=V4L2_LOST, 2=V4L2_RECOVERED,
|
||||
3=HDMI_DISCONNECTED, 4=HDMI_CONNECTED,
|
||||
5=SIGNAL_CHANGED
|
||||
```
|
||||
|
||||
### 4. C 独立进程:kvm-video
|
||||
|
||||
#### 入口点
|
||||
|
||||
```c
|
||||
// src/video/kvm_video_main.c
|
||||
int main(int argc, char** argv) {
|
||||
// 1. 解析参数(或从 socket 接收 START 命令后再初始化)
|
||||
// 2. 创建/打开共享内存
|
||||
// 3. 创建 eventfd
|
||||
// 4. 连接 Unix socket
|
||||
// 5. 等待 START 命令
|
||||
// 6. 创建 video pipeline(V4L2→RGA→MPP)
|
||||
// 7. 主循环:处理 socket 命令 + 检测 socket 断开
|
||||
|
||||
// Socket 断开 = Go 进程已退出 → 优雅退出
|
||||
// SIGTERM → 优雅退出(释放 V4L2 buffers)
|
||||
}
|
||||
```
|
||||
|
||||
#### V4L2 采集线程加固
|
||||
|
||||
```c
|
||||
// v4l2_capture.cpp — 修改
|
||||
while (cap->running) {
|
||||
ret = select(cap->fd + 1, &fds, NULL, NULL, &tv);
|
||||
if (ret < 0) {
|
||||
if (errno == EINTR) continue; // 信号中断,安全重试
|
||||
|
||||
// 瞬态错误:等待后重试
|
||||
if (errno == EAGAIN || errno == ENOMEM) {
|
||||
usleep(100000); // 100ms
|
||||
cap->capture_errors++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// 设备丢失(ENODEV/EBADF):通知 Go 并等待恢复
|
||||
report_device_event(DEVICE_EVENT_V4L2_LOST, strerror(errno));
|
||||
|
||||
// 等待设备恢复(轮询 stat + open),最多 60s
|
||||
if (wait_for_device_recovery(cap->device, 60) == 0) {
|
||||
// 重新打开设备
|
||||
close(cap->fd);
|
||||
cap->fd = open(cap->device, O_RDWR);
|
||||
if (cap->fd >= 0 && reinit_capture(cap) == 0) {
|
||||
report_device_event(DEVICE_EVENT_V4L2_RECOVERED, "");
|
||||
continue; // 恢复采集
|
||||
}
|
||||
}
|
||||
|
||||
// 恢复失败:报告错误,线程退出
|
||||
report_device_event(DEVICE_EVENT_V4L2_LOST, "recovery failed");
|
||||
break;
|
||||
}
|
||||
|
||||
if (ioctl(cap->fd, VIDIOC_DQBUF, &buf) < 0) {
|
||||
if (errno == EAGAIN) continue;
|
||||
if (errno == EIO) {
|
||||
// EIO 常见于 HDMI 信号变化期间,短暂重试
|
||||
cap->capture_errors++;
|
||||
usleep(50000);
|
||||
continue;
|
||||
}
|
||||
// 其他错误同上处理
|
||||
// ...
|
||||
}
|
||||
|
||||
// 成功:递增心跳
|
||||
atomic_fetch_add(&shm_header->heartbeat, 1);
|
||||
atomic_fetch_add(&shm_header->frame_count, 1);
|
||||
}
|
||||
```
|
||||
|
||||
### 5. Go 侧:SharedPipeline(替代 CGo NativePipeline)
|
||||
|
||||
```go
|
||||
// go/internal/videopipe/shared_pipeline.go
|
||||
type SharedPipeline struct {
|
||||
cfg Config
|
||||
running atomic.Bool
|
||||
|
||||
// IPC
|
||||
ctrlConn net.Conn // Unix socket
|
||||
ringShm *ShmRing // H.264 ring buffer (mmap)
|
||||
bgrShm *ShmBGR // BGR double buffer (mmap)
|
||||
eventFd int // H.264 通知
|
||||
bgrEventFd int // BGR 通知
|
||||
|
||||
// Process
|
||||
cmd *exec.Cmd
|
||||
pid int
|
||||
|
||||
// Callbacks (与 NativePipeline 接口一致)
|
||||
onH264 func(data []byte, pts int64, keyframe bool)
|
||||
onSnapshot func(jpegData []byte)
|
||||
|
||||
// Watchdog
|
||||
lastHeartbeat uint64
|
||||
rtpCount atomic.Int64
|
||||
}
|
||||
|
||||
// 接口兼容 — 对 main.go 透明切换
|
||||
func (p *SharedPipeline) Start() error
|
||||
func (p *SharedPipeline) Stop()
|
||||
func (p *SharedPipeline) ForceIDR() error
|
||||
func (p *SharedPipeline) UpdateBitrate(bps int) error
|
||||
func (p *SharedPipeline) UpdateFPS(fps int) error
|
||||
func (p *SharedPipeline) SetRedactRegions(regions []RedactBbox)
|
||||
func (p *SharedPipeline) IsRunning() bool
|
||||
func (p *SharedPipeline) GetRTPCount() int64
|
||||
func (p *SharedPipeline) IncrementRTPCount()
|
||||
func (p *SharedPipeline) Restart() error
|
||||
```
|
||||
|
||||
#### 核心读取循环
|
||||
|
||||
```go
|
||||
func (p *SharedPipeline) readLoop() {
|
||||
epfd := unix.EpollCreate1(0)
|
||||
unix.EpollCtl(epfd, unix.EPOLL_CTL_ADD, p.eventFd,
|
||||
&unix.EpollEvent{Events: unix.EPOLLIN, Fd: int32(p.eventFd)})
|
||||
|
||||
events := make([]unix.EpollEvent, 2)
|
||||
for p.running.Load() {
|
||||
n, _ := unix.EpollWait(epfd, events, 100) // 100ms timeout for watchdog
|
||||
if n > 0 {
|
||||
// Drain eventfd
|
||||
var buf [8]byte
|
||||
unix.Read(p.eventFd, buf[:])
|
||||
|
||||
// 读取 ring 中所有待处理条目
|
||||
for {
|
||||
entry, ok := p.ringShm.ReadNext()
|
||||
if !ok { break }
|
||||
|
||||
switch entry.Type {
|
||||
case EntryH264:
|
||||
if p.onH264 != nil {
|
||||
p.onH264(entry.Data, entry.PTS, entry.Flags&FlagKeyframe != 0)
|
||||
}
|
||||
case EntryJPEG:
|
||||
if p.onSnapshot != nil {
|
||||
p.onSnapshot(entry.Data)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Watchdog:检查心跳
|
||||
hb := atomic.LoadUint64((*uint64)(p.ringShm.HeartbeatPtr()))
|
||||
if hb == p.lastHeartbeat && p.running.Load() {
|
||||
p.stallCount++
|
||||
if p.stallCount >= 50 { // 5s (100ms × 50)
|
||||
log.Println("[SharedPipeline] Heartbeat stall, restarting child")
|
||||
p.restartChild()
|
||||
p.stallCount = 0
|
||||
}
|
||||
} else {
|
||||
p.stallCount = 0
|
||||
p.lastHeartbeat = hb
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### 进程生命周期管理
|
||||
|
||||
```go
|
||||
func (p *SharedPipeline) spawnChild() error {
|
||||
// 1. 创建共享内存
|
||||
p.ringShm = NewShmRing("/kvm-video-ring", 4*1024*1024)
|
||||
p.bgrShm = NewShmBGR("/kvm-video-bgr", p.cfg.Width, p.cfg.Height)
|
||||
|
||||
// 2. 创建 eventfd
|
||||
p.eventFd = unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC)
|
||||
p.bgrEventFd = unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC)
|
||||
|
||||
// 3. 创建 Unix socket pair
|
||||
sockFds := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM, 0)
|
||||
|
||||
// 4. 启动子进程,传递 fd
|
||||
p.cmd = exec.Command("/usr/lib/kvm/kvm-video",
|
||||
"--shm-ring", "/kvm-video-ring",
|
||||
"--shm-bgr", "/kvm-video-bgr",
|
||||
)
|
||||
p.cmd.ExtraFiles = []*os.File{
|
||||
os.NewFile(uintptr(sockFds[1]), "ctrl"),
|
||||
os.NewFile(uintptr(p.eventFd), "eventfd-h264"),
|
||||
os.NewFile(uintptr(p.bgrEventFd), "eventfd-bgr"),
|
||||
}
|
||||
|
||||
if err := p.cmd.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
p.pid = p.cmd.Process.Pid
|
||||
|
||||
// 5. 后台等待子进程退出
|
||||
go func() {
|
||||
state, _ := p.cmd.Process.Wait()
|
||||
if p.running.Load() {
|
||||
log.Printf("[SharedPipeline] Child exited: %v, auto-restarting", state)
|
||||
p.restartChild()
|
||||
}
|
||||
}()
|
||||
|
||||
// 6. 发送 START 命令
|
||||
return p.sendCommand(CmdStart, p.cfg)
|
||||
}
|
||||
```
|
||||
|
||||
### 6. USB HID 热重连
|
||||
|
||||
独立于视频管道进程隔离,在 Go 进程内解决。
|
||||
|
||||
#### DeviceLostError 类型
|
||||
|
||||
```go
|
||||
// go/internal/hid/errors.go
|
||||
type DeviceLostError struct {
|
||||
Device string
|
||||
Err error
|
||||
}
|
||||
|
||||
func isDeviceLost(err error) bool {
|
||||
return errors.Is(err, syscall.EBADF) ||
|
||||
errors.Is(err, syscall.EIO) ||
|
||||
errors.Is(err, syscall.ENXIO) ||
|
||||
errors.Is(err, syscall.ENODEV)
|
||||
}
|
||||
```
|
||||
|
||||
#### HIDManager 热重连
|
||||
|
||||
```go
|
||||
// go/internal/hid/manager.go — 新增
|
||||
func (m *HIDManager) startReopenLoop() {
|
||||
go m.reopenLoop(m.keyboard, "keyboard")
|
||||
go m.reopenLoop(m.mouse, "mouse")
|
||||
}
|
||||
|
||||
func (m *HIDManager) reopenLoop(dev HIDDevice, name string) {
|
||||
backoff := []time.Duration{500*time.Millisecond, 1*time.Second,
|
||||
2*time.Second, 5*time.Second}
|
||||
for attempt := 0; ; attempt++ {
|
||||
// 等待设备丢失信号
|
||||
<-dev.LostCh()
|
||||
|
||||
log.Printf("[HID] %s device lost, attempting reconnect", name)
|
||||
|
||||
for retry := 0; ; retry++ {
|
||||
delay := backoff[min(retry, len(backoff)-1)]
|
||||
time.Sleep(delay)
|
||||
|
||||
// 检查设备文件存在
|
||||
if _, err := os.Stat(dev.Path()); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := dev.Reopen(); err != nil {
|
||||
log.Printf("[HID] %s reopen attempt %d failed: %v", name, retry+1, err)
|
||||
continue
|
||||
}
|
||||
|
||||
log.Printf("[HID] %s reconnected after %d attempts", name, retry+1)
|
||||
// 通知 WebSocket 客户端
|
||||
m.broadcastDeviceEvent("hid_reconnected", name)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### writeWithRetry 改进
|
||||
|
||||
```go
|
||||
// mouse.go / keyboard.go — 修改
|
||||
func writeWithRetry(fd int, buf []byte, expected int, name string) error {
|
||||
for i := 0; i < maxRetries; i++ {
|
||||
n, err := syscall.Write(fd, buf)
|
||||
if err != nil {
|
||||
if err == syscall.EAGAIN || err == syscall.EWOULDBLOCK {
|
||||
// 现有退避逻辑...
|
||||
continue
|
||||
}
|
||||
// 设备丢失 — 标记并通知重连
|
||||
if isDeviceLost(err) {
|
||||
return &DeviceLostError{Device: name, Err: err}
|
||||
}
|
||||
return fmt.Errorf("failed to write %s report: %w", name, err)
|
||||
}
|
||||
if n != expected {
|
||||
return fmt.Errorf("incomplete %s write: %d/%d", name, n, expected)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s: host not polling", name)
|
||||
}
|
||||
```
|
||||
|
||||
### 7. DeviceWatchdog + systemd Watchdog
|
||||
|
||||
```go
|
||||
// go/internal/watchdog/watchdog.go
|
||||
type DeviceWatchdog struct {
|
||||
pipeline PipelineHealthChecker // SharedPipeline 或 NativePipeline
|
||||
hid *hid.HIDManager
|
||||
interval time.Duration // 5s
|
||||
sdNotify bool
|
||||
|
||||
onDeviceEvent func(event DeviceEvent) // WebSocket 推送
|
||||
}
|
||||
|
||||
type DeviceEvent struct {
|
||||
Type string `json:"type"` // "video_stall", "hid_lost", "hdmi_disconnected"
|
||||
Component string `json:"component"`
|
||||
Detail string `json:"detail"`
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
}
|
||||
|
||||
func (w *DeviceWatchdog) Run(ctx context.Context) {
|
||||
ticker := time.NewTicker(w.interval)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
allHealthy := true
|
||||
|
||||
// 检查视频管道(通过共享内存心跳)
|
||||
if !w.pipeline.IsHealthy() {
|
||||
allHealthy = false
|
||||
w.emitEvent("video_stall", "pipeline", "heartbeat timeout")
|
||||
}
|
||||
|
||||
// 检查 HID 设备
|
||||
status := w.hid.Status()
|
||||
if !status.KeyboardOpen {
|
||||
allHealthy = false
|
||||
}
|
||||
if !status.MouseOpen {
|
||||
allHealthy = false
|
||||
}
|
||||
|
||||
// systemd watchdog:只要关键组件健康就发信号
|
||||
// 视频管道不健康不算致命(会自动重启子进程)
|
||||
if w.sdNotify && (status.KeyboardOpen || status.MouseOpen) {
|
||||
daemon.SdNotify(false, daemon.SdNotifyWatchdog)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 8. 健康 API
|
||||
|
||||
```go
|
||||
// go/internal/api/health_handler.go
|
||||
// GET /api/health (无需认证)
|
||||
type HealthResponse struct {
|
||||
Status string `json:"status"` // healthy/degraded/unhealthy
|
||||
Uptime int64 `json:"uptime_seconds"`
|
||||
Components map[string]ComponentHealth `json:"components"`
|
||||
}
|
||||
|
||||
type ComponentHealth struct {
|
||||
Status string `json:"status"` // healthy/disconnected/stalled/error
|
||||
Detail string `json:"detail,omitempty"`
|
||||
LastChange int64 `json:"last_change_unix,omitempty"`
|
||||
}
|
||||
```
|
||||
|
||||
### 9. udev 规则修复
|
||||
|
||||
```bash
|
||||
# config/udev/99-kvm-video.rules — 修复
|
||||
SUBSYSTEM=="hdmirx", ACTION=="change", RUN+="/usr/lib/kvm/hdmi-hotplug.sh"
|
||||
|
||||
# config/udev/99-kvm-hid.rules — 新增热插拔
|
||||
KERNEL=="hidg[0-9]*", MODE="0660", GROUP="kvm"
|
||||
KERNEL=="hidg[0-9]*", ACTION=="add", RUN+="/usr/lib/kvm/hid-hotplug.sh add %k"
|
||||
KERNEL=="hidg[0-9]*", ACTION=="remove", RUN+="/usr/lib/kvm/hid-hotplug.sh remove %k"
|
||||
|
||||
# scripts/hdmi-hotplug.sh — 修复服务名
|
||||
SERVICE="kvm-server" # 原为 kvm-all
|
||||
|
||||
# scripts/hid-hotplug.sh — 新建
|
||||
#!/bin/bash
|
||||
ACTION="$1"
|
||||
DEVICE="$2"
|
||||
logger -t kvm-hid "HID hotplug: $ACTION $DEVICE"
|
||||
if [ "$ACTION" = "add" ]; then
|
||||
systemctl kill -s USR1 kvm-server.service 2>/dev/null || true
|
||||
fi
|
||||
```
|
||||
|
||||
### 10. systemd 服务更新
|
||||
|
||||
```ini
|
||||
# systemd/kvm-server.service — 修改
|
||||
[Service]
|
||||
WatchdogSec=30
|
||||
# 新增:SIGUSR1 处理(HID 热插拔通知)
|
||||
# Go 代码注册 signal handler → 触发 HID reopen
|
||||
```
|
||||
|
||||
```ini
|
||||
# systemd/kvm-video.service — 新建(可选,也可由 kvm-server 管理)
|
||||
# 如果想让 kvm-video 由 kvm-server 内部管理进程生命周期,则不需要此文件。
|
||||
# 推荐由 kvm-server 管理(waitpid),因为需要传递 fd 和共享内存。
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 构建系统变更
|
||||
|
||||
### CMake
|
||||
|
||||
```cmake
|
||||
# src/video/CMakeLists.txt — 新增
|
||||
add_executable(kvm-video
|
||||
kvm_video_main.c
|
||||
video_pipeline.cpp
|
||||
mpp_encoder.cpp
|
||||
v4l2_capture.cpp
|
||||
rga_converter.cpp
|
||||
shm_ring.c # 共享内存环形缓冲区
|
||||
shm_bgr.c # BGR 双缓冲
|
||||
ctrl_socket.c # Unix socket 命令处理
|
||||
)
|
||||
target_link_libraries(kvm-video rockchip_mpp rga pthread)
|
||||
```
|
||||
|
||||
### Go 构建
|
||||
|
||||
```
|
||||
# go/internal/videopipe/ 下:
|
||||
# 删除:cgo_bridge.go, cgo_exports.go, cgo_shim.c (CGo 代码)
|
||||
# 新增:shared_pipeline.go, shm_ring.go, shm_bgr.go, ctrl_client.go
|
||||
# 保留:pipeline.go (接口定义), hdmi_monitor.go (改用 socket 事件)
|
||||
```
|
||||
|
||||
### Deb 包
|
||||
|
||||
```
|
||||
kvm-server deb 新增:
|
||||
/usr/lib/kvm/kvm-video # C 视频管道进程
|
||||
/usr/lib/kvm/hid-hotplug.sh # HID 热插拔脚本
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 迁移策略
|
||||
|
||||
### 编译时切换(Build Tag)
|
||||
|
||||
```go
|
||||
// shared_pipeline.go
|
||||
//go:build !cgo_video
|
||||
|
||||
// cgo_bridge.go
|
||||
//go:build cgo_video
|
||||
```
|
||||
|
||||
默认使用 SharedPipeline(进程隔离)。`-tags cgo_video` 回退到 CGo 模式(开发调试用)。
|
||||
main.go 通过 `PipelineBackend` 接口统一调用,对 WebRTC/自适应控制/OCR 透明。
|
||||
|
||||
---
|
||||
|
||||
## 文件变更汇总
|
||||
|
||||
| 文件 | 操作 | 说明 |
|
||||
|------|------|------|
|
||||
| **C 层** | | |
|
||||
| `src/video/kvm_video_main.c` | 新建 | C 独立进程入口点 |
|
||||
| `src/video/shm_ring.c/h` | 新建 | SPSC 环形缓冲区 |
|
||||
| `src/video/shm_bgr.c/h` | 新建 | BGR 双缓冲 |
|
||||
| `src/video/ctrl_socket.c/h` | 新建 | Unix socket 命令处理 |
|
||||
| `src/video/v4l2_capture.cpp` | 修改 | 错误重试 + 设备恢复 + 心跳 |
|
||||
| `src/video/video_pipeline.cpp` | 修改 | 回调改为 shm 写入 |
|
||||
| `src/video/CMakeLists.txt` | 修改 | 新增 kvm-video target |
|
||||
| **Go 层** | | |
|
||||
| `go/internal/videopipe/shared_pipeline.go` | 新建 | SharedPipeline (替代 CGo) |
|
||||
| `go/internal/videopipe/shm_ring.go` | 新建 | 共享内存环形缓冲区 Go 端 |
|
||||
| `go/internal/videopipe/shm_bgr.go` | 新建 | BGR 双缓冲 Go 端 |
|
||||
| `go/internal/videopipe/ctrl_client.go` | 新建 | Unix socket 客户端 |
|
||||
| `go/internal/videopipe/pipeline.go` | 修改 | 提取 Pipeline 接口 |
|
||||
| `go/internal/videopipe/hdmi_monitor.go` | 修改 | 改用 socket 事件 |
|
||||
| `go/internal/hid/errors.go` | 新建 | DeviceLostError |
|
||||
| `go/internal/hid/manager.go` | 修改 | 热重连 + reopenLoop |
|
||||
| `go/internal/hid/mouse.go` | 修改 | writeWithRetry 识别设备丢失 |
|
||||
| `go/internal/hid/keyboard.go` | 修改 | 同上 + ledReadLoop 恢复 |
|
||||
| `go/internal/watchdog/watchdog.go` | 新建 | DeviceWatchdog + sd_notify |
|
||||
| `go/internal/api/health_handler.go` | 新建 | /api/health 端点 |
|
||||
| `go/internal/api/router.go` | 修改 | 注册 health 路由 |
|
||||
| `go/cmd/kvm-server/main.go` | 修改 | SharedPipeline 初始化 + watchdog |
|
||||
| **部署** | | |
|
||||
| `systemd/kvm-server.service` | 修改 | WatchdogSec=30 |
|
||||
| `config/udev/99-kvm-video.rules` | 修改 | 修复服务名 |
|
||||
| `config/udev/99-kvm-hid.rules` | 修改 | 新增热插拔规则 |
|
||||
| `scripts/hdmi-hotplug.sh` | 修改 | 修复服务名 |
|
||||
| `scripts/hid-hotplug.sh` | 新建 | HID 热插拔处理 |
|
||||
|
||||
---
|
||||
|
||||
## 验证计划
|
||||
|
||||
```bash
|
||||
# 1. C 进程独立编译和运行
|
||||
cmake -B build/video -S src/video && cmake --build build/video
|
||||
./build/video/kvm-video --help
|
||||
|
||||
# 2. Go 编译(无 CGo)
|
||||
cd go && CGO_ENABLED=0 go build ./... # 确认无 CGo 依赖
|
||||
|
||||
# 3. 集成测试:正常视频流
|
||||
# 启动 kvm-server → 浏览器连接 → 确认视频正常
|
||||
|
||||
# 4. 故障注入:杀死 C 进程
|
||||
kill -9 $(pidof kvm-video)
|
||||
# 预期:Go 检测到子进程退出 → 自动重启 → 视频恢复(<5s)
|
||||
|
||||
# 5. 故障注入:拔 HDMI 线
|
||||
# 预期:C 进程检测 V4L2 错误 → 通知 Go → 前端显示"HDMI 断开"
|
||||
# 重新插入 → C 进程恢复采集 → 视频恢复
|
||||
|
||||
# 6. 故障注入:拔 USB HID 线
|
||||
# 预期:write 返回 EBADF → DeviceLostError → reopenLoop 重试
|
||||
# 重新插入 → HID 恢复 → 前端显示"HID 重连"
|
||||
|
||||
# 7. systemd watchdog
|
||||
systemctl show kvm-server --property=WatchdogUSec
|
||||
# 预期:WatchdogUSec=30000000 (30s)
|
||||
|
||||
# 8. /api/health 端点
|
||||
curl localhost:8080/api/health | jq .
|
||||
# 预期:显示所有组件状态
|
||||
|
||||
# 9. 延迟测试
|
||||
# 对比 CGo 模式 vs SharedPipeline 模式的首帧延迟和 RTP 吞吐量
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 性能预期
|
||||
|
||||
| 指标 | CGo 模式(现有) | SharedPipeline(新) | 差异 |
|
||||
|------|----------------|---------------------|------|
|
||||
| H.264 传输延迟 | ~100-200µs (C.GoBytes) | ~50-150µs (mmap read) | 持平或更优 |
|
||||
| DMA-buf 零拷贝 | ✅ V4L2→RGA→MPP | ✅ V4L2→RGA→MPP(不变) | 相同 |
|
||||
| 进程崩溃恢复 | ❌ 整个 kvm-server 重启 (3s) | ✅ 仅重启 kvm-video (<1s) | 大幅改善 |
|
||||
| 内存隔离 | ❌ 共享地址空间 | ✅ 独立进程 | 安全 |
|
||||
| CGo 调度开销 | ~5-15µs/call | 0 (纯 Go) | 消除 |
|
||||
| 额外内存 | 0 | ~16MB shm | 可接受 |
|
||||
@@ -0,0 +1,172 @@
|
||||
# KVM WebUI 优化计划
|
||||
|
||||
> **基于 Playwright MCP 深度视觉测试 (2026-03-06)**
|
||||
> 25 张截图 × 中英文切换 × 桌面/移动端视口 × 全页面覆盖
|
||||
|
||||
## 测试覆盖矩阵
|
||||
|
||||
| 页面 | 中文 | 英文 | 移动端 | 截图 |
|
||||
|------|------|------|--------|------|
|
||||
| Login | ✅ | ✅ | - | #01 |
|
||||
| Console | ✅ | ✅ | ✅ | #02,#03,#14 |
|
||||
| Dashboard | ✅ | ✅ | ✅ | #17,#24,#25 |
|
||||
| Settings (System) | ✅ | ✅ | - | #04,#18 |
|
||||
| Settings (Users) | ✅ | - | - | #05 |
|
||||
| Settings (Account) | ✅ | - | - | #06 |
|
||||
| Settings (DDNS) | ✅ | - | - | #22 |
|
||||
| Privacy | ✅ | - | - | #07 |
|
||||
| Audit (Logs) | ✅ | ✅ | - | #15 |
|
||||
| Audit (Recordings) | ✅ | - | - | #16 |
|
||||
| Agent | ✅ | ✅ | ✅ | #19,#20,#21,#23 |
|
||||
|
||||
---
|
||||
|
||||
## 发现的问题
|
||||
|
||||
### P0 — 已修复
|
||||
|
||||
| # | 问题 | 修复 |
|
||||
|---|------|------|
|
||||
| F1 | `settings.tab_ddns` 显示原始翻译键 | 添加 `tab_ddns` 到 zh/en translation.json |
|
||||
| F2 | 密码修改 401 误触全局拦截器导致登出 | 排除 `/password` 端点的 401 重定向 |
|
||||
| F3 | 电源控制无确认弹窗 | QuickActions 添加 ConfirmDialog |
|
||||
| F4 | VideoStats 硬编码英文 | 改用 `useTranslation()` + `t()` |
|
||||
| F5 | Agent 轮询无退避 | 实现指数退避 (3s→30s) |
|
||||
| F6 | 导航栏角色名未翻译 | 使用 `t(users.role_${role})` |
|
||||
|
||||
### P1 — 需修复 (High)
|
||||
|
||||
| # | 问题 | 页面 | 状态 | 说明 |
|
||||
|---|------|------|------|------|
|
||||
| H1 | Dashboard 整页 i18n 缺失 | Dashboard | ✅ 已修复 | 添加 30+ 翻译键到 zh/en,Playwright 验证通过 |
|
||||
| H2 | DDNS 配置页 i18n 缺失 | Settings/DDNS | ✅ 已修复 | 添加 18 翻译键,provider 标签 + 字段标签全部 i18n 化 |
|
||||
| H3 | 审计日志 IP 地址列显示 `[` | Audit | ✅ 已修复 | 前端 strip IPv6 方括号,空 IP 显示 `-` |
|
||||
| H4 | Console 403 错误 | Console | 待查 | `/api/kvm/video/settings` 和 `/api/kvm/recording/status` 返回 403 |
|
||||
| H5 | LLM Config API 需 Go 重编译 | Agent | 待做 | `AgentConfigHandler` 已写入源码但未编译进二进制 |
|
||||
|
||||
### P2 — 优化建议 (Medium)
|
||||
|
||||
| # | 问题 | 页面 | 状态 | 说明 |
|
||||
|---|------|------|------|------|
|
||||
| M1 | 移动端状态栏文字重叠 | Console | ✅ 已修复 | StatusBar.css 改为 flex-wrap + clamp() 动态自适应 |
|
||||
| M2 | 导航栏 display_name + role 冗余 | 全局 | 待优化 | "System Administrator" + "系统管理员" badge 同时显示 |
|
||||
| M3 | Dashboard 导航链接未翻译 | 全局 | ✅ 已修复 | 添加 `nav.dashboard` 翻译键 |
|
||||
| M4 | 磁盘 91.6% 无告警提示 | Dashboard | 待优化 | 进度条已红色,但无文字告警 |
|
||||
|
||||
### P3 — 低优先级 (Low)
|
||||
|
||||
| # | 问题 | 页面 | 说明 |
|
||||
|---|------|------|------|
|
||||
| L1 | "KVM Console" 品牌名硬编码 | 全局 | Logo 文字不需翻译但可考虑配置化 |
|
||||
| L2 | WebRTC 重连日志过多 | Console | 每次页面切换触发断开/重连日志,console 噪音 |
|
||||
| L3 | 任务历史无分页 | Agent | 任务多时可能性能下降 |
|
||||
|
||||
---
|
||||
|
||||
## 实施计划
|
||||
|
||||
### Phase 1: Dashboard i18n (H1) — 优先级最高
|
||||
|
||||
**文件**: `src/pages/DashboardPage.tsx`
|
||||
|
||||
新增翻译键 (~40 个):
|
||||
```
|
||||
dashboard.title: "仪表盘" / "Dashboard"
|
||||
dashboard.live: "在线" / "Live"
|
||||
dashboard.hdmi: "HDMI"
|
||||
dashboard.video: "视频" / "Video"
|
||||
dashboard.hid: "HID"
|
||||
dashboard.recording: "录像" / "Recording"
|
||||
dashboard.connected: "已连接" / "Connected"
|
||||
dashboard.disconnected: "未连接" / "Disconnected"
|
||||
dashboard.streaming: "传输中" / "Streaming"
|
||||
dashboard.stopped: "已停止" / "Stopped"
|
||||
dashboard.enabled: "已启用" / "Enabled"
|
||||
dashboard.disabled: "已禁用" / "Disabled"
|
||||
dashboard.idle: "空闲" / "Idle"
|
||||
dashboard.hostname: "主机名" / "Hostname"
|
||||
dashboard.uptime: "运行时间" / "Uptime"
|
||||
dashboard.cpu_temp: "CPU 温度" / "CPU Temp"
|
||||
dashboard.version: "版本" / "Version"
|
||||
dashboard.connected_users: "在线用户" / "Connected Users"
|
||||
dashboard.goroutines: "协程数" / "Goroutines"
|
||||
dashboard.resources: "资源监控" / "Resources"
|
||||
dashboard.memory: "内存" / "Memory"
|
||||
dashboard.disk: "磁盘" / "Disk"
|
||||
dashboard.power_control: "电源控制" / "Power Control"
|
||||
dashboard.wol: "网络唤醒" / "Wake-on-LAN"
|
||||
dashboard.wake: "唤醒" / "Wake"
|
||||
dashboard.atx_power: "ATX 电源" / "ATX Power"
|
||||
dashboard.not_available: "不可用" / "Not Available"
|
||||
dashboard.atx_note: "ATX 电源控制需要 GPIO 接线。此设备未连接。" / "ATX power control requires GPIO wiring. Not connected on this device."
|
||||
```
|
||||
|
||||
### Phase 2: DDNS 配置 i18n (H2)
|
||||
|
||||
**文件**: `src/components/settings/DDNSSettings.tsx`
|
||||
|
||||
新增翻译键 (~10 个):
|
||||
```
|
||||
ddns.title: "DDNS 动态域名" / "DDNS Dynamic DNS"
|
||||
ddns.enable: "启用 DDNS" / "Enable DDNS"
|
||||
ddns.provider: "服务商" / "Provider"
|
||||
ddns.domain: "域名" / "Domain"
|
||||
ddns.interval: "更新间隔 (秒)" / "Update Interval (sec)"
|
||||
ddns.test: "测试连接" / "Test Connection"
|
||||
ddns.save: "保存" / "Save"
|
||||
```
|
||||
|
||||
### Phase 3: 审计 IP 修复 (H3)
|
||||
|
||||
**文件**: `src/pages/AuditPage.tsx`
|
||||
|
||||
修复 IP 地址列渲染:解析 `[::1]` 格式,显示为 `::1` 或 `localhost`。
|
||||
|
||||
### Phase 4: Console 403 修复 (H4)
|
||||
|
||||
**文件**: Go 后端 RBAC 或前端错误处理
|
||||
|
||||
调查 `/api/kvm/video/settings` 和 `/api/kvm/recording/status` 返回 403 的原因(可能是 RBAC 权限配置问题)。
|
||||
|
||||
### Phase 5: Go 二进制重编译 (H5)
|
||||
|
||||
```bash
|
||||
cd deps/KVM && PATH="/usr/local/go/bin:$PATH" bash scripts/build-deb.sh
|
||||
```
|
||||
|
||||
将 `AgentConfigHandler` 编译进 kvm-server 二进制,同时嵌入新前端 dist。
|
||||
|
||||
### Phase 6: 移动端响应式自适应 (M1)
|
||||
|
||||
**文件**: `src/components/console/StatusBar.tsx` 或相关 CSS
|
||||
|
||||
采用动态响应式布局,不写死断点/分辨率:
|
||||
- 使用 `flex-wrap: wrap` + `gap` 让状态项自然换行
|
||||
- 文字使用 `clamp()` 或 `min()` 动态缩放字号
|
||||
- 关键信息用 `overflow: hidden; text-overflow: ellipsis; white-space: nowrap` 防溢出
|
||||
- 容器使用百分比/`fr`/`min-content` 而非固定 px 宽度
|
||||
- 适配任意分辨率和平台(手机/平板/桌面),而非针对特定设备
|
||||
|
||||
### Phase 7: 导航栏清理 (M2, M3)
|
||||
|
||||
**文件**: `src/components/layout/Navbar.tsx`
|
||||
|
||||
- "Dashboard" 导航链接添加 i18n(`nav.dashboard`)
|
||||
- 考虑在移动端隐藏 role badge,桌面端仅显示 badge 或 display_name(二选一)
|
||||
|
||||
---
|
||||
|
||||
## 验证清单
|
||||
|
||||
每个 Phase 完成后执行:
|
||||
```bash
|
||||
npm run build # TypeScript 编译通过
|
||||
# Playwright 截图验证
|
||||
# 中文模式 + 英文模式切换检查
|
||||
# 375px 移动端视口测试
|
||||
```
|
||||
|
||||
## 截图存档
|
||||
|
||||
测试截图保存在 `deps/KVM/web/` 下:
|
||||
- `test-01-login-zh.png` ~ `test-25-dashboard-zh-i18n-missing.png`
|
||||
@@ -0,0 +1,109 @@
|
||||
# Privacy Detection Capability Evaluation
|
||||
|
||||
> Evaluation date: 2026-03-03
|
||||
> Platform: NanoPC-T6 (RK3588, 6 TOPS NPU)
|
||||
> Service: info-privacy-rs v0.1.0 (port 8001)
|
||||
|
||||
## 1. Detection Methods
|
||||
|
||||
| Method | Technology | Scope |
|
||||
|--------|-----------|-------|
|
||||
| Regex (fancy-regex) | Pattern matching with lookahead/lookbehind | Text layer: ID, phone, bank card, email, license plate |
|
||||
| NER (dictionary + regex) | Surname dictionary + location trigger words | Text layer: names, addresses |
|
||||
| RKNN OCR (PP-OCR v4/v5) | NPU-accelerated text recognition | Image → text extraction |
|
||||
| RKNN Face Detection | MediaPipe face_detection + landmark | Image: face bounding boxes |
|
||||
| Keyword Classifier | Aho-Corasick automaton | Confidential document blocking |
|
||||
|
||||
## 2. Entity Detection Accuracy
|
||||
|
||||
| Entity Type | Detection Method | Precision | Recall (est.) | Notes |
|
||||
|-------------|-----------------|-----------|---------------|-------|
|
||||
| id_card | 18-digit regex with boundary check | >99% | >95% | Excludes bank card false positives |
|
||||
| phone | 13-19 prefix, +86/-/space variants | >99% | >90% | Chinese mobile only |
|
||||
| bank_card | 15-19 digit regex | >99% | >85% | May match other long numbers |
|
||||
| email | ASCII email pattern | >99% | >95% | Standard format only |
|
||||
| license_plate | Province char + letter + 5-6 alphanumeric | >99% | >90% | Chinese plates only |
|
||||
| name | Surname dictionary + 1-3 Chinese chars | ~70% | ~50% | Misses rare surnames, over-matches common chars |
|
||||
| address | Location trigger words (省/市/区/路/号) | ~80% | ~60% | Depends on trigger word coverage |
|
||||
| face | MediaPipe RKNN + 468-point landmark | ~55% verified | ~70% detected | Side-face and occlusion challenging |
|
||||
|
||||
## 3. File Format Support
|
||||
|
||||
| Format | Text Extraction | Image Analysis | Redaction | Limitations |
|
||||
|--------|----------------|----------------|-----------|-------------|
|
||||
| DOCX | XML parsing | - | `████` replacement | None known |
|
||||
| XLSX | SharedStrings + worksheets | - | Multi-sheet replacement | None known |
|
||||
| PDF (text layer) | lopdf ASCII extraction | - | Equal-length space replacement | CIDFont (Chinese TrueType) unsupported |
|
||||
| PDF (scanned) | OCR required | Face detection | **Not implemented** | Major gap |
|
||||
| JPG/PNG/BMP | PP-OCR RKNN | Face + OCR | Black rectangle masking | Quality depends on image resolution |
|
||||
|
||||
## 4. OCR Accuracy (PP-OCR RKNN)
|
||||
|
||||
| Language | Char Accuracy | Word Accuracy | Rating |
|
||||
|----------|--------------|---------------|--------|
|
||||
| Chinese (Simplified) | 100% | 100% | Excellent |
|
||||
| English | 98.9-100% | 90-100% | Excellent |
|
||||
| Japanese | 100% | 100% | Excellent |
|
||||
| Korean | 98% | 90% | Good (67% overall) |
|
||||
| Latin script | 99.1-100% | 90-100% | Excellent |
|
||||
| Arabic | 20.8-83.7% | - | Poor (model limitation) |
|
||||
| Greek/Thai | ~0% | - | Not supported |
|
||||
|
||||
## 5. Performance (RK3588 NPU)
|
||||
|
||||
| Operation | Latency | Throughput |
|
||||
|-----------|---------|------------|
|
||||
| Face detection (short_range) | 3.73 ms | 268 FPS |
|
||||
| Face detection (full_range) | 7.88 ms | 127 FPS |
|
||||
| Face + landmark pipeline | 5.96 ms | 168 FPS |
|
||||
| OCR detection + recognition | ~15-20 ms | ~50-65 FPS |
|
||||
| DOCX analysis | <100 ms | - |
|
||||
| XLSX analysis | <200 ms | - |
|
||||
| PDF text extraction | <500 ms | - |
|
||||
| Full redaction pipeline | 1-3 sec | - |
|
||||
|
||||
## 6. Classification Logic
|
||||
|
||||
```
|
||||
Document → Keyword scan (Aho-Corasick)
|
||||
├─ Contains 保密/机密/绝密/CONFIDENTIAL/SECRET → classification: "classified", blocked: true
|
||||
└─ No keywords → Entity scan
|
||||
├─ ≥5 high-risk entities → classification: "sensitive_partial", warning
|
||||
├─ 1-4 entities → classification: "sensitive_partial"
|
||||
└─ 0 entities → classification: "normal"
|
||||
```
|
||||
|
||||
## 7. Known Gaps & Risks
|
||||
|
||||
### Critical Gaps
|
||||
1. **Scanned PDF redaction not implemented** — returns original copy
|
||||
2. **CIDFont PDF text extraction fails** — Chinese TrueType PDFs unreadable
|
||||
3. **KVM video masking placeholder only** — screenshot API doesn't apply obfuscation
|
||||
|
||||
### Medium Gaps
|
||||
4. **Chinese name detection relies on surname dictionary** — rare surnames missed
|
||||
5. **Address detection depends on trigger words** — informal addresses missed
|
||||
6. **Face detection 55% post-verification rate** — side faces and occluded faces missed
|
||||
7. **No handwriting recognition** — handwritten PII in images not detected
|
||||
|
||||
### Low Gaps
|
||||
8. **Email regex only matches ASCII** — international domain emails missed
|
||||
9. **No credit card Luhn validation** — may match non-card 16-digit numbers
|
||||
10. **No audio/video PII detection** — spoken PII in media files not covered
|
||||
|
||||
## 8. Recommendations
|
||||
|
||||
### Short-term (high impact, low effort)
|
||||
- [ ] Add Luhn checksum validation for bank card numbers
|
||||
- [ ] Expand surname dictionary with 500+ rare surnames
|
||||
- [ ] Add phone number validation (operator prefix check)
|
||||
|
||||
### Medium-term
|
||||
- [ ] Implement scanned PDF redaction (OCR bbox → image masking)
|
||||
- [ ] Add CIDFont text extraction (use poppler or mupdf binding)
|
||||
- [ ] Implement KVM video stream masking (YUV frame overlay)
|
||||
|
||||
### Long-term
|
||||
- [ ] Train custom RKNN NER model for names/addresses (replace dictionary)
|
||||
- [ ] Add international phone/ID format support
|
||||
- [ ] Audio transcription PII detection
|
||||
@@ -0,0 +1,9 @@
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["services/kvm_agent/tests", "services/privacy_gateway/tests"]
|
||||
asyncio_mode = "auto"
|
||||
addopts = "--ignore=services/kvm_agent/tests/test_integration.py -m 'not viz' -v"
|
||||
markers = [
|
||||
"integration: requires KVM hardware",
|
||||
"benchmark: performance test",
|
||||
"viz: visual debug output tests (generates PNG files)",
|
||||
]
|
||||
Executable
+150
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env bash
|
||||
# build-and-test.sh — One-click build all packages + run tests.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/build-and-test.sh # build + unit tests only
|
||||
# bash scripts/build-and-test.sh --full # build + unit + package integration
|
||||
#
|
||||
# Run from the KVM-privacy project root.
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
FULL=0
|
||||
for arg in "$@"; do
|
||||
[ "$arg" = "--full" ] && FULL=1
|
||||
done
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
pass() { echo -e "${GREEN}[PASS]${NC} $1"; }
|
||||
fail() { echo -e "${RED}[FAIL]${NC} $1"; }
|
||||
info() { echo -e "${YELLOW}[INFO]${NC} $1"; }
|
||||
|
||||
ERRORS=0
|
||||
|
||||
# ── 1. Go unit tests ──────────────────────────────────────────────────────
|
||||
info "Running Go unit tests..."
|
||||
if (cd deps/KVM/go && /usr/local/go/bin/go test ./... -count=1 -timeout 60s 2>&1); then
|
||||
pass "Go unit tests"
|
||||
else
|
||||
fail "Go unit tests"
|
||||
ERRORS=$((ERRORS + 1))
|
||||
fi
|
||||
|
||||
# ── 2. Python unit tests ──────────────────────────────────────────────────
|
||||
info "Running Python unit tests..."
|
||||
if (cd services/kvm_agent && python3 -m pytest tests/ --ignore=tests/test_integration.py -v --tb=short 2>&1); then
|
||||
pass "Python unit tests"
|
||||
else
|
||||
fail "Python unit tests"
|
||||
ERRORS=$((ERRORS + 1))
|
||||
fi
|
||||
|
||||
# ── 3. Build kvm-server deb ───────────────────────────────────────────────
|
||||
info "Building kvm-server deb..."
|
||||
if (cd deps/KVM && PATH="/usr/local/go/bin:$PATH" bash scripts/build-deb.sh 2>&1); then
|
||||
KVM_DEB=$(ls deps/KVM/dist/kvm-server_*.deb 2>/dev/null | tail -1)
|
||||
if [ -n "$KVM_DEB" ]; then
|
||||
pass "kvm-server deb: $KVM_DEB"
|
||||
else
|
||||
fail "kvm-server deb: file not found after build"
|
||||
ERRORS=$((ERRORS + 1))
|
||||
fi
|
||||
else
|
||||
fail "kvm-server deb build"
|
||||
ERRORS=$((ERRORS + 1))
|
||||
fi
|
||||
|
||||
# ── 4. Build Python debs ──────────────────────────────────────────────────
|
||||
info "Building Python debs..."
|
||||
if bash scripts/build-python-debs.sh 2>&1; then
|
||||
pass "Python debs (kvm-agent, kvm-mitm)"
|
||||
else
|
||||
fail "Python debs build"
|
||||
ERRORS=$((ERRORS + 1))
|
||||
fi
|
||||
|
||||
# ── 5. Static deb checks ──────────────────────────────────────────────────
|
||||
info "Running static deb checks..."
|
||||
for deb in deps/KVM/dist/kvm-server_*.deb build/deb/*.deb; do
|
||||
[ -f "$deb" ] || continue
|
||||
name=$(basename "$deb")
|
||||
|
||||
# Verify package info is readable
|
||||
if dpkg -I "$deb" >/dev/null 2>&1; then
|
||||
pass "dpkg -I $name"
|
||||
else
|
||||
fail "dpkg -I $name"
|
||||
ERRORS=$((ERRORS + 1))
|
||||
fi
|
||||
|
||||
# Verify package contents are listable
|
||||
if dpkg -c "$deb" >/dev/null 2>&1; then
|
||||
pass "dpkg -c $name"
|
||||
else
|
||||
fail "dpkg -c $name"
|
||||
ERRORS=$((ERRORS + 1))
|
||||
fi
|
||||
|
||||
# Service file present (skip metapackages)
|
||||
if echo "$name" | grep -qv "meta"; then
|
||||
if dpkg -c "$deb" | grep -q "\.service"; then
|
||||
pass "$name: systemd service file present"
|
||||
else
|
||||
fail "$name: NO systemd service file found"
|
||||
ERRORS=$((ERRORS + 1))
|
||||
fi
|
||||
fi
|
||||
|
||||
# No .pytest_cache or __pycache__ contamination
|
||||
if dpkg -c "$deb" | grep -q "pytest_cache"; then
|
||||
fail "$name: .pytest_cache found in package"
|
||||
ERRORS=$((ERRORS + 1))
|
||||
else
|
||||
pass "$name: no .pytest_cache contamination"
|
||||
fi
|
||||
|
||||
if dpkg -c "$deb" | grep -q "__pycache__"; then
|
||||
fail "$name: __pycache__ found in package"
|
||||
ERRORS=$((ERRORS + 1))
|
||||
else
|
||||
pass "$name: no __pycache__ contamination"
|
||||
fi
|
||||
|
||||
# Depends field not empty (skip metapackages which have only Depends on other pkgs)
|
||||
DEPENDS=$(dpkg -I "$deb" 2>/dev/null | grep "^ Depends:" || true)
|
||||
if [ -n "$DEPENDS" ]; then
|
||||
pass "$name: Depends field present"
|
||||
else
|
||||
fail "$name: Depends field missing or empty"
|
||||
ERRORS=$((ERRORS + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
# ── 6. Package integration tests (--full only) ────────────────────────────
|
||||
if [ "$FULL" = "1" ]; then
|
||||
info "Running package integration tests..."
|
||||
if bash scripts/test-packages.sh full 2>&1; then
|
||||
pass "Package integration tests"
|
||||
else
|
||||
fail "Package integration tests"
|
||||
ERRORS=$((ERRORS + 1))
|
||||
fi
|
||||
else
|
||||
info "Skipping package integration tests (use --full to enable)"
|
||||
fi
|
||||
|
||||
# ── Summary ───────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "============================================"
|
||||
if [ "$ERRORS" -eq 0 ]; then
|
||||
pass "All checks passed!"
|
||||
else
|
||||
fail "$ERRORS check(s) failed"
|
||||
exit 1
|
||||
fi
|
||||
Executable
+259
@@ -0,0 +1,259 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build all KVM-Privacy Debian packages:
|
||||
# Rust: kvm-mitm, kvm-rkllm, kvm-bridge (embed-db), kvm-npu, kvm-privacy, kvm-agent
|
||||
# Meta: kvm-meta
|
||||
#
|
||||
# Options:
|
||||
# --skip-rust Skip Rust compilation (use pre-built binaries in deb trees)
|
||||
#
|
||||
# Run from the KVM-privacy project root.
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
SKIP_RUST=0
|
||||
for arg in "$@"; do
|
||||
[ "$arg" = "--skip-rust" ] && SKIP_RUST=1
|
||||
done
|
||||
|
||||
mkdir -p build/deb
|
||||
ARCH=$(dpkg --print-architecture)
|
||||
|
||||
# ── Rust services ──────────────────────────────────────────────────────────
|
||||
if [ "$SKIP_RUST" = "0" ]; then
|
||||
echo ""
|
||||
echo "==> Building Rust services"
|
||||
|
||||
# npu-daemon
|
||||
if [ -f services/npu_daemon/Cargo.toml ]; then
|
||||
echo "==> Building npu-daemon"
|
||||
(cd services/npu_daemon && cargo build --release --jobs 2 2>&1)
|
||||
|
||||
mkdir -p debian/kvm-npu/usr/sbin
|
||||
cp services/npu_daemon/target/release/npu-daemon debian/kvm-npu/usr/sbin/
|
||||
strip debian/kvm-npu/usr/sbin/npu-daemon
|
||||
|
||||
# Config (packaged version with standard paths)
|
||||
mkdir -p debian/kvm-npu/etc/npu-daemon
|
||||
cp debian/kvm-npu/etc/npu-daemon/config.yaml debian/kvm-npu/etc/npu-daemon/ 2>/dev/null || \
|
||||
cp services/npu_daemon/deploy/config.yaml debian/kvm-npu/etc/npu-daemon/
|
||||
|
||||
# Models (OCR + face)
|
||||
mkdir -p debian/kvm-npu/usr/share/kvm-npu/models/ocr/ch
|
||||
mkdir -p debian/kvm-npu/usr/share/kvm-npu/models/ocr/en
|
||||
mkdir -p debian/kvm-npu/usr/share/kvm-npu/models/face
|
||||
cp deps/KVM/models/ocr/ppocrv4_det.rknn debian/kvm-npu/usr/share/kvm-npu/models/ocr/
|
||||
cp deps/KVM/models/ocr/ch/ppocrv4_rec.rknn debian/kvm-npu/usr/share/kvm-npu/models/ocr/ch/
|
||||
cp deps/KVM/models/ocr/ch/ppocr_keys.txt debian/kvm-npu/usr/share/kvm-npu/models/ocr/ch/
|
||||
cp deps/KVM/models/ocr/en/ppocrv4_rec.rknn debian/kvm-npu/usr/share/kvm-npu/models/ocr/en/
|
||||
cp deps/KVM/models/ocr/en/ppocr_keys.txt debian/kvm-npu/usr/share/kvm-npu/models/ocr/en/
|
||||
cp deps/info-privacy-rs/deps/mediapipe-rknn/models/face_detection_short_range_rk3588.rknn \
|
||||
debian/kvm-npu/usr/share/kvm-npu/models/face/
|
||||
|
||||
# Systemd (use packaged version)
|
||||
mkdir -p debian/kvm-npu/lib/systemd/system
|
||||
cp debian/kvm-npu/lib/systemd/system/npu-daemon.service debian/kvm-npu/lib/systemd/system/ 2>/dev/null || \
|
||||
cp services/npu_daemon/deploy/npu-daemon.service debian/kvm-npu/lib/systemd/system/
|
||||
|
||||
echo "==> npu-daemon binary ready"
|
||||
else
|
||||
echo "WARNING: services/npu_daemon/Cargo.toml not found, skipping npu-daemon"
|
||||
fi
|
||||
|
||||
# info-privacy-rs
|
||||
if [ -f deps/info-privacy-rs/Cargo.toml ]; then
|
||||
echo "==> Building info-privacy-rs"
|
||||
(cd deps/info-privacy-rs && cargo build --release --jobs 2 2>&1)
|
||||
|
||||
mkdir -p debian/kvm-privacy/usr/sbin
|
||||
cp deps/info-privacy-rs/target/release/info-privacy-rs debian/kvm-privacy/usr/sbin/
|
||||
strip debian/kvm-privacy/usr/sbin/info-privacy-rs
|
||||
|
||||
# Config
|
||||
mkdir -p debian/kvm-privacy/etc/kvm-privacy
|
||||
cp deps/info-privacy-rs/configs/pii_rules.yaml debian/kvm-privacy/etc/kvm-privacy/ 2>/dev/null || true
|
||||
cp deps/info-privacy-rs/configs/surnames.txt debian/kvm-privacy/etc/kvm-privacy/ 2>/dev/null || true
|
||||
|
||||
# Systemd (use packaged version)
|
||||
mkdir -p debian/kvm-privacy/lib/systemd/system
|
||||
cp debian/kvm-privacy/lib/systemd/system/info-privacy.service debian/kvm-privacy/lib/systemd/system/ 2>/dev/null || \
|
||||
cp deploy/systemd/info-privacy.service debian/kvm-privacy/lib/systemd/system/
|
||||
|
||||
echo "==> info-privacy-rs binary ready (no Python workers)"
|
||||
else
|
||||
echo "WARNING: deps/info-privacy-rs/Cargo.toml not found, skipping info-privacy-rs"
|
||||
fi
|
||||
|
||||
# privacy-gateway-rs → kvm-mitm
|
||||
if [ -f services/privacy-gateway-rs/Cargo.toml ]; then
|
||||
echo "==> Building privacy-gateway-rs"
|
||||
(cd services/privacy-gateway-rs && cargo build --release --jobs 2 2>&1)
|
||||
|
||||
mkdir -p debian/kvm-mitm/usr/sbin
|
||||
cp services/privacy-gateway-rs/target/release/privacy-gateway-rs debian/kvm-mitm/usr/sbin/
|
||||
strip debian/kvm-mitm/usr/sbin/privacy-gateway-rs
|
||||
|
||||
echo "==> privacy-gateway-rs binary ready"
|
||||
else
|
||||
echo "WARNING: services/privacy-gateway-rs/Cargo.toml not found, skipping privacy-gateway-rs"
|
||||
fi
|
||||
|
||||
# rkllm-server (Rust)
|
||||
if [ -f services/rkllm-server/Cargo.toml ]; then
|
||||
echo "==> Building rkllm-server (Rust)"
|
||||
(cd services/rkllm-server && cargo build --release --jobs 2 2>&1)
|
||||
|
||||
mkdir -p debian/kvm-rkllm/usr/sbin
|
||||
cp services/rkllm-server/target/release/rkllm-server debian/kvm-rkllm/usr/sbin/
|
||||
strip debian/kvm-rkllm/usr/sbin/rkllm-server
|
||||
|
||||
echo "==> rkllm-server (Rust) binary ready"
|
||||
else
|
||||
echo "WARNING: services/rkllm-server/Cargo.toml not found, skipping rkllm-server"
|
||||
fi
|
||||
|
||||
# embed-db (Rust) → kvm-bridge
|
||||
if [ -f services/embed-db-rs/Cargo.toml ]; then
|
||||
echo "==> Building embed-db"
|
||||
(cd services/embed-db-rs && cargo build --release --jobs 2 -p api 2>&1)
|
||||
|
||||
mkdir -p debian/kvm-bridge/usr/sbin
|
||||
cp services/embed-db-rs/target/release/embed-db debian/kvm-bridge/usr/sbin/
|
||||
strip debian/kvm-bridge/usr/sbin/embed-db
|
||||
|
||||
echo "==> embed-db binary ready"
|
||||
else
|
||||
echo "WARNING: services/embed-db-rs/Cargo.toml not found, skipping embed-db"
|
||||
fi
|
||||
|
||||
# kvm-agent (Rust)
|
||||
if [ -f services/kvm-agent-rs/Cargo.toml ]; then
|
||||
echo "==> Building kvm-agent (Rust)"
|
||||
(cd services/kvm-agent-rs && cargo build --release --jobs 2 -p agent-core 2>&1)
|
||||
|
||||
mkdir -p debian/kvm-agent/usr/sbin
|
||||
cp services/kvm-agent-rs/target/release/kvm-agent debian/kvm-agent/usr/sbin/
|
||||
strip debian/kvm-agent/usr/sbin/kvm-agent
|
||||
|
||||
echo "==> kvm-agent (Rust) binary ready"
|
||||
else
|
||||
echo "WARNING: services/kvm-agent-rs/Cargo.toml not found, skipping kvm-agent"
|
||||
fi
|
||||
|
||||
else
|
||||
echo "==> Skipping Rust builds (--skip-rust)"
|
||||
fi
|
||||
|
||||
# ── kvm-mitm: network scripts (kept for LAN gateway) ─────────────────────
|
||||
echo ""
|
||||
echo "==> Preparing kvm-mitm package"
|
||||
mkdir -p debian/kvm-mitm/usr/lib/kvm-mitm/network
|
||||
if [ -d deploy/network ]; then
|
||||
cp deploy/network/lan-gateway.sh debian/kvm-mitm/usr/lib/kvm-mitm/network/ 2>/dev/null || true
|
||||
cp deploy/network/dnsmasq-lan.conf debian/kvm-mitm/usr/lib/kvm-mitm/network/ 2>/dev/null || true
|
||||
chmod 755 debian/kvm-mitm/usr/lib/kvm-mitm/network/lan-gateway.sh 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# kvm-gateway service (LAN NAT setup)
|
||||
mkdir -p debian/kvm-mitm/lib/systemd/system
|
||||
if [ ! -f debian/kvm-mitm/lib/systemd/system/kvm-gateway.service ]; then
|
||||
cat > debian/kvm-mitm/lib/systemd/system/kvm-gateway.service <<'SVCEOF'
|
||||
[Unit]
|
||||
Description=KVM LAN Gateway (NAT + Transparent Proxy)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
Before=kvm-mitm.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/lib/kvm-mitm/network/lan-gateway.sh
|
||||
ExecStop=/bin/bash -c "pkill -F /run/kvm-dnsmasq.pid 2>/dev/null || true"
|
||||
Environment=KVM_WAN_IF=eth0
|
||||
Environment=KVM_LAN_IF=eth1
|
||||
Environment=KVM_LAN_IP=10.42.0.1
|
||||
Environment=KVM_LAN_SUBNET=10.42.0.0/24
|
||||
Environment=KVM_PROXY_PORT=8888
|
||||
Environment=KVM_DNSMASQ_CONF=/etc/kvm/dnsmasq-lan.conf
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
SVCEOF
|
||||
fi
|
||||
|
||||
# ── kvm-rkllm: librkllmrt.so ─────────────────────────────────────────────
|
||||
echo "==> Preparing kvm-rkllm package"
|
||||
mkdir -p debian/kvm-rkllm/usr/lib/kvm-rkllm
|
||||
RKLLM_SO="${RKLLM_LIB_SRC:-/usr/lib/librkllmrt.so}"
|
||||
if [ -f "$RKLLM_SO" ]; then
|
||||
cp "$RKLLM_SO" debian/kvm-rkllm/usr/lib/kvm-rkllm/
|
||||
echo "==> Copied librkllmrt.so from $RKLLM_SO"
|
||||
else
|
||||
echo "WARNING: librkllmrt.so not found at $RKLLM_SO"
|
||||
echo " Set RKLLM_LIB_SRC env var or package will be incomplete"
|
||||
fi
|
||||
|
||||
# ── kvm-agent (Rust binary) ───────────────────────────────────────────────
|
||||
echo "==> Preparing kvm-agent package (Rust)"
|
||||
# Ensure binary is in place (built above if --skip-rust was not set)
|
||||
mkdir -p debian/kvm-agent/usr/sbin
|
||||
# Create /usr/bin/kvm-agent symlink for backward compatibility
|
||||
mkdir -p debian/kvm-agent/usr/bin
|
||||
ln -sf /usr/sbin/kvm-agent debian/kvm-agent/usr/bin/kvm-agent
|
||||
|
||||
# ── Assemble debs ──────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "==> Assembling Debian packages"
|
||||
|
||||
# kvm-mitm (Rust)
|
||||
MITM_VER=$(grep "^Version:" debian/kvm-mitm/DEBIAN/control | awk '{print $2}')
|
||||
MITM_DEB="build/deb/kvm-mitm_${MITM_VER}_${ARCH}.deb"
|
||||
chmod 755 debian/kvm-mitm/DEBIAN/postinst debian/kvm-mitm/DEBIAN/prerm debian/kvm-mitm/DEBIAN/postrm
|
||||
dpkg-deb --build debian/kvm-mitm "$MITM_DEB"
|
||||
echo "==> kvm-mitm deb: $MITM_DEB"
|
||||
|
||||
# kvm-agent (Rust)
|
||||
AGENT_VER=$(grep "^Version:" debian/kvm-agent/DEBIAN/control | awk '{print $2}')
|
||||
AGENT_DEB="build/deb/kvm-agent_${AGENT_VER}_${ARCH}.deb"
|
||||
chmod 755 debian/kvm-agent/DEBIAN/postinst debian/kvm-agent/DEBIAN/prerm debian/kvm-agent/DEBIAN/postrm
|
||||
dpkg-deb --build debian/kvm-agent "$AGENT_DEB"
|
||||
echo "==> kvm-agent deb: $AGENT_DEB"
|
||||
|
||||
# kvm-bridge (Rust embed-db)
|
||||
BRIDGE_VER=$(grep "^Version:" debian/kvm-bridge/DEBIAN/control | awk '{print $2}')
|
||||
BRIDGE_DEB="build/deb/kvm-bridge_${BRIDGE_VER}_${ARCH}.deb"
|
||||
chmod 755 debian/kvm-bridge/DEBIAN/postinst debian/kvm-bridge/DEBIAN/prerm debian/kvm-bridge/DEBIAN/postrm
|
||||
dpkg-deb --build debian/kvm-bridge "$BRIDGE_DEB"
|
||||
echo "==> kvm-bridge deb: $BRIDGE_DEB"
|
||||
|
||||
# kvm-rkllm (Rust + librkllmrt.so)
|
||||
RKLLM_VER=$(grep "^Version:" debian/kvm-rkllm/DEBIAN/control | awk '{print $2}')
|
||||
RKLLM_DEB="build/deb/kvm-rkllm_${RKLLM_VER}_arm64.deb"
|
||||
chmod 755 debian/kvm-rkllm/DEBIAN/postinst debian/kvm-rkllm/DEBIAN/prerm debian/kvm-rkllm/DEBIAN/postrm
|
||||
dpkg-deb --build debian/kvm-rkllm "$RKLLM_DEB"
|
||||
echo "==> kvm-rkllm deb: $RKLLM_DEB"
|
||||
|
||||
# kvm-npu (Rust)
|
||||
NPU_VER=$(grep "^Version:" debian/kvm-npu/DEBIAN/control | awk '{print $2}')
|
||||
NPU_DEB="build/deb/kvm-npu_${NPU_VER}_arm64.deb"
|
||||
chmod 755 debian/kvm-npu/DEBIAN/postinst debian/kvm-npu/DEBIAN/prerm debian/kvm-npu/DEBIAN/postrm
|
||||
dpkg-deb --build debian/kvm-npu "$NPU_DEB"
|
||||
echo "==> kvm-npu deb: $NPU_DEB"
|
||||
|
||||
# kvm-privacy (Rust, no Python workers)
|
||||
PRIVACY_VER=$(grep "^Version:" debian/kvm-privacy/DEBIAN/control | awk '{print $2}')
|
||||
PRIVACY_DEB="build/deb/kvm-privacy_${PRIVACY_VER}_arm64.deb"
|
||||
chmod 755 debian/kvm-privacy/DEBIAN/postinst debian/kvm-privacy/DEBIAN/prerm debian/kvm-privacy/DEBIAN/postrm
|
||||
dpkg-deb --build debian/kvm-privacy "$PRIVACY_DEB"
|
||||
echo "==> kvm-privacy deb: $PRIVACY_DEB"
|
||||
|
||||
# kvm-meta (metapackage, no scripts)
|
||||
META_VER=$(grep "^Version:" debian/kvm-meta/DEBIAN/control | awk '{print $2}')
|
||||
META_DEB="build/deb/kvm-meta_${META_VER}_all.deb"
|
||||
dpkg-deb --build debian/kvm-meta "$META_DEB"
|
||||
echo "==> kvm-meta deb: $META_DEB"
|
||||
|
||||
echo ""
|
||||
echo "==> Done! Packages in build/deb/:"
|
||||
ls -lh build/deb/*.deb
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
build-debs.sh
|
||||
+18
-27
@@ -1,38 +1,29 @@
|
||||
#!/usr/bin/env bash
|
||||
# setup_gateway.sh — 安装 Privacy Gateway (mitmproxy + CA 证书 + systemd)
|
||||
# setup_gateway.sh — Deploy Rust Privacy Gateway + related services via DEB packages
|
||||
set -euo pipefail
|
||||
|
||||
DEVICE_IP="${DEVICE_IP:-192.168.123.181}"
|
||||
DEVICE_USER="${DEVICE_USER:-pi}"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
DEPLOY_DIR="${SCRIPT_DIR}/../deploy/systemd"
|
||||
GATEWAY_DIR="${SCRIPT_DIR}/../services/privacy_gateway"
|
||||
BUILD_DIR="${SCRIPT_DIR}/../build/deb"
|
||||
|
||||
echo "=== [1/4] 安装 mitmproxy ==="
|
||||
ssh "${DEVICE_USER}@${DEVICE_IP}" "pip3 install mitmproxy httpx cryptography 2>&1 | tail -5"
|
||||
echo "=== [1/3] Build DEB packages ==="
|
||||
bash "${SCRIPT_DIR}/build-debs.sh"
|
||||
|
||||
echo "=== [2/4] 上传 Privacy Gateway 服务文件 ==="
|
||||
ssh "${DEVICE_USER}@${DEVICE_IP}" "mkdir -p /data/project/KVM-privacy/services/privacy_gateway"
|
||||
scp "${GATEWAY_DIR}"/*.py "${DEVICE_USER}@${DEVICE_IP}:/data/project/KVM-privacy/services/privacy_gateway/"
|
||||
scp "${GATEWAY_DIR}/ai_domains.txt" "${DEVICE_USER}@${DEVICE_IP}:/data/project/KVM-privacy/services/privacy_gateway/"
|
||||
|
||||
echo "=== [3/4] 生成 CA 证书 ==="
|
||||
ssh "${DEVICE_USER}@${DEVICE_IP}" "python3 -c '
|
||||
import sys; sys.path.insert(0, \"/data/project/KVM-privacy/services/privacy_gateway\")
|
||||
from cert_manager import ensure_ca
|
||||
cert, key = ensure_ca()
|
||||
print(f\"CA 证书: {cert}\")
|
||||
'"
|
||||
|
||||
echo "=== [4/4] 安装 systemd 服务 ==="
|
||||
for svc in info-privacy mem-bridge-memory mem-bridge-router privacy-gateway; do
|
||||
if [ -f "${DEPLOY_DIR}/${svc}.service" ]; then
|
||||
scp "${DEPLOY_DIR}/${svc}.service" "${DEVICE_USER}@${DEVICE_IP}:/tmp/${svc}.service"
|
||||
ssh "${DEVICE_USER}@${DEVICE_IP}" "echo '123123' | sudo -S cp /tmp/${svc}.service /etc/systemd/system/${svc}.service"
|
||||
echo ""
|
||||
echo "=== [2/3] Upload DEB packages ==="
|
||||
for deb in kvm-mitm kvm-privacy kvm-npu kvm-bridge kvm-rkllm; do
|
||||
DEB_FILE=$(ls "${BUILD_DIR}/${deb}_"*.deb 2>/dev/null | tail -1)
|
||||
if [ -n "$DEB_FILE" ]; then
|
||||
echo " Uploading $(basename "$DEB_FILE")"
|
||||
scp "$DEB_FILE" "${DEVICE_USER}@${DEVICE_IP}:/tmp/"
|
||||
fi
|
||||
done
|
||||
ssh "${DEVICE_USER}@${DEVICE_IP}" "echo '123123' | sudo -S systemctl daemon-reload"
|
||||
|
||||
echo "=== 完成!==="
|
||||
echo "启用服务: ssh ${DEVICE_USER}@${DEVICE_IP} 'sudo systemctl enable --now privacy-gateway'"
|
||||
echo "下载 CA 证书: curl http://${DEVICE_IP}:8080/api/v1/privacy/cert -H 'Authorization: Bearer <token>' -o kvm-ca.crt"
|
||||
echo ""
|
||||
echo "=== [3/3] Install on device ==="
|
||||
echo "Run on device:"
|
||||
echo " sudo dpkg -i /tmp/kvm-npu_*.deb /tmp/kvm-privacy_*.deb /tmp/kvm-mitm_*.deb /tmp/kvm-rkllm_*.deb /tmp/kvm-bridge_*.deb"
|
||||
echo ""
|
||||
echo "Verify:"
|
||||
echo " bash tools/smoke-test.sh"
|
||||
|
||||
Executable
+436
@@ -0,0 +1,436 @@
|
||||
#!/usr/bin/env bash
|
||||
# test-packages.sh — Package integration test suite.
|
||||
#
|
||||
# Runs: install → verify → remove → reinstall → verify → purge
|
||||
# Must be executed on the target device (NanoPC-T6) with root privileges.
|
||||
#
|
||||
# Usage:
|
||||
# sudo bash scripts/test-packages.sh # verify only (assumes installed)
|
||||
# sudo bash scripts/test-packages.sh full # full install/remove/purge cycle
|
||||
# sudo bash scripts/test-packages.sh install # install + verify
|
||||
# sudo bash scripts/test-packages.sh remove # remove + verify conffiles preserved
|
||||
# sudo bash scripts/test-packages.sh purge # purge + verify clean
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
ERRORS=0
|
||||
TESTS=0
|
||||
|
||||
pass() { TESTS=$((TESTS + 1)); echo -e "${GREEN}[PASS]${NC} $1"; }
|
||||
fail() { TESTS=$((TESTS + 1)); ERRORS=$((ERRORS + 1)); echo -e "${RED}[FAIL]${NC} $1"; }
|
||||
info() { echo -e "${YELLOW}[----]${NC} $1"; }
|
||||
skip() { echo -e "${YELLOW}[SKIP]${NC} $1"; }
|
||||
|
||||
# Detect package locations
|
||||
PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
KVM_DEB=$(ls "$PROJECT_ROOT"/deps/KVM/dist/kvm-server_*.deb 2>/dev/null | tail -1 || true)
|
||||
AGENT_DEB=$(ls "$PROJECT_ROOT"/build/deb/kvm-agent_*.deb 2>/dev/null | tail -1 || true)
|
||||
MITM_DEB=$(ls "$PROJECT_ROOT"/build/deb/kvm-mitm_*.deb 2>/dev/null | tail -1 || true)
|
||||
BRIDGE_DEB=$(ls "$PROJECT_ROOT"/build/deb/kvm-bridge_*.deb 2>/dev/null | tail -1 || true)
|
||||
|
||||
MODE="${1:-verify}"
|
||||
|
||||
# ── Helper: verify kvm-server service and endpoints ──────────────────────
|
||||
verify_kvm_server() {
|
||||
local label="${1:-}"
|
||||
info "=== kvm-server verification ${label} ==="
|
||||
|
||||
# 1. kvm-server service
|
||||
if systemctl is-active --quiet kvm-server 2>/dev/null; then
|
||||
pass "kvm-server service active"
|
||||
else
|
||||
fail "kvm-server service NOT active"
|
||||
fi
|
||||
|
||||
# 2. API health endpoint
|
||||
if curl -sf --max-time 5 http://localhost:8080/api/health >/dev/null 2>&1; then
|
||||
pass "API /api/health reachable"
|
||||
else
|
||||
fail "API /api/health NOT reachable"
|
||||
fi
|
||||
|
||||
# 3. Stream stats endpoint
|
||||
if curl -sf --max-time 5 http://localhost:8080/api/v1/kvm/stream/stats >/dev/null 2>&1; then
|
||||
pass "API /api/v1/kvm/stream/stats reachable"
|
||||
else
|
||||
# May require auth — check for 401 (means server is up, auth required)
|
||||
HTTP_CODE=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://localhost:8080/api/v1/kvm/stream/stats 2>/dev/null || echo "000")
|
||||
if [ "$HTTP_CODE" = "401" ]; then
|
||||
pass "API /api/v1/kvm/stream/stats returns 401 (auth required, server up)"
|
||||
else
|
||||
fail "API /api/v1/kvm/stream/stats HTTP $HTTP_CODE"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 4. Config file exists with correct permissions
|
||||
if [ -f /etc/kvm/config.json ]; then
|
||||
pass "Config file /etc/kvm/config.json exists"
|
||||
PERMS=$(stat -c '%a' /etc/kvm/config.json 2>/dev/null || true)
|
||||
if [ "$PERMS" = "600" ] || [ "$PERMS" = "640" ] || [ "$PERMS" = "644" ]; then
|
||||
pass "Config file permissions: $PERMS"
|
||||
else
|
||||
fail "Config file permissions: $PERMS (expected 600/640/644)"
|
||||
fi
|
||||
else
|
||||
# Check for TOML config (future)
|
||||
if [ -f /etc/kvm/kvm.toml ]; then
|
||||
pass "Config file /etc/kvm/kvm.toml exists"
|
||||
else
|
||||
fail "No config file found (/etc/kvm/config.json or /etc/kvm/kvm.toml)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 5. MariaDB connectivity
|
||||
if command -v mariadb >/dev/null 2>&1; then
|
||||
if mariadb -u root -e "SELECT 1" >/dev/null 2>&1; then
|
||||
pass "MariaDB accessible"
|
||||
|
||||
# Check kvm database exists
|
||||
if mariadb -u root -e "USE kvm; SHOW TABLES" >/dev/null 2>&1; then
|
||||
TABLE_COUNT=$(mariadb -u root -N -e "USE kvm; SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='kvm'" 2>/dev/null || echo "0")
|
||||
if [ "$TABLE_COUNT" -gt 0 ]; then
|
||||
pass "MariaDB kvm database has $TABLE_COUNT tables"
|
||||
else
|
||||
fail "MariaDB kvm database has 0 tables"
|
||||
fi
|
||||
else
|
||||
skip "MariaDB kvm database not found (may not be initialized)"
|
||||
fi
|
||||
else
|
||||
skip "MariaDB not accessible (may require auth)"
|
||||
fi
|
||||
else
|
||||
skip "MariaDB client not installed"
|
||||
fi
|
||||
|
||||
# 6. journald JSON log format check
|
||||
RECENT_LOG=$(journalctl -u kvm-server -n 1 -o cat --no-pager 2>/dev/null || true)
|
||||
if [ -n "$RECENT_LOG" ]; then
|
||||
if echo "$RECENT_LOG" | python3 -c "import sys,json; json.load(sys.stdin)" 2>/dev/null; then
|
||||
pass "journald kvm-server logs are valid JSON"
|
||||
else
|
||||
# Non-JSON logs are acceptable during migration
|
||||
skip "journald kvm-server logs not yet JSON (migration in progress)"
|
||||
fi
|
||||
else
|
||||
skip "No recent kvm-server journal entries"
|
||||
fi
|
||||
|
||||
# 7. USB gadget service (soft dependency)
|
||||
if systemctl is-active --quiet kvm-usb-gadget 2>/dev/null; then
|
||||
pass "kvm-usb-gadget active (optional)"
|
||||
else
|
||||
skip "kvm-usb-gadget not active (soft dependency, OK)"
|
||||
fi
|
||||
|
||||
# 8. Package metadata
|
||||
if dpkg -s kvm-server >/dev/null 2>&1; then
|
||||
pass "dpkg -s kvm-server: package info valid"
|
||||
else
|
||||
fail "dpkg -s kvm-server: package not installed"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Helper: verify kvm-agent ─────────────────────────────────────────────
|
||||
verify_agent() {
|
||||
info "=== kvm-agent verification ==="
|
||||
|
||||
if dpkg -s kvm-agent >/dev/null 2>&1; then
|
||||
pass "dpkg -s kvm-agent: package installed"
|
||||
else
|
||||
fail "dpkg -s kvm-agent: package NOT installed"
|
||||
return
|
||||
fi
|
||||
|
||||
# Service file should exist (enabled, not necessarily running)
|
||||
if systemctl list-unit-files kvm-agent.service >/dev/null 2>&1; then
|
||||
pass "kvm-agent.service unit file present"
|
||||
else
|
||||
fail "kvm-agent.service unit file missing"
|
||||
fi
|
||||
|
||||
# Secrets env template created
|
||||
if [ -f /etc/kvm-agent/secrets.env ]; then
|
||||
pass "kvm-agent secrets.env exists"
|
||||
PERMS=$(stat -c '%a' /etc/kvm-agent/secrets.env 2>/dev/null || true)
|
||||
if [ "$PERMS" = "600" ]; then
|
||||
pass "kvm-agent secrets.env permissions: 600"
|
||||
else
|
||||
fail "kvm-agent secrets.env permissions: $PERMS (expected 600)"
|
||||
fi
|
||||
else
|
||||
fail "kvm-agent secrets.env missing"
|
||||
fi
|
||||
|
||||
# Entry point exists
|
||||
if [ -x /usr/bin/kvm-agent ]; then
|
||||
pass "kvm-agent entry point /usr/bin/kvm-agent exists"
|
||||
else
|
||||
fail "kvm-agent entry point /usr/bin/kvm-agent missing or not executable"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Helper: verify kvm-mitm ─────────────────────────────────────────────
|
||||
verify_mitm() {
|
||||
info "=== kvm-mitm verification ==="
|
||||
|
||||
if dpkg -s kvm-mitm >/dev/null 2>&1; then
|
||||
pass "dpkg -s kvm-mitm: package installed"
|
||||
else
|
||||
fail "dpkg -s kvm-mitm: package NOT installed"
|
||||
return
|
||||
fi
|
||||
|
||||
# Service file should exist
|
||||
if systemctl list-unit-files kvm-mitm.service >/dev/null 2>&1; then
|
||||
pass "kvm-mitm.service unit file present"
|
||||
else
|
||||
fail "kvm-mitm.service unit file missing"
|
||||
fi
|
||||
|
||||
# Secrets env template
|
||||
if [ -f /etc/kvm-privacy/secrets.env ]; then
|
||||
pass "kvm-mitm secrets.env exists"
|
||||
else
|
||||
fail "kvm-mitm secrets.env missing"
|
||||
fi
|
||||
|
||||
# Entry point
|
||||
if [ -x /usr/bin/kvm-mitm ]; then
|
||||
pass "kvm-mitm entry point /usr/bin/kvm-mitm exists"
|
||||
else
|
||||
fail "kvm-mitm entry point /usr/bin/kvm-mitm missing or not executable"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Helper: verify kvm-bridge ────────────────────────────────────────────
|
||||
verify_bridge() {
|
||||
info "=== kvm-bridge verification ==="
|
||||
|
||||
if dpkg -s kvm-bridge >/dev/null 2>&1; then
|
||||
pass "dpkg -s kvm-bridge: package installed"
|
||||
else
|
||||
fail "dpkg -s kvm-bridge: package NOT installed"
|
||||
return
|
||||
fi
|
||||
|
||||
# Service file (single embed-db service replaces mem-bridge-memory + router)
|
||||
if systemctl list-unit-files embed-db.service >/dev/null 2>&1; then
|
||||
pass "kvm-bridge embed-db.service unit file present"
|
||||
else
|
||||
fail "kvm-bridge embed-db.service unit file missing"
|
||||
fi
|
||||
|
||||
# Rust binary installed
|
||||
if [ -x /usr/sbin/embed-db ]; then
|
||||
pass "kvm-bridge embed-db binary installed"
|
||||
else
|
||||
fail "kvm-bridge embed-db binary missing"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Combined verify for all installed packages ───────────────────────────
|
||||
verify_system() {
|
||||
local label="${1:-}"
|
||||
|
||||
# Always verify kvm-server if installed or deb available
|
||||
if dpkg -s kvm-server >/dev/null 2>&1; then
|
||||
verify_kvm_server "$label"
|
||||
elif [ -n "$KVM_DEB" ]; then
|
||||
skip "kvm-server not installed (deb available at $KVM_DEB)"
|
||||
fi
|
||||
|
||||
# Verify optional packages if installed
|
||||
dpkg -s kvm-agent >/dev/null 2>&1 && verify_agent
|
||||
dpkg -s kvm-mitm >/dev/null 2>&1 && verify_mitm
|
||||
dpkg -s kvm-bridge >/dev/null 2>&1 && verify_bridge
|
||||
}
|
||||
|
||||
# ── Helper: verify conffiles preserved after remove ───────────────────────
|
||||
verify_remove() {
|
||||
info "=== Post-remove verification ==="
|
||||
|
||||
# Service must be stopped
|
||||
if systemctl is-active --quiet kvm-server 2>/dev/null; then
|
||||
fail "kvm-server still active after remove"
|
||||
else
|
||||
pass "kvm-server stopped after remove"
|
||||
fi
|
||||
|
||||
# Config file preserved (conffile behavior)
|
||||
if [ -f /etc/kvm/config.json ] || [ -f /etc/kvm/kvm.toml ]; then
|
||||
pass "Config file preserved after remove"
|
||||
else
|
||||
fail "Config file lost after remove"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Helper: verify clean purge ────────────────────────────────────────────
|
||||
verify_purge() {
|
||||
info "=== Post-purge verification ==="
|
||||
|
||||
if systemctl is-active --quiet kvm-server 2>/dev/null; then
|
||||
fail "kvm-server still active after purge"
|
||||
else
|
||||
pass "kvm-server stopped after purge"
|
||||
fi
|
||||
|
||||
if [ -f /etc/kvm/config.json ]; then
|
||||
fail "Config /etc/kvm/config.json not purged"
|
||||
else
|
||||
pass "Config /etc/kvm/config.json purged"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Package lifecycle helpers ────────────────────────────────────────────
|
||||
|
||||
install_pkg() {
|
||||
local deb="$1"
|
||||
local name="$2"
|
||||
if [ -z "$deb" ]; then
|
||||
skip "$name deb not found, skipping"
|
||||
return 1
|
||||
fi
|
||||
info "Installing $deb..."
|
||||
START_TIME=$(date +%s)
|
||||
dpkg -i "$deb" || apt-get -f install -y
|
||||
END_TIME=$(date +%s)
|
||||
ELAPSED=$((END_TIME - START_TIME))
|
||||
|
||||
if [ "$ELAPSED" -lt 60 ]; then
|
||||
pass "$name installation completed in ${ELAPSED}s (< 60s)"
|
||||
else
|
||||
fail "$name installation took ${ELAPSED}s (> 60s, check for hangs)"
|
||||
fi
|
||||
}
|
||||
|
||||
remove_pkg() {
|
||||
local name="$1"
|
||||
if dpkg -s "$name" >/dev/null 2>&1; then
|
||||
info "Removing $name (preserve conffiles)..."
|
||||
dpkg --remove "$name" || true
|
||||
sleep 2
|
||||
|
||||
if systemctl is-active --quiet "${name}" 2>/dev/null; then
|
||||
fail "$name still active after remove"
|
||||
else
|
||||
pass "$name stopped/removed"
|
||||
fi
|
||||
else
|
||||
skip "$name not installed, skip remove"
|
||||
fi
|
||||
}
|
||||
|
||||
purge_pkg() {
|
||||
local name="$1"
|
||||
info "Purging $name..."
|
||||
dpkg --purge "$name" 2>/dev/null || true
|
||||
sleep 1
|
||||
|
||||
if dpkg -s "$name" >/dev/null 2>&1; then
|
||||
fail "$name still in dpkg database after purge"
|
||||
else
|
||||
pass "$name purged cleanly"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Commands ──────────────────────────────────────────────────────────────
|
||||
|
||||
do_install() {
|
||||
install_pkg "$KVM_DEB" "kvm-server"
|
||||
sleep 5 # Wait for service startup
|
||||
verify_kvm_server "post-install"
|
||||
|
||||
# Install Python packages if available
|
||||
[ -n "$AGENT_DEB" ] && install_pkg "$AGENT_DEB" "kvm-agent" && verify_agent
|
||||
[ -n "$MITM_DEB" ] && install_pkg "$MITM_DEB" "kvm-mitm" && verify_mitm
|
||||
[ -n "$BRIDGE_DEB" ] && install_pkg "$BRIDGE_DEB" "kvm-bridge" && verify_bridge
|
||||
}
|
||||
|
||||
do_remove() {
|
||||
remove_pkg "kvm-bridge"
|
||||
remove_pkg "kvm-agent"
|
||||
remove_pkg "kvm-mitm"
|
||||
|
||||
info "Removing kvm-server (preserve conffiles)..."
|
||||
dpkg --remove kvm-server || true
|
||||
sleep 2
|
||||
verify_remove
|
||||
}
|
||||
|
||||
do_purge() {
|
||||
purge_pkg "kvm-bridge"
|
||||
purge_pkg "kvm-agent"
|
||||
purge_pkg "kvm-mitm"
|
||||
|
||||
info "Purging kvm-server..."
|
||||
dpkg --purge kvm-server || true
|
||||
sleep 2
|
||||
verify_purge
|
||||
}
|
||||
|
||||
# ── Main dispatch ─────────────────────────────────────────────────────────
|
||||
|
||||
case "$MODE" in
|
||||
verify)
|
||||
verify_system "current state"
|
||||
;;
|
||||
install)
|
||||
do_install
|
||||
;;
|
||||
remove)
|
||||
do_remove
|
||||
;;
|
||||
purge)
|
||||
do_purge
|
||||
;;
|
||||
full)
|
||||
info "=== FULL PACKAGE LIFECYCLE TEST ==="
|
||||
echo ""
|
||||
|
||||
# Step 1: Install
|
||||
info "[1/5] First install"
|
||||
do_install
|
||||
echo ""
|
||||
|
||||
# Step 2: Remove
|
||||
info "[2/5] Remove (preserve conffiles)"
|
||||
do_remove
|
||||
echo ""
|
||||
|
||||
# Step 3: Reinstall
|
||||
info "[3/5] Reinstall"
|
||||
do_install
|
||||
echo ""
|
||||
|
||||
# Step 4: Purge
|
||||
info "[4/5] Purge (clean removal)"
|
||||
do_purge
|
||||
echo ""
|
||||
|
||||
# Step 5: Final clean install
|
||||
info "[5/5] Final clean install"
|
||||
do_install
|
||||
echo ""
|
||||
;;
|
||||
*)
|
||||
echo "Usage: $0 {verify|install|remove|purge|full}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# ── Summary ───────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "============================================"
|
||||
echo "Tests: $TESTS Passed: $((TESTS - ERRORS)) Failed: $ERRORS"
|
||||
if [ "$ERRORS" -eq 0 ]; then
|
||||
echo -e "${GREEN}All package tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}$ERRORS test(s) failed${NC}"
|
||||
exit 1
|
||||
fi
|
||||
Generated
+4150
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,21 @@
|
||||
[workspace]
|
||||
members = [
|
||||
"crates/usearch-index",
|
||||
"crates/embedder",
|
||||
"crates/meta-store",
|
||||
"crates/search",
|
||||
"crates/memory",
|
||||
"crates/router",
|
||||
"crates/api",
|
||||
]
|
||||
resolver = "2"
|
||||
|
||||
[workspace.package]
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
license = "MIT"
|
||||
|
||||
[profile.release]
|
||||
opt-level = 3
|
||||
lto = "thin"
|
||||
strip = true
|
||||
@@ -0,0 +1,36 @@
|
||||
[package]
|
||||
name = "api"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "embed-db"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
memory = { path = "../memory" }
|
||||
router = { path = "../router" }
|
||||
embedder = { path = "../embedder" }
|
||||
meta-store = { path = "../meta-store" }
|
||||
usearch-index = { path = "../usearch-index" }
|
||||
search = { path = "../search" }
|
||||
axum = "0.8"
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
thiserror = "2"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
tower-http = { version = "0.6", features = ["cors"] }
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
futures = "0.3"
|
||||
async-trait = "0.1"
|
||||
uuid = { version = "1", features = ["v4"] }
|
||||
async-stream = "0.3"
|
||||
serde_yaml_ng = "0.10"
|
||||
tokenizers = { version = "0.21", default-features = false, features = ["fancy-regex"] }
|
||||
|
||||
[dev-dependencies]
|
||||
axum-test = "19"
|
||||
tempfile = "3"
|
||||
tokenizers = { version = "0.21", default-features = false, features = ["progressbar"] }
|
||||
@@ -0,0 +1,152 @@
|
||||
//! Service configuration.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use router::BackendConfig;
|
||||
|
||||
/// Top-level service configuration loaded from YAML or environment defaults.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct ServiceConfig {
|
||||
/// Port for the memory API server (default: 8001).
|
||||
#[serde(default = "default_memory_port")]
|
||||
pub memory_port: u16,
|
||||
|
||||
/// Port for the router API server (default: 8002).
|
||||
#[serde(default = "default_router_port")]
|
||||
pub router_port: u16,
|
||||
|
||||
/// Directory for SQLite databases and vector indices (default: ~/.embed_db).
|
||||
#[serde(default = "default_db_dir")]
|
||||
pub db_dir: PathBuf,
|
||||
|
||||
/// Path to the embedding model file.
|
||||
#[serde(default = "default_model_path")]
|
||||
pub model_path: PathBuf,
|
||||
|
||||
/// Token budget for context compression (default: 2000).
|
||||
#[serde(default = "default_token_budget")]
|
||||
pub token_budget: usize,
|
||||
|
||||
/// Complexity threshold for routing (default: 0.72).
|
||||
#[serde(default = "default_complexity_threshold")]
|
||||
pub complexity_threshold: f32,
|
||||
|
||||
/// BM25 weight for hybrid search (default: 0.3).
|
||||
#[serde(default = "default_bm25_weight")]
|
||||
pub bm25_weight: f32,
|
||||
|
||||
/// Fallback timeout in seconds (default: 10).
|
||||
#[serde(default = "default_fallback_timeout")]
|
||||
pub fallback_timeout: u64,
|
||||
|
||||
/// Backend configurations keyed by name.
|
||||
#[serde(default)]
|
||||
pub backends: HashMap<String, BackendConfig>,
|
||||
|
||||
/// Default backend name for simple queries.
|
||||
#[serde(default = "default_backend_name")]
|
||||
pub default_backend: String,
|
||||
|
||||
/// Backend name for complex queries.
|
||||
#[serde(default = "default_backend_name")]
|
||||
pub heavy_backend: String,
|
||||
|
||||
/// Ordered list of backend names for fallback.
|
||||
#[serde(default)]
|
||||
pub fallback_chain: Vec<String>,
|
||||
|
||||
/// Task-type to backend routing rules.
|
||||
#[serde(default)]
|
||||
pub routing_rules: HashMap<String, String>,
|
||||
}
|
||||
|
||||
impl Default for ServiceConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
memory_port: default_memory_port(),
|
||||
router_port: default_router_port(),
|
||||
db_dir: default_db_dir(),
|
||||
model_path: default_model_path(),
|
||||
token_budget: default_token_budget(),
|
||||
complexity_threshold: default_complexity_threshold(),
|
||||
bm25_weight: default_bm25_weight(),
|
||||
fallback_timeout: default_fallback_timeout(),
|
||||
backends: HashMap::new(),
|
||||
default_backend: default_backend_name(),
|
||||
heavy_backend: default_backend_name(),
|
||||
fallback_chain: Vec::new(),
|
||||
routing_rules: HashMap::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ServiceConfig {
|
||||
/// Load config from YAML file path, falling back to defaults for missing fields.
|
||||
pub fn from_yaml(path: &str) -> Result<Self, String> {
|
||||
let content = std::fs::read_to_string(path)
|
||||
.map_err(|e| format!("failed to read config file {}: {}", path, e))?;
|
||||
serde_yaml_ng::from_str(&content)
|
||||
.map_err(|e| format!("failed to parse config YAML: {}", e))
|
||||
}
|
||||
|
||||
/// Load from `EMBED_DB_CONFIG` env var, or return defaults.
|
||||
pub fn load() -> Self {
|
||||
match std::env::var("EMBED_DB_CONFIG") {
|
||||
Ok(path) => match Self::from_yaml(&path) {
|
||||
Ok(config) => {
|
||||
tracing::info!(path, "loaded config from YAML");
|
||||
config
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "failed to load config, using defaults");
|
||||
Self::default()
|
||||
}
|
||||
},
|
||||
Err(_) => {
|
||||
tracing::info!("no EMBED_DB_CONFIG set, using defaults");
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_memory_port() -> u16 {
|
||||
std::env::var("MEMORY_PORT")
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(8003)
|
||||
}
|
||||
|
||||
fn default_router_port() -> u16 {
|
||||
std::env::var("ROUTER_PORT")
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(8002)
|
||||
}
|
||||
fn default_db_dir() -> PathBuf {
|
||||
std::env::var("HOME")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|_| PathBuf::from("."))
|
||||
.join(".embed_db")
|
||||
}
|
||||
fn default_model_path() -> PathBuf {
|
||||
PathBuf::from("models/embedder.onnx")
|
||||
}
|
||||
fn default_token_budget() -> usize {
|
||||
2000
|
||||
}
|
||||
fn default_complexity_threshold() -> f32 {
|
||||
0.72
|
||||
}
|
||||
fn default_bm25_weight() -> f32 {
|
||||
0.3
|
||||
}
|
||||
fn default_fallback_timeout() -> u64 {
|
||||
10
|
||||
}
|
||||
fn default_backend_name() -> String {
|
||||
"local".to_string()
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
//! API crate for embed-db-rs.
|
||||
//!
|
||||
//! Provides axum route definitions for:
|
||||
//! - **Memory API** (port 8001) — turn/fact CRUD, context compression, search
|
||||
//! - **Router API** (port 8002) — OpenAI-compatible chat completions with routing
|
||||
|
||||
pub mod config;
|
||||
pub mod memory_routes;
|
||||
pub mod router_routes;
|
||||
pub mod state;
|
||||
|
||||
pub use config::ServiceConfig;
|
||||
pub use state::AppState;
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user