feat: info-privacy PII detection service with frame analysis support
- Pipeline: analyze(), redact(), analyze_image() methods - API: /analyze, /redact, /analyze/frame, /analyze/frame/base64 endpoints - Detectors: regex, NER, face (RKNN NPU) - Privacy frame route added for KVM-Privacy Hub integration
This commit is contained in:
+11
@@ -0,0 +1,11 @@
|
||||
venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
.pytest_cache/
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
.env
|
||||
tmp/
|
||||
*.db
|
||||
@@ -0,0 +1 @@
|
||||
/cache
|
||||
@@ -0,0 +1,126 @@
|
||||
# the name by which the project can be referenced within Serena
|
||||
project_name: "info-privacy"
|
||||
|
||||
|
||||
# list of languages for which language servers are started; choose from:
|
||||
# al bash clojure cpp csharp
|
||||
# csharp_omnisharp dart elixir elm erlang
|
||||
# fortran fsharp go groovy haskell
|
||||
# java julia kotlin lua markdown
|
||||
# matlab nix pascal perl php
|
||||
# php_phpactor powershell python python_jedi r
|
||||
# rego ruby ruby_solargraph rust scala
|
||||
# swift terraform toml typescript typescript_vts
|
||||
# vue yaml zig
|
||||
# (This list may be outdated. For the current list, see values of Language enum here:
|
||||
# https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py
|
||||
# For some languages, there are alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.)
|
||||
# Note:
|
||||
# - For C, use cpp
|
||||
# - For JavaScript, use typescript
|
||||
# - For Free Pascal/Lazarus, use pascal
|
||||
# Special requirements:
|
||||
# Some languages require additional setup/installations.
|
||||
# See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers
|
||||
# When using multiple languages, the first language server that supports a given file will be used for that file.
|
||||
# The first language is the default language and the respective language server will be used as a fallback.
|
||||
# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored.
|
||||
languages:
|
||||
- python
|
||||
|
||||
# the encoding used by text files in the project
|
||||
# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings
|
||||
encoding: "utf-8"
|
||||
|
||||
# The language backend to use for this project.
|
||||
# If not set, the global setting from serena_config.yml is used.
|
||||
# Valid values: LSP, JetBrains
|
||||
# Note: the backend is fixed at startup. If a project with a different backend
|
||||
# is activated post-init, an error will be returned.
|
||||
language_backend:
|
||||
|
||||
# whether to use project's .gitignore files to ignore files
|
||||
ignore_all_files_in_gitignore: true
|
||||
|
||||
# list of additional paths to ignore in this project.
|
||||
# Same syntax as gitignore, so you can use * and **.
|
||||
# Note: global ignored_paths from serena_config.yml are also applied additively.
|
||||
ignored_paths: []
|
||||
|
||||
# whether the project is in read-only mode
|
||||
# If set to true, all editing tools will be disabled and attempts to use them will result in an error
|
||||
# Added on 2025-04-18
|
||||
read_only: false
|
||||
|
||||
# list of tool names to exclude. We recommend not excluding any tools, see the readme for more details.
|
||||
# Below is the complete list of tools for convenience.
|
||||
# To make sure you have the latest list of tools, and to view their descriptions,
|
||||
# execute `uv run scripts/print_tool_overview.py`.
|
||||
#
|
||||
# * `activate_project`: Activates a project by name.
|
||||
# * `check_onboarding_performed`: Checks whether project onboarding was already performed.
|
||||
# * `create_text_file`: Creates/overwrites a file in the project directory.
|
||||
# * `delete_lines`: Deletes a range of lines within a file.
|
||||
# * `delete_memory`: Deletes a memory from Serena's project-specific memory store.
|
||||
# * `execute_shell_command`: Executes a shell command.
|
||||
# * `find_referencing_code_snippets`: Finds code snippets in which the symbol at the given location is referenced.
|
||||
# * `find_referencing_symbols`: Finds symbols that reference the symbol at the given location (optionally filtered by type).
|
||||
# * `find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type).
|
||||
# * `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes.
|
||||
# * `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file.
|
||||
# * `initial_instructions`: Gets the initial instructions for the current project.
|
||||
# Should only be used in settings where the system prompt cannot be set,
|
||||
# e.g. in clients you have no control over, like Claude Desktop.
|
||||
# * `insert_after_symbol`: Inserts content after the end of the definition of a given symbol.
|
||||
# * `insert_at_line`: Inserts content at a given line in a file.
|
||||
# * `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol.
|
||||
# * `list_dir`: Lists files and directories in the given directory (optionally with recursion).
|
||||
# * `list_memories`: Lists memories in Serena's project-specific memory store.
|
||||
# * `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building).
|
||||
# * `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation (in order to continue with the necessary context).
|
||||
# * `read_file`: Reads a file within the project directory.
|
||||
# * `read_memory`: Reads the memory with the given name from Serena's project-specific memory store.
|
||||
# * `remove_project`: Removes a project from the Serena configuration.
|
||||
# * `replace_lines`: Replaces a range of lines within a file with new content.
|
||||
# * `replace_symbol_body`: Replaces the full definition of a symbol.
|
||||
# * `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen.
|
||||
# * `search_for_pattern`: Performs a search for a pattern in the project.
|
||||
# * `summarize_changes`: Provides instructions for summarizing the changes made to the codebase.
|
||||
# * `switch_modes`: Activates modes by providing a list of their names
|
||||
# * `think_about_collected_information`: Thinking tool for pondering the completeness of collected information.
|
||||
# * `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task.
|
||||
# * `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed.
|
||||
# * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store.
|
||||
excluded_tools: []
|
||||
|
||||
# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default)
|
||||
included_optional_tools: []
|
||||
|
||||
# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools.
|
||||
# This cannot be combined with non-empty excluded_tools or included_optional_tools.
|
||||
fixed_tools: []
|
||||
|
||||
# list of mode names to that are always to be included in the set of active modes
|
||||
# The full set of modes to be activated is base_modes + default_modes.
|
||||
# If the setting is undefined, the base_modes from the global configuration (serena_config.yml) apply.
|
||||
# Otherwise, this setting overrides the global configuration.
|
||||
# Set this to [] to disable base modes for this project.
|
||||
# Set this to a list of mode names to always include the respective modes for this project.
|
||||
base_modes:
|
||||
|
||||
# list of mode names that are to be activated by default.
|
||||
# The full set of modes to be activated is base_modes + default_modes.
|
||||
# If the setting is undefined, the default_modes from the global configuration (serena_config.yml) apply.
|
||||
# Otherwise, this overrides the setting from the global configuration (serena_config.yml).
|
||||
# This setting can, in turn, be overridden by CLI parameters (--mode).
|
||||
default_modes:
|
||||
|
||||
# initial prompt for the project. It will always be given to the LLM upon activating the project
|
||||
# (contrary to the memories, which are loaded on demand).
|
||||
initial_prompt: ""
|
||||
|
||||
# time budget (seconds) per tool call for the retrieval of additional symbol information
|
||||
# such as docstrings or parameter information.
|
||||
# This overrides the corresponding setting in the global configuration; see the documentation there.
|
||||
# If null or missing, use the setting from the global configuration.
|
||||
symbol_info_budget:
|
||||
@@ -0,0 +1,118 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## Project Overview
|
||||
|
||||
基于 Rockchip RKNN NPU 的文档隐私安全处理 REST API 服务。支持 RK3588 板端和 x86 双平台。
|
||||
|
||||
**功能**：两阶段工作流——先分析检测隐私信息，用户决定后再遮罩处理。
|
||||
|
||||
## 开发环境
|
||||
|
||||
### 初始化
|
||||
|
||||
```bash
|
||||
bash setup_venv.sh
|
||||
source venv/bin/activate
|
||||
```
|
||||
|
||||
### 常用命令
|
||||
|
||||
```bash
|
||||
make test # 运行所有测试
|
||||
make test-one FILE=tests/test_api.py # 单文件测试
|
||||
make lint # 代码检查（ruff）
|
||||
make server # 启动服务（http://0.0.0.0:8000）
|
||||
make verify # 端到端验证（需先启动服务）
|
||||
make clean # 清理缓存和临时文件
|
||||
```
|
||||
|
||||
## API 端点
|
||||
|
||||
| 端点 | 说明 |
|
||||
|------|------|
|
||||
| `POST /api/v1/analyze` | 检测文档中的隐私信息，返回 JSON 报告 |
|
||||
| `POST /api/v1/redact` | 遮罩指定类型的隐私信息，返回处理后文档 |
|
||||
| `GET /api/v1/health` | 服务健康检查 |
|
||||
| `GET /api/v1/types` | 支持的隐私类型列表 |
|
||||
|
||||
### analyze 请求
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8000/api/v1/analyze \
|
||||
-F "file=@document.docx"
|
||||
```
|
||||
|
||||
### redact 请求
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8000/api/v1/redact \
|
||||
-F "file=@document.docx" \
|
||||
-F 'config={"redact_types":["id_card","phone","face"]}' \
|
||||
-o redacted.docx
|
||||
```
|
||||
|
||||
## 项目架构
|
||||
|
||||
```
|
||||
两阶段流水线：
|
||||
analyze: parser_factory → doc_classifier → [regex_detector, ner_detector, face_detector] → DetectionReport
|
||||
redact: analyze → strategy_selector → [text_redactor | image_redactor] → 处理后文档
|
||||
```
|
||||
|
||||
### 文字层策略（字节级安全）
|
||||
|
||||
- PDF（含文字层）/ Word / Excel → `TextRedactor`（删除/替换文字节点）
|
||||
- 图像 / 扫描 PDF → `ImageRedactor`（OpenCV 黑色矩形覆盖）
|
||||
- PDF 有文字层时禁止仅使用图像遮罩（防止文字层可提取）
|
||||
|
||||
### 可检测隐私类型
|
||||
|
||||
| 类型 | 检测方式 | 安全等级 |
|
||||
|------|---------|---------|
|
||||
| id_card | Regex | high |
|
||||
| phone | Regex | high |
|
||||
| bank_card | Regex | high |
|
||||
| face | mediapipe RKNN | high |
|
||||
| email | Regex | medium |
|
||||
| name | 规则词典 NER | medium |
|
||||
| address | 规则词典 NER | medium |
|
||||
| license_plate | Regex | medium |
|
||||
|
||||
## 支持的文档类型
|
||||
|
||||
| 格式 | 解析方式 |
|
||||
|------|---------|
|
||||
| PDF（文字层） | pdfminer.six |
|
||||
| PDF（扫描件） | paddle_ocr RKNN |
|
||||
| .docx | python-docx |
|
||||
| .xlsx | openpyxl |
|
||||
| JPG/PNG | paddle_ocr RKNN |
|
||||
|
||||
## 关键路径
|
||||
|
||||
| 内容 | 路径 |
|
||||
|------|------|
|
||||
| PII 规则配置 | `configs/pii_rules.yaml` |
|
||||
| 姓氏词典 | `configs/surnames.txt` |
|
||||
| paddle_ocr RKNN 模型 | `/data/rockchip/paddle_ocr/models/rknn/PP-OCRv4/` |
|
||||
| mediapipe 人脸模型 | `/data/rockchip/mediapipe/models/rknn/` |
|
||||
|
||||
## RK3588 板端部署
|
||||
|
||||
```bash
|
||||
# 同步代码
|
||||
rsync -av /data/rockchip/info-privacy/ pi@192.168.0.127:/home/pi/Desktop/info-privacy/
|
||||
|
||||
# 板端启动
|
||||
ssh pi@192.168.0.127 "clashon && clashproxy on && \
|
||||
cd /home/pi/Desktop/info-privacy && \
|
||||
bash setup_venv.sh && \
|
||||
source venv/bin/activate && \
|
||||
uvicorn info_privacy.api.main:app --host 0.0.0.0 --port 8000"
|
||||
```
|
||||
|
||||
## 设计文档
|
||||
|
||||
`docs/plans/2026-02-27-info-privacy-design.md`
|
||||
+214
@@ -0,0 +1,214 @@
|
||||
# Info-Privacy 开发指南
|
||||
|
||||
## 项目结构
|
||||
|
||||
```
|
||||
info-privacy/
|
||||
├── src/info_privacy/
|
||||
│ ├── api/
|
||||
│ │ ├── main.py # FastAPI 入口，路由注册
|
||||
│ │ ├── routes/
|
||||
│ │ │ ├── analyze.py # POST /api/v1/analyze
|
||||
│ │ │ └── redact.py # POST /api/v1/redact
|
||||
│ │ └── models.py # Pydantic 请求/响应模型
|
||||
│ ├── classifier/
|
||||
│ │ └── doc_classifier.py # 保密文件拦截（关键词匹配）
|
||||
│ ├── detectors/
|
||||
│ │ ├── regex_detector.py # Regex PII 检测
|
||||
│ │ ├── ner_detector.py # 规则词典姓名/地址检测
|
||||
│ │ └── face_detector.py # MediaPipe RKNN 人脸检测（硬件依赖）
|
||||
│ ├── parsers/
|
||||
│ │ ├── pdf_parser.py # pdfminer 文字层解析
|
||||
│ │ ├── office_parser.py # python-docx / openpyxl
|
||||
│ │ ├── image_parser.py # PaddleOCR RKNN 图像 OCR（硬件依赖）
|
||||
│ │ └── parser_factory.py # 按文件扩展名路由
|
||||
│ ├── redactors/
|
||||
│ │ ├── text_redactor.py # 文字节点替换（字节级安全）
|
||||
│ │ └── image_redactor.py # OpenCV 黑色矩形覆盖
|
||||
│ ├── models.py # 核心数据模型（TextBlock, Entity, DetectionReport）
|
||||
│ └── pipeline.py # 串联 parser → classifier → detector → redactor
|
||||
├── configs/
|
||||
│ ├── pii_rules.yaml # Regex 规则 + 保密关键词 + NER 配置
|
||||
│ └── surnames.txt # 中文姓氏词典（NER 姓名检测）
|
||||
├── tests/
|
||||
│ ├── test_models.py # 数据模型单元测试
|
||||
│ ├── test_classifier.py # 文档分类测试
|
||||
│ ├── test_detectors.py # Regex/NER/人脸检测测试
|
||||
│ ├── test_parsers.py # PDF/Office/图像解析测试
|
||||
│ ├── test_redactors.py # 文字/图像遮罩测试
|
||||
│ ├── test_pipeline.py # 端到端流水线测试
|
||||
│ ├── test_api.py # FastAPI 接口测试
|
||||
│ └── test_simulation.py # RKNN 硬件仿真测试（x86 可运行）
|
||||
├── scripts/
|
||||
│ ├── start_server.sh # 一键启动服务
|
||||
│ └── verify.py # 端到端验证脚本
|
||||
├── docs/plans/ # 设计文档（历史归档）
|
||||
├── models/ # 本地模型软链（指向 sibling 项目）
|
||||
├── tmp/ # 临时文件（已加 .gitignore）
|
||||
├── pyproject.toml
|
||||
├── Makefile
|
||||
└── setup_venv.sh
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 开发环境搭建
|
||||
|
||||
### x86 开发机
|
||||
|
||||
```bash
|
||||
# 1. 初始化虚拟环境
|
||||
bash setup_venv.sh
|
||||
source venv/bin/activate
|
||||
|
||||
# 2. 验证安装
|
||||
python -c "from info_privacy.pipeline import PrivacyPipeline; print('OK')"
|
||||
|
||||
# 3. 运行全量测试
|
||||
make test # 155 passed, 4 skipped（RKNN 相关跳过）
|
||||
make lint # ruff 代码检查
|
||||
```
|
||||
|
||||
### RK3588 板端
|
||||
|
||||
```bash
|
||||
# 同步代码（开发机执行）
|
||||
rsync -av /data/rockchip/info-privacy/ pi@192.168.0.127:/home/pi/Desktop/info-privacy/
|
||||
|
||||
# 板端环境（SSH 执行）
|
||||
cd /home/pi/Desktop/info-privacy
|
||||
bash setup_venv.sh && source venv/bin/activate
|
||||
python -c "from info_privacy.pipeline import PrivacyPipeline; print('OK')"
|
||||
```
|
||||
|
||||
板端需额外安装 `rknn-toolkit-lite2`，OpenCV/NumPy 已预装。
|
||||
|
||||
---
|
||||
|
||||
## 常用开发命令
|
||||
|
||||
```bash
|
||||
make test # 运行所有测试
|
||||
make test-one FILE=tests/test_api.py # 单文件测试
|
||||
make lint # ruff 代码检查
|
||||
make server # 启动服务（--reload 热重载）
|
||||
make verify # 端到端验证（需先启动服务）
|
||||
make clean # 清理 __pycache__ 和 tmp/
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 测试框架说明
|
||||
|
||||
### 分层测试策略
|
||||
|
||||
| 测试文件 | 类型 | 是否需要硬件 |
|
||||
|---------|------|------------|
|
||||
| `test_models.py` | 单元 | 否 |
|
||||
| `test_classifier.py` | 单元 | 否 |
|
||||
| `test_detectors.py` | 单元 | 否（人脸检测用 skip） |
|
||||
| `test_parsers.py` | 单元 | 否（图像解析用 skip） |
|
||||
| `test_redactors.py` | 单元 | 否 |
|
||||
| `test_pipeline.py` | 集成 | 否 |
|
||||
| `test_api.py` | 接口 | 否 |
|
||||
| `test_simulation.py` | 仿真 | 否（mock 替换 RKNN） |
|
||||
| `test_integration_real.py` | 集成（真实文件） | 否 |
|
||||
| `test_integration_complex.py` | 集成（边界/复杂场景） | 否 |
|
||||
|
||||
### RKNN 仿真测试
|
||||
|
||||
`test_simulation.py` 使用 `unittest.mock.patch.dict(sys.modules, ...)` 注入 mock 模块，
|
||||
在 x86 环境下验证 `FaceDetector` 和 `ImageParser` 的业务逻辑（bbox 转换、entity 构建、置信度过滤等）。
|
||||
|
||||
板端真机测试时，`test_detectors.py` 和 `test_parsers.py` 中的 skip 测试将自动运行。
|
||||
|
||||
---
|
||||
|
||||
## 扩展指南
|
||||
|
||||
### 添加新的 PII 类型（Regex）
|
||||
|
||||
1. 在 `configs/pii_rules.yaml` 的 `regex` 节下添加规则：
|
||||
```yaml
|
||||
passport:
|
||||
pattern: '[A-Z]{1,2}\d{7}'
|
||||
security_level: high
|
||||
```
|
||||
2. 在 `src/info_privacy/models.py` 的 `EntityType` 枚举中增加 `PASSPORT = "passport"`
|
||||
3. 在 `tests/test_detectors.py` 中增加对应测试用例
|
||||
|
||||
### 添加新的 PII 类型（NER）
|
||||
|
||||
1. 在 `configs/pii_rules.yaml` 的 `ner` 节下添加触发词配置
|
||||
2. 在 `src/info_privacy/detectors/ner_detector.py` 中实现匹配逻辑
|
||||
3. 添加对应 `EntityType` 枚举值
|
||||
|
||||
### 添加新的文档格式
|
||||
|
||||
1. 在 `src/info_privacy/parsers/` 下创建新的 Parser 类，实现 `parse(file_path) -> (dict, list[TextBlock])` 接口
|
||||
2. 在 `src/info_privacy/parsers/parser_factory.py` 中注册新格式的扩展名映射
|
||||
3. 新 Parser 的 `meta` 字典必须包含 `format`、`has_text_layer`、`redact_strategy` 三个字段
|
||||
|
||||
---
|
||||
|
||||
## 数据流
|
||||
|
||||
```
|
||||
[POST /analyze]
|
||||
文件 → parser_factory → TextBlock[]{text, bbox, page, layer}
|
||||
→ doc_classifier → 如果 blocked: 直接返回 DetectionReport(blocked=True)
|
||||
→ regex_detector + ner_detector + face_detector → Entity[]
|
||||
→ doc_classifier(实体密度评估) → warning?
|
||||
→ 返回 DetectionReport
|
||||
|
||||
[POST /redact]
|
||||
文件 + redact_types → analyze() → Entity[]
|
||||
→ 按 entity.layer 分流：
|
||||
layer=="text" → text_redactor（删除文字节点）
|
||||
layer=="image" → image_redactor（OpenCV 黑框覆盖）
|
||||
→ 输出文档 + X-Security-Report Header
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 关键配置文件
|
||||
|
||||
### configs/pii_rules.yaml
|
||||
|
||||
- `regex`：各类型的检测正则表达式和安全等级
|
||||
- `confidential_keywords`：保密文件拦截关键词列表
|
||||
- `ner.name.surnames_file`：中文姓氏词典路径（相对于项目根目录）
|
||||
- `ner.address.triggers`：地址检测触发词（省/市/区/路/号等）
|
||||
|
||||
### 安全等级说明
|
||||
|
||||
- `high`：遮罩时默认强制处理，体现在 `X-Security-Report` 中
|
||||
- `medium`：遮罩时按用户 `redact_types` 指定处理
|
||||
|
||||
---
|
||||
|
||||
## 常见问题
|
||||
|
||||
**Q: `make test` 显示 4 个 skipped 测试**
|
||||
|
||||
正常现象。这 4 个测试依赖 RKNN 运行时（仅 RK3588 设备可用）。x86 环境下等效的仿真测试在 `test_simulation.py` 中覆盖（15 个）。
|
||||
|
||||
**Q: 身份证号为何不会同时被标注为银行卡**
|
||||
|
||||
系统内置去重逻辑：18 位身份证号满足银行卡格式，但 `RegexDetector` 在检测完成后会过滤掉与 `id_card` 值重叠的 `bank_card` 实体，避免双重计数影响阈值判断。
|
||||
|
||||
**Q: 中文字符前的数字 PII 为何能被检测**
|
||||
|
||||
正则边界使用 `(?<!\d)` / `(?!\d)` 而非 `\b`。Python 3 的 `\b` 是 Unicode-aware，中文字符属于 `\w`，与纯数字之间不形成词边界，故改用纯数字前后断言。
|
||||
|
||||
**Q: 板端运行时 face_detector 报 ImportError**
|
||||
|
||||
确认 MediaPipe-RKNN 项目路径存在：`/data/rockchip/mediapipe/src/mediapipe_rknn/`。FaceDetector 在 `pipeline.py` 中延迟加载，首次 analyze 图像文档时才触发。
|
||||
|
||||
**Q: PDF 遮罩后用 PDF 阅读器仍能选中文字**
|
||||
|
||||
检查 PDF 是否含文字层（`meta["has_text_layer"]`）。文字层 PDF 必须用 `text_redactor`；单纯覆盖图像层无法防止文字提取。
|
||||
|
||||
**Q: 中文姓名检测误报**
|
||||
|
||||
NER 基于姓氏词典 + 上下文触发词（"姓名："、"申请人："等）。调整 `configs/pii_rules.yaml` 中 `ner.name.context_triggers` 可降低误报率。
|
||||
@@ -0,0 +1,23 @@
|
||||
.PHONY: install test test-one lint server verify clean
|
||||
|
||||
install:
|
||||
bash setup_venv.sh
|
||||
|
||||
test:
|
||||
source venv/bin/activate && pytest tests/ -v
|
||||
|
||||
test-one:
|
||||
source venv/bin/activate && pytest $(FILE) -v
|
||||
|
||||
lint:
|
||||
source venv/bin/activate && ruff check src/
|
||||
|
||||
server:
|
||||
source venv/bin/activate && uvicorn info_privacy.api.main:app --host 0.0.0.0 --port 8000 --reload
|
||||
|
||||
verify:
|
||||
source venv/bin/activate && python scripts/verify.py
|
||||
|
||||
clean:
|
||||
find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null; \
|
||||
rm -rf tmp/*.pdf tmp/*.png tmp/*.docx 2>/dev/null; true
|
||||
@@ -0,0 +1,130 @@
|
||||
# Info-Privacy
|
||||
|
||||
基于 Rockchip RKNN NPU 的文档隐私安全处理 REST API 服务，支持 RK3588 板端和 x86 双平台。
|
||||
|
||||
**两阶段工作流**：先 `analyze`（返回 JSON 检测报告）→ 用户决策 → 再 `redact`（返回处理后文档）。
|
||||
|
||||
---
|
||||
|
||||
## 支持的文档格式
|
||||
|
||||
| 格式 | 解析方式 |
|
||||
|------|---------|
|
||||
| PDF（含文字层） | pdfminer.six |
|
||||
| PDF（扫描件） | PaddleOCR RKNN |
|
||||
| .docx | python-docx |
|
||||
| .xlsx | openpyxl |
|
||||
| JPG / PNG | PaddleOCR RKNN |
|
||||
|
||||
## 可检测的隐私类型
|
||||
|
||||
| 类型 | 检测方式 | 安全等级 |
|
||||
|------|---------|---------|
|
||||
| `id_card` | Regex | high |
|
||||
| `phone` | Regex | high |
|
||||
| `bank_card` | Regex | high |
|
||||
| `face` | MediaPipe RKNN | high |
|
||||
| `email` | Regex | medium |
|
||||
| `name` | 规则词典 NER | medium |
|
||||
| `address` | 规则词典 NER | medium |
|
||||
| `license_plate` | Regex | medium |
|
||||
|
||||
---
|
||||
|
||||
## 快速开始
|
||||
|
||||
### x86 开发环境
|
||||
|
||||
```bash
|
||||
# 初始化虚拟环境
|
||||
bash setup_venv.sh
|
||||
source venv/bin/activate
|
||||
|
||||
# 启动服务
|
||||
make server # http://0.0.0.0:8000
|
||||
|
||||
# 端到端验证（需先启动服务）
|
||||
make verify
|
||||
```
|
||||
|
||||
### RK3588 板端
|
||||
|
||||
```bash
|
||||
# 同步代码
|
||||
rsync -av /data/rockchip/info-privacy/ pi@192.168.0.127:/home/pi/Desktop/info-privacy/
|
||||
|
||||
# 板端启动
|
||||
ssh pi@192.168.0.127 "clashon && clashproxy on && \
|
||||
cd /home/pi/Desktop/info-privacy && \
|
||||
bash setup_venv.sh && source venv/bin/activate && \
|
||||
uvicorn info_privacy.api.main:app --host 0.0.0.0 --port 8000"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## API 使用示例
|
||||
|
||||
### 检测隐私信息
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8000/api/v1/analyze \
|
||||
-F "file=@document.docx"
|
||||
```
|
||||
|
||||
响应：
|
||||
```json
|
||||
{
|
||||
"doc_id": "uuid",
|
||||
"classification": "normal",
|
||||
"blocked": false,
|
||||
"warning": null,
|
||||
"entities": [
|
||||
{"id": "e1", "type": "id_card", "value": "110101199001011234",
|
||||
"page": 1, "bbox": [10, 20, 200, 40], "layer": "text", "security_level": "high"}
|
||||
],
|
||||
"summary": {"id_card": 1}
|
||||
}
|
||||
```
|
||||
|
||||
### 遮罩指定类型
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8000/api/v1/redact \
|
||||
-F "file=@document.docx" \
|
||||
-F 'config={"redact_types":["id_card","phone"]}' \
|
||||
-o redacted.docx
|
||||
```
|
||||
|
||||
响应 Body 为处理后文档，Header `X-Security-Report` 包含遮罩统计。
|
||||
|
||||
### 其他端点
|
||||
|
||||
```bash
|
||||
GET /api/v1/health # 健康检查
|
||||
GET /api/v1/types # 支持的隐私类型列表
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 遮罩安全保证
|
||||
|
||||
系统根据文档类型**自动选择最安全策略**，用户无需指定：
|
||||
|
||||
| 文档类型 | 遮罩方式 | 安全保证 |
|
||||
|---------|---------|---------|
|
||||
| PDF（文字层）/ Word / Excel | 删除文字节点（text_replace） | 字节级安全 |
|
||||
| 图像 / 扫描 PDF | OpenCV 黑色矩形覆盖（image_mask） | 图像级安全 |
|
||||
|
||||
> **重要**：PDF 含文字层时，即使 `redact_types` 包含 `face`，也强制使用 text_replace 防止文字层信息泄露。
|
||||
|
||||
---
|
||||
|
||||
## RKNN 模型路径
|
||||
|
||||
| 模型 | 路径 |
|
||||
|------|------|
|
||||
| PaddleOCR 检测 | `/data/rockchip/paddle_ocr/models/rknn/PP-OCRv4/det/` |
|
||||
| PaddleOCR 识别 | `/data/rockchip/paddle_ocr/models/rknn/PP-OCRv4/rec/` |
|
||||
| MediaPipe 人脸 | `/data/rockchip/mediapipe/models/rknn/` |
|
||||
|
||||
详见 [DEVELOP.md](DEVELOP.md) · [RELEASE.md](RELEASE.md)
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
# 发布记录
|
||||
|
||||
## v0.1.1 (2026-02-27)
|
||||
|
||||
### Bug 修复
|
||||
|
||||
- **Regex 边界修复**：`id_card` / `bank_card` 正则将 `\b` 改为 `(?<!\d)` / `(?!\d)`，修复中文字符紧邻数字时 PII 漏检问题（Python 3 `\b` 为 Unicode-aware）
|
||||
- **身份证/银行卡去重**：`RegexDetector` 新增去重逻辑，18 位身份证号不再被同时识别为银行卡，消除实体数量虚增导致阈值误判
|
||||
- **DOCX 表格单元格遮罩**：`TextRedactor._redact_docx` 补全表格单元格遍历，修复隐藏在表格中的 PII 无法遮罩的问题
|
||||
- **DOCX 表格单元格解析**：`OfficeParser._parse_docx` 补全表格单元格遍历，修复表格中 PII 漏检问题
|
||||
|
||||
### 测试增强（155 passed，4 skipped）
|
||||
|
||||
| 测试文件 | 新增测试 | 覆盖内容 |
|
||||
|---------|---------|---------|
|
||||
| `test_detectors.py` | +10 | 银行卡检测、车牌检测、身份证/银行卡去重、末位X、security_level、多实体同块、NER 组合 |
|
||||
| `test_classifier.py` | +4 | 全部 10 个保密关键词（parametrize）、阈值边界（4/5）、中风险不触发警告 |
|
||||
| `test_api.py` | +5 | X-Security-Report 头、xlsx 格式、无 PII 返回 normal、summary 一致性、保密文档返回 403 |
|
||||
| `test_pipeline.py` | +4 | summary 契约验证、无 PII 空实体、多实体触发警告、多类型同时遮罩 |
|
||||
| `test_redactors.py` | +4 | Excel 遮罩、DOCX 表格单元格遮罩、████遮罩字符验证、text层实体不触发图像遮罩 |
|
||||
|
||||
---
|
||||
|
||||
## v0.1.0 (2026-02-27)
|
||||
|
||||
### 新功能
|
||||
|
||||
**API 服务（FastAPI）：**
|
||||
- `POST /api/v1/analyze`：文档隐私检测，返回 JSON 报告（实体列表 + 分类 + 摘要）
|
||||
- `POST /api/v1/redact`：按指定类型遮罩，返回处理后文档（格式与输入一致）
|
||||
- `GET /api/v1/health`：服务健康检查
|
||||
- `GET /api/v1/types`：支持的隐私类型列表
|
||||
|
||||
**文档解析（双平台）：**
|
||||
- PDF 文字层解析（pdfminer.six）
|
||||
- Word / Excel 解析（python-docx / openpyxl）
|
||||
- 图像 OCR（PaddleOCR RKNN，RK3588 硬件）
|
||||
- Parser Factory 按文件扩展名自动路由
|
||||
|
||||
**隐私检测：**
|
||||
- Regex 检测：身份证、手机号、银行卡、邮箱、车牌（共 5 类）
|
||||
- 规则词典 NER：中文姓名（姓氏词典 + 上下文）、地址（触发词匹配）
|
||||
- 人脸检测：MediaPipe RKNN（RK3588 硬件，pipeline 延迟加载）
|
||||
- 文档分类前置拦截：保密文件关键词检测，`classified` 类直接返回 403
|
||||
|
||||
**遮罩处理（字节级安全）：**
|
||||
- TextRedactor：删除 docx/xlsx 文字节点，字节级安全
|
||||
- ImageRedactor：OpenCV 黑色矩形覆盖图像层实体
|
||||
- 策略自动选择：含文字层 PDF 强制 text_replace，防止文字层泄露
|
||||
|
||||
**测试覆盖（初始 50 passed，4 skipped）：**
|
||||
- 8 个测试文件，覆盖数据模型 / 分类器 / 检测器 / 解析器 / 遮罩器 / 流水线 / API 接口
|
||||
- `test_simulation.py`：15 个 RKNN 仿真测试（mock 替换硬件依赖，x86 全量可运行）
|
||||
|
||||
### 支持平台
|
||||
|
||||
| 平台 | Python | 测试状态 |
|
||||
|------|--------|---------|
|
||||
| x86 / Ubuntu 22.04+ | 3.10 – 3.12 | 46 passed（4 RKNN skip） |
|
||||
| RK3588 / Ubuntu 24.04 | 3.12.3 | 50 passed（含 RKNN）|
|
||||
|
||||
### 依赖版本
|
||||
|
||||
| 依赖 | 版本要求 |
|
||||
|------|---------|
|
||||
| fastapi | ≥ 0.110.0 |
|
||||
| uvicorn | ≥ 0.29.0 |
|
||||
| pdfminer.six | ≥ 20221105 |
|
||||
| pypdf | ≥ 4.0.0 |
|
||||
| python-docx | ≥ 1.1.0 |
|
||||
| openpyxl | ≥ 3.1.0 |
|
||||
| opencv-python | ≥ 4.5.5 |
|
||||
| numpy | ≥ 1.24.0 |
|
||||
| rknn-toolkit-lite2 | 2.3.2（仅 RK3588）|
|
||||
@@ -0,0 +1,58 @@
|
||||
# PII 正则规则
|
||||
regex:
|
||||
id_card:
|
||||
pattern: '(?<!\d)([1-9]\d{5})(19|20)\d{2}(0[1-9]|1[0-2])(0[1-9]|[12]\d|3[01])\d{3}[\dX](?!\d)'
|
||||
security_level: high
|
||||
phone:
|
||||
pattern: '(?<!\d)((\+?86[-\s]?)?1[3-9]\d{9})(?!\d)'
|
||||
security_level: high
|
||||
bank_card:
|
||||
pattern: '(?<!\d)([1-9]\d{15,18})(?!\d)'
|
||||
security_level: high
|
||||
email:
|
||||
pattern: '\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b'
|
||||
security_level: medium
|
||||
license_plate:
|
||||
pattern: '[京津沪渝冀豫云辽黑湘皖鲁新苏浙赣鄂桂甘晋蒙陕吉闽贵粤川青藏琼宁夏][A-Z][A-Z0-9]{5,6}'
|
||||
security_level: medium
|
||||
|
||||
# 保密关键词
|
||||
confidential_keywords:
|
||||
- 机密
|
||||
- 绝密
|
||||
- 保密
|
||||
- 内部文件
|
||||
- 内部资料
|
||||
- CONFIDENTIAL
|
||||
- SECRET
|
||||
- TOP SECRET
|
||||
- RESTRICTED
|
||||
- FOR INTERNAL USE ONLY
|
||||
|
||||
# OCR + 人脸检测模型路径（Rust 子进程调用）
|
||||
rknn:
|
||||
ocr:
|
||||
det_model: /data/rockchip/paddle_ocr/models/rknn/PP-OCRv4/det/ppocrv4_det_ch_int8.rknn
|
||||
rec_model: /data/rockchip/paddle_ocr/models/rknn/PP-OCRv4/rec/ppocrv4_rec_ch_fp16.rknn
|
||||
dict_path: /data/rockchip/rknn_model_zoo/examples/PPOCR/PPOCR-System/model/ppocr_keys_v1.txt
|
||||
target: rk3588
|
||||
face:
|
||||
model: /data/rockchip/mediapipe/models/rknn/face_detection_short_range_rk3588.rknn
|
||||
|
||||
# 中文姓名词典规则
|
||||
ner:
|
||||
name:
|
||||
security_level: medium
|
||||
surnames_file: configs/surnames.txt
|
||||
address:
|
||||
security_level: medium
|
||||
triggers:
|
||||
- 省
|
||||
- 市
|
||||
- 区
|
||||
- 县
|
||||
- 街道
|
||||
- 路
|
||||
- 号
|
||||
- 小区
|
||||
- 村
|
||||
@@ -0,0 +1,101 @@
|
||||
# 常见中文姓氏
|
||||
李
|
||||
王
|
||||
张
|
||||
刘
|
||||
陈
|
||||
杨
|
||||
黄
|
||||
赵
|
||||
吴
|
||||
周
|
||||
徐
|
||||
孙
|
||||
马
|
||||
朱
|
||||
胡
|
||||
郭
|
||||
何
|
||||
高
|
||||
林
|
||||
郑
|
||||
谢
|
||||
罗
|
||||
梁
|
||||
宋
|
||||
唐
|
||||
许
|
||||
邓
|
||||
韩
|
||||
冯
|
||||
曹
|
||||
彭
|
||||
曾
|
||||
萧
|
||||
田
|
||||
董
|
||||
潘
|
||||
袁
|
||||
蔡
|
||||
蒋
|
||||
余
|
||||
于
|
||||
杜
|
||||
叶
|
||||
程
|
||||
苏
|
||||
魏
|
||||
吕
|
||||
丁
|
||||
任
|
||||
卢
|
||||
姚
|
||||
沈
|
||||
钟
|
||||
姜
|
||||
崔
|
||||
谭
|
||||
陆
|
||||
汪
|
||||
范
|
||||
廖
|
||||
石
|
||||
金
|
||||
韦
|
||||
贾
|
||||
夏
|
||||
付
|
||||
方
|
||||
邹
|
||||
熊
|
||||
白
|
||||
孟
|
||||
秦
|
||||
邱
|
||||
侯
|
||||
江
|
||||
尹
|
||||
薛
|
||||
闫
|
||||
段
|
||||
雷
|
||||
龙
|
||||
黎
|
||||
史
|
||||
陶
|
||||
贺
|
||||
顾
|
||||
毛
|
||||
郝
|
||||
龚
|
||||
邵
|
||||
万
|
||||
钱
|
||||
严
|
||||
覃
|
||||
武
|
||||
戴
|
||||
莫
|
||||
孔
|
||||
向
|
||||
汤
|
||||
@@ -0,0 +1,406 @@
|
||||
# 专利技术交底书
|
||||
|
||||
**项目名称**：基于 RKNN NPU 的多格式文档隐私信息检测与安全遮罩处理系统
|
||||
**文档日期**：2026-02-27
|
||||
**技术领域**：信息安全 / 文档处理 / 边缘计算
|
||||
**技术负责人**：待填写
|
||||
|
||||
---
|
||||
|
||||
## 一、技术先进性评估
|
||||
|
||||
### 1.1 现有技术缺陷
|
||||
|
||||
| 现有方案 | 技术缺陷 |
|
||||
|---------|---------|
|
||||
| Adobe Acrobat PDF 编辑 | 仅支持 PDF，图像覆盖不清除文字层，文字层仍可被复制提取 |
|
||||
| AWS Comprehend / 百度文字识别 | 云端 SaaS，敏感文档不可出境；仅检测不遮罩；无多格式支持 |
|
||||
| Microsoft Presidio | 纯 x86 软件，无 NPU 硬件加速；不支持图像/扫描件；无遮罩输出 |
|
||||
| 通用 OCR + 正则 | 不处理文档原生文字层；无 PDF 内容流级别遮罩；缺乏文档分类拦截 |
|
||||
| 传统 PDF 字节替换 | 忽略 PDF 多层压缩滤波器，在压缩流上直接替换字节会破坏文件结构 |
|
||||
|
||||
### 1.2 本系统核心创新点
|
||||
|
||||
本系统在以下四个技术维度具有明显的先进性：
|
||||
|
||||
**① 分层实体管理与自适应安全遮罩策略**
|
||||
现有技术将文字遮罩和图像遮罩视为独立操作；本系统通过"层标记"（`layer: text | image`）统一管理实体来源，自动检测文档是否含文字层，强制选择最安全遮罩路径，杜绝文字层信息泄露。
|
||||
|
||||
**② PDF 内容流多层滤波器安全替换**
|
||||
现有 PDF 遮罩工具在字节层直接替换，无法处理叠加 ASCII85Decode + FlateDecode 的内容流。本系统采用完全解压→等长空格替换→单层重压缩的处理流程，保证结构完整性的同时实现字节级内容清除。
|
||||
|
||||
**③ 面向中文语义的复合 PII 检测**
|
||||
针对中文字符与数字混排时 Unicode `\b` 边界失效问题，采用纯数字前后断言（`(?<!\d)` / `(?!\d)`）代替词边界；结合姓氏词典驱动的轻量 NER，实现无 ML 模型依赖的中文姓名/地址识别，以及 id_card/bank_card 跨类型去重机制。
|
||||
|
||||
**④ 边缘端 RKNN NPU 多模态推理集成**
|
||||
在 RK3588 嵌入式设备上集成 PaddleOCR（检测+识别两阶段）和 MediaPipe 人脸检测的 RKNN 量化模型，实现离线、本地化、硬件加速的图像隐私检测，无需连接云端。
|
||||
|
||||
---
|
||||
|
||||
## 二、发明名称
|
||||
|
||||
**面向多格式文档的两阶段隐私信息自适应检测与字节级安全遮罩方法及系统**
|
||||
|
||||
---
|
||||
|
||||
## 三、技术领域
|
||||
|
||||
本发明属于信息安全与文档处理技术领域，具体涉及多格式电子文档（PDF、Word、Excel、图像）中个人隐私信息（PII）的自动检测、分类评估及字节级安全遮罩处理方法。
|
||||
|
||||
---
|
||||
|
||||
## 四、背景技术
|
||||
|
||||
随着数字办公普及，含有个人隐私信息的电子文档在企业内部流转量大幅增加。《个人信息保护法》《数据安全法》等法规对文档中的身份证号、手机号、人脸图像等 PII 提出了严格的合规处理要求。
|
||||
|
||||
现有技术存在以下不足：
|
||||
|
||||
1. **文字层泄露问题**：对含文字层 PDF 仅做图像覆盖，无法阻止通过复制粘贴或搜索提取隐私信息。
|
||||
2. **PDF 结构破坏问题**：直接在 PDF 二进制文件中替换字节，忽略内容流的多层压缩编码（如 ASCII85Decode + FlateDecode），导致输出文件损坏或替换失效。
|
||||
3. **单格式局限**：现有工具多针对单一文档格式，无法跨 PDF/Word/Excel/图像提供一致的 PII 处理能力。
|
||||
4. **云端依赖**：主流 PII 检测服务为云端 SaaS，含敏感信息的文档无法上传外部服务器，与隐私保护目标相悖。
|
||||
5. **图像文档盲区**：针对扫描件和图像的 OCR + PII 检测缺乏嵌入式硬件加速，在边缘设备上实时性不足。
|
||||
|
||||
---
|
||||
|
||||
## 五、发明内容
|
||||
|
||||
### 5.1 发明目标
|
||||
|
||||
提供一种多格式文档隐私信息的两阶段处理方法，该方法：
|
||||
- 支持 PDF（文字层/扫描件）、Word、Excel、图像等格式
|
||||
- 自动选择最安全遮罩策略，防止文字层信息泄露
|
||||
- 实现 PDF 内容流的完整解码-替换-重压缩流程
|
||||
- 支持嵌入式 NPU 硬件加速的图像隐私检测
|
||||
- 全流程本地化处理，无需连接外部服务器
|
||||
|
||||
### 5.2 技术方案
|
||||
|
||||
#### 5.2.1 总体架构（两阶段工作流）
|
||||
|
||||
本发明将文档隐私处理分为**分析阶段**和**遮罩阶段**两步，用户在中间进行决策：
|
||||
|
||||
```
|
||||
[分析阶段]
|
||||
输入文档
|
||||
↓ 解析器工厂（按扩展名路由）
|
||||
TextBlock[] {text, bbox, page, layer}
|
||||
↓ 文档分类器（Step 0：保密拦截）
|
||||
[若 classified] → 返回 403，中止
|
||||
↓ 三层检测器并行运行（Step 1）
|
||||
Entity[] {type, value, bbox, page, layer, security_level}
|
||||
↓ 文档分类器（Step 3：密度评估）
|
||||
DetectionReport {entities, classification, warning}
|
||||
|
||||
[用户决策]
|
||||
选择要遮罩的 PII 类型 → redact_types
|
||||
|
||||
[遮罩阶段]
|
||||
重新分析 → 筛选目标实体 → 按 layer 分流
|
||||
text 层 → TextRedactor（字节级）
|
||||
image 层 → ImageRedactor（像素级）
|
||||
→ 输出安全文档 + X-Security-Report
|
||||
```
|
||||
|
||||
两阶段设计允许用户在看到检测报告后自主决定遮罩范围，避免过度或不足遮罩。
|
||||
|
||||
#### 5.2.2 分层实体管理与自适应策略选择（核心发明一）
|
||||
|
||||
**问题**：PDF 文档同时包含可提取的文字层和渲染后的图像层。仅对图像层做像素覆盖，无法清除文字层中的隐私信息。
|
||||
|
||||
**方案**：
|
||||
|
||||
1. 解析阶段：每个 `TextBlock` 携带 `layer` 字段（`"text"` 或 `"image"`），标记内容来源（pdfminer 提取 vs OCR 识别）。
|
||||
|
||||
2. PDF 文字层检测：`PdfParser` 通过 pdfminer 解析，若成功提取到文本块则记录 `has_text_layer=True`，并将 `redact_strategy` 置为 `"text_replace"`。
|
||||
|
||||
3. 自适应策略选择规则：
|
||||
|
||||
```
|
||||
if 文档含文字层 (has_text_layer=True):
|
||||
强制使用 TextRedactor（清除文字节点）
|
||||
即使 redact_types 包含 face，也先做文字层遮罩
|
||||
else:
|
||||
使用 ImageRedactor（像素覆盖）
|
||||
```
|
||||
|
||||
4. 混合遮罩：同一遮罩操作中，文字实体和图像实体分流处理，文字层遮罩结果作为图像层遮罩的输入（管道式）。
|
||||
|
||||
**技术效果**：彻底消除文字层泄露风险；`X-Security-Report` 明确标注 `security_guarantee: "byte_level"` 或 `"image_level"`，用户可验证遮罩强度。
|
||||
|
||||
#### 5.2.3 PDF 内容流多层滤波器解码替换方法（核心发明二）
|
||||
|
||||
**问题**：PDF 内容流可能叠加多种压缩滤波器（如 `/ASCII85Decode /FlateDecode`）。在压缩字节上直接搜索替换 UTF-8 字符串无效；使用 pypdf 内置 `set_data()` 仅支持单层 FlateDecode。
|
||||
|
||||
**方案**：采用"完全解压 → 明文替换 → 单层重压缩"三步流程：
|
||||
|
||||
```python
|
||||
# Step 1：完全解压（自动处理所有 filter 层）
|
||||
decoded = stream_obj.get_data()
|
||||
|
||||
# Step 2：在明文字节中等长替换
|
||||
for val in values:
|
||||
encoded = val.encode("utf-8")
|
||||
decoded = decoded.replace(encoded, b" " * len(encoded))
|
||||
|
||||
# Step 3：手动 zlib 压缩 + 更新 filter 为单层 FlateDecode
|
||||
if decoded_modified != decoded_original:
|
||||
stream_obj._data = zlib.compress(decoded_modified)
|
||||
stream_obj[NameObject("/Filter")] = NameObject("/FlateDecode")
|
||||
stream_obj.pop(NameObject("/DecodeParms"), None)
|
||||
```
|
||||
|
||||
关键设计决策：
|
||||
- **等长替换**（用等量空格代替目标字符串）：内容流字节偏移量不变，避免破坏下游对象的字节偏移引用
|
||||
- **单层输出**：消除原始多层 filter，简化文件结构，提高兼容性
|
||||
- **异常跳过**：对无法解码的流（图像等二进制数据）静默跳过，避免破坏非文字内容
|
||||
|
||||
**技术效果**：完整支持 ASCII85+FlateDecode、LZWDecode+FlateDecode 等多种叠加 filter 组合；输出文件可被所有标准 PDF 阅读器正常打开；字节级安全保证，不可通过 PDF 工具恢复被遮罩内容。
|
||||
|
||||
#### 5.2.4 面向中文语义的复合 PII 检测方法（核心发明三）
|
||||
|
||||
**问题一（Unicode 边界）**：Python 3 的 `\b` 词边界断言为 Unicode-aware，中文字符（CJK Unified Ideographs）属于 `\w`，导致"证件号110101..."中 '号' 与 '1' 之间无边界，身份证正则漏检。
|
||||
|
||||
**方案**：以数字专属断言替代通用词边界：
|
||||
- `(?<!\d)` 替代 `\b`（左侧）：仅要求左侧非数字字符
|
||||
- `(?!\d)` 替代 `\b`（右侧）：仅要求右侧非数字字符
|
||||
|
||||
适用规则：`id_card`、`bank_card`、`phone`
|
||||
|
||||
**问题二（跨类型重叠）**：18 位中国身份证号格式满足 16-19 位银行卡检测规则，同一数值被重复分类，导致实体密度虚增触发误警告。
|
||||
|
||||
**方案**：检测完成后应用去重过滤：
|
||||
```python
|
||||
id_card_values = {e.value for e in entities if e.type == EntityType.ID_CARD}
|
||||
entities = [e for e in entities
|
||||
if not (e.type == EntityType.BANK_CARD and e.value in id_card_values)]
|
||||
```
|
||||
|
||||
**问题三（中文姓名）**：中文命名实体识别（NER）通常依赖大型 ML 模型，不适合嵌入式部署。
|
||||
|
||||
**方案**：采用姓氏词典驱动的轻量规则 NER：
|
||||
- 维护 100+ 常见中文姓氏词典（`configs/surnames.txt`，可扩展）
|
||||
- 编译模式：`(姓氏列表)|后接 1-3 个中文字符 + 上下文边界`
|
||||
- 地址检测：基于触发词表（省/市/区/路/号等），支持 YAML 配置扩展
|
||||
|
||||
**技术效果**：无需 ML 模型即可实现中文 PII 检测；彻底消除 Unicode 边界导致的漏检；去重机制确保实体密度评估准确性。
|
||||
|
||||
#### 5.2.5 边缘端 RKNN NPU 多模态隐私检测（核心发明四）
|
||||
|
||||
**方案**：在 RK3588 嵌入式设备上集成两个 RKNN 量化模型：
|
||||
|
||||
1. **PaddleOCR RKNN（图像文字识别）**：
|
||||
- 检测模型（PP-OCRv4 det，INT8 量化）：输出文字区域四边形坐标
|
||||
- 识别模型（PP-OCRv4 rec，FP16 量化）：批量文字识别
|
||||
- 旋转文本校正：`get_rotate_crop_image()` 对倾斜文字框进行仿射变换后识别
|
||||
- 置信度过滤：识别置信度 < 0.5 的结果丢弃
|
||||
|
||||
2. **MediaPipe 人脸检测 RKNN**：
|
||||
- 模型：`face_detection_short_range_rk3588.rknn`（Short-Range 近景人脸）
|
||||
- 延迟加载：首次需要时才初始化 RKNN 运行时，节省内存
|
||||
- 输出：像素级人脸框坐标，直接用于 OpenCV 图像遮罩
|
||||
|
||||
3. **跨平台兼容**：通过 `sys.modules` 动态注入，x86 环境使用 mock 或 RKNN-Toolkit2 仿真，RK3588 环境使用 RKNN-Toolkit-Lite2 硬件推理，API 接口完全一致。
|
||||
|
||||
**技术效果**：离线本地化处理，无数据出境风险；RKNN INT8 量化使 OCR 吞吐量相比 CPU 推理提升 3-5×；人脸检测与文字 PII 检测统一在同一 pipeline 中处理。
|
||||
|
||||
---
|
||||
|
||||
## 六、权利要求（草案）
|
||||
|
||||
### 独立权利要求
|
||||
|
||||
**权利要求 1**（方法权利要求）
|
||||
|
||||
一种多格式文档隐私信息的两阶段安全处理方法，其特征在于，包括以下步骤：
|
||||
|
||||
（1）**文档解析步骤**：接收输入文档，根据文档格式调用相应解析器，提取文本块集合，每个文本块包含文本内容、边界框坐标、页码及层标记，所述层标记区分文字层（原生文字提取）与图像层（光学字符识别）；
|
||||
|
||||
（2）**分类拦截步骤**：对文本块进行保密标记关键词扫描，若发现预定义保密标记则立即中止并返回拦截响应；
|
||||
|
||||
（3）**多层检测步骤**：对通过拦截检查的文档，并行运行至少两种检测器：基于规则的正则表达式检测器用于数值型 PII，规则词典命名实体识别检测器用于语义型 PII；对图像层文档额外运行神经网络人脸检测器；
|
||||
|
||||
（4）**策略选择步骤**：根据文档是否包含可提取文字层，自动选择遮罩策略：含文字层文档强制采用文字节点替换策略，纯图像文档采用像素覆盖策略；
|
||||
|
||||
（5）**分层遮罩步骤**：依据用户指定的遮罩类型，对文字层实体执行字节级文字节点替换，对图像层实体执行像素级黑色矩形覆盖，两种操作管道式衔接；
|
||||
|
||||
（6）**输出步骤**：返回与输入格式一致的遮罩后文档，并附带包含遮罩统计及安全保证等级的安全报告。
|
||||
|
||||
**权利要求 2**（PDF 处理方法权利要求）
|
||||
|
||||
根据权利要求 1 所述的方法，其特征在于，所述文字节点替换策略中针对 PDF 格式的处理步骤包括：
|
||||
|
||||
（a）读取 PDF 页面内容流对象；
|
||||
|
||||
（b）对内容流应用完整解码操作，自动处理叠加的多层压缩滤波器，获得明文内容流字节序列；
|
||||
|
||||
（c）在明文字节序列中对目标隐私字符串执行等长空格字节替换；
|
||||
|
||||
（d）对修改后的明文字节序列重新进行 Deflate 压缩，以单层 FlateDecode 滤波器写回内容流；
|
||||
|
||||
（e）移除原有多层滤波器参数，保持 PDF 内部字节偏移引用完整性。
|
||||
|
||||
**权利要求 3**（中文 PII 检测方法权利要求）
|
||||
|
||||
根据权利要求 1 所述的方法，其特征在于，所述正则表达式检测器中针对中文文档的改进包括：
|
||||
|
||||
（a）对数值型 PII 正则规则，以数字专属负向后顾断言 `(?<!\d)` 和负向前瞻断言 `(?!\d)` 替代 Unicode 词边界断言 `\b`，解决中文字符与数字混排时词边界失效问题；
|
||||
|
||||
（b）对同一数值可能匹配多种 PII 类型的情形，实施跨类型去重：以更具体的匹配类型（身份证）优先，过滤与之重叠的宽泛匹配类型（银行卡）。
|
||||
|
||||
**权利要求 4**（系统权利要求）
|
||||
|
||||
一种多格式文档隐私信息安全处理系统，其特征在于，包括：
|
||||
|
||||
- **解析器工厂模块**：根据文档扩展名路由至对应解析器，输出统一格式的文本块集合；
|
||||
- **文档分类模块**：实现两级分类评估，包括基于关键词的硬拦截和基于实体密度的软警告；
|
||||
- **隐私检测模块**：包含正则检测器、规则词典 NER 检测器和神经网络人脸检测器三个子模块；
|
||||
- **策略选择模块**：根据文档文字层特征自动确定遮罩策略；
|
||||
- **遮罩执行模块**：包含文字层遮罩子模块和图像层遮罩子模块，支持管道式组合；
|
||||
- **安全报告模块**：输出遮罩统计、策略类型及安全保证等级。
|
||||
|
||||
### 从属权利要求
|
||||
|
||||
**权利要求 5**
|
||||
根据权利要求 1 所述的方法，其特征在于，所述规则词典命名实体识别检测器对中文姓名的检测步骤为：从姓氏词典文件动态加载姓氏集合，编译正则模式匹配"姓氏 + 1 至 3 个连续中文字符"结构，并通过文本上下文边界约束降低误识别率。
|
||||
|
||||
**权利要求 6**
|
||||
根据权利要求 4 所述的系统，其特征在于，所述神经网络人脸检测器在嵌入式 NPU 设备上运行经量化优化的 RKNN 格式神经网络模型，并采用延迟初始化策略，仅在首次处理图像文档时加载模型至 NPU，减少系统启动开销。
|
||||
|
||||
**权利要求 7**
|
||||
根据权利要求 1 所述的方法，其特征在于，还包括跨平台兼容步骤：神经网络推理模块通过运行时动态模块注入机制，在嵌入式 NPU 设备上使用硬件推理运行时，在通用 x86 设备上使用软件仿真运行时，两种环境下暴露相同接口。
|
||||
|
||||
---
|
||||
|
||||
## 七、说明书附图（逻辑结构）
|
||||
|
||||
### 图 1：系统总体架构
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ REST API 层 (FastAPI) │
|
||||
│ POST /analyze POST /redact │
|
||||
└────────────────────┬──────────────────┬─────────────────┘
|
||||
│ │
|
||||
▼ ▼
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ PrivacyPipeline │
|
||||
│ │
|
||||
│ ┌──────────────┐ ┌──────────────┐ │
|
||||
│ │ ParserFactory │ │ DocClassifier│ │
|
||||
│ │ PDF → PDF │ │ keyword │ │
|
||||
│ │ image → OCR │ │ density │ │
|
||||
│ │ docx/xlsx │ └──────────────┘ │
|
||||
│ └──────┬───────┘ │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ ┌──────────────────────────────────┐ │
|
||||
│ │ 检测器层 (并行) │ │
|
||||
│ │ RegexDetector NERDetector │ │
|
||||
│ │ (id/phone/ (name/address) │ │
|
||||
│ │ bank/email) │ │
|
||||
│ │ FaceDetector(RKNN) │ │
|
||||
│ └──────────────────────────────────┘ │
|
||||
│ │
|
||||
│ ┌──────────────────────────────────┐ │
|
||||
│ │ 遮罩器层 (管道式) │ │
|
||||
│ │ TextRedactor ImageRedactor │ │
|
||||
│ │ (PDF/DOCX/XLSX) (PNG/JPG/PDF) │ │
|
||||
│ └──────────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### 图 2：PDF 内容流遮罩流程
|
||||
|
||||
```
|
||||
PDF 原始文件
|
||||
│
|
||||
▼
|
||||
读取 /Contents 流对象
|
||||
│
|
||||
▼
|
||||
stream.get_data()
|
||||
┌──────────────────────┐
|
||||
│ 自动解码所有滤波器 │
|
||||
│ ASCII85 + FlateDec │
|
||||
│ → 明文 PS 操作指令 │
|
||||
└──────────┬───────────┘
|
||||
│
|
||||
▼
|
||||
等长空格替换目标字符串
|
||||
"110101199001011234"
|
||||
→ " "（18空格）
|
||||
│
|
||||
▼
|
||||
zlib.compress(明文字节)
|
||||
stream._data = 压缩结果
|
||||
stream["/Filter"] = "/FlateDecode"
|
||||
│
|
||||
▼
|
||||
PdfWriter.write() → 输出文件
|
||||
```
|
||||
|
||||
### 图 3：自适应遮罩策略决策
|
||||
|
||||
```
|
||||
文档输入
|
||||
│
|
||||
▼
|
||||
┌───────────────┐
|
||||
│ 文字层检测 │
|
||||
│ pdfminer │
|
||||
└───────┬───────┘
|
||||
│
|
||||
┌───────┴───────┐
|
||||
│ │
|
||||
有文字层 无文字层
|
||||
│ │
|
||||
▼ ▼
|
||||
TextRedactor ImageRedactor
|
||||
(字节级安全) (像素级安全)
|
||||
DOCX/XLSX PNG/JPG
|
||||
PDF(text) PDF(scan)
|
||||
│ │
|
||||
└───────┬───────┘
|
||||
▼
|
||||
X-Security-Report
|
||||
{strategy, security_guarantee}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 八、说明书摘要
|
||||
|
||||
本发明公开了一种面向多格式文档的两阶段隐私信息自适应检测与字节级安全遮罩处理方法及系统。系统将隐私处理分为分析和遮罩两个独立阶段，中间允许用户基于检测报告进行决策。分析阶段采用保密关键词拦截、正则表达式、规则词典 NER 和 RKNN NPU 加速的神经网络人脸检测构成的三层检测体系；每个检测实体携带层标记区分文字来源与图像来源。遮罩阶段根据文档是否包含可提取文字层自动选择最安全遮罩策略：含文字层文档强制执行文字节点替换（字节级）；对 PDF 格式采用完整滤波器解码-等长替换-单层重压缩的内容流处理流程，支持多层叠加压缩滤波器；纯图像文档采用 OpenCV 像素级覆盖。系统支持在 Rockchip RK3588 嵌入式 NPU 硬件和 x86 通用处理器双平台部署，全流程本地化处理，无需连接外部服务器。
|
||||
|
||||
**关键词**：隐私信息保护、文档遮罩、PDF 内容流、RKNN NPU、两阶段处理、自适应策略
|
||||
|
||||
---
|
||||
|
||||
## 九、对比现有技术的实质性特点
|
||||
|
||||
| 技术特征 | 本发明 | 现有技术 |
|
||||
|---------|--------|---------|
|
||||
| PDF 多层滤波器处理 | ✅ 完整解码所有 filter，等长替换，单层重压缩 | ❌ 直接字节替换（破坏结构）或仅支持单层 filter |
|
||||
| 文字层泄露防护 | ✅ 自动检测文字层，含文字层强制 text_replace | ❌ 图像遮罩不清除文字层 |
|
||||
| 实体层标记 | ✅ text/image 双层，同一类型区分来源 | ❌ 无来源区分，统一处理 |
|
||||
| 中文 Unicode 边界 | ✅ `(?<!\d)` 数字专属断言 | ❌ `\b` 在中文环境失效 |
|
||||
| 跨类型 PII 去重 | ✅ id_card 值从 bank_card 中过滤 | ❌ 重复计数导致误警告 |
|
||||
| 嵌入式 NPU 加速 | ✅ RK3588 RKNN INT8/FP16 量化推理 | ❌ 仅支持云端或 x86 CPU |
|
||||
| 全本地化处理 | ✅ 无外部接口依赖 | ❌ 大多依赖云端 API |
|
||||
| 多格式一致处理 | ✅ PDF/DOCX/XLSX/图像 统一 pipeline | ❌ 格式孤立，无统一流程 |
|
||||
|
||||
---
|
||||
|
||||
## 十、附件索引
|
||||
|
||||
| 文件 | 说明 |
|
||||
|------|------|
|
||||
| `src/info_privacy/pipeline.py` | 两阶段 pipeline 核心实现 |
|
||||
| `src/info_privacy/redactors/text_redactor.py` | PDF 内容流遮罩实现（核心发明二） |
|
||||
| `src/info_privacy/detectors/regex_detector.py` | Unicode 边界 + 去重实现（核心发明三） |
|
||||
| `src/info_privacy/parsers/pdf_parser.py` | 文字层自动检测实现（核心发明一） |
|
||||
| `src/info_privacy/parsers/image_parser.py` | RKNN OCR 集成（核心发明四） |
|
||||
| `src/info_privacy/detectors/face_detector.py` | RKNN 人脸检测集成（核心发明四） |
|
||||
| `configs/pii_rules.yaml` | 配置驱动 PII 规则（权利要求 5 附件） |
|
||||
| `tests/test_integration_complex.py` | 关键技术点验证测试用例 |
|
||||
@@ -0,0 +1,229 @@
|
||||
# Info-Privacy 系统设计文档
|
||||
|
||||
**日期**: 2026-02-27
|
||||
**版本**: v1.0
|
||||
**平台**: RK3588 / x86 兼容
|
||||
|
||||
---
|
||||
|
||||
## 项目定位
|
||||
|
||||
基于 Rockchip RKNN NPU 的文档隐私安全处理服务。读取输入文档，检测其中的隐私信息，用户决定对哪些类型进行遮罩处理，输出与输入格式一致的安全文档。
|
||||
|
||||
**两阶段工作流**：先分析（analyze）→ 用户决策 → 再遮罩（redact）
|
||||
|
||||
---
|
||||
|
||||
## 支持的文档类型
|
||||
|
||||
| 类型 | 解析方式 |
|
||||
|------|---------|
|
||||
| PDF（含文字层） | pdfminer.six 提取文字 + 坐标 |
|
||||
| PDF（扫描件） | 转图像 → paddle_ocr RKNN |
|
||||
| Word (.docx) | python-docx |
|
||||
| Excel (.xlsx) | openpyxl |
|
||||
| 图像 (JPG/PNG) | paddle_ocr RKNN |
|
||||
|
||||
---
|
||||
|
||||
## 文档智能分类（前置拦截）
|
||||
|
||||
**analyze 第 0 步**，优先级最高：
|
||||
|
||||
- 检测保密标记：文字关键词（机密/绝密/保密/内部/CONFIDENTIAL/SECRET）、页眉页脚水印、保密章图像
|
||||
- `classified`：直接拦截，返回 403 + 拦截原因，不进入 PII 检测
|
||||
- `sensitive_partial`：高密度敏感实体，放行 + 警告提示
|
||||
- `normal`：正常放行
|
||||
|
||||
---
|
||||
|
||||
## 隐私信息类型
|
||||
|
||||
| type | 检测方式 | 安全等级 |
|
||||
|------|---------|---------|
|
||||
| `id_card` | Regex（18位+校验） | high |
|
||||
| `phone` | Regex（+86/1xx） | high |
|
||||
| `bank_card` | Regex | high |
|
||||
| `face` | mediapipe RKNN | high |
|
||||
| `email` | Regex | medium |
|
||||
| `name` | 规则词典 NER | medium |
|
||||
| `address` | 规则词典 NER | medium |
|
||||
| `license_plate` | Regex | medium |
|
||||
| `confidential_mark` | 关键词+图像检测 | 触发拦截 |
|
||||
|
||||
---
|
||||
|
||||
## 遮罩策略（系统自动选择最安全方案）
|
||||
|
||||
用户只需指定 `redact_types`，方法由系统自动决定：
|
||||
|
||||
| 文档层类型 | 自动使用方案 | 安全保证 |
|
||||
|-----------|------------|---------|
|
||||
| 文字层（PDF/Word/Excel） | text_remove（删除文字节点） | ✅ 字节级 |
|
||||
| 图像层（图像文档/扫描PDF） | image_mask（黑色矩形覆盖） | ◑ 图像级 |
|
||||
| PDF 文字层 + 图像遮罩 | **禁止**，强制 text_remove | ✅ 字节级 |
|
||||
|
||||
---
|
||||
|
||||
## API 设计
|
||||
|
||||
### POST /api/v1/analyze
|
||||
|
||||
**输入**: `multipart/form-data`，字段 `file`
|
||||
|
||||
**响应**:
|
||||
```json
|
||||
{
|
||||
"doc_id": "uuid",
|
||||
"classification": "classified | sensitive_partial | normal",
|
||||
"blocked": false,
|
||||
"block_reason": null,
|
||||
"warning": "文档含多处高风险实体，建议全量处理",
|
||||
"doc_meta": {
|
||||
"format": "pdf",
|
||||
"has_text_layer": true,
|
||||
"redact_strategy": "text_replace"
|
||||
},
|
||||
"entities": [
|
||||
{
|
||||
"id": "e1",
|
||||
"type": "id_card",
|
||||
"value": "110101199001011234",
|
||||
"location": {
|
||||
"page": 1,
|
||||
"bbox": [x1, y1, x2, y2],
|
||||
"layer": "text"
|
||||
},
|
||||
"security_level": "high"
|
||||
}
|
||||
],
|
||||
"summary": {"id_card": 2, "phone": 3, "face": 1}
|
||||
}
|
||||
```
|
||||
|
||||
### POST /api/v1/redact
|
||||
|
||||
**输入**: `multipart/form-data`
|
||||
- `file`: 原始文档
|
||||
- `config`: JSON `{"redact_types": ["id_card", "phone", "face"]}`（未列出类型不处理）
|
||||
|
||||
**响应**:
|
||||
- Body: 处理后文档（`application/octet-stream`，格式与输入一致）
|
||||
- Header `X-Security-Report`: JSON 安全报告
|
||||
|
||||
### GET /api/v1/types
|
||||
|
||||
返回系统支持的隐私类型与当前模型状态。
|
||||
|
||||
### GET /api/v1/health
|
||||
|
||||
服务健康检查。
|
||||
|
||||
---
|
||||
|
||||
## 项目结构
|
||||
|
||||
```
|
||||
info-privacy/
|
||||
├── src/info_privacy/
|
||||
│ ├── api/
|
||||
│ │ ├── main.py # FastAPI app 入口
|
||||
│ │ ├── routes/
|
||||
│ │ │ ├── analyze.py
|
||||
│ │ │ └── redact.py
|
||||
│ │ └── models.py # Pydantic 模型
|
||||
│ ├── classifier/
|
||||
│ │ └── doc_classifier.py # 文档分类（保密标记检测）
|
||||
│ ├── detectors/
|
||||
│ │ ├── regex_detector.py # Regex PII 检测
|
||||
│ │ ├── ner_detector.py # 规则词典 NER
|
||||
│ │ └── face_detector.py # mediapipe RKNN
|
||||
│ ├── parsers/
|
||||
│ │ ├── pdf_parser.py # pdfminer 文字层解析
|
||||
│ │ ├── image_parser.py # paddle_ocr RKNN
|
||||
│ │ ├── office_parser.py # python-docx / openpyxl
|
||||
│ │ └── parser_factory.py # 按文件类型分发
|
||||
│ ├── redactors/
|
||||
│ │ ├── text_redactor.py # 文字节点替换（字节级）
|
||||
│ │ └── image_redactor.py # OpenCV 图像遮罩
|
||||
│ └── pipeline.py # parser→classifier→detector→redactor 串联
|
||||
├── scripts/
|
||||
│ ├── start_server.sh
|
||||
│ └── verify.py
|
||||
├── tests/
|
||||
│ ├── test_detectors.py
|
||||
│ ├── test_parsers.py
|
||||
│ ├── test_redactors.py
|
||||
│ └── test_api.py
|
||||
├── configs/
|
||||
│ └── pii_rules.yaml # 正则规则 + 词典（可扩展）
|
||||
├── models/ # RKNN 模型（软链到 sibling 项目）
|
||||
├── tmp/
|
||||
├── venv/
|
||||
├── pyproject.toml
|
||||
├── requirements.txt
|
||||
├── Makefile
|
||||
├── README.md
|
||||
├── DEVELOP.md
|
||||
└── RELEASE.md
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 核心数据流
|
||||
|
||||
```
|
||||
[POST /analyze]
|
||||
上传文档
|
||||
→ parser_factory → TextBlock[]{text, bbox, page, layer} + images
|
||||
→ doc_classifier → classification（blocked? warning?）
|
||||
→ detectors（并行）
|
||||
├── regex_detector → Entity[]
|
||||
├── ner_detector → Entity[]
|
||||
└── face_detector → Entity[]
|
||||
→ 返回 DetectionReport
|
||||
|
||||
[POST /redact]
|
||||
上传文档 + config{redact_types}
|
||||
→ 重新解析 → TextBlock[] + images
|
||||
→ 重新检测 → Entity[]
|
||||
→ strategy_selector:
|
||||
entity.layer=="text" → text_redactor.remove(entity)
|
||||
entity.layer=="image" → image_redactor.mask(entity)
|
||||
→ 重组文档 → 输出文件 + SecurityReport
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 关键依赖（RK3588 兼容）
|
||||
|
||||
| 依赖 | 版本 | ARM64 |
|
||||
|------|------|-------|
|
||||
| fastapi | ≥0.110 | ✅ |
|
||||
| uvicorn | ≥0.29 | ✅ |
|
||||
| pdfminer.six | ≥20221105 | ✅ 纯Python |
|
||||
| pypdf | ≥4.0 | ✅ 纯Python |
|
||||
| python-docx | ≥1.1 | ✅ 纯Python |
|
||||
| openpyxl | ≥3.1 | ✅ 纯Python |
|
||||
| opencv-python | 4.13.0 | ✅ 已安装 |
|
||||
| paddle_ocr（复用） | — | ✅ 已有RKNN模型 |
|
||||
| mediapipe_rknn（复用） | — | ✅ 已有RKNN模型 |
|
||||
|
||||
---
|
||||
|
||||
## 部署
|
||||
|
||||
**x86 开发机:**
|
||||
```bash
|
||||
python3 -m venv venv && source venv/bin/activate
|
||||
pip install -e ".[dev]"
|
||||
make server # uvicorn on :8000
|
||||
```
|
||||
|
||||
**RK3588 板端:**
|
||||
```bash
|
||||
ssh pi@192.168.0.127 "clashon && clashproxy on && \
|
||||
cd /home/pi/Desktop/info-privacy && \
|
||||
source venv/bin/activate && \
|
||||
uvicorn info_privacy.api.main:app --host 0.0.0.0 --port 8000"
|
||||
```
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,38 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=68.0", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "info-privacy"
|
||||
version = "0.1.0"
|
||||
description = "Document privacy redaction service for Rockchip RKNN"
|
||||
requires-python = ">=3.9"
|
||||
dependencies = [
|
||||
"fastapi>=0.110.0",
|
||||
"uvicorn>=0.29.0",
|
||||
"python-multipart>=0.0.9",
|
||||
"pdfminer.six>=20221105",
|
||||
"pypdf>=4.0.0",
|
||||
"python-docx>=1.1.0",
|
||||
"openpyxl>=3.1.0",
|
||||
"opencv-python>=4.5.5.64",
|
||||
"numpy>=1.24.0",
|
||||
"pyyaml>=6.0",
|
||||
"pyclipper>=1.3.0",
|
||||
"shapely>=2.0.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=8.0.0",
|
||||
"pytest-asyncio>=0.23.0",
|
||||
"httpx>=0.27.0",
|
||||
"ruff>=0.4.0",
|
||||
]
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["src"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
asyncio_mode = "auto"
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "$SCRIPT_DIR/.."
|
||||
source venv/bin/activate
|
||||
uvicorn info_privacy.api.main:app --host 0.0.0.0 --port 8000
|
||||
Executable
+73
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env python3
|
||||
"""端到端验证脚本：在本地启动的服务上测试完整流程。"""
|
||||
import sys
|
||||
import json
|
||||
import io
|
||||
|
||||
try:
|
||||
import docx
|
||||
import requests
|
||||
except ImportError as e:
|
||||
print(f"缺少依赖: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
BASE = "http://localhost:8000/api/v1"
|
||||
|
||||
|
||||
def check(name, cond, detail=""):
|
||||
mark = "✓" if cond else "✗"
|
||||
print(f" {mark} {name}" + (f": {detail}" if detail else ""))
|
||||
if not cond:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def make_docx_bytes(text: str) -> bytes:
|
||||
doc = docx.Document()
|
||||
doc.add_paragraph(text)
|
||||
buf = io.BytesIO()
|
||||
doc.save(buf)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
print("=== Info-Privacy 端到端验证 ===\n")
|
||||
|
||||
# 1. health
|
||||
try:
|
||||
r = requests.get(f"{BASE}/health", timeout=5)
|
||||
check("health 接口", r.status_code == 200)
|
||||
except requests.exceptions.ConnectionError:
|
||||
print(" ✗ 服务未启动，请先运行: make server")
|
||||
sys.exit(1)
|
||||
|
||||
# 2. types
|
||||
r = requests.get(f"{BASE}/types")
|
||||
check("types 接口", r.status_code == 200)
|
||||
check("types 包含 id_card", "id_card" in r.json().get("entity_types", []))
|
||||
|
||||
# 3. analyze - 正常文档
|
||||
docx_bytes = make_docx_bytes("身份证：110101199001011234 电话：13812345678")
|
||||
r = requests.post(f"{BASE}/analyze", files={"file": ("test.docx", docx_bytes)})
|
||||
check("analyze 状态码", r.status_code == 200)
|
||||
data = r.json()
|
||||
check("analyze 未拦截", not data["blocked"])
|
||||
types_found = {e["type"] for e in data["entities"]}
|
||||
check("检测到 id_card 或 phone", "id_card" in types_found or "phone" in types_found, str(types_found))
|
||||
|
||||
# 4. analyze - 保密文件
|
||||
secret_bytes = make_docx_bytes("【机密】本文件严禁外传")
|
||||
r = requests.post(f"{BASE}/analyze", files={"file": ("secret.docx", secret_bytes)})
|
||||
check("机密文件被拦截", r.json()["blocked"])
|
||||
|
||||
# 5. redact
|
||||
docx_bytes = make_docx_bytes("身份证：110101199001011234 地址：北京市朝阳区")
|
||||
cfg = json.dumps({"redact_types": ["id_card"]})
|
||||
r = requests.post(f"{BASE}/redact",
|
||||
files={"file": ("test.docx", docx_bytes)},
|
||||
data={"config": cfg})
|
||||
check("redact 状态码", r.status_code == 200)
|
||||
result = docx.Document(io.BytesIO(r.content))
|
||||
all_text = " ".join(p.text for p in result.paragraphs)
|
||||
check("身份证已遮罩", "110101199001011234" not in all_text)
|
||||
check("地址保留", "北京" in all_text)
|
||||
|
||||
print("\n=== 所有验证通过 ✓ ===")
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
command -v python3 >/dev/null || { echo "错误: python3 未找到"; exit 1; }
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate
|
||||
pip install --upgrade pip
|
||||
pip install -e ".[dev]"
|
||||
echo "venv ready. Run: source venv/bin/activate"
|
||||
@@ -0,0 +1,27 @@
|
||||
# src/info_privacy/api/main.py
|
||||
from fastapi import FastAPI
|
||||
from info_privacy.api.routes.analyze import router as analyze_router
|
||||
from info_privacy.api.routes.frame import router as frame_router
|
||||
from info_privacy.api.routes.redact import router as redact_router
|
||||
|
||||
app = FastAPI(title="Info-Privacy API", version="0.1.0")
|
||||
|
||||
app.include_router(analyze_router, prefix="/api/v1")
|
||||
app.include_router(frame_router, prefix="/api/v1")
|
||||
app.include_router(redact_router, prefix="/api/v1")
|
||||
|
||||
|
||||
@app.get("/api/v1/health")
|
||||
def health():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/api/v1/types")
|
||||
def types():
|
||||
return {
|
||||
"entity_types": [
|
||||
"id_card", "phone", "bank_card", "email",
|
||||
"license_plate", "name", "address", "face",
|
||||
],
|
||||
"note": "redact_strategy is auto-selected (text_replace for text layer, image_mask for images)"
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
# src/info_privacy/api/models.py
|
||||
from __future__ import annotations
|
||||
from typing import Optional
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class EntityOut(BaseModel):
|
||||
id: str
|
||||
type: str
|
||||
value: Optional[str]
|
||||
page: int
|
||||
bbox: list[float]
|
||||
layer: str
|
||||
security_level: str
|
||||
|
||||
|
||||
class DocMeta(BaseModel):
|
||||
format: str
|
||||
has_text_layer: bool
|
||||
redact_strategy: str
|
||||
|
||||
|
||||
class AnalyzeResponse(BaseModel):
|
||||
doc_id: str
|
||||
classification: str
|
||||
blocked: bool
|
||||
block_reason: Optional[str]
|
||||
warning: Optional[str]
|
||||
doc_meta: DocMeta
|
||||
entities: list[EntityOut]
|
||||
summary: dict[str, int]
|
||||
|
||||
|
||||
class RedactConfig(BaseModel):
|
||||
redact_types: list[str]
|
||||
@@ -0,0 +1,39 @@
|
||||
# src/info_privacy/api/routes/analyze.py
|
||||
import tempfile
|
||||
import os
|
||||
from fastapi import APIRouter, File, UploadFile, HTTPException
|
||||
from info_privacy.api.models import AnalyzeResponse, EntityOut, DocMeta
|
||||
from info_privacy.pipeline import PrivacyPipeline
|
||||
|
||||
router = APIRouter()
|
||||
_pipeline = PrivacyPipeline()
|
||||
|
||||
|
||||
@router.post("/analyze", response_model=AnalyzeResponse)
|
||||
async def analyze(file: UploadFile = File(...)):
|
||||
suffix = os.path.splitext(file.filename or "")[1].lower() or ".tmp"
|
||||
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp:
|
||||
tmp.write(await file.read())
|
||||
tmp_path = tmp.name
|
||||
|
||||
try:
|
||||
report = _pipeline.analyze(tmp_path)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
finally:
|
||||
os.unlink(tmp_path)
|
||||
|
||||
return AnalyzeResponse(
|
||||
doc_id=report.doc_id,
|
||||
classification=report.classification.value,
|
||||
blocked=report.blocked,
|
||||
block_reason=report.block_reason,
|
||||
warning=report.warning,
|
||||
doc_meta=DocMeta(**report.doc_meta),
|
||||
entities=[EntityOut(
|
||||
id=e.id, type=e.type.value, value=e.value,
|
||||
page=e.page, bbox=e.bbox, layer=e.layer,
|
||||
security_level=e.security_level,
|
||||
) for e in report.entities],
|
||||
summary=report.summary,
|
||||
)
|
||||
@@ -0,0 +1,84 @@
|
||||
# src/info_privacy/api/routes/frame.py
|
||||
"""视频帧 PII 分析端点 —— 接受 base64 编码图像或 multipart 上传。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import io
|
||||
|
||||
import numpy as np
|
||||
from fastapi import APIRouter, File, HTTPException, UploadFile
|
||||
from pydantic import BaseModel
|
||||
|
||||
from info_privacy.api.models import AnalyzeResponse, DocMeta, EntityOut
|
||||
from info_privacy.pipeline import PrivacyPipeline
|
||||
|
||||
router = APIRouter()
|
||||
_pipeline = PrivacyPipeline()
|
||||
|
||||
|
||||
class FrameBase64Request(BaseModel):
|
||||
"""Base64 编码图像帧请求体。"""
|
||||
image: str # base64-encoded JPEG/PNG bytes
|
||||
format: str = "jpeg"
|
||||
|
||||
|
||||
@router.post("/analyze/frame", response_model=AnalyzeResponse)
|
||||
async def analyze_frame_upload(file: UploadFile = File(...)):
|
||||
"""接受 multipart 图像文件上传，返回 PII 检测报告。
|
||||
|
||||
适用于 Privacy Gateway 上传附件预处理。
|
||||
图像数据在内存处理，不写磁盘（临时文件由 pipeline 内部管理）。
|
||||
"""
|
||||
try:
|
||||
import cv2
|
||||
data = await file.read()
|
||||
arr = np.frombuffer(data, dtype=np.uint8)
|
||||
img = cv2.imdecode(arr, cv2.IMREAD_COLOR)
|
||||
if img is None:
|
||||
raise HTTPException(status_code=400, detail="无法解码图像文件")
|
||||
report = _pipeline.analyze_image(img)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"图像分析失败: {e}")
|
||||
|
||||
return _build_response(report)
|
||||
|
||||
|
||||
@router.post("/analyze/frame/base64", response_model=AnalyzeResponse)
|
||||
async def analyze_frame_base64(req: FrameBase64Request):
|
||||
"""接受 base64 编码图像，返回 PII 检测报告。
|
||||
|
||||
适用于 KVM OCR Monitor 将视频帧直接传递（无磁盘 I/O）。
|
||||
"""
|
||||
try:
|
||||
import cv2
|
||||
raw = base64.b64decode(req.image)
|
||||
arr = np.frombuffer(raw, dtype=np.uint8)
|
||||
img = cv2.imdecode(arr, cv2.IMREAD_COLOR)
|
||||
if img is None:
|
||||
raise HTTPException(status_code=400, detail="base64 解码后无法识别为图像")
|
||||
report = _pipeline.analyze_image(img)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"图像分析失败: {e}")
|
||||
|
||||
return _build_response(report)
|
||||
|
||||
|
||||
def _build_response(report) -> AnalyzeResponse:
|
||||
return AnalyzeResponse(
|
||||
doc_id=report.doc_id,
|
||||
classification=report.classification.value,
|
||||
blocked=report.blocked,
|
||||
block_reason=report.block_reason,
|
||||
warning=report.warning,
|
||||
doc_meta=DocMeta(**report.doc_meta),
|
||||
entities=[EntityOut(
|
||||
id=e.id, type=e.type.value, value=e.value,
|
||||
page=e.page, bbox=e.bbox, layer=e.layer,
|
||||
security_level=e.security_level,
|
||||
) for e in report.entities],
|
||||
summary=report.summary,
|
||||
)
|
||||
@@ -0,0 +1,45 @@
|
||||
# src/info_privacy/api/routes/redact.py
|
||||
import tempfile
|
||||
import os
|
||||
import json
|
||||
from fastapi import APIRouter, File, Form, UploadFile, HTTPException
|
||||
from fastapi.responses import FileResponse
|
||||
from info_privacy.api.models import RedactConfig
|
||||
from info_privacy.pipeline import PrivacyPipeline
|
||||
|
||||
router = APIRouter()
|
||||
_pipeline = PrivacyPipeline()
|
||||
|
||||
|
||||
@router.post("/redact")
|
||||
async def redact(
|
||||
file: UploadFile = File(...),
|
||||
config: str = Form(...),
|
||||
):
|
||||
try:
|
||||
cfg = RedactConfig.model_validate_json(config)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=422, detail=f"config 格式错误: {e}")
|
||||
|
||||
suffix = os.path.splitext(file.filename or "")[1].lower() or ".tmp"
|
||||
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp:
|
||||
tmp.write(await file.read())
|
||||
in_path = tmp.name
|
||||
|
||||
out_path = in_path + "_redacted" + suffix
|
||||
|
||||
try:
|
||||
security_report = _pipeline.redact(in_path, cfg.redact_types, out_path)
|
||||
except ValueError as e:
|
||||
os.unlink(in_path)
|
||||
raise HTTPException(status_code=403, detail=str(e))
|
||||
finally:
|
||||
if os.path.exists(in_path):
|
||||
os.unlink(in_path)
|
||||
|
||||
return FileResponse(
|
||||
out_path,
|
||||
media_type="application/octet-stream",
|
||||
filename=f"redacted_{file.filename}",
|
||||
headers={"X-Security-Report": json.dumps(security_report, ensure_ascii=False)},
|
||||
)
|
||||
@@ -0,0 +1,91 @@
|
||||
# src/info_privacy/classifier/doc_classifier.py
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import yaml
|
||||
|
||||
from info_privacy.models import Classification, Entity, TextBlock
|
||||
|
||||
|
||||
SENSITIVE_PARTIAL_THRESHOLD = 5 # 超过N个高风险实体触发警告
|
||||
|
||||
|
||||
@dataclass
|
||||
class ClassificationResult:
|
||||
classification: Classification
|
||||
blocked: bool
|
||||
block_reason: Optional[str]
|
||||
warning: Optional[str]
|
||||
|
||||
|
||||
def _build_automaton(keywords: list[str]):
|
||||
"""构建 Aho-Corasick 自动机；不可用时降级为普通列表。"""
|
||||
try:
|
||||
import ahocorasick
|
||||
A = ahocorasick.Automaton()
|
||||
for kw in keywords:
|
||||
A.add_word(kw, kw)
|
||||
A.make_automaton()
|
||||
return A
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
|
||||
class DocClassifier:
|
||||
def __init__(self, config_path: str = "configs/pii_rules.yaml"):
|
||||
cfg = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
|
||||
raw = [kw.upper() for kw in cfg.get("confidential_keywords", [])]
|
||||
self._keywords = raw
|
||||
# Aho-Corasick 单次扫描 O(n+m)，降级时用 O(n·m) list
|
||||
self._automaton = _build_automaton(raw)
|
||||
|
||||
def classify(
|
||||
self,
|
||||
blocks: list[TextBlock],
|
||||
entities: Optional[list[Entity]] = None,
|
||||
) -> ClassificationResult:
|
||||
# 优先检查保密标记
|
||||
hit = self._find_keyword(blocks)
|
||||
if hit is not None:
|
||||
kw, page = hit
|
||||
return ClassificationResult(
|
||||
classification=Classification.CLASSIFIED,
|
||||
blocked=True,
|
||||
block_reason=f"检测到保密标记：\u201c{kw}\u201d（第{page}页）",
|
||||
warning=None,
|
||||
)
|
||||
|
||||
# 高密度敏感实体检查
|
||||
if entities:
|
||||
high_risk = [e for e in entities if e.security_level == "high"]
|
||||
if len(high_risk) >= SENSITIVE_PARTIAL_THRESHOLD:
|
||||
return ClassificationResult(
|
||||
classification=Classification.SENSITIVE_PARTIAL,
|
||||
blocked=False,
|
||||
block_reason=None,
|
||||
warning=f"文档含 {len(high_risk)} 处高风险隐私实体，建议全量处理",
|
||||
)
|
||||
|
||||
return ClassificationResult(
|
||||
classification=Classification.NORMAL,
|
||||
blocked=False,
|
||||
block_reason=None,
|
||||
warning=None,
|
||||
)
|
||||
|
||||
def _find_keyword(self, blocks: list[TextBlock]) -> Optional[tuple[str, int]]:
|
||||
"""返回首个命中的 (keyword, page)，未命中返回 None。"""
|
||||
if self._automaton is not None:
|
||||
for block in blocks:
|
||||
text_upper = block.text.upper()
|
||||
for _, kw in self._automaton.iter(text_upper):
|
||||
return kw, block.page
|
||||
else:
|
||||
for block in blocks:
|
||||
text_upper = block.text.upper()
|
||||
for kw in self._keywords:
|
||||
if kw in text_upper:
|
||||
return kw, block.page
|
||||
return None
|
||||
@@ -0,0 +1,45 @@
|
||||
# src/info_privacy/detectors/face_detector.py
|
||||
"""人脸检测器，复用 mediapipe_rknn FaceDetection。
|
||||
|
||||
参考: /data/rockchip/mediapipe/src/mediapipe_rknn/solutions/face_detection.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
|
||||
from info_privacy.models import Entity, EntityType
|
||||
|
||||
_MEDIAPIPE_SRC = Path("/data/rockchip/mediapipe/src")
|
||||
_DEFAULT_MODEL = "/data/rockchip/mediapipe/models/rknn/face_detection_short_range_rk3588.rknn"
|
||||
|
||||
|
||||
class FaceDetector:
|
||||
def __init__(self, model_path: str = _DEFAULT_MODEL):
|
||||
if str(_MEDIAPIPE_SRC) not in sys.path:
|
||||
sys.path.insert(0, str(_MEDIAPIPE_SRC))
|
||||
from mediapipe_rknn.solutions import FaceDetection
|
||||
self._detector = FaceDetection(model_path=model_path)
|
||||
self._detector.load()
|
||||
|
||||
def detect_in_image(self, img: np.ndarray, page: int = 1) -> list[Entity]:
|
||||
"""在图像 numpy 数组中检测所有人脸，返回 Entity 列表。"""
|
||||
results = self._detector.detect(img)
|
||||
entities: list[Entity] = []
|
||||
if results is None:
|
||||
return entities
|
||||
for det in results:
|
||||
bbox = det.bbox # [x1, y1, x2, y2] 像素坐标
|
||||
entities.append(Entity(
|
||||
id=str(uuid.uuid4()),
|
||||
type=EntityType.FACE,
|
||||
value=None,
|
||||
page=page,
|
||||
bbox=[float(v) for v in bbox],
|
||||
layer="image",
|
||||
security_level="high",
|
||||
))
|
||||
return entities
|
||||
@@ -0,0 +1,57 @@
|
||||
# src/info_privacy/detectors/ner_detector.py
|
||||
import re
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from info_privacy.models import Entity, EntityType, TextBlock
|
||||
|
||||
|
||||
class NERDetector:
|
||||
def __init__(self, config_path: str = "configs/pii_rules.yaml"):
|
||||
cfg = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
|
||||
ner_cfg = cfg.get("ner", {})
|
||||
|
||||
surnames_file = ner_cfg.get("name", {}).get("surnames_file", "configs/surnames.txt")
|
||||
surnames = [
|
||||
line.strip() for line in Path(surnames_file).read_text(encoding="utf-8").splitlines()
|
||||
if line.strip() and not line.startswith("#")
|
||||
]
|
||||
surname_pattern = "|".join(re.escape(s) for s in surnames)
|
||||
# 姓氏 + 2-4个汉字姓名
|
||||
self._name_re = re.compile(
|
||||
rf'(?<![^\s，。、：；\(（])({surname_pattern})[\u4e00-\u9fff]{{1,3}}(?![^\s，。、：；\)）])'
|
||||
)
|
||||
|
||||
# 地址：含省/市/区/路/号等关键词的连续中文片段
|
||||
addr_triggers = ner_cfg.get("address", {}).get("triggers", ["省", "市", "区", "路", "号"])
|
||||
trigger_pat = "|".join(re.escape(t) for t in addr_triggers)
|
||||
self._addr_re = re.compile(
|
||||
rf'[\u4e00-\u9fff\d]{{2,30}}(?:{trigger_pat})[\u4e00-\u9fff\d]{{0,20}}'
|
||||
)
|
||||
|
||||
def detect(self, blocks: list[TextBlock]) -> list[Entity]:
|
||||
entities: list[Entity] = []
|
||||
for block in blocks:
|
||||
for match in self._name_re.finditer(block.text):
|
||||
entities.append(Entity(
|
||||
id=str(uuid.uuid4()),
|
||||
type=EntityType.NAME,
|
||||
value=match.group(),
|
||||
page=block.page,
|
||||
bbox=block.bbox,
|
||||
layer=block.layer,
|
||||
security_level="medium",
|
||||
))
|
||||
for match in self._addr_re.finditer(block.text):
|
||||
entities.append(Entity(
|
||||
id=str(uuid.uuid4()),
|
||||
type=EntityType.ADDRESS,
|
||||
value=match.group(),
|
||||
page=block.page,
|
||||
bbox=block.bbox,
|
||||
layer=block.layer,
|
||||
security_level="medium",
|
||||
))
|
||||
return entities
|
||||
@@ -0,0 +1,41 @@
|
||||
import re
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from info_privacy.models import Entity, EntityType, TextBlock
|
||||
|
||||
|
||||
class RegexDetector:
|
||||
def __init__(self, config_path: str = "configs/pii_rules.yaml"):
|
||||
cfg = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
|
||||
self._rules = {
|
||||
EntityType(k): {
|
||||
"pattern": re.compile(v["pattern"]),
|
||||
"security_level": v["security_level"],
|
||||
}
|
||||
for k, v in cfg["regex"].items()
|
||||
}
|
||||
|
||||
def detect(self, blocks: list[TextBlock]) -> list[Entity]:
|
||||
entities: list[Entity] = []
|
||||
for block in blocks:
|
||||
for etype, rule in self._rules.items():
|
||||
for match in rule["pattern"].finditer(block.text):
|
||||
entities.append(Entity(
|
||||
id=str(uuid.uuid4()),
|
||||
type=etype,
|
||||
value=match.group(),
|
||||
page=block.page,
|
||||
bbox=block.bbox,
|
||||
layer=block.layer,
|
||||
security_level=rule["security_level"],
|
||||
))
|
||||
# 身份证号满足银行卡格式（18位），去重：已匹配为 id_card 的值不再记为 bank_card
|
||||
id_card_values = {e.value for e in entities if e.type == EntityType.ID_CARD}
|
||||
entities = [
|
||||
e for e in entities
|
||||
if not (e.type == EntityType.BANK_CARD and e.value in id_card_values)
|
||||
]
|
||||
return entities
|
||||
@@ -0,0 +1,61 @@
|
||||
# src/info_privacy/models.py
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class EntityType(str, Enum):
|
||||
ID_CARD = "id_card"
|
||||
PHONE = "phone"
|
||||
BANK_CARD = "bank_card"
|
||||
EMAIL = "email"
|
||||
LICENSE_PLATE = "license_plate"
|
||||
NAME = "name"
|
||||
ADDRESS = "address"
|
||||
FACE = "face"
|
||||
CONFIDENTIAL_MARK = "confidential_mark"
|
||||
|
||||
|
||||
class Classification(str, Enum):
|
||||
CLASSIFIED = "classified"
|
||||
SENSITIVE_PARTIAL = "sensitive_partial"
|
||||
NORMAL = "normal"
|
||||
|
||||
|
||||
@dataclass
|
||||
class TextBlock:
|
||||
text: str
|
||||
bbox: list[float] # [x1, y1, x2, y2]
|
||||
page: int
|
||||
layer: str # "text" | "image"
|
||||
confidence: float = 1.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class Entity:
|
||||
id: str
|
||||
type: EntityType
|
||||
value: Optional[str]
|
||||
page: int
|
||||
bbox: list[float]
|
||||
layer: str # "text" | "image"
|
||||
security_level: str # "high" | "medium" | "low"
|
||||
|
||||
|
||||
@dataclass
|
||||
class DetectionReport:
|
||||
doc_id: str
|
||||
classification: Classification
|
||||
blocked: bool
|
||||
block_reason: Optional[str]
|
||||
warning: Optional[str]
|
||||
doc_meta: dict
|
||||
entities: list[Entity]
|
||||
|
||||
@property
|
||||
def summary(self) -> dict[str, int]:
|
||||
counts: dict[str, int] = {}
|
||||
for e in self.entities:
|
||||
counts[e.type.value] = counts.get(e.type.value, 0) + 1
|
||||
return counts
|
||||
@@ -0,0 +1,99 @@
|
||||
# src/info_privacy/parsers/image_parser.py
|
||||
"""图像 OCR 解析器，基于 rknn_model_zoo PaddleOCR 推理。
|
||||
|
||||
参考: /data/rockchip/rknn_model_zoo/examples/PPOCR/PPOCR-System/python/
|
||||
平台: RK3588（rknn-toolkit-lite2）或 x86 仿真（rknn-toolkit2）
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import sys
|
||||
import numpy as np
|
||||
import cv2
|
||||
from pathlib import Path
|
||||
|
||||
from info_privacy.models import TextBlock
|
||||
|
||||
_PPOCR_DIR = Path("/data/rockchip/rknn_model_zoo/examples/PPOCR/PPOCR-System/python")
|
||||
|
||||
|
||||
class ImageParser:
|
||||
def __init__(
|
||||
self,
|
||||
det_model_path: str,
|
||||
rec_model_path: str,
|
||||
dict_path: str,
|
||||
target: str = "rk3588",
|
||||
):
|
||||
if str(_PPOCR_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(_PPOCR_DIR))
|
||||
|
||||
import ppocr_det as predict_det
|
||||
import ppocr_rec as predict_rec
|
||||
|
||||
class Args:
|
||||
pass
|
||||
|
||||
args = Args()
|
||||
args.det_model_path = det_model_path
|
||||
args.rec_model_path = rec_model_path
|
||||
args.dict_path = dict_path
|
||||
args.target = target
|
||||
|
||||
self._detector = predict_det.TextDetector(args)
|
||||
self._recognizer = predict_rec.TextRecognizer(args)
|
||||
|
||||
def parse(self, file_path: str) -> tuple[dict, list[TextBlock]]:
|
||||
img = cv2.imread(file_path)
|
||||
if img is None:
|
||||
raise ValueError(f"无法读取图像: {file_path}")
|
||||
blocks = self._run_ocr(img, page=1)
|
||||
meta = {
|
||||
"format": "image",
|
||||
"has_text_layer": False,
|
||||
"redact_strategy": "image_mask",
|
||||
}
|
||||
return meta, blocks
|
||||
|
||||
def run_on_image(self, img: np.ndarray, page: int = 1) -> list[TextBlock]:
|
||||
"""供 PDF 扫描件调用（page 已知）。"""
|
||||
return self._run_ocr(img, page)
|
||||
|
||||
def _run_ocr(self, img: np.ndarray, page: int) -> list[TextBlock]:
|
||||
if str(_PPOCR_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(_PPOCR_DIR))
|
||||
import ppocr_det as predict_det
|
||||
|
||||
ori_im = img.copy()
|
||||
dt_boxes = self._detector.run(img)
|
||||
if dt_boxes is None or len(dt_boxes) == 0:
|
||||
return []
|
||||
|
||||
img_crop_list = []
|
||||
for box in sorted(dt_boxes, key=lambda b: (b[0][1], b[0][0])):
|
||||
crop = predict_det.get_rotate_crop_image(ori_im, box)
|
||||
img_crop_list.append(crop)
|
||||
|
||||
rec_res = self._recognizer.run(img_crop_list)
|
||||
|
||||
blocks: list[TextBlock] = []
|
||||
for box, rec in zip(dt_boxes, rec_res):
|
||||
# rec_res 的每项可能是 (text, score) 或 [(text, score)]
|
||||
if isinstance(rec, (list, tuple)) and len(rec) > 0:
|
||||
if isinstance(rec[0], (list, tuple)):
|
||||
text, score = rec[0]
|
||||
else:
|
||||
text, score = rec[0], rec[1] if len(rec) > 1 else 1.0
|
||||
else:
|
||||
continue
|
||||
if float(score) < 0.5:
|
||||
continue
|
||||
xs = box[:, 0]
|
||||
ys = box[:, 1]
|
||||
bbox = [float(xs.min()), float(ys.min()), float(xs.max()), float(ys.max())]
|
||||
blocks.append(TextBlock(
|
||||
text=str(text),
|
||||
bbox=bbox,
|
||||
page=page,
|
||||
layer="image",
|
||||
confidence=float(score),
|
||||
))
|
||||
return blocks
|
||||
@@ -0,0 +1,74 @@
|
||||
# src/info_privacy/parsers/office_parser.py
|
||||
from __future__ import annotations
|
||||
from pathlib import Path
|
||||
|
||||
from info_privacy.models import TextBlock
|
||||
|
||||
|
||||
class OfficeParser:
|
||||
def parse(self, file_path: str) -> tuple[dict, list[TextBlock]]:
|
||||
suffix = Path(file_path).suffix.lower()
|
||||
if suffix == ".docx":
|
||||
return self._parse_docx(file_path)
|
||||
elif suffix in (".xlsx", ".xls"):
|
||||
return self._parse_excel(file_path)
|
||||
raise ValueError(f"不支持的 Office 格式: {suffix}")
|
||||
|
||||
def _parse_docx(self, path: str) -> tuple[dict, list[TextBlock]]:
|
||||
import docx
|
||||
doc = docx.Document(path)
|
||||
blocks: list[TextBlock] = []
|
||||
slot = 0 # 用于生成 bbox 的行号计数器
|
||||
|
||||
def _add(text: str) -> None:
|
||||
nonlocal slot
|
||||
stripped = text.strip()
|
||||
if not stripped:
|
||||
return
|
||||
blocks.append(TextBlock(
|
||||
text=stripped,
|
||||
bbox=[0, slot * 20, 500, (slot + 1) * 20],
|
||||
page=1,
|
||||
layer="text",
|
||||
))
|
||||
slot += 1
|
||||
|
||||
# 正文段落
|
||||
for para in doc.paragraphs:
|
||||
_add(para.text)
|
||||
|
||||
# 表格单元格（按行→列顺序）
|
||||
for table in doc.tables:
|
||||
for row in table.rows:
|
||||
for cell in row.cells:
|
||||
for para in cell.paragraphs:
|
||||
_add(para.text)
|
||||
|
||||
meta = {
|
||||
"format": "docx",
|
||||
"has_text_layer": True,
|
||||
"redact_strategy": "text_replace",
|
||||
}
|
||||
return meta, blocks
|
||||
|
||||
def _parse_excel(self, path: str) -> tuple[dict, list[TextBlock]]:
|
||||
import openpyxl
|
||||
wb = openpyxl.load_workbook(path, read_only=True, data_only=True)
|
||||
blocks: list[TextBlock] = []
|
||||
for sheet_idx, ws in enumerate(wb.worksheets):
|
||||
for row_idx, row in enumerate(ws.iter_rows(values_only=True)):
|
||||
text = " ".join(str(c) for c in row if c is not None).strip()
|
||||
if not text:
|
||||
continue
|
||||
blocks.append(TextBlock(
|
||||
text=text,
|
||||
bbox=[0, row_idx * 20, 500, (row_idx + 1) * 20],
|
||||
page=sheet_idx + 1,
|
||||
layer="text",
|
||||
))
|
||||
meta = {
|
||||
"format": "xlsx",
|
||||
"has_text_layer": True,
|
||||
"redact_strategy": "text_replace",
|
||||
}
|
||||
return meta, blocks
|
||||
@@ -0,0 +1,34 @@
|
||||
# src/info_privacy/parsers/parser_factory.py
|
||||
from __future__ import annotations
|
||||
from pathlib import Path
|
||||
from info_privacy.parsers.pdf_parser import PdfParser
|
||||
from info_privacy.parsers.image_parser import ImageParser
|
||||
from info_privacy.parsers.office_parser import OfficeParser
|
||||
|
||||
_RKNN_DET = "/data/rockchip/paddle_ocr/models/rknn/PP-OCRv4/det/ppocrv4_det_ch_int8.rknn"
|
||||
_RKNN_REC = "/data/rockchip/paddle_ocr/models/rknn/PP-OCRv4/rec/ppocrv4_rec_ch_fp16.rknn"
|
||||
_DICT = "/data/rockchip/paddle_ocr/data/dicts/ppocr_keys_v1.txt"
|
||||
|
||||
|
||||
class ParserFactory:
|
||||
def __init__(self, det_model: str = _RKNN_DET, rec_model: str = _RKNN_REC, dict_path: str = _DICT):
|
||||
self._det = det_model
|
||||
self._rec = rec_model
|
||||
self._dict = dict_path
|
||||
self._image_parser: ImageParser | None = None
|
||||
|
||||
def get_parser(self, file_path: str) -> PdfParser | ImageParser | OfficeParser:
|
||||
suffix = Path(file_path).suffix.lower()
|
||||
if suffix == ".pdf":
|
||||
return PdfParser()
|
||||
elif suffix in (".jpg", ".jpeg", ".png", ".tiff", ".bmp"):
|
||||
return self._get_image_parser()
|
||||
elif suffix in (".docx", ".xlsx", ".xls"):
|
||||
return OfficeParser()
|
||||
raise ValueError(f"不支持的文件格式: {suffix}")
|
||||
|
||||
def _get_image_parser(self) -> ImageParser:
|
||||
# 延迟初始化（RKNN加载慢）
|
||||
if self._image_parser is None:
|
||||
self._image_parser = ImageParser(self._det, self._rec, self._dict)
|
||||
return self._image_parser
|
||||
@@ -0,0 +1,51 @@
|
||||
# src/info_privacy/parsers/pdf_parser.py
|
||||
from __future__ import annotations
|
||||
from pathlib import Path
|
||||
|
||||
from pdfminer.converter import PDFPageAggregator
|
||||
from pdfminer.layout import LAParams, LTTextBox
|
||||
from pdfminer.pdfdocument import PDFDocument
|
||||
from pdfminer.pdfinterp import PDFPageInterpreter, PDFResourceManager
|
||||
from pdfminer.pdfpage import PDFPage
|
||||
from pdfminer.pdfparser import PDFParser as _PDFParser
|
||||
|
||||
from info_privacy.models import TextBlock
|
||||
|
||||
|
||||
class PdfParser:
|
||||
def parse(self, file_path: str) -> tuple[dict, list[TextBlock]]:
|
||||
path = Path(file_path)
|
||||
blocks: list[TextBlock] = []
|
||||
has_text_layer = False
|
||||
|
||||
rsrcmgr = PDFResourceManager()
|
||||
laparams = LAParams(line_margin=0.5, char_margin=2.0)
|
||||
device = PDFPageAggregator(rsrcmgr, laparams=laparams)
|
||||
interpreter = PDFPageInterpreter(rsrcmgr, device)
|
||||
|
||||
with open(path, "rb") as f:
|
||||
parser = _PDFParser(f)
|
||||
doc = PDFDocument(parser)
|
||||
for page_num, page in enumerate(PDFPage.create_pages(doc), start=1):
|
||||
interpreter.process_page(page)
|
||||
layout = device.get_result()
|
||||
for element in layout:
|
||||
if isinstance(element, LTTextBox):
|
||||
text = element.get_text().strip()
|
||||
if not text:
|
||||
continue
|
||||
has_text_layer = True
|
||||
x0, y0, x1, y1 = element.bbox
|
||||
blocks.append(TextBlock(
|
||||
text=text,
|
||||
bbox=[x0, y0, x1, y1],
|
||||
page=page_num,
|
||||
layer="text",
|
||||
))
|
||||
|
||||
meta = {
|
||||
"format": "pdf",
|
||||
"has_text_layer": has_text_layer,
|
||||
"redact_strategy": "text_replace" if has_text_layer else "image_mask",
|
||||
}
|
||||
return meta, blocks
|
||||
@@ -0,0 +1,146 @@
|
||||
# src/info_privacy/pipeline.py
|
||||
from __future__ import annotations
|
||||
import shutil
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
from info_privacy.models import Classification, DetectionReport, EntityType, TextBlock
|
||||
from info_privacy.classifier.doc_classifier import DocClassifier
|
||||
from info_privacy.detectors.regex_detector import RegexDetector
|
||||
from info_privacy.detectors.ner_detector import NERDetector
|
||||
from info_privacy.parsers.parser_factory import ParserFactory
|
||||
from info_privacy.redactors.text_redactor import TextRedactor
|
||||
from info_privacy.redactors.image_redactor import ImageRedactor
|
||||
|
||||
_CONFIG = "configs/pii_rules.yaml"
|
||||
|
||||
|
||||
class PrivacyPipeline:
|
||||
def __init__(self, config_path: str = _CONFIG):
|
||||
self._factory = ParserFactory()
|
||||
self._classifier = DocClassifier(config_path)
|
||||
self._regex = RegexDetector(config_path)
|
||||
self._ner = NERDetector(config_path)
|
||||
self._text_redactor = TextRedactor()
|
||||
self._image_redactor = ImageRedactor()
|
||||
self._face_detector = None # 延迟加载（RKNN 初始化慢）
|
||||
# 检测器并行执行池（regex/NER 均无共享状态，线程安全）
|
||||
self._detector_pool = ThreadPoolExecutor(max_workers=2)
|
||||
|
||||
def analyze(self, file_path: str) -> DetectionReport:
|
||||
parser = self._factory.get_parser(file_path)
|
||||
meta, blocks = parser.parse(file_path)
|
||||
|
||||
# Step 0: 保密文件快速拦截
|
||||
clf = self._classifier.classify(blocks)
|
||||
if clf.blocked:
|
||||
return DetectionReport(
|
||||
doc_id=str(uuid.uuid4()),
|
||||
classification=clf.classification,
|
||||
blocked=True,
|
||||
block_reason=clf.block_reason,
|
||||
warning=None,
|
||||
doc_meta=meta,
|
||||
entities=[],
|
||||
)
|
||||
|
||||
# Step 1: regex + NER 并行检测（两者独立无共享状态）
|
||||
f_regex = self._detector_pool.submit(self._regex.detect, blocks)
|
||||
f_ner = self._detector_pool.submit(self._ner.detect, blocks)
|
||||
entities = f_regex.result() + f_ner.result()
|
||||
|
||||
# Step 2: 图像文档额外做人脸检测（NPU，与上面 CPU 检测错开）
|
||||
if meta.get("redact_strategy") == "image_mask":
|
||||
import cv2
|
||||
img = cv2.imread(file_path)
|
||||
if img is not None:
|
||||
try:
|
||||
entities += self._get_face_detector().detect_in_image(img, page=1)
|
||||
except Exception:
|
||||
pass # RKNN 不可用时跳过人脸检测
|
||||
|
||||
# Step 3: 再次评估分类（有 entities 了）
|
||||
clf = self._classifier.classify(blocks, entities=entities)
|
||||
|
||||
return DetectionReport(
|
||||
doc_id=str(uuid.uuid4()),
|
||||
classification=clf.classification,
|
||||
blocked=False,
|
||||
block_reason=None,
|
||||
warning=clf.warning,
|
||||
doc_meta=meta,
|
||||
entities=entities,
|
||||
)
|
||||
|
||||
def redact(self, file_path: str, redact_types: list[str], out_path: str) -> dict:
|
||||
"""遮罩指定类型实体，自动选最安全策略，返回安全报告。
|
||||
复用 analyze 结果，避免重复解析和检测。
|
||||
"""
|
||||
report = self.analyze(file_path)
|
||||
return self.redact_from_report(file_path, report, redact_types, out_path)
|
||||
|
||||
def redact_from_report(
|
||||
self,
|
||||
file_path: str,
|
||||
report: DetectionReport,
|
||||
redact_types: list[str],
|
||||
out_path: str,
|
||||
) -> dict:
|
||||
"""基于已有 analyze 报告执行遮罩，避免重复 analyze 开销。"""
|
||||
if report.blocked:
|
||||
raise ValueError(f"文件被拦截，无法遮罩: {report.block_reason}")
|
||||
|
||||
target_types = {EntityType(t) for t in redact_types}
|
||||
to_redact = [e for e in report.entities if e.type in target_types]
|
||||
|
||||
text_entities = [e for e in to_redact if e.layer == "text"]
|
||||
image_entities = [e for e in to_redact if e.layer == "image"]
|
||||
|
||||
strategy = report.doc_meta.get("redact_strategy", "image_mask")
|
||||
|
||||
if text_entities:
|
||||
self._text_redactor.redact(file_path, text_entities, out_path)
|
||||
if image_entities:
|
||||
src = out_path if text_entities else file_path
|
||||
self._image_redactor.redact(src, image_entities, out_path)
|
||||
|
||||
if not text_entities and not image_entities:
|
||||
shutil.copy(file_path, out_path)
|
||||
|
||||
return {
|
||||
"redacted": {t.value: sum(1 for e in to_redact if e.type == t) for t in target_types},
|
||||
"strategy_used": strategy,
|
||||
"security_guarantee": "byte_level" if strategy == "text_replace" else "image_level",
|
||||
}
|
||||
|
||||
def analyze_image(self, img) -> DetectionReport:
|
||||
"""视频帧 PII 检测（供 KVM 视频流遮罩和 Privacy Gateway 使用）。
|
||||
|
||||
Args:
|
||||
img: numpy ndarray (BGR, HWC) - 来自 OpenCV 或 base64 解码的图像帧
|
||||
|
||||
Returns:
|
||||
DetectionReport - 与 analyze() 相同格式，doc_meta 包含 redact_strategy: image_mask
|
||||
"""
|
||||
import tempfile
|
||||
import cv2
|
||||
import os
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
|
||||
try:
|
||||
cv2.imwrite(tmp_path, img)
|
||||
return self.analyze(tmp_path)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _get_face_detector(self):
|
||||
if self._face_detector is None:
|
||||
from info_privacy.detectors.face_detector import FaceDetector
|
||||
self._face_detector = FaceDetector()
|
||||
return self._face_detector
|
||||
@@ -0,0 +1,36 @@
|
||||
# src/info_privacy/redactors/image_redactor.py
|
||||
from __future__ import annotations
|
||||
from pathlib import Path
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
from info_privacy.models import Entity
|
||||
|
||||
|
||||
class ImageRedactor:
|
||||
def redact(self, file_path: str, entities: list[Entity], out_path: str) -> None:
|
||||
suffix = Path(file_path).suffix.lower()
|
||||
if suffix == ".pdf":
|
||||
self._redact_pdf_image(file_path, entities, out_path)
|
||||
return
|
||||
|
||||
img = cv2.imread(file_path)
|
||||
if img is None:
|
||||
raise ValueError(f"无法读取图像: {file_path}")
|
||||
|
||||
for entity in entities:
|
||||
if entity.layer != "image":
|
||||
continue
|
||||
x1, y1, x2, y2 = (int(v) for v in entity.bbox)
|
||||
# numpy slice 赋值与 cv2.rectangle 速度相当，无需额外依赖
|
||||
img[y1:y2, x1:x2] = 0
|
||||
|
||||
cv2.imwrite(out_path, img)
|
||||
|
||||
def _redact_pdf_image(self, path: str, entities: list[Entity], out_path: str) -> None:
|
||||
"""扫描型 PDF：使用 pypdf 复制页面，不支持图像层遮罩。
|
||||
对于扫描 PDF，需先用 pdf2image 渲染再遮罩，此处记录警告并直接复制。
|
||||
"""
|
||||
import shutil
|
||||
shutil.copy(path, out_path)
|
||||
@@ -0,0 +1,113 @@
|
||||
# src/info_privacy/redactors/text_redactor.py
|
||||
"""文字层遮罩器：直接修改文档文字节点，字节级安全。
|
||||
|
||||
策略：
|
||||
- DOCX: python-docx 段落 run 中将目标字符串替换为 ████
|
||||
- PDF (含文字层): pypdf 写出后做字节级替换
|
||||
- XLSX: openpyxl 单元格值替换
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from info_privacy.models import Entity
|
||||
|
||||
_MASK = "████"
|
||||
|
||||
|
||||
class TextRedactor:
|
||||
def redact(self, file_path: str, entities: list[Entity], out_path: str) -> None:
|
||||
suffix = Path(file_path).suffix.lower()
|
||||
if suffix == ".docx":
|
||||
self._redact_docx(file_path, entities, out_path)
|
||||
elif suffix == ".pdf":
|
||||
self._redact_pdf(file_path, entities, out_path)
|
||||
elif suffix in (".xlsx", ".xls"):
|
||||
self._redact_excel(file_path, entities, out_path)
|
||||
else:
|
||||
raise ValueError(f"TextRedactor 不支持: {suffix}")
|
||||
|
||||
def _redact_docx(self, path: str, entities: list[Entity], out: str) -> None:
|
||||
import docx
|
||||
doc = docx.Document(path)
|
||||
values = {e.value for e in entities if e.value and e.layer == "text"}
|
||||
|
||||
def _redact_paragraphs(paragraphs) -> None:
|
||||
for para in paragraphs:
|
||||
for run in para.runs:
|
||||
for val in values:
|
||||
if val in run.text:
|
||||
run.text = run.text.replace(val, _MASK)
|
||||
|
||||
_redact_paragraphs(doc.paragraphs)
|
||||
for table in doc.tables:
|
||||
for row in table.rows:
|
||||
for cell in row.cells:
|
||||
_redact_paragraphs(cell.paragraphs)
|
||||
doc.save(out)
|
||||
|
||||
def _redact_pdf(self, path: str, entities: list[Entity], out: str) -> None:
|
||||
"""PDF 内容流级替换：解压全部 filter→替换目标字节→纯 FlateDecode 写回。
|
||||
|
||||
PDF 内容流可能叠加多层 filter（如 ASCII85Decode + FlateDecode），
|
||||
不能在原始文件字节上直接做字符串替换。正确流程：
|
||||
1. get_data() 解码所有 filter，得到明文内容流字节
|
||||
2. 在明文中替换目标值（等长空格覆盖）
|
||||
3. 手动 zlib 压缩并以单层 FlateDecode filter 写回
|
||||
"""
|
||||
import shutil
|
||||
import zlib
|
||||
|
||||
import pypdf
|
||||
from pypdf.generic import ArrayObject, NameObject
|
||||
|
||||
values = {e.value for e in entities if e.value and e.layer == "text"}
|
||||
if not values:
|
||||
shutil.copy(path, out)
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(path)
|
||||
writer = pypdf.PdfWriter()
|
||||
|
||||
for page in reader.pages:
|
||||
contents = page.get("/Contents")
|
||||
if contents is not None:
|
||||
stream_refs = (
|
||||
list(contents) if isinstance(contents, ArrayObject)
|
||||
else [contents]
|
||||
)
|
||||
for ref in stream_refs:
|
||||
stream_obj = ref.get_object()
|
||||
try:
|
||||
decoded = stream_obj.get_data() # 解压所有 filter
|
||||
except Exception:
|
||||
continue # 无法解码的流（图像等）跳过
|
||||
|
||||
modified = decoded
|
||||
for val in values:
|
||||
encoded = val.encode("utf-8")
|
||||
modified = modified.replace(encoded, b" " * len(encoded))
|
||||
|
||||
if modified != decoded:
|
||||
# 重新压缩并替换 filter 为单层 FlateDecode
|
||||
stream_obj._data = zlib.compress(modified)
|
||||
stream_obj[NameObject("/Filter")] = NameObject("/FlateDecode")
|
||||
stream_obj.pop(NameObject("/DecodeParms"), None)
|
||||
|
||||
writer.add_page(page)
|
||||
|
||||
with open(out, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
def _redact_excel(self, path: str, entities: list[Entity], out: str) -> None:
|
||||
import openpyxl
|
||||
wb = openpyxl.load_workbook(path)
|
||||
values = {e.value for e in entities if e.value and e.layer == "text"}
|
||||
for ws in wb.worksheets:
|
||||
for row in ws.iter_rows():
|
||||
for cell in row:
|
||||
if cell.value and isinstance(cell.value, str):
|
||||
for val in values:
|
||||
if val in cell.value:
|
||||
cell.value = cell.value.replace(val, _MASK)
|
||||
wb.save(out)
|
||||
@@ -0,0 +1,145 @@
|
||||
# tests/test_api.py
|
||||
import pytest
|
||||
import docx
|
||||
import io
|
||||
import json
|
||||
import openpyxl
|
||||
from fastapi.testclient import TestClient
|
||||
from info_privacy.api.main import app
|
||||
|
||||
client = TestClient(app)
|
||||
|
||||
|
||||
def test_health():
|
||||
r = client.get("/api/v1/health")
|
||||
assert r.status_code == 200
|
||||
assert r.json()["status"] == "ok"
|
||||
|
||||
|
||||
def test_types():
|
||||
r = client.get("/api/v1/types")
|
||||
assert r.status_code == 200
|
||||
assert "id_card" in r.json()["entity_types"]
|
||||
|
||||
|
||||
def test_analyze_docx(tmp_path):
|
||||
doc = docx.Document()
|
||||
doc.add_paragraph("身份证：110101199001011234 电话：13812345678")
|
||||
p = tmp_path / "test.docx"
|
||||
doc.save(str(p))
|
||||
with open(p, "rb") as f:
|
||||
r = client.post("/api/v1/analyze",
|
||||
files={"file": ("test.docx", f, "application/octet-stream")})
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
assert data["blocked"] is False
|
||||
types = {e["type"] for e in data["entities"]}
|
||||
assert "id_card" in types or "phone" in types
|
||||
|
||||
|
||||
def test_analyze_classified(tmp_path):
|
||||
doc = docx.Document()
|
||||
doc.add_paragraph("【机密】内部文件")
|
||||
p = tmp_path / "secret.docx"
|
||||
doc.save(str(p))
|
||||
with open(p, "rb") as f:
|
||||
r = client.post("/api/v1/analyze",
|
||||
files={"file": ("secret.docx", f, "application/octet-stream")})
|
||||
assert r.status_code == 200
|
||||
assert r.json()["blocked"] is True
|
||||
|
||||
|
||||
def test_redact_docx(tmp_path):
|
||||
doc = docx.Document()
|
||||
doc.add_paragraph("身份证：110101199001011234 地址：北京市")
|
||||
p = tmp_path / "test.docx"
|
||||
doc.save(str(p))
|
||||
cfg = json.dumps({"redact_types": ["id_card"]})
|
||||
with open(p, "rb") as f:
|
||||
r = client.post("/api/v1/redact",
|
||||
files={"file": ("test.docx", f, "application/octet-stream")},
|
||||
data={"config": cfg})
|
||||
assert r.status_code == 200
|
||||
result_doc = docx.Document(io.BytesIO(r.content))
|
||||
all_text = " ".join(para.text for para in result_doc.paragraphs)
|
||||
assert "110101199001011234" not in all_text
|
||||
|
||||
|
||||
def test_redact_returns_x_security_report_header(tmp_path):
|
||||
"""redact 响应头应包含 X-Security-Report，内含 JSON 格式遮罩统计。"""
|
||||
doc = docx.Document()
|
||||
doc.add_paragraph("身份证：110101199001011234")
|
||||
p = tmp_path / "test.docx"
|
||||
doc.save(str(p))
|
||||
cfg = json.dumps({"redact_types": ["id_card"]})
|
||||
with open(p, "rb") as f:
|
||||
r = client.post("/api/v1/redact",
|
||||
files={"file": ("test.docx", f, "application/octet-stream")},
|
||||
data={"config": cfg})
|
||||
assert r.status_code == 200
|
||||
assert "x-security-report" in r.headers
|
||||
report = json.loads(r.headers["x-security-report"])
|
||||
assert isinstance(report, dict)
|
||||
|
||||
|
||||
def test_analyze_xlsx(tmp_path):
|
||||
"""analyze 端点应支持 .xlsx 格式并返回检测结果。"""
|
||||
wb = openpyxl.Workbook()
|
||||
ws = wb.active
|
||||
ws["A1"] = "姓名"
|
||||
ws["B1"] = "身份证"
|
||||
ws["A2"] = "张三"
|
||||
ws["B2"] = "110101199001011234"
|
||||
p = tmp_path / "test.xlsx"
|
||||
wb.save(str(p))
|
||||
with open(p, "rb") as f:
|
||||
r = client.post("/api/v1/analyze",
|
||||
files={"file": ("test.xlsx", f, "application/octet-stream")})
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
assert any(e["type"] == "id_card" for e in data["entities"])
|
||||
|
||||
|
||||
def test_analyze_no_pii_returns_normal(tmp_path):
|
||||
"""不含任何 PII 的文档应返回 normal 分类且 blocked=False。"""
|
||||
doc = docx.Document()
|
||||
doc.add_paragraph("本季度销售额同比增长15%，各项指标均达标。")
|
||||
p = tmp_path / "clean.docx"
|
||||
doc.save(str(p))
|
||||
with open(p, "rb") as f:
|
||||
r = client.post("/api/v1/analyze",
|
||||
files={"file": ("clean.docx", f, "application/octet-stream")})
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
assert data["classification"] == "normal"
|
||||
assert data["blocked"] is False
|
||||
|
||||
|
||||
def test_analyze_summary_matches_entities(tmp_path):
|
||||
"""analyze 响应中 summary 各类型计数应与 entities 列表精确一致。"""
|
||||
doc = docx.Document()
|
||||
doc.add_paragraph("身份证：110101199001011234 电话：13812345678")
|
||||
p = tmp_path / "test.docx"
|
||||
doc.save(str(p))
|
||||
with open(p, "rb") as f:
|
||||
r = client.post("/api/v1/analyze",
|
||||
files={"file": ("test.docx", f, "application/octet-stream")})
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
for etype, count in data["summary"].items():
|
||||
actual = sum(1 for e in data["entities"] if e["type"] == etype)
|
||||
assert actual == count, f"{etype}: summary={count}，实际 entities={actual}"
|
||||
|
||||
|
||||
def test_redact_classified_returns_403(tmp_path):
|
||||
"""对保密文档执行 redact 应返回 403。"""
|
||||
doc = docx.Document()
|
||||
doc.add_paragraph("【机密】内部绝密资料")
|
||||
p = tmp_path / "secret.docx"
|
||||
doc.save(str(p))
|
||||
cfg = json.dumps({"redact_types": ["id_card"]})
|
||||
with open(p, "rb") as f:
|
||||
r = client.post("/api/v1/redact",
|
||||
files={"file": ("secret.docx", f, "application/octet-stream")},
|
||||
data={"config": cfg})
|
||||
assert r.status_code == 403
|
||||
@@ -0,0 +1,85 @@
|
||||
# tests/test_classifier.py
|
||||
import pytest
|
||||
from info_privacy.models import TextBlock, Classification, Entity, EntityType
|
||||
from info_privacy.classifier.doc_classifier import DocClassifier
|
||||
|
||||
def test_classified_by_keyword():
|
||||
classifier = DocClassifier("configs/pii_rules.yaml")
|
||||
blocks = [TextBlock(text="【机密】本文件仅供内部使用", bbox=[0,0,200,20], page=1, layer="text")]
|
||||
result = classifier.classify(blocks)
|
||||
assert result.classification == Classification.CLASSIFIED
|
||||
assert result.blocked is True
|
||||
assert result.block_reason is not None
|
||||
|
||||
def test_sensitive_partial_many_entities():
|
||||
from info_privacy.models import Entity, EntityType
|
||||
classifier = DocClassifier("configs/pii_rules.yaml")
|
||||
entities = [
|
||||
Entity(id=str(i), type=EntityType.ID_CARD, value="x", page=1,
|
||||
bbox=[0,0,1,1], layer="text", security_level="high")
|
||||
for i in range(6)
|
||||
]
|
||||
result = classifier.classify([], entities=entities)
|
||||
assert result.classification == Classification.SENSITIVE_PARTIAL
|
||||
assert result.warning is not None
|
||||
|
||||
def test_normal_document():
|
||||
classifier = DocClassifier("configs/pii_rules.yaml")
|
||||
blocks = [TextBlock(text="本季度销售报告", bbox=[0,0,200,20], page=1, layer="text")]
|
||||
result = classifier.classify(blocks)
|
||||
assert result.classification == Classification.NORMAL
|
||||
assert result.blocked is False
|
||||
|
||||
|
||||
_ALL_KEYWORDS = [
|
||||
"机密", "绝密", "保密", "内部文件", "内部资料",
|
||||
"CONFIDENTIAL", "SECRET", "TOP SECRET", "RESTRICTED", "FOR INTERNAL USE ONLY",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("keyword", _ALL_KEYWORDS)
|
||||
def test_all_confidential_keywords_trigger_blocked(keyword):
|
||||
"""pii_rules.yaml 中每个保密关键词都应触发 classified 拦截。"""
|
||||
classifier = DocClassifier("configs/pii_rules.yaml")
|
||||
blocks = [TextBlock(text=f"标题：{keyword}", bbox=[0,0,300,20], page=1, layer="text")]
|
||||
result = classifier.classify(blocks)
|
||||
assert result.classification == Classification.CLASSIFIED, f"关键词 '{keyword}' 未触发拦截"
|
||||
assert result.blocked is True
|
||||
assert result.block_reason is not None
|
||||
|
||||
|
||||
def _make_high_risk_entities(n: int) -> list[Entity]:
|
||||
return [
|
||||
Entity(id=str(i), type=EntityType.PHONE, value="13800000000",
|
||||
page=1, bbox=[0, 0, 1, 1], layer="text", security_level="high")
|
||||
for i in range(n)
|
||||
]
|
||||
|
||||
|
||||
def test_exactly_4_high_risk_no_warning():
|
||||
"""4 个高风险实体：低于阈值，不触发 sensitive_partial 警告。"""
|
||||
classifier = DocClassifier("configs/pii_rules.yaml")
|
||||
result = classifier.classify([], entities=_make_high_risk_entities(4))
|
||||
assert result.classification == Classification.NORMAL
|
||||
assert result.warning is None
|
||||
|
||||
|
||||
def test_exactly_5_high_risk_triggers_warning():
|
||||
"""5 个高风险实体：恰好达到阈值，应触发 sensitive_partial 警告。"""
|
||||
classifier = DocClassifier("configs/pii_rules.yaml")
|
||||
result = classifier.classify([], entities=_make_high_risk_entities(5))
|
||||
assert result.classification == Classification.SENSITIVE_PARTIAL
|
||||
assert result.warning is not None
|
||||
|
||||
|
||||
def test_medium_entities_alone_do_not_trigger_warning():
|
||||
"""10 个中风险实体（email）不应触发 sensitive_partial 警告。"""
|
||||
classifier = DocClassifier("configs/pii_rules.yaml")
|
||||
entities = [
|
||||
Entity(id=str(i), type=EntityType.EMAIL, value="x@x.com",
|
||||
page=1, bbox=[0, 0, 1, 1], layer="text", security_level="medium")
|
||||
for i in range(10)
|
||||
]
|
||||
result = classifier.classify([], entities=entities)
|
||||
assert result.classification == Classification.NORMAL
|
||||
assert result.warning is None
|
||||
@@ -0,0 +1,185 @@
|
||||
import pytest
|
||||
from info_privacy.models import TextBlock, EntityType
|
||||
from info_privacy.detectors.regex_detector import RegexDetector
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def detector():
|
||||
return RegexDetector("configs/pii_rules.yaml")
|
||||
|
||||
|
||||
def test_detect_id_card(detector):
|
||||
block = TextBlock(text="姓名：张三 身份证：110101199001011234", bbox=[0, 0, 100, 20], page=1, layer="text")
|
||||
entities = detector.detect([block])
|
||||
types = [e.type for e in entities]
|
||||
assert EntityType.ID_CARD in types
|
||||
|
||||
|
||||
def test_detect_phone(detector):
|
||||
block = TextBlock(text="联系电话：13812345678", bbox=[0, 0, 100, 20], page=1, layer="text")
|
||||
entities = detector.detect([block])
|
||||
types = [e.type for e in entities]
|
||||
assert EntityType.PHONE in types
|
||||
|
||||
|
||||
def test_detect_email(detector):
|
||||
block = TextBlock(text="邮箱：test@example.com", bbox=[0, 0, 100, 20], page=1, layer="text")
|
||||
entities = detector.detect([block])
|
||||
assert any(e.type == EntityType.EMAIL for e in entities)
|
||||
|
||||
|
||||
def test_no_false_positive_short_number(detector):
|
||||
block = TextBlock(text="编号：12345", bbox=[0, 0, 100, 20], page=1, layer="text")
|
||||
entities = detector.detect([block])
|
||||
assert not any(e.type == EntityType.ID_CARD for e in entities)
|
||||
|
||||
|
||||
def test_entity_bbox_matches_block(detector):
|
||||
block = TextBlock(text="电话：13812345678", bbox=[10, 20, 200, 40], page=2, layer="text")
|
||||
entities = detector.detect([block])
|
||||
assert entities[0].page == 2
|
||||
assert entities[0].bbox == [10, 20, 200, 40]
|
||||
|
||||
|
||||
# NER 检测器测试
|
||||
from info_privacy.detectors.ner_detector import NERDetector
|
||||
|
||||
@pytest.fixture
|
||||
def ner():
|
||||
return NERDetector("configs/pii_rules.yaml")
|
||||
|
||||
def test_detect_name(ner):
|
||||
block = TextBlock(text="申请人：张伟，联系地址如下", bbox=[0,0,200,20], page=1, layer="text")
|
||||
entities = ner.detect([block])
|
||||
assert any(e.type == EntityType.NAME for e in entities)
|
||||
|
||||
def test_detect_address(ner):
|
||||
block = TextBlock(text="住址：北京市朝阳区建国路88号", bbox=[0,0,300,20], page=1, layer="text")
|
||||
entities = ner.detect([block])
|
||||
assert any(e.type == EntityType.ADDRESS for e in entities)
|
||||
|
||||
def test_no_name_without_surname(ner):
|
||||
block = TextBlock(text="操作系统版本3.1", bbox=[0,0,200,20], page=1, layer="text")
|
||||
entities = ner.detect([block])
|
||||
assert not any(e.type == EntityType.NAME for e in entities)
|
||||
|
||||
|
||||
def test_ner_no_address_false_positive(ner):
|
||||
"""普通正文不含地址触发词，不应检测到 address。"""
|
||||
block = TextBlock(text="本季度产品销售额增长20%", bbox=[0,0,300,20], page=1, layer="text")
|
||||
entities = ner.detect([block])
|
||||
assert not any(e.type == EntityType.ADDRESS for e in entities)
|
||||
|
||||
|
||||
def test_ner_name_and_address_same_block(ner):
|
||||
"""同一段文本中姓名和地址应同时被检测。"""
|
||||
block = TextBlock(
|
||||
text="申请人：王芳，地址：上海市浦东新区张江路100号",
|
||||
bbox=[0, 0, 400, 20], page=1, layer="text",
|
||||
)
|
||||
entities = ner.detect([block])
|
||||
types = {e.type for e in entities}
|
||||
assert EntityType.NAME in types
|
||||
assert EntityType.ADDRESS in types
|
||||
|
||||
|
||||
# RegexDetector 新增覆盖
|
||||
|
||||
def test_detect_bank_card_16_digit(detector):
|
||||
"""16 位纯银行卡号（非身份证格式）应被识别为 bank_card。"""
|
||||
block = TextBlock(text="卡号：6222021302001002", bbox=[0,0,200,20], page=1, layer="text")
|
||||
entities = detector.detect([block])
|
||||
types = [e.type for e in entities]
|
||||
assert EntityType.BANK_CARD in types
|
||||
assert EntityType.ID_CARD not in types
|
||||
|
||||
|
||||
def test_id_card_not_double_counted_as_bank_card(detector):
|
||||
"""18 位身份证号不应同时被识别为 bank_card（去重逻辑）。"""
|
||||
block = TextBlock(text="证件号：110101199001011234", bbox=[0,0,200,20], page=1, layer="text")
|
||||
entities = detector.detect([block])
|
||||
id_entities = [e for e in entities if e.type == EntityType.ID_CARD]
|
||||
bank_overlap = [e for e in entities
|
||||
if e.type == EntityType.BANK_CARD and e.value == "110101199001011234"]
|
||||
assert len(id_entities) == 1
|
||||
assert len(bank_overlap) == 0
|
||||
|
||||
|
||||
def test_detect_id_card_ending_in_x(detector):
|
||||
"""末位为 X 的身份证号应被正确识别，且 value 保留末位 X。"""
|
||||
block = TextBlock(text="证件：11010119900101110X", bbox=[0,0,200,20], page=1, layer="text")
|
||||
entities = detector.detect([block])
|
||||
id_entities = [e for e in entities if e.type == EntityType.ID_CARD]
|
||||
assert len(id_entities) == 1
|
||||
assert id_entities[0].value.endswith("X")
|
||||
|
||||
|
||||
def test_detect_license_plate(detector):
|
||||
"""有效的中国车牌应被识别为 license_plate。"""
|
||||
block = TextBlock(text="车辆：京A12345", bbox=[0,0,200,20], page=1, layer="text")
|
||||
entities = detector.detect([block])
|
||||
assert any(e.type == EntityType.LICENSE_PLATE for e in entities)
|
||||
|
||||
|
||||
def test_entity_security_level_high(detector):
|
||||
"""高风险类型 id_card 的 security_level 应为 high。"""
|
||||
block = TextBlock(text="110101199001011234", bbox=[0,0,200,20], page=1, layer="text")
|
||||
entities = detector.detect([block])
|
||||
id_e = next(e for e in entities if e.type == EntityType.ID_CARD)
|
||||
assert id_e.security_level == "high"
|
||||
|
||||
|
||||
def test_entity_security_level_medium(detector):
|
||||
"""中风险类型 email 的 security_level 应为 medium。"""
|
||||
block = TextBlock(text="user@example.com", bbox=[0,0,200,20], page=1, layer="text")
|
||||
entities = detector.detect([block])
|
||||
email_e = next(e for e in entities if e.type == EntityType.EMAIL)
|
||||
assert email_e.security_level == "medium"
|
||||
|
||||
|
||||
def test_multiple_entities_same_block(detector):
|
||||
"""同一文本块中同时含手机号和邮箱，两者均应被检测。"""
|
||||
block = TextBlock(
|
||||
text="联系方式：13812345678 / admin@corp.com",
|
||||
bbox=[0, 0, 400, 20], page=1, layer="text",
|
||||
)
|
||||
entities = detector.detect([block])
|
||||
types = {e.type for e in entities}
|
||||
assert EntityType.PHONE in types
|
||||
assert EntityType.EMAIL in types
|
||||
|
||||
|
||||
# 人脸检测器测试
|
||||
import os
|
||||
import numpy as np
|
||||
import cv2
|
||||
|
||||
FACE_MODEL = "/data/rockchip/mediapipe/models/rknn/face_detection_short_range_rk3588.rknn"
|
||||
|
||||
def _rknn_available():
|
||||
try:
|
||||
import rknn
|
||||
return True
|
||||
except ImportError:
|
||||
try:
|
||||
import rknnlite
|
||||
return True
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
def test_face_detector_skip_if_no_model():
|
||||
if not os.path.exists(FACE_MODEL) or not _rknn_available():
|
||||
pytest.skip("人脸RKNN模型不存在或 RKNN 运行时不可用")
|
||||
from info_privacy.detectors.face_detector import FaceDetector
|
||||
detector = FaceDetector(FACE_MODEL)
|
||||
assert detector is not None
|
||||
|
||||
def test_face_detector_returns_entities():
|
||||
if not os.path.exists(FACE_MODEL) or not _rknn_available():
|
||||
pytest.skip("人脸RKNN模型不存在或 RKNN 运行时不可用")
|
||||
from info_privacy.detectors.face_detector import FaceDetector
|
||||
detector = FaceDetector(FACE_MODEL)
|
||||
# 白色空图像，无人脸
|
||||
img = np.ones((300, 300, 3), dtype=np.uint8) * 255
|
||||
entities = detector.detect_in_image(img, page=1)
|
||||
assert isinstance(entities, list)
|
||||
@@ -0,0 +1,598 @@
|
||||
# tests/test_integration_complex.py
|
||||
"""高复杂度集成测试：实体值精确性、多页 PDF、表格解析、格式变体、
|
||||
边界条件、误报拒绝、幂等性、阈值精确、跨 sheet 定位等。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import cv2
|
||||
import docx as _docx
|
||||
import numpy as np
|
||||
import openpyxl
|
||||
import pytest
|
||||
from pdfminer.high_level import extract_text
|
||||
from reportlab.lib.pagesizes import A4
|
||||
from reportlab.pdfbase import pdfmetrics
|
||||
from reportlab.pdfbase.ttfonts import TTFont
|
||||
from reportlab.pdfgen import canvas
|
||||
|
||||
from info_privacy.models import EntityType
|
||||
from info_privacy.pipeline import PrivacyPipeline
|
||||
|
||||
# ─── 字体 ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
_FONT_PATH = "/usr/share/fonts/truetype/arphic/uming.ttc"
|
||||
_FONT_NAME = "UMingC" # 用不同名防止重复注册冲突
|
||||
|
||||
if os.path.exists(_FONT_PATH) and _FONT_NAME not in pdfmetrics.getRegisteredFontNames():
|
||||
pdfmetrics.registerFont(TTFont(_FONT_NAME, _FONT_PATH, subfontIndex=0))
|
||||
|
||||
_needs_font = pytest.mark.skipif(
|
||||
not os.path.exists(_FONT_PATH), reason="uming.ttc 不存在"
|
||||
)
|
||||
|
||||
# ─── 测试数据 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
PERSONS = [
|
||||
{"id": "110101199001011234", "phone": "13812345678", "name": "张伟",
|
||||
"email": "zhang.wei@corp.com", "bank": "6222021234567890123"},
|
||||
{"id": "310101198505051234", "phone": "13900001111", "name": "李娜",
|
||||
"email": "li.na@test.org", "bank": "6222021111111111111"},
|
||||
{"id": "440101200003031234", "phone": "15900002222", "name": "王芳",
|
||||
"email": "wang.fang@example.net", "bank": "6222023333333333333"},
|
||||
{"id": "320101199707071234", "phone": "18700003333", "name": "赵磊",
|
||||
"email": "zhao.lei@sample.cn", "bank": "6222024444444444444"},
|
||||
{"id": "510101196808081234", "phone": "13700004444", "name": "陈静",
|
||||
"email": "chen.jing@demo.com", "bank": "6222025555555555555"},
|
||||
]
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def pipeline():
|
||||
return PrivacyPipeline()
|
||||
|
||||
|
||||
# ─── 工具函数 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def _make_multipage_pdf(path: str, pages: list[list[str]]) -> None:
|
||||
"""生成多页 PDF，每个子列表为一页的行列表。"""
|
||||
c = canvas.Canvas(path, pagesize=A4)
|
||||
c.setFont(_FONT_NAME, 14)
|
||||
for page_lines in pages:
|
||||
y = 750
|
||||
for line in page_lines:
|
||||
c.drawString(50, y, line)
|
||||
y -= 30
|
||||
c.showPage()
|
||||
c.save()
|
||||
|
||||
|
||||
def _docx_text(path: str) -> str:
|
||||
doc = _docx.Document(path)
|
||||
parts = [p.text for p in doc.paragraphs]
|
||||
for table in doc.tables:
|
||||
for row in table.rows:
|
||||
for cell in row.cells:
|
||||
parts.extend(p.text for p in cell.paragraphs)
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def _xlsx_text(path: str) -> str:
|
||||
wb = openpyxl.load_workbook(path, data_only=True)
|
||||
parts = []
|
||||
for ws in wb.worksheets:
|
||||
for row in ws.iter_rows(values_only=True):
|
||||
parts.extend(str(c) for c in row if c is not None)
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
# ─── 1. 实体值精确性（不只检查类型，验证 value 字段）────────────────────────────
|
||||
|
||||
def test_entity_value_id_card_exact(tmp_path, pipeline):
|
||||
doc = _docx.Document()
|
||||
doc.add_paragraph(f"身份证号：{PERSONS[0]['id']}")
|
||||
p = str(tmp_path / "t.docx")
|
||||
doc.save(p)
|
||||
report = pipeline.analyze(p)
|
||||
id_entities = [e for e in report.entities if e.type == EntityType.ID_CARD]
|
||||
assert len(id_entities) >= 1
|
||||
values = {e.value for e in id_entities}
|
||||
assert PERSONS[0]["id"] in values, f"身份证值不对，得到：{values}"
|
||||
|
||||
|
||||
def test_entity_value_phone_exact(tmp_path, pipeline):
|
||||
doc = _docx.Document()
|
||||
doc.add_paragraph(f"联系电话：{PERSONS[0]['phone']}")
|
||||
p = str(tmp_path / "t.docx")
|
||||
doc.save(p)
|
||||
report = pipeline.analyze(p)
|
||||
phones = [e for e in report.entities if e.type == EntityType.PHONE]
|
||||
assert any(e.value == PERSONS[0]["phone"] for e in phones), \
|
||||
f"手机号值不对，得到：{[e.value for e in phones]}"
|
||||
|
||||
|
||||
def test_entity_value_email_exact(tmp_path, pipeline):
|
||||
doc = _docx.Document()
|
||||
doc.add_paragraph(f"邮箱：{PERSONS[0]['email']}")
|
||||
p = str(tmp_path / "t.docx")
|
||||
doc.save(p)
|
||||
report = pipeline.analyze(p)
|
||||
emails = [e for e in report.entities if e.type == EntityType.EMAIL]
|
||||
assert any(e.value == PERSONS[0]["email"] for e in emails)
|
||||
|
||||
|
||||
# ─── 2. PDF 多页：entity.page 字段正确 ──────────────────────────────────────────
|
||||
|
||||
@_needs_font
|
||||
def test_pdf_multipage_entity_page_numbers(tmp_path, pipeline):
|
||||
"""3 页 PDF，每页一个身份证，entity.page 应分别为 1/2/3。"""
|
||||
pdf = str(tmp_path / "multi.pdf")
|
||||
_make_multipage_pdf(pdf, [
|
||||
[f"第一页 身份证：{PERSONS[0]['id']}"],
|
||||
[f"第二页 身份证：{PERSONS[1]['id']}"],
|
||||
[f"第三页 身份证：{PERSONS[2]['id']}"],
|
||||
])
|
||||
report = pipeline.analyze(pdf)
|
||||
id_entities = [e for e in report.entities if e.type == EntityType.ID_CARD]
|
||||
pages = {e.page for e in id_entities}
|
||||
assert len(pages) == 3, f"应检测到 3 个不同页码，实际 pages={pages}"
|
||||
assert pages == {1, 2, 3}, f"页码不对：{pages}"
|
||||
|
||||
|
||||
@_needs_font
|
||||
def test_pdf_multipage_redact_all_pages(tmp_path, pipeline):
|
||||
"""多页 PDF 遮罩后每页的身份证都消失。"""
|
||||
pdf = str(tmp_path / "multi.pdf")
|
||||
out = str(tmp_path / "out.pdf")
|
||||
_make_multipage_pdf(pdf, [
|
||||
[f"页1 {PERSONS[0]['id']}"],
|
||||
[f"页2 {PERSONS[1]['id']}"],
|
||||
])
|
||||
pipeline.redact(pdf, redact_types=["id_card"], out_path=out)
|
||||
result_text = extract_text(out)
|
||||
assert PERSONS[0]["id"] not in result_text, "第1页身份证未被遮罩"
|
||||
assert PERSONS[1]["id"] not in result_text, "第2页身份证未被遮罩"
|
||||
|
||||
|
||||
@_needs_font
|
||||
def test_pdf_multipage_preserves_non_pii_lines(tmp_path, pipeline):
|
||||
"""多页 PDF 遮罩后非 PII 内容保留。"""
|
||||
pdf = str(tmp_path / "multi.pdf")
|
||||
out = str(tmp_path / "out.pdf")
|
||||
# 第 2 页用纯 ASCII 锚点：pdfminer 对跨页 CJK 字形解码不稳定，
|
||||
# ASCII 内容始终可靠，足以验证非 PII 内容未被误删。
|
||||
_make_multipage_pdf(pdf, [
|
||||
["合同编号：HT-2026-001", f"甲方身份证：{PERSONS[0]['id']}"],
|
||||
["REF-NOREDACT-P2"],
|
||||
])
|
||||
pipeline.redact(pdf, redact_types=["id_card"], out_path=out)
|
||||
result_text = extract_text(out)
|
||||
assert PERSONS[0]["id"] not in result_text
|
||||
assert "HT-2026-001" in result_text, "第1页非PII文本被误删"
|
||||
assert "REF-NOREDACT-P2" in result_text, "第2页非PII文本被误删"
|
||||
|
||||
|
||||
# ─── 3. DOCX 表格单元格解析 ──────────────────────────────────────────────────────
|
||||
|
||||
def test_docx_table_detects_id_card(tmp_path, pipeline):
|
||||
"""PII 藏在表格单元格中应被正常检测。"""
|
||||
doc = _docx.Document()
|
||||
table = doc.add_table(rows=2, cols=3)
|
||||
table.cell(0, 0).text = "姓名"
|
||||
table.cell(0, 1).text = "身份证号"
|
||||
table.cell(0, 2).text = "手机号"
|
||||
table.cell(1, 0).text = PERSONS[0]["name"]
|
||||
table.cell(1, 1).text = PERSONS[0]["id"]
|
||||
table.cell(1, 2).text = PERSONS[0]["phone"]
|
||||
p = str(tmp_path / "table.docx")
|
||||
doc.save(p)
|
||||
report = pipeline.analyze(p)
|
||||
types = {e.type for e in report.entities}
|
||||
assert EntityType.ID_CARD in types, "表格中的身份证未被检测"
|
||||
assert EntityType.PHONE in types, "表格中的手机号未被检测"
|
||||
|
||||
|
||||
def test_docx_table_redact_removes_value(tmp_path, pipeline):
|
||||
"""对含表格的 DOCX 执行遮罩，表格内的 PII 也被删除。"""
|
||||
doc = _docx.Document()
|
||||
table = doc.add_table(rows=3, cols=2)
|
||||
for i, p in enumerate(PERSONS[:3]):
|
||||
table.cell(i, 0).text = p["name"]
|
||||
table.cell(i, 1).text = p["id"]
|
||||
path = str(tmp_path / "table.docx")
|
||||
doc.save(path)
|
||||
out = str(tmp_path / "out.docx")
|
||||
pipeline.redact(path, redact_types=["id_card"], out_path=out)
|
||||
text = _docx_text(out)
|
||||
for p in PERSONS[:3]:
|
||||
assert p["id"] not in text, f"{p['id']} 未被遮罩"
|
||||
|
||||
|
||||
def test_docx_table_paragraph_both_detected(tmp_path, pipeline):
|
||||
"""正文段落 + 表格都有 PII 时，两处均被检测到。"""
|
||||
doc = _docx.Document()
|
||||
doc.add_paragraph(f"合同方：{PERSONS[0]['name']} 身份证：{PERSONS[0]['id']}")
|
||||
table = doc.add_table(rows=1, cols=2)
|
||||
table.cell(0, 0).text = PERSONS[1]["name"]
|
||||
table.cell(0, 1).text = PERSONS[1]["id"]
|
||||
path = str(tmp_path / "mixed.docx")
|
||||
doc.save(path)
|
||||
report = pipeline.analyze(path)
|
||||
id_values = {e.value for e in report.entities if e.type == EntityType.ID_CARD}
|
||||
assert PERSONS[0]["id"] in id_values, "段落中身份证未检测"
|
||||
assert PERSONS[1]["id"] in id_values, "表格中身份证未检测"
|
||||
|
||||
|
||||
# ─── 4. XLSX 跨 sheet 检测与 entity.page ─────────────────────────────────────────
|
||||
|
||||
@pytest.fixture
|
||||
def multi_sheet_xlsx(tmp_path):
|
||||
wb = openpyxl.Workbook()
|
||||
ws1 = wb.active
|
||||
ws1.title = "员工信息"
|
||||
ws1.append(["姓名", "身份证"])
|
||||
for p in PERSONS[:2]:
|
||||
ws1.append([p["name"], p["id"]])
|
||||
|
||||
ws2 = wb.create_sheet("联系方式")
|
||||
ws2.append(["姓名", "手机", "邮箱"])
|
||||
for p in PERSONS[2:4]:
|
||||
ws2.append([p["name"], p["phone"], p["email"]])
|
||||
|
||||
path = str(tmp_path / "multi_sheet.xlsx")
|
||||
wb.save(path)
|
||||
return path
|
||||
|
||||
|
||||
def test_xlsx_multisheet_detects_pii_in_both(multi_sheet_xlsx, pipeline):
|
||||
report = pipeline.analyze(multi_sheet_xlsx)
|
||||
types = {e.type for e in report.entities}
|
||||
assert EntityType.ID_CARD in types, "Sheet1 身份证未检测"
|
||||
assert EntityType.PHONE in types, "Sheet2 手机号未检测"
|
||||
assert EntityType.EMAIL in types, "Sheet2 邮箱未检测"
|
||||
|
||||
|
||||
def test_xlsx_multisheet_page_numbers(multi_sheet_xlsx, pipeline):
|
||||
"""Sheet1 的实体 page=1，Sheet2 的实体 page=2。"""
|
||||
report = pipeline.analyze(multi_sheet_xlsx)
|
||||
id_pages = {e.page for e in report.entities if e.type == EntityType.ID_CARD}
|
||||
phone_pages = {e.page for e in report.entities if e.type == EntityType.PHONE}
|
||||
assert 1 in id_pages, f"Sheet1 的身份证 page 应为 1，实际：{id_pages}"
|
||||
assert 2 in phone_pages, f"Sheet2 的手机号 page 应为 2，实际：{phone_pages}"
|
||||
|
||||
|
||||
def test_xlsx_multisheet_redact_across_sheets(multi_sheet_xlsx, tmp_path, pipeline):
|
||||
out = str(tmp_path / "out.xlsx")
|
||||
pipeline.redact(multi_sheet_xlsx, redact_types=["id_card", "phone"], out_path=out)
|
||||
text = _xlsx_text(out)
|
||||
for p in PERSONS[:2]:
|
||||
assert p["id"] not in text, f"Sheet1 身份证 {p['id']} 未被遮罩"
|
||||
for p in PERSONS[2:4]:
|
||||
assert p["phone"] not in text, f"Sheet2 手机 {p['phone']} 未被遮罩"
|
||||
|
||||
|
||||
# ─── 5. 手机号格式变体 ────────────────────────────────────────────────────────────
|
||||
|
||||
@pytest.mark.parametrize("phone_str,label", [
|
||||
("13812345678", "纯数字"),
|
||||
("+8613812345678", "+86前缀"),
|
||||
("86-13812345678", "86-前缀"),
|
||||
("+86 13812345678", "+86空格"),
|
||||
("86 13812345678", "86空格"),
|
||||
])
|
||||
def test_phone_format_variants(tmp_path, pipeline, phone_str, label):
|
||||
doc = _docx.Document()
|
||||
doc.add_paragraph(f"联系人手机：{phone_str}")
|
||||
p = str(tmp_path / "t.docx")
|
||||
doc.save(p)
|
||||
report = pipeline.analyze(p)
|
||||
phones = [e for e in report.entities if e.type == EntityType.PHONE]
|
||||
assert len(phones) >= 1, f"格式 '{label}' 未被检测到手机号"
|
||||
|
||||
|
||||
# ─── 6. 误报拒绝：不应匹配的模式 ─────────────────────────────────────────────────
|
||||
|
||||
@pytest.mark.parametrize("text,no_type,label", [
|
||||
("编号：12345", EntityType.ID_CARD, "5位短数字"),
|
||||
("这是第1990年的事", EntityType.ID_CARD, "含年份数字"),
|
||||
("共有15人参会", EntityType.PHONE, "2位数字"),
|
||||
("192.168.0.127端口8080", EntityType.PHONE, "IP地址"),
|
||||
("请联系test@", EntityType.EMAIL, "不完整邮箱"),
|
||||
("版本号：1.2.3.4", EntityType.EMAIL, "版本号"),
|
||||
("合同金额：1234567890元", EntityType.BANK_CARD, "10位金额"),
|
||||
])
|
||||
def test_no_false_positive(tmp_path, pipeline, text, no_type, label):
|
||||
doc = _docx.Document()
|
||||
doc.add_paragraph(text)
|
||||
p = str(tmp_path / "t.docx")
|
||||
doc.save(p)
|
||||
report = pipeline.analyze(p)
|
||||
matched = [e for e in report.entities if e.type == no_type]
|
||||
assert len(matched) == 0, f"'{label}' 误报了 {no_type.value}：{[e.value for e in matched]}"
|
||||
|
||||
|
||||
# ─── 7. 行内 PII（嵌在句子中间）─────────────────────────────────────────────────────
|
||||
|
||||
def test_inline_pii_phone_in_sentence(tmp_path, pipeline):
|
||||
"""手机号嵌在一个完整中文句子中也应被检测到。"""
|
||||
doc = _docx.Document()
|
||||
doc.add_paragraph(f"请于明日拨打{PERSONS[0]['phone']}与客服确认到访时间。")
|
||||
p = str(tmp_path / "t.docx")
|
||||
doc.save(p)
|
||||
report = pipeline.analyze(p)
|
||||
phones = [e for e in report.entities if e.type == EntityType.PHONE]
|
||||
assert len(phones) >= 1, "句子中间的手机号未检测"
|
||||
|
||||
|
||||
def test_inline_pii_multiple_on_same_line(tmp_path, pipeline):
|
||||
"""同一段落含身份证+手机+邮箱，三者都要被检测到。"""
|
||||
doc = _docx.Document()
|
||||
doc.add_paragraph(
|
||||
f"甲方{PERSONS[0]['name']}（证件号{PERSONS[0]['id']}，"
|
||||
f"电话{PERSONS[0]['phone']}，"
|
||||
f"邮箱{PERSONS[0]['email']}）确认签字。"
|
||||
)
|
||||
p = str(tmp_path / "t.docx")
|
||||
doc.save(p)
|
||||
report = pipeline.analyze(p)
|
||||
types = {e.type for e in report.entities}
|
||||
assert EntityType.ID_CARD in types, "行内身份证未检测"
|
||||
assert EntityType.PHONE in types, "行内手机号未检测"
|
||||
assert EntityType.EMAIL in types, "行内邮箱未检测"
|
||||
|
||||
|
||||
def test_two_phones_same_paragraph(tmp_path, pipeline):
|
||||
"""同一段落包含 2 个不同手机号，两个都被检测到。"""
|
||||
doc = _docx.Document()
|
||||
doc.add_paragraph(f"联系人A：{PERSONS[0]['phone']}，联系人B：{PERSONS[1]['phone']}")
|
||||
p = str(tmp_path / "t.docx")
|
||||
doc.save(p)
|
||||
report = pipeline.analyze(p)
|
||||
phones = {e.value for e in report.entities if e.type == EntityType.PHONE}
|
||||
assert PERSONS[0]["phone"] in phones, "第1个手机号未检测"
|
||||
assert PERSONS[1]["phone"] in phones, "第2个手机号未检测"
|
||||
|
||||
|
||||
# ─── 8. 遮罩幂等性：redact → re-analyze 验证 ─────────────────────────────────────
|
||||
|
||||
def test_redact_then_reanalyze_no_id_card(tmp_path, pipeline):
|
||||
"""遮罩身份证后，再次 analyze 应检测不到身份证。"""
|
||||
doc = _docx.Document()
|
||||
doc.add_paragraph(f"身份证：{PERSONS[0]['id']}，电话：{PERSONS[0]['phone']}")
|
||||
p = str(tmp_path / "orig.docx")
|
||||
doc.save(p)
|
||||
|
||||
out = str(tmp_path / "redacted.docx")
|
||||
pipeline.redact(p, redact_types=["id_card"], out_path=out)
|
||||
|
||||
re_report = pipeline.analyze(out)
|
||||
id_entities = [e for e in re_report.entities if e.type == EntityType.ID_CARD]
|
||||
assert len(id_entities) == 0, f"遮罩后仍检测到身份证：{[e.value for e in id_entities]}"
|
||||
|
||||
|
||||
def test_redact_then_reanalyze_other_types_survive(tmp_path, pipeline):
|
||||
"""遮罩身份证后，其他类型（手机号）不受影响，仍可被检测。"""
|
||||
doc = _docx.Document()
|
||||
doc.add_paragraph(f"身份证：{PERSONS[0]['id']}，电话：{PERSONS[0]['phone']}")
|
||||
p = str(tmp_path / "orig.docx")
|
||||
doc.save(p)
|
||||
|
||||
out = str(tmp_path / "redacted.docx")
|
||||
pipeline.redact(p, redact_types=["id_card"], out_path=out)
|
||||
|
||||
re_report = pipeline.analyze(out)
|
||||
phones = [e for e in re_report.entities if e.type == EntityType.PHONE]
|
||||
assert len(phones) >= 1, "遮罩身份证后手机号不应消失"
|
||||
|
||||
|
||||
def test_redact_all_types_empty_reanalyze(tmp_path, pipeline):
|
||||
"""全量遮罩后，re-analyze 应无任何高危实体。"""
|
||||
doc = _docx.Document()
|
||||
doc.add_paragraph(
|
||||
f"姓名：{PERSONS[0]['name']} 身份证：{PERSONS[0]['id']} "
|
||||
f"电话：{PERSONS[0]['phone']} 邮箱：{PERSONS[0]['email']} "
|
||||
f"银行卡：{PERSONS[0]['bank']}"
|
||||
)
|
||||
p = str(tmp_path / "orig.docx")
|
||||
doc.save(p)
|
||||
|
||||
out = str(tmp_path / "redacted.docx")
|
||||
pipeline.redact(
|
||||
p,
|
||||
redact_types=["id_card", "phone", "email", "bank_card", "name"],
|
||||
out_path=out,
|
||||
)
|
||||
|
||||
re_report = pipeline.analyze(out)
|
||||
high_risk = [e for e in re_report.entities if e.security_level == "high"]
|
||||
assert len(high_risk) == 0, f"全量遮罩后仍有高危实体：{[e.type for e in high_risk]}"
|
||||
|
||||
|
||||
# ─── 9. 阈值边界：sensitive_partial 触发精确值 ────────────────────────────────────
|
||||
|
||||
def test_threshold_below_no_warning(tmp_path, pipeline):
|
||||
"""4 个高危实体（< 5）→ 不触发 sensitive_partial。"""
|
||||
doc = _docx.Document()
|
||||
for p in PERSONS[:4]:
|
||||
doc.add_paragraph(f"身份证：{p['id']}")
|
||||
path = str(tmp_path / "t.docx")
|
||||
doc.save(path)
|
||||
report = pipeline.analyze(path)
|
||||
assert report.warning is None, f"4个高危实体不应触发警告，实际：{report.warning}"
|
||||
assert report.classification.value == "normal"
|
||||
|
||||
|
||||
def test_threshold_at_boundary_triggers_warning(tmp_path, pipeline):
|
||||
"""5 个高危实体（= 阈值）→ 触发 sensitive_partial 警告。"""
|
||||
doc = _docx.Document()
|
||||
for p in PERSONS[:5]:
|
||||
doc.add_paragraph(f"身份证：{p['id']}")
|
||||
path = str(tmp_path / "t.docx")
|
||||
doc.save(path)
|
||||
report = pipeline.analyze(path)
|
||||
assert report.warning is not None, "5个高危实体应触发警告"
|
||||
assert report.classification.value == "sensitive_partial"
|
||||
|
||||
|
||||
def test_threshold_above_warning_message_has_count(tmp_path, pipeline):
|
||||
"""警告信息中应包含实体数量。"""
|
||||
doc = _docx.Document()
|
||||
for p in PERSONS: # 5人
|
||||
doc.add_paragraph(f"身份证：{p['id']}")
|
||||
path = str(tmp_path / "t.docx")
|
||||
doc.save(path)
|
||||
report = pipeline.analyze(path)
|
||||
assert report.warning is not None
|
||||
assert "5" in report.warning, f"警告信息中应包含数量 5，实际：{report.warning}"
|
||||
|
||||
|
||||
# ─── 10. 遮罩计数精确性 ───────────────────────────────────────────────────────────
|
||||
|
||||
def test_redact_count_matches_detection(tmp_path, pipeline):
|
||||
"""security_report['redacted']['id_card'] 应等于检测到的数量。"""
|
||||
doc = _docx.Document()
|
||||
for p in PERSONS[:3]:
|
||||
doc.add_paragraph(f"身份证：{p['id']}")
|
||||
path = str(tmp_path / "t.docx")
|
||||
doc.save(path)
|
||||
out = str(tmp_path / "out.docx")
|
||||
|
||||
analyze_report = pipeline.analyze(path)
|
||||
detected_count = sum(1 for e in analyze_report.entities if e.type == EntityType.ID_CARD)
|
||||
|
||||
security_report = pipeline.redact(path, redact_types=["id_card"], out_path=out)
|
||||
assert security_report["redacted"]["id_card"] == detected_count, \
|
||||
f"遮罩计数 {security_report['redacted']['id_card']} ≠ 检测数 {detected_count}"
|
||||
|
||||
|
||||
def test_redact_count_zero_for_unspecified_type(tmp_path, pipeline):
|
||||
"""未遮罩的类型在 security_report['redacted'] 中计数应为 0。"""
|
||||
doc = _docx.Document()
|
||||
doc.add_paragraph(f"身份证：{PERSONS[0]['id']}，电话：{PERSONS[0]['phone']}")
|
||||
path = str(tmp_path / "t.docx")
|
||||
doc.save(path)
|
||||
out = str(tmp_path / "out.docx")
|
||||
report = pipeline.redact(path, redact_types=["phone"], out_path=out)
|
||||
# id_card 未在 redact_types 中，对应 key 不应出现或为 0
|
||||
assert report["redacted"].get("id_card", 0) == 0
|
||||
|
||||
|
||||
# ─── 11. 5 人记录大文档：全检测 + 全遮罩 ─────────────────────────────────────────
|
||||
|
||||
@pytest.fixture
|
||||
def five_person_docx(tmp_path):
|
||||
doc = _docx.Document()
|
||||
for p in PERSONS:
|
||||
doc.add_paragraph(
|
||||
f"姓名：{p['name']} 身份证：{p['id']} "
|
||||
f"电话：{p['phone']} 邮箱：{p['email']} 银行卡：{p['bank']}"
|
||||
)
|
||||
path = str(tmp_path / "five.docx")
|
||||
doc.save(path)
|
||||
return path
|
||||
|
||||
|
||||
def test_five_persons_detect_all_id_cards(five_person_docx, pipeline):
|
||||
report = pipeline.analyze(five_person_docx)
|
||||
id_values = {e.value for e in report.entities if e.type == EntityType.ID_CARD}
|
||||
for p in PERSONS:
|
||||
assert p["id"] in id_values, f"未检测到 {p['name']} 的身份证"
|
||||
|
||||
|
||||
def test_five_persons_detect_all_phones(five_person_docx, pipeline):
|
||||
report = pipeline.analyze(five_person_docx)
|
||||
phone_values = {e.value for e in report.entities if e.type == EntityType.PHONE}
|
||||
for p in PERSONS:
|
||||
assert p["phone"] in phone_values, f"未检测到 {p['name']} 的手机号"
|
||||
|
||||
|
||||
def test_five_persons_full_redact(five_person_docx, tmp_path, pipeline):
|
||||
out = str(tmp_path / "out.docx")
|
||||
pipeline.redact(
|
||||
five_person_docx,
|
||||
redact_types=["id_card", "phone", "email", "bank_card"],
|
||||
out_path=out,
|
||||
)
|
||||
from info_privacy.parsers.office_parser import OfficeParser
|
||||
_, blocks = OfficeParser().parse(out)
|
||||
full_text = " ".join(b.text for b in blocks)
|
||||
for p in PERSONS:
|
||||
assert p["id"] not in full_text, f"{p['name']} 的身份证未被遮罩"
|
||||
assert p["phone"] not in full_text, f"{p['name']} 的手机号未被遮罩"
|
||||
assert p["email"] not in full_text, f"{p['name']} 的邮箱未被遮罩"
|
||||
assert p["bank"] not in full_text, f"{p['name']} 的银行卡未被遮罩"
|
||||
|
||||
|
||||
def test_five_persons_name_preserved_after_id_redact(five_person_docx, tmp_path, pipeline):
|
||||
"""遮罩身份证后，姓名（未被遮罩类型）仍保留。"""
|
||||
out = str(tmp_path / "out.docx")
|
||||
pipeline.redact(five_person_docx, redact_types=["id_card"], out_path=out)
|
||||
text = _docx_text(out)
|
||||
for p in PERSONS:
|
||||
assert p["name"] in text, f"{p['name']} 不应被删除（未遮罩姓名类型）"
|
||||
|
||||
|
||||
# ─── 12. 图像：bbox 边界越界处理 ──────────────────────────────────────────────────
|
||||
|
||||
@pytest.fixture
|
||||
def blank_png(tmp_path):
|
||||
img = np.ones((200, 400, 3), dtype=np.uint8) * 200
|
||||
p = str(tmp_path / "blank.png")
|
||||
cv2.imwrite(p, img)
|
||||
return p
|
||||
|
||||
|
||||
def test_image_redactor_bbox_clipped_at_boundary(blank_png, tmp_path):
|
||||
"""bbox 超出图像边界时不应抛出异常，输出仍为有效图像。"""
|
||||
from info_privacy.redactors.image_redactor import ImageRedactor
|
||||
from info_privacy.models import Entity, EntityType
|
||||
|
||||
out = str(tmp_path / "out.png")
|
||||
entity = Entity(
|
||||
id="e1", type=EntityType.FACE, value=None,
|
||||
page=1, bbox=[350.0, 150.0, 500.0, 300.0], # 超出 400×200 的边界
|
||||
layer="image", security_level="high",
|
||||
)
|
||||
ImageRedactor().redact(blank_png, [entity], out)
|
||||
result = cv2.imread(out)
|
||||
assert result is not None, "越界 bbox 导致输出图像损坏"
|
||||
orig = cv2.imread(blank_png)
|
||||
assert result.shape == orig.shape, "越界 bbox 后图像尺寸变了"
|
||||
|
||||
|
||||
def test_image_redactor_minimum_bbox(blank_png, tmp_path):
|
||||
"""极小 bbox（1×1 像素）应能正常处理。"""
|
||||
from info_privacy.redactors.image_redactor import ImageRedactor
|
||||
from info_privacy.models import Entity, EntityType
|
||||
|
||||
out = str(tmp_path / "out.png")
|
||||
entity = Entity(
|
||||
id="e1", type=EntityType.FACE, value=None,
|
||||
page=1, bbox=[100.0, 100.0, 101.0, 101.0],
|
||||
layer="image", security_level="high",
|
||||
)
|
||||
ImageRedactor().redact(blank_png, [entity], out)
|
||||
result = cv2.imread(out)
|
||||
assert result is not None
|
||||
assert result[100, 100].tolist() == [0, 0, 0], "1×1 遮罩区域不是黑色"
|
||||
|
||||
|
||||
def test_image_redactor_overlapping_bboxes(blank_png, tmp_path):
|
||||
"""两个重叠的 bbox 都被遮罩（重叠区域仍为黑）。"""
|
||||
from info_privacy.redactors.image_redactor import ImageRedactor
|
||||
from info_privacy.models import Entity, EntityType
|
||||
|
||||
out = str(tmp_path / "out.png")
|
||||
entities = [
|
||||
Entity(id="e1", type=EntityType.FACE, value=None,
|
||||
page=1, bbox=[50.0, 50.0, 150.0, 150.0], layer="image", security_level="high"),
|
||||
Entity(id="e2", type=EntityType.FACE, value=None,
|
||||
page=1, bbox=[100.0, 100.0, 200.0, 180.0], layer="image", security_level="high"),
|
||||
]
|
||||
ImageRedactor().redact(blank_png, entities, out)
|
||||
result = cv2.imread(out)
|
||||
# 两个独立区域均为黑
|
||||
assert result[60, 60].tolist() == [0, 0, 0], "第1个 bbox 未遮罩"
|
||||
assert result[170, 170].tolist() == [0, 0, 0], "第2个 bbox 未遮罩"
|
||||
# 重叠区域也为黑
|
||||
assert result[120, 120].tolist() == [0, 0, 0], "重叠区域未遮罩"
|
||||
@@ -0,0 +1,436 @@
|
||||
# tests/test_integration_real.py
|
||||
"""真实数据集成测试：生成含 PII 的真实文档，验证检测 + 遮罩全链路。
|
||||
|
||||
格式覆盖：
|
||||
PDF（文字层）/ DOCX / XLSX / PNG 图像遮罩
|
||||
保密文件拦截 / 多类型混合检测
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
|
||||
import cv2
|
||||
import docx as _docx
|
||||
import numpy as np
|
||||
import openpyxl
|
||||
import pytest
|
||||
from pdfminer.high_level import extract_text
|
||||
from reportlab.lib.pagesizes import A4
|
||||
from reportlab.pdfbase import pdfmetrics
|
||||
from reportlab.pdfbase.ttfonts import TTFont
|
||||
from reportlab.pdfgen import canvas
|
||||
|
||||
from info_privacy.pipeline import PrivacyPipeline
|
||||
from info_privacy.models import EntityType
|
||||
|
||||
# ─── 字体注册 ─────────────────────────────────────────────────────────────────
|
||||
|
||||
_FONT_PATH = "/usr/share/fonts/truetype/arphic/uming.ttc"
|
||||
_FONT_NAME = "UMing"
|
||||
|
||||
if os.path.exists(_FONT_PATH):
|
||||
pdfmetrics.registerFont(TTFont(_FONT_NAME, _FONT_PATH, subfontIndex=0))
|
||||
_HAS_FONT = True
|
||||
else:
|
||||
_HAS_FONT = False
|
||||
|
||||
_needs_font = pytest.mark.skipif(not _HAS_FONT, reason="uming.ttc 字体不存在")
|
||||
|
||||
# ─── 测试数据常量 ─────────────────────────────────────────────────────────────
|
||||
|
||||
ID_CARD = "110101199001011234"
|
||||
PHONE = "13812345678"
|
||||
BANK = "6222021234567890123"
|
||||
EMAIL = "zhang.wei@example.com"
|
||||
PLATE = "京A12345"
|
||||
NAME_CTX = "申请人：张伟，"
|
||||
ADDR_CTX = "地址：北京市朝阳区建国路88号"
|
||||
|
||||
PII_PARAGRAPH = (
|
||||
f"申请人：张伟，身份证号：{ID_CARD}\n"
|
||||
f"联系电话：{PHONE}，邮箱：{EMAIL}\n"
|
||||
f"银行卡：{BANK}，车牌：{PLATE}\n"
|
||||
f"{ADDR_CTX}"
|
||||
)
|
||||
|
||||
# ─── 工具函数 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def _make_pdf(path: str, lines: list[str]) -> None:
|
||||
"""用 reportlab 生成含中文文字层的 PDF。"""
|
||||
c = canvas.Canvas(path, pagesize=A4)
|
||||
c.setFont(_FONT_NAME, 14)
|
||||
y = 750
|
||||
for line in lines:
|
||||
c.drawString(50, y, line)
|
||||
y -= 30
|
||||
c.save()
|
||||
|
||||
|
||||
def _pdf_text(path: str) -> str:
|
||||
return extract_text(path)
|
||||
|
||||
|
||||
def _docx_text(path: str) -> str:
|
||||
doc = _docx.Document(path)
|
||||
return "\n".join(p.text for p in doc.paragraphs)
|
||||
|
||||
|
||||
def _xlsx_text(path: str) -> str:
|
||||
wb = openpyxl.load_workbook(path, data_only=True)
|
||||
parts = []
|
||||
for ws in wb.worksheets:
|
||||
for row in ws.iter_rows(values_only=True):
|
||||
parts.extend(str(c) for c in row if c is not None)
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def pipeline():
|
||||
return PrivacyPipeline()
|
||||
|
||||
|
||||
# ─── PDF（文字层）测试 ────────────────────────────────────────────────────────
|
||||
|
||||
@_needs_font
|
||||
def test_pdf_analyze_detects_id_card(tmp_path, pipeline):
|
||||
pdf = str(tmp_path / "pii.pdf")
|
||||
_make_pdf(pdf, [
|
||||
f"申请人：张伟",
|
||||
f"身份证号：{ID_CARD}",
|
||||
f"联系电话：{PHONE}",
|
||||
f"邮箱：{EMAIL}",
|
||||
])
|
||||
report = pipeline.analyze(pdf)
|
||||
assert report.blocked is False
|
||||
types = {e.type for e in report.entities}
|
||||
assert EntityType.ID_CARD in types, f"未检测到身份证，实体：{types}"
|
||||
|
||||
|
||||
@_needs_font
|
||||
def test_pdf_analyze_detects_phone_and_email(tmp_path, pipeline):
|
||||
pdf = str(tmp_path / "pii.pdf")
|
||||
_make_pdf(pdf, [f"电话：{PHONE}", f"邮箱：{EMAIL}"])
|
||||
report = pipeline.analyze(pdf)
|
||||
types = {e.type for e in report.entities}
|
||||
assert EntityType.PHONE in types, f"未检测到手机号，实体：{types}"
|
||||
assert EntityType.EMAIL in types, f"未检测到邮箱，实体：{types}"
|
||||
|
||||
|
||||
@_needs_font
|
||||
def test_pdf_analyze_detects_bank_and_plate(tmp_path, pipeline):
|
||||
pdf = str(tmp_path / "pii.pdf")
|
||||
_make_pdf(pdf, [f"银行卡：{BANK}", f"车牌：{PLATE}"])
|
||||
report = pipeline.analyze(pdf)
|
||||
types = {e.type for e in report.entities}
|
||||
assert EntityType.BANK_CARD in types, f"未检测到银行卡，实体：{types}"
|
||||
assert EntityType.LICENSE_PLATE in types, f"未检测到车牌，实体：{types}"
|
||||
|
||||
|
||||
@_needs_font
|
||||
def test_pdf_redact_removes_id_card(tmp_path, pipeline):
|
||||
pdf = str(tmp_path / "pii.pdf")
|
||||
out = str(tmp_path / "redacted.pdf")
|
||||
_make_pdf(pdf, [f"身份证号：{ID_CARD}", f"电话：{PHONE}"])
|
||||
pipeline.redact(pdf, redact_types=["id_card"], out_path=out)
|
||||
result_text = _pdf_text(out)
|
||||
assert ID_CARD not in result_text, "身份证号未被遮罩"
|
||||
assert "电话" in result_text, "无关内容被误删"
|
||||
|
||||
|
||||
@_needs_font
|
||||
def test_pdf_redact_multi_type(tmp_path, pipeline):
|
||||
pdf = str(tmp_path / "pii.pdf")
|
||||
out = str(tmp_path / "redacted.pdf")
|
||||
_make_pdf(pdf, [f"身份证：{ID_CARD}", f"电话：{PHONE}", f"邮箱：{EMAIL}"])
|
||||
pipeline.redact(pdf, redact_types=["id_card", "phone", "email"], out_path=out)
|
||||
result_text = _pdf_text(out)
|
||||
assert ID_CARD not in result_text
|
||||
assert PHONE not in result_text
|
||||
assert EMAIL not in result_text
|
||||
|
||||
|
||||
@_needs_font
|
||||
def test_pdf_classified_blocked(tmp_path, pipeline):
|
||||
pdf = str(tmp_path / "secret.pdf")
|
||||
_make_pdf(pdf, ["【机密】本文件仅供内部使用", "禁止外传"])
|
||||
report = pipeline.analyze(pdf)
|
||||
assert report.blocked is True, "保密文件未被拦截"
|
||||
assert report.classification.value == "classified"
|
||||
|
||||
|
||||
@_needs_font
|
||||
def test_pdf_normal_no_false_positive(tmp_path, pipeline):
|
||||
pdf = str(tmp_path / "normal.pdf")
|
||||
_make_pdf(pdf, ["第一季度销售报告", "本季度营业额同比增长12%"])
|
||||
report = pipeline.analyze(pdf)
|
||||
assert report.blocked is False
|
||||
# 不应有高置信度 PII
|
||||
high_risk = [e for e in report.entities if e.security_level == "high"]
|
||||
assert len(high_risk) == 0, f"正常文档误报高风险实体：{[e.type for e in high_risk]}"
|
||||
|
||||
|
||||
# ─── DOCX 测试 ────────────────────────────────────────────────────────────────
|
||||
|
||||
@pytest.fixture
|
||||
def pii_docx(tmp_path):
|
||||
doc = _docx.Document()
|
||||
doc.add_paragraph(f"申请人：张伟 身份证：{ID_CARD}")
|
||||
doc.add_paragraph(f"联系电话：{PHONE} 邮箱：{EMAIL}")
|
||||
doc.add_paragraph(f"银行卡号：{BANK}")
|
||||
doc.add_paragraph(f"住址：北京市朝阳区建国路88号")
|
||||
p = str(tmp_path / "pii.docx")
|
||||
doc.save(p)
|
||||
return p
|
||||
|
||||
|
||||
def test_docx_analyze_detects_id_card(pii_docx, pipeline):
|
||||
report = pipeline.analyze(pii_docx)
|
||||
types = {e.type for e in report.entities}
|
||||
assert EntityType.ID_CARD in types, f"未检测到身份证，实体：{types}"
|
||||
|
||||
|
||||
def test_docx_analyze_detects_phone(pii_docx, pipeline):
|
||||
report = pipeline.analyze(pii_docx)
|
||||
types = {e.type for e in report.entities}
|
||||
assert EntityType.PHONE in types, f"未检测到手机号，实体：{types}"
|
||||
|
||||
|
||||
def test_docx_analyze_detects_name(pii_docx, pipeline):
|
||||
report = pipeline.analyze(pii_docx)
|
||||
types = {e.type for e in report.entities}
|
||||
assert EntityType.NAME in types, f"未检测到姓名，实体：{types}"
|
||||
|
||||
|
||||
def test_docx_analyze_detects_address(pii_docx, pipeline):
|
||||
report = pipeline.analyze(pii_docx)
|
||||
types = {e.type for e in report.entities}
|
||||
assert EntityType.ADDRESS in types, f"未检测到地址，实体：{types}"
|
||||
|
||||
|
||||
def test_docx_summary_counts(pii_docx, pipeline):
|
||||
report = pipeline.analyze(pii_docx)
|
||||
assert report.summary.get("id_card", 0) >= 1
|
||||
assert report.summary.get("phone", 0) >= 1
|
||||
|
||||
|
||||
def test_docx_redact_id_card(pii_docx, tmp_path, pipeline):
|
||||
out = str(tmp_path / "out.docx")
|
||||
pipeline.redact(pii_docx, redact_types=["id_card"], out_path=out)
|
||||
text = _docx_text(out)
|
||||
assert ID_CARD not in text, "身份证号未被遮罩"
|
||||
assert "张伟" in text, "姓名被误删"
|
||||
assert PHONE in text, "手机号被误删"
|
||||
|
||||
|
||||
def test_docx_redact_multi_type(pii_docx, tmp_path, pipeline):
|
||||
out = str(tmp_path / "out.docx")
|
||||
pipeline.redact(pii_docx, redact_types=["id_card", "phone", "bank_card"], out_path=out)
|
||||
text = _docx_text(out)
|
||||
assert ID_CARD not in text
|
||||
assert PHONE not in text
|
||||
assert BANK not in text
|
||||
|
||||
|
||||
def test_docx_redact_preserves_unspecified(pii_docx, tmp_path, pipeline):
|
||||
"""未指定的类型不应被遮罩。"""
|
||||
out = str(tmp_path / "out.docx")
|
||||
pipeline.redact(pii_docx, redact_types=["id_card"], out_path=out)
|
||||
text = _docx_text(out)
|
||||
assert EMAIL in text, "邮箱不应被遮罩"
|
||||
assert BANK in text, "银行卡不应被遮罩"
|
||||
|
||||
|
||||
def test_docx_classified_blocked(tmp_path, pipeline):
|
||||
doc = _docx.Document()
|
||||
doc.add_paragraph("【绝密】本文件属于国家机密")
|
||||
p = str(tmp_path / "secret.docx")
|
||||
doc.save(p)
|
||||
report = pipeline.analyze(p)
|
||||
assert report.blocked is True
|
||||
assert len(report.entities) == 0, "拦截文件不应返回实体列表"
|
||||
|
||||
|
||||
def test_docx_meta_fields(pii_docx, pipeline):
|
||||
report = pipeline.analyze(pii_docx)
|
||||
assert report.doc_meta["format"] == "docx"
|
||||
assert report.doc_meta["has_text_layer"] is True
|
||||
assert report.doc_meta["redact_strategy"] == "text_replace"
|
||||
|
||||
|
||||
# ─── XLSX 测试 ────────────────────────────────────────────────────────────────
|
||||
|
||||
@pytest.fixture
|
||||
def pii_xlsx(tmp_path):
|
||||
wb = openpyxl.Workbook()
|
||||
ws = wb.active
|
||||
ws.title = "人员信息"
|
||||
ws.append(["姓名", "身份证号", "手机号", "邮箱", "银行卡"])
|
||||
ws.append(["张伟", ID_CARD, PHONE, EMAIL, BANK])
|
||||
ws.append(["李娜", "310101199505051234", "13900001111", "li@test.com", "6222021111111111111"])
|
||||
p = str(tmp_path / "pii.xlsx")
|
||||
wb.save(p)
|
||||
return p
|
||||
|
||||
|
||||
def test_xlsx_analyze_detects_pii(pii_xlsx, pipeline):
|
||||
report = pipeline.analyze(pii_xlsx)
|
||||
types = {e.type for e in report.entities}
|
||||
assert EntityType.ID_CARD in types
|
||||
assert EntityType.PHONE in types
|
||||
|
||||
|
||||
def test_xlsx_analyze_two_id_cards(pii_xlsx, pipeline):
|
||||
report = pipeline.analyze(pii_xlsx)
|
||||
id_count = sum(1 for e in report.entities if e.type == EntityType.ID_CARD)
|
||||
assert id_count >= 2, f"应检测到至少 2 个身份证，实际：{id_count}"
|
||||
|
||||
|
||||
def test_xlsx_redact_removes_id_cards(pii_xlsx, tmp_path, pipeline):
|
||||
out = str(tmp_path / "out.xlsx")
|
||||
pipeline.redact(pii_xlsx, redact_types=["id_card"], out_path=out)
|
||||
text = _xlsx_text(out)
|
||||
assert ID_CARD not in text, "身份证号未被遮罩"
|
||||
assert "310101199505051234" not in text, "第二个身份证号未被遮罩"
|
||||
|
||||
|
||||
def test_xlsx_redact_preserves_headers(pii_xlsx, tmp_path, pipeline):
|
||||
out = str(tmp_path / "out.xlsx")
|
||||
pipeline.redact(pii_xlsx, redact_types=["id_card"], out_path=out)
|
||||
wb = openpyxl.load_workbook(out, data_only=True)
|
||||
ws = wb.active
|
||||
headers = [cell.value for cell in ws[1]]
|
||||
assert "姓名" in headers, "列头被误删"
|
||||
assert "身份证号" in headers, "列头被误删"
|
||||
|
||||
|
||||
def test_xlsx_meta_fields(pii_xlsx, pipeline):
|
||||
report = pipeline.analyze(pii_xlsx)
|
||||
assert report.doc_meta["format"] == "xlsx"
|
||||
assert report.doc_meta["redact_strategy"] == "text_replace"
|
||||
|
||||
|
||||
# ─── PNG 图像遮罩测试（不依赖 RKNN，直接测试 ImageRedactor）────────────────────
|
||||
|
||||
@pytest.fixture
|
||||
def pii_png(tmp_path):
|
||||
"""生成包含文字的 PNG（用于测试图像遮罩，OCR 阶段跳过）。"""
|
||||
img = np.ones((400, 800, 3), dtype=np.uint8) * 240
|
||||
cv2.putText(img, f"ID: {ID_CARD}", (20, 80),
|
||||
cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 0, 0), 2)
|
||||
cv2.putText(img, f"TEL: {PHONE}", (20, 160),
|
||||
cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 0, 0), 2)
|
||||
cv2.rectangle(img, (200, 220), (400, 320), (180, 200, 220), -1) # 模拟人脸区域
|
||||
p = str(tmp_path / "pii.png")
|
||||
cv2.imwrite(p, img)
|
||||
return p
|
||||
|
||||
|
||||
def test_png_image_redactor_black_box(pii_png, tmp_path):
|
||||
"""ImageRedactor 对指定 bbox 覆盖纯黑矩形。"""
|
||||
from info_privacy.redactors.image_redactor import ImageRedactor
|
||||
from info_privacy.models import Entity, EntityType
|
||||
|
||||
out = str(tmp_path / "out.png")
|
||||
entity = Entity(
|
||||
id="f1", type=EntityType.FACE, value=None,
|
||||
page=1, bbox=[200.0, 220.0, 400.0, 320.0],
|
||||
layer="image", security_level="high",
|
||||
)
|
||||
ImageRedactor().redact(pii_png, [entity], out)
|
||||
|
||||
result = cv2.imread(out)
|
||||
region = result[220:320, 200:400]
|
||||
assert region.max() == 0, "遮罩区域应为纯黑"
|
||||
|
||||
|
||||
def test_png_image_redactor_preserves_size(pii_png, tmp_path):
|
||||
"""遮罩后图像尺寸不变。"""
|
||||
from info_privacy.redactors.image_redactor import ImageRedactor
|
||||
from info_privacy.models import Entity, EntityType
|
||||
|
||||
out = str(tmp_path / "out.png")
|
||||
entity = Entity(
|
||||
id="f2", type=EntityType.FACE, value=None,
|
||||
page=1, bbox=[20.0, 40.0, 300.0, 100.0],
|
||||
layer="image", security_level="high",
|
||||
)
|
||||
ImageRedactor().redact(pii_png, [entity], out)
|
||||
orig = cv2.imread(pii_png)
|
||||
result = cv2.imread(out)
|
||||
assert orig.shape == result.shape
|
||||
|
||||
|
||||
def test_png_image_redactor_outside_bbox_intact(pii_png, tmp_path):
|
||||
"""遮罩区域外的像素不应被修改。"""
|
||||
from info_privacy.redactors.image_redactor import ImageRedactor
|
||||
from info_privacy.models import Entity, EntityType
|
||||
|
||||
out = str(tmp_path / "out.png")
|
||||
# 只遮罩右下角
|
||||
entity = Entity(
|
||||
id="f3", type=EntityType.FACE, value=None,
|
||||
page=1, bbox=[600.0, 300.0, 780.0, 380.0],
|
||||
layer="image", security_level="high",
|
||||
)
|
||||
ImageRedactor().redact(pii_png, [entity], out)
|
||||
orig = cv2.imread(pii_png)
|
||||
result = cv2.imread(out)
|
||||
# 左上角应与原图一致
|
||||
np.testing.assert_array_equal(
|
||||
orig[0:50, 0:50], result[0:50, 0:50],
|
||||
err_msg="非遮罩区域像素被修改",
|
||||
)
|
||||
|
||||
|
||||
def test_png_image_redactor_multiple_entities(pii_png, tmp_path):
|
||||
"""多个 entity 都被遮罩。"""
|
||||
from info_privacy.redactors.image_redactor import ImageRedactor
|
||||
from info_privacy.models import Entity, EntityType
|
||||
|
||||
out = str(tmp_path / "out.png")
|
||||
entities = [
|
||||
Entity(id="e1", type=EntityType.FACE, value=None,
|
||||
page=1, bbox=[20.0, 40.0, 400.0, 110.0], layer="image", security_level="high"),
|
||||
Entity(id="e2", type=EntityType.FACE, value=None,
|
||||
page=1, bbox=[200.0, 220.0, 400.0, 320.0], layer="image", security_level="high"),
|
||||
]
|
||||
ImageRedactor().redact(pii_png, entities, out)
|
||||
result = cv2.imread(out)
|
||||
r1 = result[40:110, 20:400]
|
||||
r2 = result[220:320, 200:400]
|
||||
assert r1.max() == 0, "第一个遮罩区域未被覆盖"
|
||||
assert r2.max() == 0, "第二个遮罩区域未被覆盖"
|
||||
|
||||
|
||||
# ─── 综合 summary / warning 测试 ─────────────────────────────────────────────
|
||||
|
||||
def test_sensitive_partial_warning_many_entities(tmp_path, pipeline):
|
||||
"""大量高危实体触发 sensitive_partial 警告。"""
|
||||
doc = _docx.Document()
|
||||
for i in range(7):
|
||||
n = int(ID_CARD[:17]) + i
|
||||
doc.add_paragraph(f"身份证：{n}X")
|
||||
p = str(tmp_path / "many.docx")
|
||||
doc.save(p)
|
||||
report = pipeline.analyze(p)
|
||||
if report.warning:
|
||||
# 有警告时，分类应为 sensitive_partial
|
||||
assert report.classification.value == "sensitive_partial"
|
||||
|
||||
|
||||
def test_redact_returns_security_report(tmp_path, pipeline):
|
||||
"""redact() 返回字典包含 redacted 和 strategy_used 字段。"""
|
||||
doc = _docx.Document()
|
||||
doc.add_paragraph(f"身份证：{ID_CARD}")
|
||||
p = str(tmp_path / "t.docx")
|
||||
doc.save(p)
|
||||
out = str(tmp_path / "out.docx")
|
||||
report = pipeline.redact(p, redact_types=["id_card"], out_path=out)
|
||||
assert "redacted" in report
|
||||
assert "strategy_used" in report
|
||||
assert report["strategy_used"] == "text_replace"
|
||||
assert report["security_guarantee"] == "byte_level"
|
||||
@@ -0,0 +1,46 @@
|
||||
# tests/test_models.py
|
||||
from info_privacy.models import TextBlock, Entity, DetectionReport, EntityType
|
||||
|
||||
def test_entity_type_enum():
|
||||
assert EntityType.ID_CARD.value == "id_card"
|
||||
assert EntityType.FACE.value == "face"
|
||||
|
||||
def test_text_block_creation():
|
||||
block = TextBlock(
|
||||
text="张三，身份证：110101199001011234",
|
||||
bbox=[10, 20, 200, 40],
|
||||
page=1,
|
||||
layer="text",
|
||||
)
|
||||
assert block.layer == "text"
|
||||
|
||||
def test_entity_creation():
|
||||
entity = Entity(
|
||||
id="e1",
|
||||
type=EntityType.ID_CARD,
|
||||
value="110101199001011234",
|
||||
page=1,
|
||||
bbox=[10, 20, 200, 40],
|
||||
layer="text",
|
||||
security_level="high",
|
||||
)
|
||||
assert entity.type == EntityType.ID_CARD
|
||||
|
||||
def test_detection_report_summary():
|
||||
from info_privacy.models import Classification
|
||||
report = DetectionReport(
|
||||
doc_id="test-uuid",
|
||||
classification=Classification.NORMAL,
|
||||
blocked=False,
|
||||
block_reason=None,
|
||||
warning=None,
|
||||
doc_meta={"format": "pdf", "has_text_layer": True, "redact_strategy": "text_replace"},
|
||||
entities=[
|
||||
Entity(id="e1", type=EntityType.ID_CARD, value="x",
|
||||
page=1, bbox=[0,0,1,1], layer="text", security_level="high"),
|
||||
Entity(id="e2", type=EntityType.PHONE, value="y",
|
||||
page=1, bbox=[0,0,1,1], layer="text", security_level="high"),
|
||||
],
|
||||
)
|
||||
assert report.summary["id_card"] == 1
|
||||
assert report.summary["phone"] == 1
|
||||
@@ -0,0 +1,142 @@
|
||||
# tests/test_parsers.py
|
||||
import pytest
|
||||
from pathlib import Path
|
||||
from info_privacy.parsers.pdf_parser import PdfParser
|
||||
|
||||
@pytest.fixture
|
||||
def pdf_path():
|
||||
p = Path("tmp/sample_text.pdf")
|
||||
if not p.exists():
|
||||
pytest.skip("需先运行生成脚本")
|
||||
return str(p)
|
||||
|
||||
def test_pdf_has_text_layer(pdf_path):
|
||||
parser = PdfParser()
|
||||
meta, blocks = parser.parse(pdf_path)
|
||||
assert meta["has_text_layer"] is True
|
||||
assert meta["format"] == "pdf"
|
||||
assert len(blocks) > 0
|
||||
|
||||
def test_pdf_blocks_have_text(pdf_path):
|
||||
parser = PdfParser()
|
||||
_, blocks = parser.parse(pdf_path)
|
||||
all_text = " ".join(b.text for b in blocks)
|
||||
assert "身份证" in all_text or "张伟" in all_text
|
||||
|
||||
def test_pdf_blocks_layer_is_text(pdf_path):
|
||||
parser = PdfParser()
|
||||
_, blocks = parser.parse(pdf_path)
|
||||
assert all(b.layer == "text" for b in blocks)
|
||||
|
||||
def test_pdf_blocks_have_bbox(pdf_path):
|
||||
parser = PdfParser()
|
||||
_, blocks = parser.parse(pdf_path)
|
||||
for b in blocks:
|
||||
assert len(b.bbox) == 4
|
||||
assert b.bbox[2] > b.bbox[0] # x2 > x1
|
||||
|
||||
|
||||
# Office 解析器测试
|
||||
import docx as _docx
|
||||
from info_privacy.parsers.office_parser import OfficeParser
|
||||
|
||||
@pytest.fixture
|
||||
def sample_docx(tmp_path):
|
||||
doc = _docx.Document()
|
||||
doc.add_paragraph("申请人：李明，身份证：310101198501011234")
|
||||
doc.add_paragraph("手机：13999998888")
|
||||
p = str(tmp_path / "sample.docx")
|
||||
doc.save(p)
|
||||
return p
|
||||
|
||||
def test_office_parser_docx(sample_docx):
|
||||
parser = OfficeParser()
|
||||
meta, blocks = parser.parse(sample_docx)
|
||||
assert meta["format"] == "docx"
|
||||
assert meta["has_text_layer"] is True
|
||||
assert meta["redact_strategy"] == "text_replace"
|
||||
all_text = " ".join(b.text for b in blocks)
|
||||
assert "李明" in all_text or "身份证" in all_text
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 图像 OCR 解析器测试（Task 7）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
import os
|
||||
|
||||
FACE_MODEL = "/data/rockchip/paddle_ocr/models/rknn/PP-OCRv4/det/ppocrv4_det_ch_int8.rknn"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_image(tmp_path):
|
||||
img = np.ones((200, 600, 3), dtype=np.uint8) * 255
|
||||
cv2.putText(img, "ID: 110101199001011234", (10, 50),
|
||||
cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 0), 2)
|
||||
p = str(tmp_path / "sample.png")
|
||||
cv2.imwrite(p, img)
|
||||
return p
|
||||
|
||||
|
||||
def _rknn_available() -> bool:
|
||||
"""检查 RKNN 运行时是否可用（仅 RK3588 设备上可用）。"""
|
||||
try:
|
||||
import importlib.util
|
||||
return importlib.util.find_spec("rknn") is not None or importlib.util.find_spec("rknnlite") is not None
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def test_image_parser_returns_blocks(sample_image):
|
||||
det_model = "/data/rockchip/paddle_ocr/models/rknn/PP-OCRv4/det/ppocrv4_det_ch_int8.rknn"
|
||||
rec_model = "/data/rockchip/paddle_ocr/models/rknn/PP-OCRv4/rec/ppocrv4_rec_ch_fp16.rknn"
|
||||
dict_file = "/data/rockchip/paddle_ocr/data/dicts/ppocr_keys_v1.txt"
|
||||
if not os.path.exists(det_model):
|
||||
pytest.skip("RKNN模型不存在，跳过")
|
||||
if not _rknn_available():
|
||||
pytest.skip("rknn/rknnlite 不可用（非 RK3588 设备），跳过")
|
||||
from info_privacy.parsers.image_parser import ImageParser
|
||||
parser = ImageParser(det_model_path=det_model, rec_model_path=rec_model, dict_path=dict_file)
|
||||
meta, blocks = parser.parse(sample_image)
|
||||
assert meta["format"] == "image"
|
||||
assert meta["has_text_layer"] is False
|
||||
assert meta["redact_strategy"] == "image_mask"
|
||||
assert isinstance(blocks, list)
|
||||
|
||||
|
||||
def test_image_parser_block_layer(sample_image):
|
||||
det_model = "/data/rockchip/paddle_ocr/models/rknn/PP-OCRv4/det/ppocrv4_det_ch_int8.rknn"
|
||||
if not os.path.exists(det_model):
|
||||
pytest.skip("RKNN模型不存在")
|
||||
if not _rknn_available():
|
||||
pytest.skip("rknn/rknnlite 不可用（非 RK3588 设备），跳过")
|
||||
rec_model = "/data/rockchip/paddle_ocr/models/rknn/PP-OCRv4/rec/ppocrv4_rec_ch_fp16.rknn"
|
||||
dict_file = "/data/rockchip/paddle_ocr/data/dicts/ppocr_keys_v1.txt"
|
||||
from info_privacy.parsers.image_parser import ImageParser
|
||||
parser = ImageParser(det_model_path=det_model, rec_model_path=rec_model, dict_path=dict_file)
|
||||
_, blocks = parser.parse(sample_image)
|
||||
for b in blocks:
|
||||
assert b.layer == "image"
|
||||
|
||||
|
||||
# Parser Factory 测试
|
||||
from info_privacy.parsers.parser_factory import ParserFactory
|
||||
|
||||
def test_factory_routes_pdf(pdf_path):
|
||||
factory = ParserFactory()
|
||||
parser = factory.get_parser(pdf_path)
|
||||
from info_privacy.parsers.pdf_parser import PdfParser
|
||||
assert isinstance(parser, PdfParser)
|
||||
|
||||
def test_factory_routes_docx(sample_docx, tmp_path):
|
||||
factory = ParserFactory()
|
||||
parser = factory.get_parser(sample_docx)
|
||||
from info_privacy.parsers.office_parser import OfficeParser
|
||||
assert isinstance(parser, OfficeParser)
|
||||
|
||||
def test_factory_unsupported_raises():
|
||||
factory = ParserFactory()
|
||||
with pytest.raises(ValueError, match="不支持"):
|
||||
factory.get_parser("file.xyz")
|
||||
@@ -0,0 +1,89 @@
|
||||
# tests/test_pipeline.py
|
||||
import pytest
|
||||
import docx
|
||||
from info_privacy.pipeline import PrivacyPipeline
|
||||
|
||||
@pytest.fixture
|
||||
def pipeline():
|
||||
return PrivacyPipeline()
|
||||
|
||||
@pytest.fixture
|
||||
def sample_docx_path(tmp_path):
|
||||
doc = docx.Document()
|
||||
doc.add_paragraph("姓名：张伟 身份证：110101199001011234 电话：13812345678")
|
||||
p = tmp_path / "test.docx"
|
||||
doc.save(str(p))
|
||||
return str(p)
|
||||
|
||||
def test_analyze_docx(pipeline, sample_docx_path):
|
||||
report = pipeline.analyze(sample_docx_path)
|
||||
assert report.blocked is False
|
||||
assert len(report.entities) > 0
|
||||
types = {e.type.value for e in report.entities}
|
||||
assert "id_card" in types or "phone" in types
|
||||
|
||||
def test_analyze_classified_doc(pipeline, tmp_path):
|
||||
doc = docx.Document()
|
||||
doc.add_paragraph("【机密】本文件仅供内部使用")
|
||||
p = tmp_path / "secret.docx"
|
||||
doc.save(str(p))
|
||||
report = pipeline.analyze(str(p))
|
||||
assert report.blocked is True
|
||||
assert report.classification.value == "classified"
|
||||
|
||||
def test_redact_docx_removes_entity(pipeline, sample_docx_path, tmp_path):
|
||||
out = str(tmp_path / "out.docx")
|
||||
pipeline.redact(sample_docx_path, redact_types=["id_card"], out_path=out)
|
||||
doc = docx.Document(out)
|
||||
all_text = " ".join(p.text for p in doc.paragraphs)
|
||||
assert "110101199001011234" not in all_text
|
||||
assert "张伟" in all_text # 未指定遮罩 name，保留
|
||||
|
||||
|
||||
def test_analyze_returns_summary_dict(pipeline, sample_docx_path):
|
||||
"""DetectionReport.summary 各类型计数应与 entities 列表精确对应。"""
|
||||
report = pipeline.analyze(sample_docx_path)
|
||||
for etype, count in report.summary.items():
|
||||
actual = sum(1 for e in report.entities if e.type.value == etype)
|
||||
assert actual == count, f"{etype}: summary={count}，entities={actual}"
|
||||
|
||||
|
||||
def test_analyze_no_pii_empty_entities(pipeline, tmp_path):
|
||||
"""无 PII 的文档：entities 为空，分类为 normal，无 warning。"""
|
||||
doc = docx.Document()
|
||||
doc.add_paragraph("季度汇报：各项指标均达标。")
|
||||
p = tmp_path / "clean.docx"
|
||||
doc.save(str(p))
|
||||
report = pipeline.analyze(str(p))
|
||||
assert len(report.entities) == 0
|
||||
assert report.classification.value == "normal"
|
||||
assert report.warning is None
|
||||
|
||||
|
||||
def test_analyze_many_entities_triggers_warning(pipeline, tmp_path):
|
||||
"""5 个以上高风险实体应在 DetectionReport 中生成 warning。"""
|
||||
doc = docx.Document()
|
||||
ids = [
|
||||
"110101199001011234", "110101199001011235", "110101199001011236",
|
||||
"110101199001011237", "110101199001011238",
|
||||
]
|
||||
doc.add_paragraph(" ".join(ids))
|
||||
p = tmp_path / "many.docx"
|
||||
doc.save(str(p))
|
||||
report = pipeline.analyze(str(p))
|
||||
assert report.warning is not None
|
||||
|
||||
|
||||
def test_redact_multi_type_all_removed(pipeline, tmp_path):
|
||||
"""同时遮罩 id_card/phone/email 三种类型，各类值均应从文档消失。"""
|
||||
doc = docx.Document()
|
||||
doc.add_paragraph("身份证：110101199001011234 电话：13812345678 邮箱：user@corp.com")
|
||||
p = tmp_path / "multi.docx"
|
||||
doc.save(str(p))
|
||||
out = str(tmp_path / "out.docx")
|
||||
pipeline.redact(str(p), redact_types=["id_card", "phone", "email"], out_path=out)
|
||||
result = docx.Document(out)
|
||||
text = " ".join(para.text for para in result.paragraphs)
|
||||
assert "110101199001011234" not in text
|
||||
assert "13812345678" not in text
|
||||
assert "user@corp.com" not in text
|
||||
@@ -0,0 +1,162 @@
|
||||
# tests/test_redactors.py
|
||||
import pytest
|
||||
import docx
|
||||
from pathlib import Path
|
||||
from info_privacy.models import Entity, EntityType
|
||||
from info_privacy.redactors.text_redactor import TextRedactor
|
||||
|
||||
@pytest.fixture
|
||||
def sample_docx_path(tmp_path):
|
||||
doc = docx.Document()
|
||||
doc.add_paragraph("身份证：310101198501011234 手机：13999998888")
|
||||
p = tmp_path / "test.docx"
|
||||
doc.save(str(p))
|
||||
return str(p)
|
||||
|
||||
@pytest.fixture
|
||||
def entity_id_card():
|
||||
return Entity(
|
||||
id="e1", type=EntityType.ID_CARD,
|
||||
value="310101198501011234",
|
||||
page=1, bbox=[0,0,100,20], layer="text", security_level="high"
|
||||
)
|
||||
|
||||
def test_redact_docx_removes_value(sample_docx_path, entity_id_card, tmp_path):
|
||||
out_path = str(tmp_path / "out.docx")
|
||||
redactor = TextRedactor()
|
||||
redactor.redact(sample_docx_path, [entity_id_card], out_path)
|
||||
doc = docx.Document(out_path)
|
||||
all_text = " ".join(p.text for p in doc.paragraphs)
|
||||
assert "310101198501011234" not in all_text
|
||||
|
||||
def test_redact_docx_preserves_other_text(sample_docx_path, entity_id_card, tmp_path):
|
||||
out_path = str(tmp_path / "out.docx")
|
||||
redactor = TextRedactor()
|
||||
redactor.redact(sample_docx_path, [entity_id_card], out_path)
|
||||
doc = docx.Document(out_path)
|
||||
all_text = " ".join(p.text for p in doc.paragraphs)
|
||||
assert "手机" in all_text
|
||||
|
||||
|
||||
# 图像遮罩器测试
|
||||
import cv2
|
||||
import numpy as np
|
||||
from info_privacy.redactors.image_redactor import ImageRedactor
|
||||
|
||||
@pytest.fixture
|
||||
def sample_image_path(tmp_path):
|
||||
img = np.ones((200, 400, 3), dtype=np.uint8) * 255
|
||||
cv2.putText(img, "ID: 123456789", (10, 100), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,0), 2)
|
||||
p = str(tmp_path / "test.png")
|
||||
cv2.imwrite(p, img)
|
||||
return p
|
||||
|
||||
@pytest.fixture
|
||||
def entity_face():
|
||||
return Entity(id="f1", type=EntityType.FACE, value=None,
|
||||
page=1, bbox=[10, 80, 200, 120], layer="image", security_level="high")
|
||||
|
||||
def test_image_redactor_black_box(sample_image_path, entity_face, tmp_path):
|
||||
out = str(tmp_path / "out.png")
|
||||
redactor = ImageRedactor()
|
||||
redactor.redact(sample_image_path, [entity_face], out)
|
||||
result = cv2.imread(out)
|
||||
# 遮罩区域应为纯黑
|
||||
region = result[80:120, 10:200]
|
||||
assert region.max() == 0
|
||||
|
||||
def test_image_redactor_output_same_size(sample_image_path, entity_face, tmp_path):
|
||||
out = str(tmp_path / "out.png")
|
||||
redactor = ImageRedactor()
|
||||
redactor.redact(sample_image_path, [entity_face], out)
|
||||
orig = cv2.imread(sample_image_path)
|
||||
result = cv2.imread(out)
|
||||
assert orig.shape == result.shape
|
||||
|
||||
|
||||
def test_redact_excel_removes_value(tmp_path):
|
||||
"""Excel 单元格中的 PII 应被 TextRedactor 替换为遮罩字符。"""
|
||||
import openpyxl
|
||||
wb = openpyxl.Workbook()
|
||||
ws = wb.active
|
||||
ws["A1"] = "身份证"
|
||||
ws["A2"] = "310101198501011234"
|
||||
p = str(tmp_path / "test.xlsx")
|
||||
wb.save(p)
|
||||
|
||||
entity = Entity(
|
||||
id="e1", type=EntityType.ID_CARD, value="310101198501011234",
|
||||
page=1, bbox=[0, 0, 100, 20], layer="text", security_level="high",
|
||||
)
|
||||
out = str(tmp_path / "out.xlsx")
|
||||
TextRedactor().redact(p, [entity], out)
|
||||
|
||||
wb_out = openpyxl.load_workbook(out)
|
||||
values = [cell.value for row in wb_out.active.iter_rows() for cell in row if cell.value]
|
||||
assert "310101198501011234" not in values
|
||||
assert any("████" in str(v) for v in values), "遮罩字符 ████ 未出现"
|
||||
|
||||
|
||||
def test_redact_docx_table_cell_removes_value(tmp_path):
|
||||
"""DOCX 表格单元格中的 PII 应被正确遮罩（非仅正文段落）。"""
|
||||
import docx as _docx
|
||||
doc = _docx.Document()
|
||||
table = doc.add_table(rows=2, cols=2)
|
||||
table.cell(0, 0).text = "姓名"
|
||||
table.cell(0, 1).text = "身份证"
|
||||
table.cell(1, 0).text = "张三"
|
||||
table.cell(1, 1).text = "310101198501011234"
|
||||
p = str(tmp_path / "table.docx")
|
||||
doc.save(p)
|
||||
|
||||
entity = Entity(
|
||||
id="e1", type=EntityType.ID_CARD, value="310101198501011234",
|
||||
page=1, bbox=[0, 0, 100, 20], layer="text", security_level="high",
|
||||
)
|
||||
out = str(tmp_path / "out.docx")
|
||||
TextRedactor().redact(p, [entity], out)
|
||||
|
||||
doc_out = _docx.Document(out)
|
||||
cell_texts = [
|
||||
cell.text for table in doc_out.tables
|
||||
for row in table.rows for cell in row.cells
|
||||
]
|
||||
assert "310101198501011234" not in cell_texts
|
||||
|
||||
|
||||
def test_redact_docx_mask_uses_filled_blocks(tmp_path):
|
||||
"""遮罩后目标值应被替换为全角方块字符 ████。"""
|
||||
import docx as _docx
|
||||
doc = _docx.Document()
|
||||
doc.add_paragraph("手机：13812345678")
|
||||
p = str(tmp_path / "test.docx")
|
||||
doc.save(p)
|
||||
|
||||
entity = Entity(
|
||||
id="e1", type=EntityType.PHONE, value="13812345678",
|
||||
page=1, bbox=[0, 0, 100, 20], layer="text", security_level="high",
|
||||
)
|
||||
out = str(tmp_path / "out.docx")
|
||||
TextRedactor().redact(p, [entity], out)
|
||||
|
||||
doc_out = _docx.Document(out)
|
||||
all_text = " ".join(para.text for para in doc_out.paragraphs)
|
||||
assert "████" in all_text
|
||||
|
||||
|
||||
def test_image_redactor_skips_text_layer_entities(tmp_path):
|
||||
"""ImageRedactor 对 layer='text' 的实体不应执行图像遮罩。"""
|
||||
img = np.ones((200, 400, 3), dtype=np.uint8) * 255
|
||||
p = str(tmp_path / "test.png")
|
||||
cv2.imwrite(p, img)
|
||||
|
||||
text_entity = Entity(
|
||||
id="e1", type=EntityType.ID_CARD, value="123",
|
||||
page=1, bbox=[10, 10, 100, 50], layer="text", security_level="high",
|
||||
)
|
||||
out = str(tmp_path / "out.png")
|
||||
ImageRedactor().redact(p, [text_entity], out)
|
||||
|
||||
orig = cv2.imread(p)
|
||||
result = cv2.imread(out)
|
||||
assert np.array_equal(orig, result), "text 层实体不应触发图像遮罩"
|
||||
@@ -0,0 +1,267 @@
|
||||
# tests/test_simulation.py
|
||||
"""仿真测试：用 unittest.mock 替换 RKNN 硬件依赖，在 x86 环境全量可运行。
|
||||
|
||||
覆盖范围：
|
||||
- FaceDetector：detect_in_image 业务逻辑（bbox 转换、实体构建、空结果处理）
|
||||
- ImageParser：OCR 结果解析（TextBlock 构建、置信度过滤、meta 字段）
|
||||
"""
|
||||
import sys
|
||||
import numpy as np
|
||||
import pytest
|
||||
import cv2
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
# ─── 工具函数 ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def _make_detection(bbox: list) -> MagicMock:
|
||||
"""创建带 bbox 属性的模拟人脸检测结果。"""
|
||||
det = MagicMock()
|
||||
det.bbox = bbox
|
||||
return det
|
||||
|
||||
|
||||
def _make_face_solutions_mock(detections=None):
|
||||
"""返回模拟 mediapipe_rknn.solutions 模块及其内部 detector 实例。"""
|
||||
mock_instance = MagicMock()
|
||||
mock_instance.detect.return_value = detections if detections is not None else []
|
||||
|
||||
mock_solutions = MagicMock()
|
||||
mock_solutions.FaceDetection.return_value = mock_instance
|
||||
|
||||
return mock_solutions, mock_instance
|
||||
|
||||
|
||||
def _make_ppocr_mocks(dt_boxes=None, rec_results=None):
|
||||
"""返回模拟 ppocr_det / ppocr_rec 模块对及内部实例。"""
|
||||
mock_det = MagicMock()
|
||||
mock_det_inst = MagicMock()
|
||||
mock_det_inst.run.return_value = dt_boxes # None 或 box list
|
||||
mock_det.TextDetector.return_value = mock_det_inst
|
||||
mock_det.get_rotate_crop_image.return_value = np.zeros((32, 100, 3), dtype=np.uint8)
|
||||
|
||||
mock_rec = MagicMock()
|
||||
mock_rec_inst = MagicMock()
|
||||
mock_rec_inst.run.return_value = rec_results or []
|
||||
mock_rec.TextRecognizer.return_value = mock_rec_inst
|
||||
|
||||
return mock_det, mock_rec
|
||||
|
||||
|
||||
def _write_blank_image(path: str, h: int = 100, w: int = 300) -> None:
|
||||
cv2.imwrite(path, np.ones((h, w, 3), dtype=np.uint8) * 255)
|
||||
|
||||
|
||||
# ─── FaceDetector 仿真测试 ─────────────────────────────────────────────────────
|
||||
|
||||
@pytest.fixture
|
||||
def face_detector_no_face():
|
||||
mock_solutions, _ = _make_face_solutions_mock(detections=[])
|
||||
patches = {
|
||||
"mediapipe_rknn": MagicMock(),
|
||||
"mediapipe_rknn.solutions": mock_solutions,
|
||||
}
|
||||
with patch.dict(sys.modules, patches):
|
||||
from info_privacy.detectors.face_detector import FaceDetector
|
||||
yield FaceDetector("/fake/model.rknn")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def face_detector_two_faces():
|
||||
dets = [
|
||||
_make_detection([10.0, 20.0, 100.0, 150.0]),
|
||||
_make_detection([200.0, 50.0, 350.0, 200.0]),
|
||||
]
|
||||
mock_solutions, _ = _make_face_solutions_mock(detections=dets)
|
||||
patches = {
|
||||
"mediapipe_rknn": MagicMock(),
|
||||
"mediapipe_rknn.solutions": mock_solutions,
|
||||
}
|
||||
with patch.dict(sys.modules, patches):
|
||||
from info_privacy.detectors.face_detector import FaceDetector
|
||||
yield FaceDetector("/fake/model.rknn")
|
||||
|
||||
|
||||
def test_face_no_detection_returns_empty_list(face_detector_no_face):
|
||||
img = np.ones((300, 300, 3), dtype=np.uint8) * 255
|
||||
entities = face_detector_no_face.detect_in_image(img, page=1)
|
||||
assert isinstance(entities, list)
|
||||
assert len(entities) == 0
|
||||
|
||||
|
||||
def test_face_two_detections_returns_two_entities(face_detector_two_faces):
|
||||
from info_privacy.models import EntityType
|
||||
img = np.zeros((400, 500, 3), dtype=np.uint8)
|
||||
entities = face_detector_two_faces.detect_in_image(img, page=2)
|
||||
assert len(entities) == 2
|
||||
|
||||
|
||||
def test_face_entity_fields(face_detector_two_faces):
|
||||
"""每个 Entity 的 type / layer / security_level / page 必须正确。"""
|
||||
from info_privacy.models import EntityType
|
||||
img = np.zeros((300, 300, 3), dtype=np.uint8)
|
||||
entities = face_detector_two_faces.detect_in_image(img, page=3)
|
||||
for e in entities:
|
||||
assert e.type == EntityType.FACE
|
||||
assert e.layer == "image"
|
||||
assert e.security_level == "high"
|
||||
assert e.page == 3
|
||||
assert len(e.bbox) == 4
|
||||
|
||||
|
||||
def test_face_entity_bbox_is_float(face_detector_two_faces):
|
||||
"""bbox 坐标必须转换为 float，保证 JSON 序列化不报错。"""
|
||||
img = np.zeros((300, 300, 3), dtype=np.uint8)
|
||||
entities = face_detector_two_faces.detect_in_image(img, page=1)
|
||||
for e in entities:
|
||||
for v in e.bbox:
|
||||
assert isinstance(v, float)
|
||||
|
||||
|
||||
def test_face_detect_returns_none_gives_empty():
|
||||
"""detect() 返回 None 时 detect_in_image 应安全返回 []。"""
|
||||
mock_solutions, mock_inst = _make_face_solutions_mock()
|
||||
mock_inst.detect.return_value = None
|
||||
patches = {
|
||||
"mediapipe_rknn": MagicMock(),
|
||||
"mediapipe_rknn.solutions": mock_solutions,
|
||||
}
|
||||
with patch.dict(sys.modules, patches):
|
||||
from info_privacy.detectors.face_detector import FaceDetector
|
||||
detector = FaceDetector("/fake/model.rknn")
|
||||
result = detector.detect_in_image(np.zeros((200, 200, 3), dtype=np.uint8), page=1)
|
||||
assert result == []
|
||||
|
||||
|
||||
def test_face_entity_id_unique(face_detector_two_faces):
|
||||
"""两张人脸的 Entity.id 必须不同（uuid4 生成）。"""
|
||||
img = np.zeros((300, 300, 3), dtype=np.uint8)
|
||||
entities = face_detector_two_faces.detect_in_image(img, page=1)
|
||||
ids = [e.id for e in entities]
|
||||
assert len(set(ids)) == len(ids)
|
||||
|
||||
|
||||
# ─── ImageParser 仿真测试 ──────────────────────────────────────────────────────
|
||||
|
||||
@pytest.fixture
|
||||
def parser_no_text():
|
||||
"""OCR 检测无文字区域（dt_boxes=None）的 ImageParser。"""
|
||||
mock_det, mock_rec = _make_ppocr_mocks(dt_boxes=None)
|
||||
patches = {"ppocr_det": mock_det, "ppocr_rec": mock_rec}
|
||||
with patch.dict(sys.modules, patches):
|
||||
from info_privacy.parsers.image_parser import ImageParser
|
||||
yield ImageParser("/f/det.rknn", "/f/rec.rknn", "/f/dict.txt")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser_with_id_card():
|
||||
"""OCR 识别出身份证号文字区域的 ImageParser。"""
|
||||
boxes = [np.array([[10, 10], [200, 10], [200, 30], [10, 30]], dtype=np.float32)]
|
||||
recs = [("身份证：110101199001011234", 0.95)]
|
||||
mock_det, mock_rec = _make_ppocr_mocks(dt_boxes=boxes, rec_results=recs)
|
||||
patches = {"ppocr_det": mock_det, "ppocr_rec": mock_rec}
|
||||
with patch.dict(sys.modules, patches):
|
||||
from info_privacy.parsers.image_parser import ImageParser
|
||||
yield ImageParser("/f/det.rknn", "/f/rec.rknn", "/f/dict.txt")
|
||||
|
||||
|
||||
def test_image_parser_meta_format(parser_no_text, tmp_path):
|
||||
img_path = str(tmp_path / "t.png")
|
||||
_write_blank_image(img_path)
|
||||
meta, _ = parser_no_text.parse(img_path)
|
||||
assert meta["format"] == "image"
|
||||
assert meta["has_text_layer"] is False
|
||||
assert meta["redact_strategy"] == "image_mask"
|
||||
|
||||
|
||||
def test_image_parser_no_boxes_returns_empty(parser_no_text, tmp_path):
|
||||
img_path = str(tmp_path / "t.png")
|
||||
_write_blank_image(img_path)
|
||||
_, blocks = parser_no_text.parse(img_path)
|
||||
assert blocks == []
|
||||
|
||||
|
||||
def test_image_parser_blocks_layer_is_image(parser_with_id_card, tmp_path):
|
||||
img_path = str(tmp_path / "t.png")
|
||||
_write_blank_image(img_path)
|
||||
_, blocks = parser_with_id_card.parse(img_path)
|
||||
assert len(blocks) >= 1
|
||||
for b in blocks:
|
||||
assert b.layer == "image"
|
||||
|
||||
|
||||
def test_image_parser_block_text_content(parser_with_id_card, tmp_path):
|
||||
img_path = str(tmp_path / "t.png")
|
||||
_write_blank_image(img_path)
|
||||
_, blocks = parser_with_id_card.parse(img_path)
|
||||
texts = " ".join(b.text for b in blocks)
|
||||
assert "身份证" in texts
|
||||
|
||||
|
||||
def test_image_parser_block_bbox_valid(parser_with_id_card, tmp_path):
|
||||
img_path = str(tmp_path / "t.png")
|
||||
_write_blank_image(img_path)
|
||||
_, blocks = parser_with_id_card.parse(img_path)
|
||||
for b in blocks:
|
||||
assert len(b.bbox) == 4
|
||||
assert b.bbox[2] > b.bbox[0] # x2 > x1
|
||||
assert b.bbox[3] > b.bbox[1] # y2 > y1
|
||||
|
||||
|
||||
def test_image_parser_low_confidence_filtered(tmp_path):
|
||||
"""置信度 < 0.5 的 OCR 结果不得生成 TextBlock。"""
|
||||
boxes = [np.array([[10, 10], [200, 10], [200, 30], [10, 30]], dtype=np.float32)]
|
||||
recs = [("模糊文字", 0.3)]
|
||||
mock_det, mock_rec = _make_ppocr_mocks(dt_boxes=boxes, rec_results=recs)
|
||||
patches = {"ppocr_det": mock_det, "ppocr_rec": mock_rec}
|
||||
with patch.dict(sys.modules, patches):
|
||||
from info_privacy.parsers.image_parser import ImageParser
|
||||
parser = ImageParser("/f/det.rknn", "/f/rec.rknn", "/f/dict.txt")
|
||||
img_path = str(tmp_path / "t.png")
|
||||
_write_blank_image(img_path)
|
||||
_, blocks = parser.parse(img_path)
|
||||
assert len(blocks) == 0
|
||||
|
||||
|
||||
def test_image_parser_boundary_confidence(tmp_path):
|
||||
"""置信度恰好 = 0.5 的结果应保留（>= 0.5 视为有效）。"""
|
||||
boxes = [np.array([[10, 10], [200, 10], [200, 30], [10, 30]], dtype=np.float32)]
|
||||
recs = [("临界文字", 0.5)]
|
||||
mock_det, mock_rec = _make_ppocr_mocks(dt_boxes=boxes, rec_results=recs)
|
||||
patches = {"ppocr_det": mock_det, "ppocr_rec": mock_rec}
|
||||
with patch.dict(sys.modules, patches):
|
||||
from info_privacy.parsers.image_parser import ImageParser
|
||||
parser = ImageParser("/f/det.rknn", "/f/rec.rknn", "/f/dict.txt")
|
||||
img_path = str(tmp_path / "t.png")
|
||||
_write_blank_image(img_path)
|
||||
_, blocks = parser.parse(img_path)
|
||||
assert len(blocks) == 1
|
||||
|
||||
|
||||
def test_image_parser_run_on_image_page(parser_with_id_card):
|
||||
"""run_on_image() 供 PDF 扫描件调用时，page 参数必须正确透传。"""
|
||||
img = np.ones((100, 300, 3), dtype=np.uint8) * 255
|
||||
blocks = parser_with_id_card.run_on_image(img, page=5)
|
||||
assert all(b.page == 5 for b in blocks)
|
||||
|
||||
|
||||
def test_image_parser_multiple_boxes(tmp_path):
|
||||
"""多个文字区域均能正确生成对应 TextBlock。"""
|
||||
boxes = [
|
||||
np.array([[10, 10], [200, 10], [200, 30], [10, 30]], dtype=np.float32),
|
||||
np.array([[10, 50], [300, 50], [300, 70], [10, 70]], dtype=np.float32),
|
||||
]
|
||||
recs = [("姓名：张三", 0.9), ("电话：13812345678", 0.88)]
|
||||
mock_det, mock_rec = _make_ppocr_mocks(dt_boxes=boxes, rec_results=recs)
|
||||
patches = {"ppocr_det": mock_det, "ppocr_rec": mock_rec}
|
||||
with patch.dict(sys.modules, patches):
|
||||
from info_privacy.parsers.image_parser import ImageParser
|
||||
parser = ImageParser("/f/det.rknn", "/f/rec.rknn", "/f/dict.txt")
|
||||
img_path = str(tmp_path / "t.png")
|
||||
_write_blank_image(img_path, h=200, w=400)
|
||||
_, blocks = parser.parse(img_path)
|
||||
assert len(blocks) == 2
|
||||
texts = {b.text for b in blocks}
|
||||
assert "姓名：张三" in texts
|
||||
assert "电话：13812345678" in texts
|
||||
Reference in New Issue
Block a user