翻译模块

This commit is contained in:
zhongluofeng committed 2026-09-17 17:32:20 +08:00
1 parent 79c43d5340
commit f6c1cc250e
47 files changed
+10822 -104

No files matched your search

+532
View File
@@ -0,0 +1,532 @@
//! 翻译历史(SQLite)。
//!
//! 存储约定:`{app_data_dir}/translate/history.db`。去重键是
//! `(source_text, to_lang, engine_id)` —— 同一段文字、同一个目标语言、同一个引擎
//! 只保留一条,重复翻译只更新时间与译文。**不含 `via`**:划词翻过的句子再用主面板翻,
//! 是同一件事,拆成两条只会让历史变得难搜。
//!
//! 收藏条目不参与容量淘汰(`prune_to_max`)——用户明确说「留着」的东西,
//! 不该因为新记录挤进来而消失。
//!
//! 搜索走 FTS5 三元组索引(`history_fts`),详见 `FTS_MIN_CHARS` 与 `fts_phrase` 的说明。
use std::path::Path;
use std::sync::Mutex;
use rusqlite::{params, Connection};
use serde::Serialize;
use specta::Type;
/// 库结构版本。
///
/// 与本值不等的库在打开时**整库重建**(见 `History::new`)。现阶段模块仍在开发、
/// 未实装,历史属于可丢弃数据,因此不做增量迁移——维护一堆迁移分支、还要处理
/// 「迁移到一半失败」留下的半新半旧库,成本远高于丢掉几条测试记录。
/// **实装之后再改结构就必须换成真正的迁移。**
const SCHEMA_VERSION: i64 = 2;
/// 走 FTS 索引所需的最小字符数。
///
/// trigram 分词器把文本切成连续 3 字符的 n-gram,索引里不存在长度小于 3 的片段,
/// 因此 1~2 个字符的 MATCH **不会报错,只会静默返回空结果**。「条件明明对却搜不到」
/// 比「慢一点」糟糕得多(中文里两字词又恰恰最常见),所以短词回退到 LIKE 全表扫描:
/// 此时无论走哪条路都谈不上选择性,扫描是可接受的代价。
const FTS_MIN_CHARS: usize = 3;
/// 一条历史记录
#[derive(Debug, Clone, Serialize, Type)]
#[serde(rename_all = "camelCase")]
pub struct HistoryItem {
pub id: i64,
/// 毫秒时间戳
pub ts: i64,
pub from_lang: String,
pub to_lang: String,
pub engine_id: String,
pub engine_name: String,
pub source_text: String,
pub result_text: String,
/// 来源:"manual" | "selection" | "clipboard" | "screenshot" | "preview"
pub via: String,
pub favorited: bool,
pub latency_ms: i64,
}
pub struct History {
conn: Mutex<Connection>,
}
/// 全部建表语句。结构版本变化时整库重建,所以这里不需要考虑兼容旧结构。
///
/// `history_fts` 是**外部内容表**(`content='history'`):索引只存倒排表,
/// 正文仍只留在 `history` 里一份,不做双份存储。代价是它不会自己感知主表变化,
/// 必须靠下面三个触发器手动同步——漏了任何一个,索引就会和主表静默错位。
const SCHEMA_SQL: &str = "
CREATE TABLE IF NOT EXISTS history (
id INTEGER PRIMARY KEY,
ts INTEGER NOT NULL,
from_lang TEXT NOT NULL,
to_lang TEXT NOT NULL,
engine_id TEXT NOT NULL,
engine_name TEXT NOT NULL DEFAULT '',
source_text TEXT NOT NULL,
result_text TEXT NOT NULL,
via TEXT NOT NULL DEFAULT 'manual',
favorited INTEGER NOT NULL DEFAULT 0,
latency_ms INTEGER NOT NULL DEFAULT 0,
UNIQUE(source_text, to_lang, engine_id)
);
CREATE INDEX IF NOT EXISTS idx_history_ts ON history(ts DESC);
CREATE INDEX IF NOT EXISTS idx_history_fav ON history(favorited, ts DESC);
CREATE VIRTUAL TABLE IF NOT EXISTS history_fts USING fts5(
source_text, result_text,
content='history', content_rowid='id',
tokenize='trigram'
);
CREATE TRIGGER IF NOT EXISTS history_fts_ai AFTER INSERT ON history BEGIN
INSERT INTO history_fts(rowid, source_text, result_text)
VALUES (new.id, new.source_text, new.result_text);
END;
CREATE TRIGGER IF NOT EXISTS history_fts_ad AFTER DELETE ON history BEGIN
INSERT INTO history_fts(history_fts, rowid, source_text, result_text)
VALUES ('delete', old.id, old.source_text, old.result_text);
END;
CREATE TRIGGER IF NOT EXISTS history_fts_au AFTER UPDATE ON history BEGIN
INSERT INTO history_fts(history_fts, rowid, source_text, result_text)
VALUES ('delete', old.id, old.source_text, old.result_text);
INSERT INTO history_fts(rowid, source_text, result_text)
VALUES (new.id, new.source_text, new.result_text);
END;
";
impl History {
pub fn new(dir: &Path) -> Result<Self, String> {
std::fs::create_dir_all(dir).map_err(|e| format!("创建翻译目录失败: {e}"))?;
let conn =
Connection::open(dir.join("history.db")).map_err(|e| format!("打开历史库失败: {e}"))?;
conn.execute_batch("PRAGMA journal_mode = WAL;")
.map_err(|e| format!("初始化历史库失败: {e}"))?;
let version: i64 = conn
.query_row("PRAGMA user_version", [], |row| row.get(0))
.map_err(|e| format!("读取历史库版本失败: {e}"))?;
if version != SCHEMA_VERSION {
// 先删 FTS 表再删主表:触发器挂在主表上,会跟着一起消失
conn.execute_batch(
"DROP TABLE IF EXISTS history_fts;
DROP TABLE IF EXISTS history;",
)
.map_err(|e| format!("重建历史库失败: {e}"))?;
}
conn.execute_batch(SCHEMA_SQL)
.map_err(|e| format!("初始化历史表失败: {e}"))?;
conn.execute_batch(&format!("PRAGMA user_version = {SCHEMA_VERSION};"))
.map_err(|e| format!("写入历史库版本失败: {e}"))?;
Ok(Self {
conn: Mutex::new(conn),
})
}
fn conn(&self) -> std::sync::MutexGuard<'_, Connection> {
self.conn.lock().unwrap_or_else(|e| e.into_inner())
}
/// 记录一次翻译(去重:命中则更新时间与译文)。
pub fn record(
&self,
from_lang: &str,
to_lang: &str,
engine_id: &str,
engine_name: &str,
source_text: &str,
result_text: &str,
via: &str,
latency_ms: i64,
) -> Result<(), String> {
if source_text.trim().is_empty() || result_text.trim().is_empty() {
return Ok(());
}
let now = chrono::Utc::now().timestamp_millis();
self.conn()
.execute(
"INSERT INTO history (ts, from_lang, to_lang, engine_id, engine_name,
source_text, result_text, via, favorited, latency_ms)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, 0, ?9)
ON CONFLICT(source_text, to_lang, engine_id)
DO UPDATE SET ts = excluded.ts, result_text = excluded.result_text,
via = excluded.via, latency_ms = excluded.latency_ms",
params![
now,
from_lang,
to_lang,
engine_id,
engine_name,
source_text,
result_text,
via,
latency_ms
],
)
.map_err(|e| format!("写入历史失败: {e}"))?;
Ok(())
}
/// 列表。`query` 非空时按原文/译文模糊匹配。
///
/// 两种 SQL 形态互斥,但**参数个数与顺序固定为 (limit, offset, ?3)**,
/// 这样绑定点不需要跟着分支走:
/// - 关键词 ≥ `FTS_MIN_CHARS` 走 FTS5 索引(`history_fts MATCH`);
/// - 更短(或没有关键词)走 LIKE。
///
/// 两条路径都是子串语义、都忽略 ASCII 大小写,行为一致。
pub fn list(
&self,
offset: i64,
limit: i64,
query: Option<&str>,
favorited_only: bool,
) -> Result<Vec<HistoryItem>, String> {
let term = query.map(str::trim).filter(|s| !s.is_empty());
let use_fts = term.is_some_and(|t| t.chars().count() >= FTS_MIN_CHARS);
let mut sql = String::new();
if use_fts {
// 列必须带 h. 前缀:source_text / result_text 在两张表里同名,不加限定会歧义。
sql.push_str(
"SELECT h.id, h.ts, h.from_lang, h.to_lang, h.engine_id, h.engine_name,
h.source_text, h.result_text, h.via, h.favorited, h.latency_ms
FROM history_fts f JOIN history h ON h.id = f.rowid
WHERE history_fts MATCH ?3",
);
if favorited_only {
sql.push_str(" AND h.favorited = 1");
}
sql.push_str(" ORDER BY h.ts DESC LIMIT ?1 OFFSET ?2");
} else {
sql.push_str(
"SELECT id, ts, from_lang, to_lang, engine_id, engine_name, source_text,
result_text, via, favorited, latency_ms FROM history WHERE ",
);
if favorited_only {
sql.push_str("favorited = 1 AND ");
}
// 无关键词时 pattern 为 NULL,`?3 IS NULL` 让条件恒真。
// 这样参数个数固定,避免「有/无 ?3」两种形态下绑定索引不一致。
// ESCAPE '\':SQLite 的 LIKE 默认没有转义符,不写这个子句
// escape_like 对 %/_ 的转义就是无效代码。
sql.push_str(
"(?3 IS NULL OR source_text LIKE ?3 ESCAPE '\\' OR result_text LIKE ?3 ESCAPE '\\')
ORDER BY ts DESC LIMIT ?1 OFFSET ?2",
);
}
let param3 = term.map(|t| {
if use_fts {
fts_phrase(t)
} else {
format!("%{}%", escape_like(t))
}
});
let conn = self.conn();
let mut stmt = conn.prepare(&sql).map_err(|e| format!("查询历史失败: {e}"))?;
let rows = stmt
.query_map(params![limit, offset, param3], row_to_item)
.map_err(|e| format!("查询历史失败: {e}"))?;
let mut items = Vec::new();
for row in rows {
items.push(row.map_err(|e| format!("读取历史失败: {e}"))?);
}
Ok(items)
}
pub fn delete(&self, id: i64) -> Result<(), String> {
self.conn()
.execute("DELETE FROM history WHERE id = ?1", params![id])
.map_err(|e| format!("删除历史失败: {e}"))?;
Ok(())
}
pub fn clear(&self) -> Result<(), String> {
self.conn()
.execute("DELETE FROM history", [])
.map_err(|e| format!("清空历史失败: {e}"))?;
Ok(())
}
pub fn set_favorited(&self, id: i64, favorited: bool) -> Result<(), String> {
self.conn()
.execute(
"UPDATE history SET favorited = ?1 WHERE id = ?2",
params![favorited as i64, id],
)
.map_err(|e| format!("更新收藏失败: {e}"))?;
Ok(())
}
/// 容量淘汰:超出上限时优先删最旧的非收藏记录。
pub fn prune_to_max(&self, max_items: i64) {
if max_items <= 0 {
return;
}
let Ok(conn) = self.conn.lock() else {
return;
};
let _ = conn.execute(
"DELETE FROM history WHERE id IN (
SELECT id FROM history WHERE favorited = 0
ORDER BY ts DESC LIMIT -1 OFFSET ?1
)",
params![max_items],
);
}
}
fn escape_like(input: &str) -> String {
input.replace('\\', "\\\\").replace('%', "\\%").replace('_', "\\_")
}
/// 把用户输入包成一个 FTS5 短语查询。
///
/// **必须包引号**:FTS5 的 MATCH 参数有自己的一套查询语法,`-`、`*`、`(`、`^`
/// 以及 `AND`/`OR`/`NOT` 都会被当作操作符——用户搜 `a-b` 会被解释成「含 a 但不含 b」,
/// 搜 `(` 之类则直接抛语法错误。整串加双引号后退化成「按字面顺序出现的短语」,
/// 与原本 LIKE 子串语义对齐;引号内的 `"` 用双写转义。
fn fts_phrase(term: &str) -> String {
format!("\"{}\"", term.replace('"', "\"\""))
}
fn row_to_item(row: &rusqlite::Row<'_>) -> rusqlite::Result<HistoryItem> {
Ok(HistoryItem {
id: row.get(0)?,
ts: row.get(1)?,
from_lang: row.get(2)?,
to_lang: row.get(3)?,
engine_id: row.get(4)?,
engine_name: row.get(5)?,
source_text: row.get(6)?,
result_text: row.get(7)?,
via: row.get(8)?,
favorited: row.get::<_, i64>(9)? != 0,
latency_ms: row.get(10)?,
})
}
#[cfg(test)]
mod tests {
use super::*;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering};
/// 每个用例一个独立目录。`History` 持有连接,测试结束前无法删目录(Windows 会拒绝),
/// 因此这里只保证不互相踩,不留清理——落在系统临时目录里是可以接受的代价。
fn open(tag: &str) -> (History, PathBuf) {
static SEQ: AtomicUsize = AtomicUsize::new(0);
let n = SEQ.fetch_add(1, Ordering::Relaxed);
let dir = std::env::temp_dir().join(format!(
"thing-history-test-{}-{tag}-{n}",
std::process::id()
));
let _ = std::fs::remove_dir_all(&dir);
let history = History::new(&dir).expect("建库失败");
(history, dir)
}
/// 一组覆盖两种分词行为的样例:
/// 中日英混排、含 FTS 元字符(`-` `*` `"`)、含 LIKE 元字符(`%`)、含 FTS 操作符关键字。
const SOURCE_JA: &str = "日本語の翻訳を確認します";
const RESULT_JA: &str = "确认日本语翻译";
const SOURCE_EN: &str = "return the formatted date string";
const RESULT_EN: &str = "返回格式化后的日期字符串";
const SOURCE_JA2: &str = "すべての項目に入力をしてください";
const RESULT_JA2: &str = "请在所有项目中输入。";
const SOURCE_META: &str = "tail with \"quote\" and *star and a-b";
const SOURCE_PCT: &str = "100% done";
const SOURCE_KW: &str = "a AND b";
fn seed(h: &History) {
let rows = [
(SOURCE_JA, RESULT_JA, "manual"),
(SOURCE_EN, RESULT_EN, "clipboard"),
(SOURCE_JA2, RESULT_JA2, "selection"),
(SOURCE_META, "x", "manual"),
(SOURCE_PCT, "y", "manual"),
(SOURCE_KW, "z", "manual"),
];
for (i, (src, res, via)) in rows.iter().enumerate() {
h.record("auto", "zh-Hans", "mock", "Mock", src, res, via, i as i64)
.expect("写入失败");
}
}
/// 用原生 LIKE 算出的基准集合,作为 FTS 路径的正确性参照。
fn like_baseline(dir: &Path, term: &str, favorited_only: bool) -> Vec<i64> {
let conn = Connection::open(dir.join("history.db")).unwrap();
let pattern = format!("%{}%", escape_like(term));
let sql = if favorited_only {
"SELECT id FROM history WHERE favorited = 1
AND (source_text LIKE ?1 ESCAPE '\\' OR result_text LIKE ?1 ESCAPE '\\')"
} else {
"SELECT id FROM history WHERE
source_text LIKE ?1 ESCAPE '\\' OR result_text LIKE ?1 ESCAPE '\\'"
};
let mut stmt = conn.prepare(sql).unwrap();
let rows = stmt.query_map(params![pattern], |r| r.get::<_, i64>(0)).unwrap();
rows.map(|r| r.unwrap()).collect()
}
/// 按集合比较,忽略顺序:`record()` 用 wall clock 打时间戳,同一用例内的记录
/// ts 可能相同,`ORDER BY ts DESC` 的先后不稳定,比顺序会随机失败。
fn sorted_ids(h: &History, term: &str, favorited_only: bool) -> Vec<i64> {
let mut ids: Vec<i64> = h
.list(0, 100, Some(term), favorited_only)
.expect("查询失败")
.iter()
.map(|i| i.id)
.collect();
ids.sort_unstable();
ids
}
fn sorted(v: Vec<i64>) -> Vec<i64> {
let mut v = v;
v.sort_unstable();
v
}
/// 两种 SQL 形态必须给出**完全一致**的结果。
///
/// 这条测试的核心是钉住长度分流:trigram 索引里没有短于 3 字符的片段,
/// 对 1~2 字的关键词 `MATCH` 不报错、只返回空集。少了回退分支,中文两字词
/// (最常见的一类查询)就会「条件明明对却搜不到」。
#[test]
fn fts_and_like_paths_agree_across_query_lengths() {
let (h, dir) = open("paths-agree");
seed(&h);
let ids: Vec<i64> = h.list(0, 100, None, false).unwrap().iter().map(|i| i.id).collect();
assert_eq!(ids.len(), 6, "样例数据应为 6 条(去重键各不相同)");
h.set_favorited(ids[1], true).unwrap();
let terms = [
"确", // 1 字 → LIKE 回退
"确认", // 2 字 → LIKE 回退(中文里最常见,最容易踩坑的长度)
"fo", // 2 字符 ASCII → LIKE 回退
"确认日", // 3 字 → FTS
"日本语翻译", // 5 字 → FTS
"form", // 4 字符 → FTS,且是 formatted 的子串
"the formatted",
"zzz", // 无命中
"100%", // LIKE 元字符:两条路径都必须按字面处理
"a-b", // FTS 里 `-` 是操作符,必须被短语引号中和
"*star",
"with \"quote\"",
"AND", // FTS 关键字,必须按字面匹配而不是当运算符
"(((",
];
for term in terms {
for favorited_only in [false, true] {
assert_eq!(
sorted_ids(&h, term, favorited_only),
sorted(like_baseline(&dir, term, favorited_only)),
"关键词 {term:?}(favoritedOnly={favorited_only})两条路径结果不一致"
);
}
}
}
/// 外部内容表不会自动感知主表变化,全靠触发器。漏一个就会让索引与正文静默错位
/// ——表现是「刚翻过的句子搜不到」或「已删掉的记录还能搜出来」,都很难归因。
#[test]
fn triggers_keep_fts_in_sync_with_upsert_and_delete() {
let (h, _dir) = open("trigger-sync");
h.record("ja", "zh-Hans", "mock", "Mock", SOURCE_JA, RESULT_JA, "manual", 1)
.unwrap();
assert_eq!(sorted_ids(&h, "日本语翻译", false).len(), 1, "新记录应可搜到");
// 去重键命中 → 走 ON CONFLICT DO UPDATE,应触发 update 触发器
h.record("ja", "zh-Hans", "mock", "Mock", SOURCE_JA, "换了译文的说法", "manual", 2)
.unwrap();
assert_eq!(h.list(0, 100, None, false).unwrap().len(), 1, "应仍是同一条记录");
assert_eq!(sorted_ids(&h, "换了译文的说法", false).len(), 1, "更新后新译文应可搜到");
assert!(sorted_ids(&h, "日本语翻译", false).is_empty(), "更新后旧译文不应再命中");
let id = h.list(0, 100, None, false).unwrap()[0].id;
h.delete(id).unwrap();
assert!(sorted_ids(&h, "换了译文的说法", false).is_empty(), "删除后不应再命中");
h.record("ja", "zh-Hans", "mock", "Mock", SOURCE_JA, RESULT_JA, "manual", 3)
.unwrap();
h.clear().unwrap();
assert!(h.list(0, 100, None, false).unwrap().is_empty());
assert!(sorted_ids(&h, "换了译文的说法", false).is_empty(), "清空后索引应为空");
}
/// FTS 索引必须与主表行数一致。用 FTS5 自带的完整性检查暴露静默错位。
#[test]
fn fts_index_passes_integrity_check() {
let (h, _dir) = open("integrity");
seed(&h);
let conn = h.conn();
conn.execute("INSERT INTO history_fts(history_fts) VALUES('integrity-check')", [])
.expect("FTS 索引与主表不一致");
}
/// 收藏条目不参与容量淘汰(既有约定,顺手一起守住)。
#[test]
fn prune_drops_oldest_unfavorited_only() {
let (h, _dir) = open("prune");
for i in 0..6 {
// 原文必须各不相同:去重键含 source_text,同一段文字只会留一条
let src = format!("{SOURCE_JA}{i}");
h.record("auto", "zh-Hans", "mock", "Mock", &src, RESULT_JA, "manual", i)
.unwrap();
}
let ids: Vec<i64> = h.list(0, 100, None, false).unwrap().iter().map(|i| i.id).collect();
assert_eq!(ids.len(), 6);
let favorite = ids[2];
h.set_favorited(favorite, true).unwrap();
h.prune_to_max(3);
let left: Vec<i64> = h.list(0, 100, None, false).unwrap().iter().map(|i| i.id).collect();
assert!(left.contains(&favorite), "收藏条目被淘汰了");
assert_eq!(left.len(), 4, "上限 3 之外只该多留那条收藏,实际剩 {left:?}");
}
/// 结构版本变化时整库重建。这条是「现在允许丢数据」这个前提的守卫:
/// 一旦有人把 SCHEMA_VERSION 忘了改,旧库会带着不兼容的结构跑下去。
#[test]
fn stale_schema_is_dropped_and_rebuilt() {
let (h, dir) = open("schema-reset");
seed(&h);
assert_eq!(h.list(0, 100, None, false).unwrap().len(), 6);
drop(h);
{
let conn = Connection::open(dir.join("history.db")).unwrap();
conn.execute("INSERT INTO history(ts, from_lang, to_lang, engine_id, engine_name, source_text, result_text, via, favorited, latency_ms)
VALUES (1,'auto','zh-Hans','mock','Mock','stale row','stale row','manual',0,0)", []).unwrap();
conn.execute_batch("PRAGMA user_version = 1;").unwrap();
}
let rebuilt = History::new(&dir).expect("重建失败");
assert!(
rebuilt.list(0, 100, None, false).unwrap().is_empty(),
"版本不匹配时旧数据应被清掉"
);
assert!(sorted_ids(&rebuilt, "stale row", false).is_empty(), "重建后不应残留旧索引");
}
/// 短语化必须中和 FTS5 的查询语法,否则用户搜 `a-b` 会被解释成「含 a 但不含 b」,
/// 搜 `(` 之类则直接抛语法错误。
#[test]
fn fts_phrase_quotes_and_escapes() {
assert_eq!(fts_phrase("a-b"), "\"a-b\"");
assert_eq!(fts_phrase("say \"hi\""), "\"say \"\"hi\"\"\"");
assert_eq!(fts_phrase("("), "\"(\"");
}
}