feat: add memory crate for embed-db-rs (Phase 8B-4)

TurnManager + FactManager + Compressor + time decay: - TurnManager: embed → USearch add → SQLite store, semantic search - FactManager: fact CRUD with hybrid (semantic + BM25) search - Compressor: 5-layer priority token budget (facts 25%, long 25%, short 35%) - Time decay: exp(-lambda * age_hours), default lambda=0.05 - Token counting: len/4 approximation - 18 tests passing Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-08 09:24:28 +00:00
parent cd00c79262
commit da8ecdce42
10 changed files with 1251 additions and 0 deletions
@@ -229,6 +229,19 @@ dependencies = [
 "static_assertions",
 ]

+[[package]]
+name = "console"
+version = "0.15.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"
+dependencies = [
+ "encode_unicode",
+ "libc",
+ "once_cell",
+ "unicode-width",
+ "windows-sys 0.59.0",
+]
+
 [[package]]
 name = "core-foundation"
 version = "0.10.1"
@@ -503,6 +516,12 @@ dependencies = [
 "tracing",
 ]

+[[package]]
+name = "encode_unicode"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
+
 [[package]]
 name = "equivalent"
 version = "1.0.2"
@@ -754,6 +773,19 @@ dependencies = [
 "serde_core",
 ]

+[[package]]
+name = "indicatif"
+version = "0.17.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235"
+dependencies = [
+ "console",
+ "number_prefix",
+ "portable-atomic",
+ "unicode-width",
+ "web-time",
+]
+
 [[package]]
 name = "instant"
 version = "0.1.13"
@@ -954,6 +986,22 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "memory"
+version = "0.1.0"
+dependencies = [
+ "embedder",
+ "meta-store",
+ "search",
+ "serde",
+ "serde_json",
+ "tempfile",
+ "thiserror 2.0.18",
+ "tokenizers",
+ "tracing",
+ "usearch-index",
+]
+
 [[package]]
 name = "meta-store"
 version = "0.1.0"
@@ -1087,6 +1135,12 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "number_prefix"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
+
 [[package]]
 name = "once_cell"
 version = "1.21.3"
@@ -1921,6 +1975,7 @@ dependencies = [
 "esaxx-rs",
 "fancy-regex",
 "getrandom 0.3.4",
+ "indicatif",
 "itertools 0.14.0",
 "log",
 "macro_rules_attribute",
@@ -2215,6 +2270,16 @@ dependencies = [
 "wasm-bindgen",
 ]

+[[package]]
+name = "web-time"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "webpki-root-certs"
 version = "1.0.6"
@@ -4,6 +4,7 @@ members = [
    "crates/embedder",
    "crates/meta-store",
    "crates/search",
+    "crates/memory",
 ]
 resolver = "2"

@@ -0,0 +1,18 @@
+[package]
+name = "memory"
+version.workspace = true
+edition.workspace = true
+
+[dependencies]
+embedder = { path = "../embedder" }
+meta-store = { path = "../meta-store" }
+usearch-index = { path = "../usearch-index" }
+search = { path = "../search" }
+thiserror = "2"
+tracing = "0.1"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+
+[dev-dependencies]
+tempfile = "3"
+tokenizers = { version = "0.21", default-features = false, features = ["progressbar"] }
@@ -0,0 +1,292 @@
+//! 5-layer priority token budget compressor.
+//!
+//! Compresses memory entries (facts, working memory, long-term, short-term)
+//! into a token-budgeted XML-formatted context string for LLM prompts.
+
+use crate::token::count_tokens;
+use crate::types::CompressEntry;
+
+/// Budget allocation breakdown.
+#[derive(Debug, Clone)]
+pub struct BudgetInfo {
+    /// Tokens allocated to facts.
+    pub facts: usize,
+    /// Tokens allocated to long-term memory.
+    pub long_term: usize,
+    /// Tokens allocated to short-term history.
+    pub short_term: usize,
+    /// Total tokens actually used.
+    pub used: usize,
+}
+
+/// Result of the compression pipeline.
+#[derive(Debug, Clone)]
+pub struct CompressResult {
+    /// Included fact entries.
+    pub facts: Vec<CompressEntry>,
+    /// Included long-term memory entries.
+    pub long_term: Vec<CompressEntry>,
+    /// Included short-term history entries.
+    pub short_term: Vec<CompressEntry>,
+    /// XML-formatted output string combining all layers.
+    pub formatted: String,
+    /// Total tokens used across all layers.
+    pub tokens_used: usize,
+    /// Per-layer budget allocation info.
+    pub budget_info: BudgetInfo,
+}
+
+/// Compresses multi-layer memory into a token-budgeted context string.
+///
+/// Budget allocation (proportional to total budget):
+/// - Facts: 25%
+/// - Long-term: 25%
+/// - Short-term: 35%
+/// - Working memory: unconditionally included (no budget cut), remaining 15% reserved
+pub struct Compressor {
+    /// Total token budget (default: 2000).
+    budget: usize,
+}
+
+impl Compressor {
+    /// Create a new compressor with the given total token budget.
+    pub fn new(budget: usize) -> Self {
+        Self { budget }
+    }
+
+    /// Compress entries from all memory layers into a budgeted result.
+    ///
+    /// # Arguments
+    ///
+    /// * `facts` — extracted knowledge facts (budget: 25%)
+    /// * `working_memory` — current task context, unconditionally included
+    /// * `long_term` — semantically retrieved older entries (budget: 25%)
+    /// * `short_term` — recent conversation turns (budget: 35%)
+    pub fn compress(
+        &self,
+        facts: &[CompressEntry],
+        working_memory: &[CompressEntry],
+        long_term: &[CompressEntry],
+        short_term: &[CompressEntry],
+    ) -> CompressResult {
+        let facts_budget = self.budget * 25 / 100;
+        let long_budget = self.budget * 25 / 100;
+        let short_budget = self.budget * 35 / 100;
+
+        let selected_facts = select_within_budget(facts, facts_budget);
+        let selected_long = select_within_budget(long_term, long_budget);
+        let selected_short = select_within_budget(short_term, short_budget);
+
+        // Working memory is unconditionally included
+        let selected_working: Vec<CompressEntry> = working_memory.to_vec();
+
+        // Calculate actual token usage
+        let facts_used: usize = selected_facts.iter().map(|e| count_tokens(&e.content)).sum();
+        let long_used: usize = selected_long.iter().map(|e| count_tokens(&e.content)).sum();
+        let short_used: usize = selected_short.iter().map(|e| count_tokens(&e.content)).sum();
+        let working_used: usize = selected_working
+            .iter()
+            .map(|e| count_tokens(&e.content))
+            .sum();
+        let total_used = facts_used + long_used + short_used + working_used;
+
+        // Format as XML-style sections
+        let formatted = format_sections(
+            &selected_facts,
+            &selected_working,
+            &selected_short,
+            &selected_long,
+        );
+
+        CompressResult {
+            facts: selected_facts,
+            long_term: selected_long,
+            short_term: selected_short,
+            formatted,
+            tokens_used: total_used,
+            budget_info: BudgetInfo {
+                facts: facts_budget,
+                long_term: long_budget,
+                short_term: short_budget,
+                used: total_used,
+            },
+        }
+    }
+}
+
+impl Default for Compressor {
+    fn default() -> Self {
+        Self::new(2000)
+    }
+}
+
+/// Select entries from the input until the token budget is exhausted.
+///
+/// Entries are taken in order (caller should pre-sort by relevance).
+fn select_within_budget(entries: &[CompressEntry], budget: usize) -> Vec<CompressEntry> {
+    let mut selected = Vec::new();
+    let mut used = 0;
+
+    for entry in entries {
+        let tokens = count_tokens(&entry.content);
+        if used + tokens > budget && !selected.is_empty() {
+            break;
+        }
+        selected.push(entry.clone());
+        used += tokens;
+        if used >= budget {
+            break;
+        }
+    }
+
+    selected
+}
+
+/// Format the selected entries as XML-style sections.
+fn format_sections(
+    facts: &[CompressEntry],
+    working_memory: &[CompressEntry],
+    short_term: &[CompressEntry],
+    long_term: &[CompressEntry],
+) -> String {
+    let mut parts = Vec::new();
+
+    if !facts.is_empty() {
+        let mut section = String::from("<facts>\n");
+        for entry in facts {
+            section.push_str(&format!("{}: {}\n", entry.role, entry.content));
+        }
+        section.push_str("</facts>");
+        parts.push(section);
+    }
+
+    if !working_memory.is_empty() {
+        let mut section = String::from("<working_memory>\n");
+        for entry in working_memory {
+            section.push_str(&format!("{}: {}\n", entry.role, entry.content));
+        }
+        section.push_str("</working_memory>");
+        parts.push(section);
+    }
+
+    if !short_term.is_empty() {
+        let mut section = String::from("<history>\n");
+        for entry in short_term {
+            section.push_str(&format!("{}: {}\n", entry.role, entry.content));
+        }
+        section.push_str("</history>");
+        parts.push(section);
+    }
+
+    if !long_term.is_empty() {
+        let mut section = String::from("<knowledge>\n");
+        for entry in long_term {
+            section.push_str(&format!("{}: {}\n", entry.role, entry.content));
+        }
+        section.push_str("</knowledge>");
+        parts.push(section);
+    }
+
+    parts.join("\n")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn make_entry(role: &str, content: &str, score: f64) -> CompressEntry {
+        CompressEntry {
+            role: role.to_string(),
+            content: content.to_string(),
+            score,
+        }
+    }
+
+    #[test]
+    fn test_compress_basic() {
+        let compressor = Compressor::new(2000);
+        let facts = vec![make_entry("science", "The sky is blue", 0.9)];
+        let working = vec![make_entry("user", "Current task context", 1.0)];
+        let long_term = vec![make_entry("user", "Earlier conversation about weather", 0.7)];
+        let short_term = vec![make_entry("assistant", "I can help with that", 0.8)];
+
+        let result = compressor.compress(&facts, &working, &long_term, &short_term);
+
+        assert!(!result.formatted.is_empty());
+        assert!(result.formatted.contains("<facts>"));
+        assert!(result.formatted.contains("</facts>"));
+        assert!(result.formatted.contains("<working_memory>"));
+        assert!(result.formatted.contains("</working_memory>"));
+        assert!(result.formatted.contains("<history>"));
+        assert!(result.formatted.contains("</history>"));
+        assert!(result.formatted.contains("<knowledge>"));
+        assert!(result.formatted.contains("</knowledge>"));
+        assert!(result.formatted.contains("The sky is blue"));
+        assert!(result.formatted.contains("Current task context"));
+
+        assert!(!result.facts.is_empty());
+        assert!(!result.long_term.is_empty());
+        assert!(!result.short_term.is_empty());
+        assert!(result.tokens_used > 0);
+    }
+
+    #[test]
+    fn test_compress_budget_limits() {
+        // Budget = 100 tokens → facts_budget=25, long=25, short=35
+        let compressor = Compressor::new(100);
+
+        // Each entry is roughly 50 chars → ~12 tokens
+        let facts = vec![
+            make_entry("cat1", "A".repeat(50).as_str(), 0.9),
+            make_entry("cat2", "B".repeat(50).as_str(), 0.8),
+            make_entry("cat3", "C".repeat(50).as_str(), 0.7),
+        ];
+
+        let result = compressor.compress(&facts, &[], &[], &[]);
+
+        // With facts_budget=25 tokens, and each entry ~12 tokens,
+        // at most 2 entries should fit (12+12=24 ≤ 25)
+        assert!(
+            result.facts.len() <= 2,
+            "should cut entries exceeding budget, got {} entries",
+            result.facts.len()
+        );
+    }
+
+    #[test]
+    fn test_compress_working_memory_unconditional() {
+        // Very small budget
+        let compressor = Compressor::new(10);
+
+        // Working memory is large but should be included unconditionally
+        let working = vec![make_entry(
+            "system",
+            &"X".repeat(200), // ~50 tokens, way over budget
+            1.0,
+        )];
+
+        let result = compressor.compress(&[], &working, &[], &[]);
+
+        // Working memory must still appear in formatted output
+        assert!(
+            result.formatted.contains("<working_memory>"),
+            "working memory should be included even when over budget"
+        );
+        assert!(result.tokens_used > 0);
+    }
+
+    #[test]
+    fn test_compress_empty() {
+        let compressor = Compressor::new(2000);
+        let result = compressor.compress(&[], &[], &[], &[]);
+
+        assert!(
+            result.formatted.is_empty(),
+            "all empty should produce empty formatted string"
+        );
+        assert!(result.facts.is_empty());
+        assert!(result.long_term.is_empty());
+        assert!(result.short_term.is_empty());
+        assert_eq!(result.tokens_used, 0);
+    }
+}
@@ -0,0 +1,216 @@
+//! Knowledge fact management with vector and hybrid search.
+//!
+//! Ties together [`FactStore`] (SQLite metadata), [`VectorIndex`] (HNSW search),
+//! and [`Embedder`] (text embedding) into a unified API for CRUD operations
+//! on extracted knowledge facts with semantic search.
+
+use std::sync::Arc;
+
+use tracing::debug;
+
+use embedder::Embedder;
+use meta_store::{FactRow, FactStore};
+use search::HybridScorer;
+use usearch_index::VectorIndex;
+
+use crate::token::count_tokens;
+use crate::types::ScoredFact;
+use crate::MemoryError;
+
+/// Manages knowledge facts with embedded vector search.
+///
+/// Uses a separate [`VectorIndex`] from turns (facts and turns live in
+/// independent vector spaces for cleaner retrieval).
+pub struct FactManager {
+    fact_store: FactStore,
+    vector_index: VectorIndex,
+    embedder: Arc<Embedder>,
+}
+
+impl FactManager {
+    /// Create a new FactManager.
+    ///
+    /// * `fact_store` — SQLite-backed fact metadata store.
+    /// * `vector_index` — HNSW vector index for semantic search over facts.
+    /// * `embedder` — shared text embedder (same instance as TurnManager).
+    pub fn new(
+        fact_store: FactStore,
+        vector_index: VectorIndex,
+        embedder: Arc<Embedder>,
+    ) -> Self {
+        Self {
+            fact_store,
+            vector_index,
+            embedder,
+        }
+    }
+
+    /// Add a knowledge fact.
+    ///
+    /// 1. Embeds the content.
+    /// 2. Adds to the vector index.
+    /// 3. Stores metadata in SQLite.
+    ///
+    /// Returns the fact's row ID.
+    pub fn add_fact(
+        &mut self,
+        content: &str,
+        category: &str,
+        entity: Option<&str>,
+        tags: &[String],
+        source_session: &str,
+    ) -> Result<i64, MemoryError> {
+        // Determine next faiss_id by counting existing facts
+        let existing_count = self.fact_store.count()? as i64;
+        let faiss_id = existing_count; // Simple monotonic assignment
+
+        // Embed the content
+        let embeddings = self.embedder.embed_passages(&[content])?;
+        let vec = embeddings
+            .into_iter()
+            .next()
+            .ok_or_else(|| MemoryError::Embed("empty embedding result".to_string()))?;
+
+        // Add to vector index
+        self.vector_index.add(faiss_id as u64, &vec)?;
+
+        let _ = count_tokens(content); // future use for token tracking
+
+        // Store in SQLite with default confidence and no conflict
+        let fact_id = self.fact_store.add_fact(
+            content,
+            category,
+            entity,
+            tags,
+            source_session,
+            faiss_id,
+            1.0, // default confidence
+            false,
+        )?;
+
+        debug!(fact_id, faiss_id, category, "fact added to memory");
+        Ok(fact_id)
+    }
+
+    /// Semantic search for facts by query text.
+    ///
+    /// Returns scored facts sorted by similarity (highest first).
+    pub fn semantic_search(
+        &self,
+        query: &str,
+        top_k: usize,
+    ) -> Result<Vec<ScoredFact>, MemoryError> {
+        let query_vec = self.embedder.embed_query(query)?;
+        let (ids, distances) = self.vector_index.search(&query_vec, top_k)?;
+
+        if ids.is_empty() {
+            return Ok(vec![]);
+        }
+
+        let faiss_ids: Vec<i64> = ids.iter().map(|&id| id as i64).collect();
+        let facts = self.fact_store.get_by_faiss_ids(&faiss_ids)?;
+
+        // Build lookup map
+        let mut fact_map: std::collections::HashMap<i64, FactRow> = facts
+            .into_iter()
+            .map(|f| (f.faiss_id, f))
+            .collect();
+
+        let mut results = Vec::new();
+        for (key, dist) in ids.iter().zip(distances.iter()) {
+            let faiss_id = *key as i64;
+            if let Some(fact) = fact_map.remove(&faiss_id) {
+                let score = 1.0 - dist;
+                results.push(ScoredFact { fact, score });
+            }
+        }
+
+        Ok(results)
+    }
+
+    /// Hybrid search combining semantic similarity and BM25 text matching.
+    ///
+    /// Uses [`HybridScorer`] to fuse vector and BM25 scores with the given
+    /// BM25 weight (0.0 = pure semantic, 1.0 = pure BM25).
+    pub fn hybrid_search(
+        &self,
+        query: &str,
+        top_k: usize,
+        bm25_weight: f32,
+    ) -> Result<Vec<ScoredFact>, MemoryError> {
+        // 1. Semantic search
+        let query_vec = self.embedder.embed_query(query)?;
+        let (sem_ids, sem_distances) = self.vector_index.search(&query_vec, top_k)?;
+
+        // Convert distances to similarity scores for the hybrid scorer
+        let sem_scores: Vec<f32> = sem_distances.iter().map(|d| 1.0 - d).collect();
+
+        // 2. BM25 search via FactStore's FTS5
+        let bm25_results_raw = self.fact_store.bm25_search(query, top_k)?;
+
+        // Convert FactSearchResult to search::BM25SearchResult
+        let bm25_results: Vec<search::BM25SearchResult> = bm25_results_raw
+            .iter()
+            .map(|r| search::BM25SearchResult {
+                id: r.faiss_id,
+                score: r.bm25_score as f32,
+                path: String::new(),
+            })
+            .collect();
+
+        // 3. Fuse scores
+        let scorer = HybridScorer::new(bm25_weight);
+        let fused = scorer.fuse(&sem_ids, &sem_scores, &bm25_results);
+
+        // 4. Retrieve full fact metadata for top results
+        let fused_faiss_ids: Vec<i64> = fused.iter().take(top_k).map(|r| r.id as i64).collect();
+        let facts = self.fact_store.get_by_faiss_ids(&fused_faiss_ids)?;
+
+        let mut fact_map: std::collections::HashMap<i64, FactRow> =
+            facts.into_iter().map(|f| (f.faiss_id, f)).collect();
+
+        let mut results = Vec::new();
+        for hr in fused.iter().take(top_k) {
+            let faiss_id = hr.id as i64;
+            if let Some(fact) = fact_map.remove(&faiss_id) {
+                results.push(ScoredFact {
+                    fact,
+                    score: hr.score,
+                });
+            }
+        }
+
+        Ok(results)
+    }
+
+    /// List facts with optional category and entity filters.
+    pub fn list_facts(
+        &self,
+        category: Option<&str>,
+        entity: Option<&str>,
+    ) -> Result<Vec<FactRow>, MemoryError> {
+        Ok(self.fact_store.list_facts(category, entity)?)
+    }
+
+    /// Update a fact's content and confidence.
+    pub fn update_fact(
+        &self,
+        fact_id: i64,
+        content: &str,
+        confidence: f64,
+    ) -> Result<(), MemoryError> {
+        Ok(self.fact_store.update_fact(fact_id, content, confidence)?)
+    }
+
+    /// Delete a fact by ID.
+    pub fn delete_fact(&mut self, fact_id: i64) -> Result<(), MemoryError> {
+        // Note: vector index entry is tombstoned but not removed
+        // (USearch doesn't support efficient deletion by content).
+        Ok(self.fact_store.delete_fact(fact_id)?)
+    }
+
+    /// Count total facts.
+    pub fn count(&self) -> Result<usize, MemoryError> {
+        Ok(self.fact_store.count()?)
+    }
+}
@@ -0,0 +1,258 @@
+//! Memory management crate for embed-db-rs.
+//!
+//! Ties together [`embedder`], [`meta_store`], [`usearch_index`], and [`search`]
+//! into a unified memory system with:
+//!
+//! - **TurnManager** — conversation turn storage with vector search
+//! - **FactManager** — knowledge fact CRUD with hybrid (semantic + BM25) search
+//! - **Compressor** — 5-layer priority token budget compression
+//! - **Time decay** — exponential score decay based on entry age
+
+pub mod compressor;
+pub mod fact_manager;
+pub mod time_decay;
+pub mod token;
+pub mod turn_manager;
+pub mod types;
+
+// Re-export main public types
+pub use compressor::{BudgetInfo, CompressResult, Compressor};
+pub use fact_manager::FactManager;
+pub use time_decay::{apply_time_decay, DEFAULT_LAMBDA};
+pub use token::count_tokens;
+pub use turn_manager::TurnManager;
+pub use types::{CompressEntry, ScoredEntry, ScoredFact, ScoredTurn};
+
+/// Errors returned by the memory crate.
+#[derive(Debug, thiserror::Error)]
+pub enum MemoryError {
+    /// SQLite metadata store error.
+    #[error("store error: {0}")]
+    Store(#[from] meta_store::MetaStoreError),
+
+    /// Vector index error.
+    #[error("vector index error: {0}")]
+    VectorIndex(#[from] usearch_index::Error),
+
+    /// Embedding error.
+    #[error("embedding error: {0}")]
+    Embed(String),
+
+    /// Search error.
+    #[error("search error: {0}")]
+    Search(#[from] search::SearchError),
+}
+
+impl From<embedder::EmbedError> for MemoryError {
+    fn from(e: embedder::EmbedError) -> Self {
+        MemoryError::Embed(e.to_string())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::io::Write;
+    use std::path::PathBuf;
+    use std::sync::Arc;
+
+    use embedder::{EmbedConfig, Embedder, MockBackend};
+    use meta_store::{FactStore, TurnStore};
+    use usearch_index::VectorIndex;
+
+    use crate::fact_manager::FactManager;
+    use crate::turn_manager::TurnManager;
+
+    const DIM: usize = 4;
+    const SEQ_LEN: usize = 8;
+
+    /// Create a test tokenizer with a minimal vocab.
+    fn make_test_tokenizer(seq_len: usize) -> tokenizers::Tokenizer {
+        use tokenizers::models::wordpiece::WordPiece;
+        use tokenizers::pre_tokenizers::whitespace::Whitespace;
+
+        let dir = tempfile::tempdir().unwrap();
+        let vocab_path = dir.path().join("vocab.txt");
+        let mut f = std::fs::File::create(&vocab_path).unwrap();
+        for token in [
+            "[PAD]", "[UNK]", "[CLS]", "[SEP]", "query", ":", "passage",
+            "hello", "world", "test", "text", "the", "a", "fact", "user",
+            "weather", "science", "rust", "programming", "memory",
+        ] {
+            writeln!(f, "{}", token).unwrap();
+        }
+        drop(f);
+
+        let wp = WordPiece::from_file(vocab_path.to_str().unwrap())
+            .unk_token("[UNK]".to_string())
+            .build()
+            .unwrap();
+
+        let mut tok = tokenizers::Tokenizer::new(wp);
+        tok.with_pre_tokenizer(Some(Whitespace::default()));
+        tok.with_padding(Some(tokenizers::PaddingParams {
+            strategy: tokenizers::PaddingStrategy::Fixed(seq_len),
+            pad_id: 0,
+            pad_token: "[PAD]".to_string(),
+            ..Default::default()
+        }));
+        tok.with_truncation(Some(tokenizers::TruncationParams {
+            max_length: seq_len,
+            ..Default::default()
+        }))
+        .unwrap();
+
+        // Leak the tempdir so files remain accessible
+        std::mem::forget(dir);
+        tok
+    }
+
+    fn make_test_embedder() -> Arc<Embedder> {
+        let config = EmbedConfig {
+            model_path: PathBuf::from("test.onnx"),
+            seq_len: SEQ_LEN,
+            batch_size: 32,
+            emb_dim: DIM,
+        };
+        let backend = Box::new(MockBackend::new(DIM));
+        let tokenizer = make_test_tokenizer(SEQ_LEN);
+        Arc::new(Embedder::new(backend, tokenizer, config))
+    }
+
+    // ─── TurnManager tests ───
+
+    #[test]
+    fn test_add_turn() {
+        let dir = tempfile::tempdir().unwrap();
+        let db_path = dir.path().join("turns.db");
+        let turn_store = TurnStore::open(&db_path).unwrap();
+        let vector_index = VectorIndex::new(DIM, 100, usearch_index::ScalarKind::F32).unwrap();
+        let embedder = make_test_embedder();
+
+        let mut tm = TurnManager::new(turn_store, vector_index, embedder);
+
+        let id = tm.add_turn("sess1", "user", "hello world").unwrap();
+        assert!(id > 0);
+
+        let turns = tm.list_turns("sess1").unwrap();
+        assert_eq!(turns.len(), 1);
+        assert_eq!(turns[0].content, "hello world");
+        assert_eq!(turns[0].role, "user");
+    }
+
+    #[test]
+    fn test_semantic_search() {
+        let dir = tempfile::tempdir().unwrap();
+        let db_path = dir.path().join("turns.db");
+        let turn_store = TurnStore::open(&db_path).unwrap();
+        let vector_index = VectorIndex::new(DIM, 100, usearch_index::ScalarKind::F32).unwrap();
+        let embedder = make_test_embedder();
+
+        let mut tm = TurnManager::new(turn_store, vector_index, embedder);
+
+        tm.add_turn("sess1", "user", "hello world").unwrap();
+        tm.add_turn("sess1", "assistant", "test text").unwrap();
+        tm.add_turn("sess1", "user", "rust programming").unwrap();
+
+        let results = tm.semantic_search("hello", 5).unwrap();
+        // MockBackend returns deterministic vectors based on input,
+        // so we should get results back
+        assert!(!results.is_empty(), "should find results via semantic search");
+    }
+
+    #[test]
+    fn test_list_turns() {
+        let dir = tempfile::tempdir().unwrap();
+        let db_path = dir.path().join("turns.db");
+        let turn_store = TurnStore::open(&db_path).unwrap();
+        let vector_index = VectorIndex::new(DIM, 100, usearch_index::ScalarKind::F32).unwrap();
+        let embedder = make_test_embedder();
+
+        let mut tm = TurnManager::new(turn_store, vector_index, embedder);
+
+        tm.add_turn("sess1", "user", "turn 1").unwrap();
+        tm.add_turn("sess1", "assistant", "turn 2").unwrap();
+        tm.add_turn("sess2", "user", "turn 3").unwrap();
+
+        assert_eq!(tm.list_turns("sess1").unwrap().len(), 2);
+        assert_eq!(tm.list_turns("sess2").unwrap().len(), 1);
+    }
+
+    #[test]
+    fn test_delete_session() {
+        let dir = tempfile::tempdir().unwrap();
+        let db_path = dir.path().join("turns.db");
+        let turn_store = TurnStore::open(&db_path).unwrap();
+        let vector_index = VectorIndex::new(DIM, 100, usearch_index::ScalarKind::F32).unwrap();
+        let embedder = make_test_embedder();
+
+        let mut tm = TurnManager::new(turn_store, vector_index, embedder);
+
+        tm.add_turn("sess1", "user", "hello").unwrap();
+        tm.add_turn("sess1", "assistant", "world").unwrap();
+
+        tm.delete_session("sess1").unwrap();
+        assert_eq!(tm.list_turns("sess1").unwrap().len(), 0);
+    }
+
+    // ─── FactManager tests ───
+
+    #[test]
+    fn test_add_fact() {
+        let dir = tempfile::tempdir().unwrap();
+        let db_path = dir.path().join("facts.db");
+        let fact_store = FactStore::open(&db_path).unwrap();
+        let vector_index = VectorIndex::new(DIM, 100, usearch_index::ScalarKind::F32).unwrap();
+        let embedder = make_test_embedder();
+
+        let mut fm = FactManager::new(fact_store, vector_index, embedder);
+
+        let id = fm
+            .add_fact("the sky is blue", "science", None, &[], "sess1")
+            .unwrap();
+        assert!(id > 0);
+        assert_eq!(fm.count().unwrap(), 1);
+    }
+
+    #[test]
+    fn test_semantic_search_facts() {
+        let dir = tempfile::tempdir().unwrap();
+        let db_path = dir.path().join("facts.db");
+        let fact_store = FactStore::open(&db_path).unwrap();
+        let vector_index = VectorIndex::new(DIM, 100, usearch_index::ScalarKind::F32).unwrap();
+        let embedder = make_test_embedder();
+
+        let mut fm = FactManager::new(fact_store, vector_index, embedder);
+
+        fm.add_fact("rust is a programming language", "science", None, &[], "s1")
+            .unwrap();
+        fm.add_fact("the weather is sunny", "weather", None, &[], "s1")
+            .unwrap();
+
+        let results = fm.semantic_search("programming", 5).unwrap();
+        assert!(!results.is_empty(), "should find facts via semantic search");
+    }
+
+    #[test]
+    fn test_list_facts_filtered() {
+        let dir = tempfile::tempdir().unwrap();
+        let db_path = dir.path().join("facts.db");
+        let fact_store = FactStore::open(&db_path).unwrap();
+        let vector_index = VectorIndex::new(DIM, 100, usearch_index::ScalarKind::F32).unwrap();
+        let embedder = make_test_embedder();
+
+        let mut fm = FactManager::new(fact_store, vector_index, embedder);
+
+        fm.add_fact("fact about science", "science", None, &[], "s1")
+            .unwrap();
+        fm.add_fact("fact about weather", "weather", None, &[], "s1")
+            .unwrap();
+        fm.add_fact("another science fact", "science", None, &[], "s1")
+            .unwrap();
+
+        let science_facts = fm.list_facts(Some("science"), None).unwrap();
+        assert_eq!(science_facts.len(), 2);
+
+        let all_facts = fm.list_facts(None, None).unwrap();
+        assert_eq!(all_facts.len(), 3);
+    }
+}
@@ -0,0 +1,160 @@
+//! Time-based score decay for memory entries.
+//!
+//! Applies exponential decay to scores based on entry age, so that newer
+//! entries are preferred over older ones when relevance scores are similar.
+
+use crate::types::ScoredEntry;
+
+/// Apply exponential time decay to a list of scored entries.
+///
+/// For each entry:
+///   `age_hours = (now - entry.created_at) / 3600.0`
+///   `entry.score *= exp(-lambda * age_hours)`
+///
+/// After applying decay, entries are sorted by score descending (highest first).
+///
+/// # Arguments
+///
+/// * `entries` — mutable slice of scored entries to decay in-place.
+/// * `now` — current time as seconds since UNIX epoch (parameterized for testability).
+/// * `lambda` — decay rate. Default recommendation: 0.05.
+///              At lambda=0.05, a 24-hour-old entry retains ~30% of its score.
+///              Use 0.0 to disable decay entirely.
+pub fn apply_time_decay(entries: &mut [ScoredEntry], now: f64, lambda: f64) {
+    for entry in entries.iter_mut() {
+        let age_hours = (now - entry.created_at) / 3600.0;
+        // Clamp negative ages to zero (future timestamps shouldn't boost scores)
+        let age_hours = age_hours.max(0.0);
+        entry.score *= (-lambda * age_hours).exp();
+    }
+
+    // Sort by score descending
+    entries.sort_by(|a, b| {
+        b.score
+            .partial_cmp(&a.score)
+            .unwrap_or(std::cmp::Ordering::Equal)
+    });
+}
+
+/// Default decay rate (lambda).
+pub const DEFAULT_LAMBDA: f64 = 0.05;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_time_decay_recent_high_score() {
+        let now = 1000.0 * 3600.0; // arbitrary "now"
+        let mut entries = vec![
+            // Recent entry (1 hour ago)
+            ScoredEntry {
+                score: 1.0,
+                created_at: now - 3600.0,
+            },
+            // Old entry (24 hours ago)
+            ScoredEntry {
+                score: 1.0,
+                created_at: now - 24.0 * 3600.0,
+            },
+        ];
+
+        apply_time_decay(&mut entries, now, DEFAULT_LAMBDA);
+
+        // Recent entry should have higher score after decay
+        assert!(
+            entries[0].score > entries[1].score,
+            "recent entry ({}) should score higher than old entry ({})",
+            entries[0].score,
+            entries[1].score
+        );
+        // Recent entry: exp(-0.05 * 1) ≈ 0.951
+        assert!(
+            (entries[0].score - (-0.05_f64).exp()).abs() < 1e-6,
+            "recent score should be ~0.951, got {}",
+            entries[0].score
+        );
+    }
+
+    #[test]
+    fn test_time_decay_old_low_score() {
+        let now = 1000.0 * 3600.0;
+        let mut entries = vec![ScoredEntry {
+            score: 1.0,
+            created_at: now - 48.0 * 3600.0, // 48 hours old
+        }];
+
+        apply_time_decay(&mut entries, now, DEFAULT_LAMBDA);
+
+        // exp(-0.05 * 48) = exp(-2.4) ≈ 0.0907
+        let expected = (-0.05 * 48.0_f64).exp();
+        assert!(
+            (entries[0].score - expected).abs() < 1e-6,
+            "48h old entry should have score ~{expected}, got {}",
+            entries[0].score
+        );
+        assert!(
+            entries[0].score < 0.1,
+            "48h old entry should be significantly decayed"
+        );
+    }
+
+    #[test]
+    fn test_time_decay_zero_lambda() {
+        let now = 1000.0 * 3600.0;
+        let mut entries = vec![
+            ScoredEntry {
+                score: 0.8,
+                created_at: now - 100.0 * 3600.0,
+            },
+            ScoredEntry {
+                score: 0.5,
+                created_at: now - 1.0 * 3600.0,
+            },
+        ];
+
+        apply_time_decay(&mut entries, now, 0.0);
+
+        // With lambda=0, scores should not change
+        // entries are sorted descending, so 0.8 comes first
+        assert!(
+            (entries[0].score - 0.8).abs() < 1e-6,
+            "no decay with lambda=0"
+        );
+        assert!(
+            (entries[1].score - 0.5).abs() < 1e-6,
+            "no decay with lambda=0"
+        );
+    }
+
+    #[test]
+    fn test_time_decay_sorts_descending() {
+        let now = 1000.0 * 3600.0;
+        let mut entries = vec![
+            ScoredEntry {
+                score: 0.3,
+                created_at: now - 1.0 * 3600.0,
+            },
+            ScoredEntry {
+                score: 0.9,
+                created_at: now - 1.0 * 3600.0,
+            },
+            ScoredEntry {
+                score: 0.6,
+                created_at: now - 1.0 * 3600.0,
+            },
+        ];
+
+        apply_time_decay(&mut entries, now, DEFAULT_LAMBDA);
+
+        // All same age, so relative ordering is by original score
+        for w in entries.windows(2) {
+            assert!(
+                w[0].score >= w[1].score,
+                "should be sorted descending: {} < {}",
+                w[0].score,
+                w[1].score
+            );
+        }
+    }
+}
@@ -0,0 +1,45 @@
+//! Simple token counting approximation.
+//!
+//! Uses the heuristic `len / 4` (roughly 4 bytes per token for English text).
+//! This avoids a heavy tiktoken dependency while being accurate enough for
+//! budget allocation.
+
+/// Estimate the number of tokens in `text`.
+///
+/// Uses `text.len() / 4` with a minimum of 1 (empty string counts as 1 token
+/// to avoid zero-budget entries).
+pub fn count_tokens(text: &str) -> usize {
+    std::cmp::max(1, text.len() / 4)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_count_tokens() {
+        // "hello world" = 11 chars → 11/4 = 2
+        assert_eq!(count_tokens("hello world"), 2);
+        // 8 chars → 2
+        assert_eq!(count_tokens("12345678"), 2);
+        // 4 chars → 1
+        assert_eq!(count_tokens("abcd"), 1);
+        // 100 chars → 25
+        let long_text = "a".repeat(100);
+        assert_eq!(count_tokens(&long_text), 25);
+    }
+
+    #[test]
+    fn test_count_tokens_empty() {
+        // Empty string → minimum 1
+        assert_eq!(count_tokens(""), 1);
+    }
+
+    #[test]
+    fn test_count_tokens_short() {
+        // 1-3 chars → max(1, 0) = 1
+        assert_eq!(count_tokens("a"), 1);
+        assert_eq!(count_tokens("ab"), 1);
+        assert_eq!(count_tokens("abc"), 1);
+    }
+}
@@ -0,0 +1,159 @@
+//! Conversation turn management with vector search.
+//!
+//! Ties together [`TurnStore`] (SQLite metadata), [`VectorIndex`] (HNSW search),
+//! and [`Embedder`] (text embedding) into a unified API for adding and
+//! searching conversation turns.
+
+use std::sync::Arc;
+
+use tracing::debug;
+
+use embedder::Embedder;
+use meta_store::{SessionInfo, TurnRow, TurnStore};
+use usearch_index::VectorIndex;
+
+use crate::token::count_tokens;
+use crate::types::ScoredTurn;
+use crate::MemoryError;
+
+/// Manages conversation turns with embedded vector search.
+///
+/// Each turn is:
+/// 1. Embedded into a vector via the shared [`Embedder`]
+/// 2. Added to the [`VectorIndex`] for similarity search
+/// 3. Stored in [`TurnStore`] (SQLite) for metadata queries
+pub struct TurnManager {
+    turn_store: TurnStore,
+    vector_index: VectorIndex,
+    embedder: Arc<Embedder>,
+}
+
+impl TurnManager {
+    /// Create a new TurnManager.
+    ///
+    /// * `turn_store` — SQLite-backed turn/session metadata store.
+    /// * `vector_index` — HNSW vector index for semantic search over turns.
+    /// * `embedder` — shared text embedder (typically via `Arc`).
+    pub fn new(
+        turn_store: TurnStore,
+        vector_index: VectorIndex,
+        embedder: Arc<Embedder>,
+    ) -> Self {
+        Self {
+            turn_store,
+            vector_index,
+            embedder,
+        }
+    }
+
+    /// Add a conversation turn.
+    ///
+    /// 1. Gets the next available FAISS ID from the turn store.
+    /// 2. Embeds the content using the passage prefix.
+    /// 3. Adds the embedding to the vector index.
+    /// 4. Stores the turn metadata in SQLite.
+    ///
+    /// Returns the turn's row ID.
+    pub fn add_turn(
+        &mut self,
+        session_id: &str,
+        role: &str,
+        content: &str,
+    ) -> Result<i64, MemoryError> {
+        let faiss_id = self.turn_store.next_faiss_id()?;
+        let token_count = count_tokens(content) as i64;
+
+        // Embed the content
+        let embeddings = self.embedder.embed_passages(&[content])?;
+        let vec = embeddings
+            .into_iter()
+            .next()
+            .ok_or_else(|| MemoryError::Embed("empty embedding result".to_string()))?;
+
+        // Add to vector index
+        self.vector_index.add(faiss_id as u64, &vec)?;
+
+        // Store in SQLite
+        let turn_id =
+            self.turn_store
+                .add_turn(session_id, role, content, Some(faiss_id), token_count)?;
+
+        debug!(turn_id, faiss_id, session_id, role, "turn added to memory");
+        Ok(turn_id)
+    }
+
+    /// Search turns by semantic similarity.
+    ///
+    /// 1. Embeds the query text.
+    /// 2. Searches the vector index for nearest neighbors.
+    /// 3. Enriches results with turn metadata from SQLite.
+    ///
+    /// Returns scored turns sorted by similarity (highest first).
+    pub fn semantic_search(
+        &self,
+        query: &str,
+        top_k: usize,
+    ) -> Result<Vec<ScoredTurn>, MemoryError> {
+        let query_vec = self.embedder.embed_query(query)?;
+        let (ids, distances) = self.vector_index.search(&query_vec, top_k)?;
+
+        if ids.is_empty() {
+            return Ok(vec![]);
+        }
+
+        // Convert u64 keys to i64 faiss_ids for SQLite lookup
+        let faiss_ids: Vec<i64> = ids.iter().map(|&id| id as i64).collect();
+        let turns = self.turn_store.get_by_faiss_ids(&faiss_ids)?;
+
+        // Build a lookup map: faiss_id → TurnRow
+        let mut turn_map: std::collections::HashMap<i64, TurnRow> = turns
+            .into_iter()
+            .filter_map(|t| t.faiss_id.map(|fid| (fid, t)))
+            .collect();
+
+        // Combine with scores, preserving search order
+        let mut results = Vec::new();
+        for (key, dist) in ids.iter().zip(distances.iter()) {
+            let faiss_id = *key as i64;
+            if let Some(turn) = turn_map.remove(&faiss_id) {
+                // Convert distance to similarity score.
+                // For IP metric: distance = 1 - dot(a,b), so similarity = 1 - distance.
+                let score = 1.0 - dist;
+                results.push(ScoredTurn { turn, score });
+            }
+        }
+
+        Ok(results)
+    }
+
+    /// List all turns for a session (chronological order).
+    pub fn list_turns(&self, session_id: &str) -> Result<Vec<TurnRow>, MemoryError> {
+        Ok(self.turn_store.list_turns(session_id)?)
+    }
+
+    /// Get the N most recent turns for a session (chronological order).
+    pub fn get_recent_turns(
+        &self,
+        session_id: &str,
+        n: usize,
+    ) -> Result<Vec<TurnRow>, MemoryError> {
+        Ok(self.turn_store.get_recent_turns(session_id, n)?)
+    }
+
+    /// Delete a session and all its turns.
+    pub fn delete_session(&self, session_id: &str) -> Result<(), MemoryError> {
+        // Note: we don't remove vectors from the index — tombstoned entries
+        // in USearch are acceptable and will be ignored on search miss.
+        Ok(self.turn_store.delete_session(session_id)?)
+    }
+
+    /// List all sessions with their turn counts.
+    pub fn list_sessions(&self) -> Result<Vec<SessionInfo>, MemoryError> {
+        Ok(self.turn_store.list_sessions()?)
+    }
+
+    /// Count total turns across all sessions.
+    pub fn count_turns(&self) -> Result<usize, MemoryError> {
+        Ok(self.turn_store.count_turns()?)
+    }
+}
@@ -0,0 +1,37 @@
+//! Shared types for the memory crate.
+
+use meta_store::{FactRow, TurnRow};
+use serde::{Deserialize, Serialize};
+
+/// A conversation turn enriched with a similarity score.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ScoredTurn {
+    pub turn: TurnRow,
+    pub score: f32,
+}
+
+/// A knowledge fact enriched with a similarity score.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ScoredFact {
+    pub fact: FactRow,
+    pub score: f32,
+}
+
+/// An entry for the compressor pipeline.
+#[derive(Debug, Clone)]
+pub struct CompressEntry {
+    /// Role (for turns) or category (for facts).
+    pub role: String,
+    /// The text content.
+    pub content: String,
+    /// Relevance score (semantic similarity or BM25).
+    pub score: f64,
+}
+
+/// A generic scored entry with timestamp, used for time decay.
+pub struct ScoredEntry {
+    /// Mutable score that will be decayed.
+    pub score: f64,
+    /// Creation timestamp as seconds since UNIX epoch.
+    pub created_at: f64,
+}