Skip to main content

coven/
coven.rs

1//! Top-level API: open one handle and drive rows, blobs, sync, and
2//! membership through it.
3
4use std::num::NonZeroUsize;
5use std::path::{Path, PathBuf};
6use std::sync::Arc;
7
8use rusqlite::Connection;
9use tracing::{debug, warn};
10
11use crate::blob::local_files::LocalBlobError;
12use crate::blob::{BlobRef, BlobTransitionObserver};
13use crate::clock::{ClockRef, SystemClock};
14use crate::config::{Config, HomeStorage};
15use crate::custody::KeyCustody;
16use crate::database::{Database, DbError, OpenError};
17use crate::handle::CovenHandle;
18use crate::identity_custody::IdentityCustody;
19use crate::keys::StoreKeys;
20use crate::migration::{Migration, MigrationError};
21use crate::store_dir::PathTokenError;
22use crate::sync::hlc::UpdatedAtStamper;
23use crate::sync::session::SyncedTable;
24use crate::sync::sync_manager::ConfigProvider;
25use coven_core::{WriteId, WriteReceipt};
26
27pub type CovenResult<T> = Result<T, CovenError>;
28
29const LOCAL_STAGE_MARKER: &str = ".coven-stage-";
30
31#[derive(Debug, thiserror::Error)]
32pub enum CovenError {
33    #[error("database error: {0}")]
34    Database(#[from] DbError),
35    #[error("migration error: {0}")]
36    Migration(MigrationError),
37    #[error("sqlite error: {0}")]
38    Sqlite(#[from] rusqlite::Error),
39    #[error("blob error: {0}")]
40    Blob(String),
41    #[error("unsafe blob path: {0}")]
42    UnsafeBlobPath(#[from] PathTokenError),
43    #[error("malformed local path: {0}")]
44    MalformedPath(String),
45    #[error("the write SQL closure panicked")]
46    WriteClosurePanicked,
47    #[error("synced_tables must be set before opening a coven store")]
48    MissingSyncedTables,
49    #[error("migrations must be set before opening a coven store")]
50    MissingMigrations,
51    #[error("candidate resolution failed: {0}")]
52    CandidateResolution(String),
53    #[error("blob_tombstone_grace must be a positive duration")]
54    InvalidBlobTombstoneGrace,
55    #[error("browsable cloud storage cannot be used with scoped table {table:?}")]
56    BrowsableStorageWithScopedTable { table: String },
57    #[error("blob {namespace}/{id} is still referenced by a row after the write")]
58    BlobStillReferenced { namespace: String, id: String },
59    #[error("blob {namespace}/{id} is already referenced by a row")]
60    BlobAlreadyReferenced { namespace: String, id: String },
61    #[error("blob {namespace}/{id} is owned by an unpublished write")]
62    BlobOwnedByPendingWrite { namespace: String, id: String },
63    #[error("store is already open: {}", store_dir.display())]
64    AlreadyOpen { store_dir: PathBuf },
65    #[error("I/O error: {0}")]
66    Io(#[from] std::io::Error),
67}
68
69impl From<OpenError> for CovenError {
70    fn from(value: OpenError) -> Self {
71        match value {
72            OpenError::Migration(e) => CovenError::Migration(e),
73            OpenError::Db(e) => CovenError::Database(e),
74        }
75    }
76}
77
78impl From<LocalBlobError> for CovenError {
79    fn from(value: LocalBlobError) -> Self {
80        CovenError::Blob(value.to_string())
81    }
82}
83
84#[derive(Clone)]
85pub struct CovenConfig(ConfigProvider);
86
87impl CovenConfig {
88    fn current(&self) -> Config {
89        (self.0)()
90    }
91
92    fn provider(&self) -> ConfigProvider {
93        self.0.clone()
94    }
95}
96
97impl From<Config> for CovenConfig {
98    fn from(value: Config) -> Self {
99        let config = value;
100        Self(Arc::new(move || config.clone()))
101    }
102}
103
104impl<F> From<F> for CovenConfig
105where
106    F: Fn() -> Config + Send + Sync + 'static,
107{
108    fn from(value: F) -> Self {
109        Self(Arc::new(value))
110    }
111}
112
113pub struct Coven;
114
115impl Coven {
116    pub fn builder(config: impl Into<CovenConfig>) -> CovenBuilder {
117        let config = config.into();
118        let current = config.current();
119        CovenBuilder {
120            config,
121            synced_tables: None,
122            migrations: None,
123            blob_tombstone_grace: crate::blob::delete::BLOB_TOMBSTONE_GRACE,
124            max_concurrent_uploads: NonZeroUsize::MIN,
125            max_concurrent_downloads: NonZeroUsize::MIN,
126            clock: Arc::new(SystemClock),
127            key_service: StoreKeys::new(current.store_id),
128            key_custody: KeyCustody::Keyring,
129            identity_custody: IdentityCustody::Keyring,
130            cloudkit_ops: None,
131            observer: None,
132        }
133    }
134}
135
136pub struct CovenBuilder {
137    config: CovenConfig,
138    synced_tables: Option<Vec<SyncedTable>>,
139    migrations: Option<Vec<Migration>>,
140    blob_tombstone_grace: chrono::Duration,
141    max_concurrent_uploads: NonZeroUsize,
142    max_concurrent_downloads: NonZeroUsize,
143    clock: ClockRef,
144    key_service: StoreKeys,
145    key_custody: KeyCustody,
146    identity_custody: IdentityCustody,
147    cloudkit_ops: Option<Arc<dyn crate::storage::cloud::cloudkit::CloudKitOps>>,
148    observer: Option<Arc<dyn BlobTransitionObserver>>,
149}
150
151/// The single-writer store lock: an exclusive advisory lock on
152/// `<store>/.coven-lock`, held for the life of a full [`open`](CovenBuilder::open)
153/// handle (and its running sync loop). A second full open of the same store is
154/// refused with [`CovenError::AlreadyOpen`] while the lock is held — the invariant
155/// that keeps two writers from racing the same db and blob store.
156///
157/// # Read-only opens take no lock
158///
159/// [`open_read_only`](CovenBuilder::open_read_only) deliberately does **not** touch
160/// this lock. The lock is exclusive, so a shared lock on the same file would block
161/// against a writer that already holds it (and vice versa) — a reader could never
162/// coexist with the writer it exists to read alongside. But a read-only open needs
163/// no lock at all: the lock guards against a second *writer*, and a read-only handle
164/// holds a `SQLITE_OPEN_READONLY` connection that cannot write. So a read-only open
165/// skips the guard entirely. Cross-process safety comes from WAL mode (a reader sees
166/// committed rows while the writer commits more), not from this lock; the blob cache
167/// a reader may populate is per-device scratch written atomically (temp + rename), so
168/// a reader and the writer touching the same cache file never tear it. This lets one
169/// writer and any number of read-only readers coexist on one store.
170pub(crate) struct StoreOpenGuard {
171    _file: std::fs::File,
172}
173
174impl StoreOpenGuard {
175    /// Acquire the guard for a test, panicking on refusal.
176    #[cfg(test)]
177    pub(crate) fn acquire_for_test(store_dir: &crate::store_dir::StoreDir) -> std::sync::Arc<Self> {
178        std::sync::Arc::new(Self::acquire(store_dir).expect("acquire store open guard"))
179    }
180
181    pub(crate) fn acquire(store_dir: &crate::store_dir::StoreDir) -> CovenResult<Self> {
182        let db_path = store_dir.db_path();
183        let Some(dir) = db_path.parent() else {
184            return Err(CovenError::MalformedPath(format!(
185                "store database path has no parent: {}",
186                db_path.display()
187            )));
188        };
189        std::fs::create_dir_all(dir)?;
190        let file = std::fs::OpenOptions::new()
191            .read(true)
192            .write(true)
193            .create(true)
194            .truncate(false)
195            .open(dir.join(".coven-lock"))?;
196        match file.try_lock() {
197            Ok(()) => Ok(Self { _file: file }),
198            Err(std::fs::TryLockError::WouldBlock) => Err(CovenError::AlreadyOpen {
199                store_dir: dir.to_path_buf(),
200            }),
201            Err(std::fs::TryLockError::Error(error)) => Err(CovenError::Io(error)),
202        }
203    }
204}
205
206impl CovenBuilder {
207    pub fn synced_tables(mut self, tables: Vec<SyncedTable>) -> Self {
208        self.synced_tables = Some(tables);
209        self
210    }
211
212    /// How long a deleted blob is kept after its tombstone is written before the
213    /// tombstone GC erases it: the cross-device convergence window. Defaults to
214    /// [`crate::blob::delete::BLOB_TOMBSTONE_GRACE`]. Must be positive — a
215    /// zero-or-negative grace is refused at [`open`](Self::open), since it would
216    /// let the GC erase a blob a lagging peer still references.
217    pub fn blob_tombstone_grace(mut self, grace: chrono::Duration) -> Self {
218        self.blob_tombstone_grace = grace;
219        self
220    }
221
222    /// How many blob uploads the sync cycle's upload drain runs at once. Defaults
223    /// to one (one at a time). A [`NonZeroUsize`] so a zero — which would leave the
224    /// drain admitting nothing and never completing — cannot be set.
225    pub fn max_concurrent_uploads(mut self, n: NonZeroUsize) -> Self {
226        self.max_concurrent_uploads = n;
227        self
228    }
229
230    /// How many blob downloads a [`pin`](CovenHandle::pin) call fetches at once.
231    /// Defaults to one (one at a time). A [`NonZeroUsize`] so a zero — which would
232    /// leave the pin loop admitting nothing and never completing — cannot be set.
233    pub fn max_concurrent_downloads(mut self, n: NonZeroUsize) -> Self {
234        self.max_concurrent_downloads = n;
235        self
236    }
237
238    /// The host's synced-schema migration ladder, applied over `PRAGMA
239    /// user_version` at open. The top version is the wire `schema_version` every
240    /// changeset is stamped with.
241    pub fn migrations(mut self, migrations: Vec<Migration>) -> Self {
242        self.migrations = Some(migrations);
243        self
244    }
245
246    pub fn clock(mut self, clock: ClockRef) -> Self {
247        self.clock = clock;
248        self
249    }
250
251    pub fn key_service(mut self, key_service: StoreKeys) -> Self {
252        self.key_service = key_service;
253        self
254    }
255
256    /// How the store's master key is protected: the OS keyring (the
257    /// default), a passphrase-wrapped file, an in-memory session value, or a
258    /// host's own [`MasterKeyCustody`](crate::MasterKeyCustody) implementation.
259    /// coven builds every cipher internally from what this custody supplies —
260    /// the host never touches a crypto type.
261    pub fn key_custody(mut self, custody: KeyCustody) -> Self {
262        self.key_custody = custody;
263        self
264    }
265
266    /// How this store's device-signing identity is protected: the OS keyring
267    /// (the default), a passphrase-wrapped file, an in-memory session value,
268    /// or a host's own
269    /// [`DeviceIdentityCustody`](crate::DeviceIdentityCustody) implementation.
270    /// Selected next to [`key_custody`](Self::key_custody) — the identity is
271    /// scoped to this store, established as part of creating, joining, or
272    /// restoring it (see [`CovenHandle::initialize_identity`]).
273    pub fn identity_custody(mut self, custody: IdentityCustody) -> Self {
274        self.identity_custody = custody;
275        self
276    }
277
278    pub fn cloudkit_ops(
279        mut self,
280        ops: Arc<dyn crate::storage::cloud::cloudkit::CloudKitOps>,
281    ) -> Self {
282        self.cloudkit_ops = Some(ops);
283        self
284    }
285
286    pub fn apply_cloudkit_ops(
287        mut self,
288        ops: Option<Arc<dyn crate::storage::cloud::cloudkit::CloudKitOps>>,
289    ) -> Self {
290        self.cloudkit_ops = ops;
291        self
292    }
293
294    pub fn observer(mut self, observer: Arc<dyn BlobTransitionObserver>) -> Self {
295        self.observer = Some(observer);
296        self
297    }
298
299    /// Open the store, returning the [`CovenHandle`].
300    ///
301    /// Opening performs no keyring interaction: it opens the database, runs
302    /// migrations, and resolves the master-key custody selection to a value
303    /// (constructing the trait object, never calling its `unlock`) — a locked
304    /// agent (no OS keyring session, no established master key or device
305    /// identity) can `open()` a store and use it fully for rows and Local
306    /// blobs. The first read of any key happens lazily, at the specific call
307    /// that needs it ([`CovenHandle::connect_sync`],
308    /// [`CovenHandle::master_key_fingerprint`], and similar).
309    pub fn open(self) -> CovenResult<CovenHandle> {
310        let config = self.config.current();
311        let tables = self.synced_tables.ok_or(CovenError::MissingSyncedTables)?;
312        let migrations = self.migrations.ok_or(CovenError::MissingMigrations)?;
313        validate_storage_scope(&config, &tables)?;
314        if self.blob_tombstone_grace <= chrono::Duration::zero() {
315            return Err(CovenError::InvalidBlobTombstoneGrace);
316        }
317        let db_path = config.store_dir.db_path();
318        let provider = self.config.provider();
319        let store_dir = config.store_dir.clone();
320        let transfer_limits = crate::blob::TransferLimits {
321            uploads: self.max_concurrent_uploads,
322            downloads: self.max_concurrent_downloads,
323        };
324        let open_guard = Arc::new(StoreOpenGuard::acquire(&store_dir)?);
325        remove_orphaned_local_blob_temps(&store_dir, std::time::SystemTime::now())?;
326        let (db, stamper) = Database::open(
327            &db_path,
328            tables.clone(),
329            self.blob_tombstone_grace,
330            transfer_limits,
331            config.device_id.clone(),
332            &migrations,
333        )?;
334        // A second, read-only connection on the same WAL database, opened after the
335        // writer completed its migrations so the schema exists. It backs
336        // [`CovenHandle::sql_read`]: a pure read runs here on its own connection
337        // thread, concurrent with the writer rather than queued behind it, and
338        // attaches no changeset session. Opening it fails `open()` loudly — there is
339        // no full handle without its read path.
340        let read_db = Database::open_read_only(
341            &db_path,
342            tables,
343            self.blob_tombstone_grace,
344            transfer_limits,
345            config.device_id.clone(),
346            &migrations,
347        )?;
348        let key_custody = self.key_custody.resolve(&config.store_id, &store_dir);
349        let identity_custody = self.identity_custody.resolve(&config.store_id, &store_dir);
350        Ok(CovenHandle::new(
351            db,
352            read_db,
353            stamper,
354            store_dir,
355            provider,
356            self.key_service,
357            key_custody,
358            identity_custody,
359            self.clock,
360            self.cloudkit_ops,
361            self.observer,
362            open_guard,
363        ))
364    }
365
366    /// Open the store read-only for a same-store secondary reader: a separate
367    /// process (or a second handle) that must read rows and blobs while another
368    /// handle holds the full [`open`](Self::open). Returns a [`CovenReadHandle`],
369    /// whose surface is reads only — SQL queries and blob reads — with no write,
370    /// sync, migration, or stamp API by construction.
371    ///
372    /// Unlike [`open`](Self::open) this takes no store lock (see
373    /// [`StoreOpenGuard`]): it succeeds while a writer holds the exclusive lock,
374    /// and any number of read-only opens coexist. It opens a `SQLITE_OPEN_READONLY`
375    /// connection against the schema on disk, running no migration ladder — but it
376    /// refuses a db a newer binary migrated past what this binary supports
377    /// ([`CovenError::Migration`] with [`MigrationError::SchemaTooNew`]), the same
378    /// policy the writer enforces. It runs no orphan-temp cleanup either (that is a
379    /// write the lock-holding writer owns).
380    ///
381    /// Cross-process reads are safe because the writer opens the db in WAL mode; a
382    /// blob read that misses locally fetches from the cloud into the per-device
383    /// cache (files written atomically), which is device scratch and touches no
384    /// synced state.
385    pub fn open_read_only(self) -> CovenResult<crate::read_handle::CovenReadHandle> {
386        let config = self.config.current();
387        let tables = self.synced_tables.ok_or(CovenError::MissingSyncedTables)?;
388        let migrations = self.migrations.ok_or(CovenError::MissingMigrations)?;
389        validate_storage_scope(&config, &tables)?;
390        let db_path = config.store_dir.db_path();
391        let provider = self.config.provider();
392        let store_dir = config.store_dir.clone();
393        // No StoreOpenGuard and no orphan-temp cleanup: both are writer concerns
394        // (see StoreOpenGuard). A reader must not take the exclusive lock the
395        // writer holds, nor write the filesystem the writer owns.
396        let db = Database::open_read_only(
397            &db_path,
398            tables,
399            self.blob_tombstone_grace,
400            crate::blob::TransferLimits {
401                uploads: self.max_concurrent_uploads,
402                downloads: self.max_concurrent_downloads,
403            },
404            config.device_id.clone(),
405            &migrations,
406        )?;
407        let key_custody = self.key_custody.resolve(&config.store_id, &store_dir);
408        let identity_custody = self.identity_custody.resolve(&config.store_id, &store_dir);
409        Ok(crate::read_handle::CovenReadHandle::new(
410            db,
411            store_dir,
412            provider,
413            self.key_service,
414            key_custody,
415            identity_custody,
416            self.clock,
417            self.cloudkit_ops,
418        ))
419    }
420}
421
422fn validate_storage_scope(config: &Config, tables: &[SyncedTable]) -> CovenResult<()> {
423    if config.cloud_home.storage == HomeStorage::Browsable {
424        if let Some(table) = tables
425            .iter()
426            .find(|table| table.audience_column().is_some())
427        {
428            return Err(CovenError::BrowsableStorageWithScopedTable {
429                table: table.name().to_string(),
430            });
431        }
432    }
433    Ok(())
434}
435
436/// Host SQL inside one journaled write transaction.
437///
438/// The context exposes SQL operations, but never the underlying SQLite
439/// connection or transaction. In particular, a host cannot remove coven's SQL
440/// authorizer and address its attached gate baseline:
441///
442/// ```compile_fail
443/// # use coven::{CovenError, CovenHandle};
444/// # async fn cannot_remove_guard(handle: &CovenHandle) -> Result<(), CovenError> {
445/// handle.sql(|sql| {
446///     sql.tx().authorizer(
447///         None::<fn(coven::rusqlite::hooks::AuthContext<'_>)
448///             -> coven::rusqlite::hooks::Authorization>,
449///     )?;
450///     Ok(())
451/// }).await?;
452/// # Ok(())
453/// # }
454/// ```
455pub struct SqlContext<'ctx, 'conn> {
456    tx: &'ctx rusqlite::Transaction<'conn>,
457    stamper: UpdatedAtStamper,
458}
459
460impl<'ctx, 'conn> SqlContext<'ctx, 'conn> {
461    pub(crate) fn new(tx: &'ctx rusqlite::Transaction<'conn>, stamper: UpdatedAtStamper) -> Self {
462        Self { tx, stamper }
463    }
464
465    /// Execute one SQL statement with bound parameters.
466    pub fn execute<P>(&self, sql: &str, params: P) -> rusqlite::Result<usize>
467    where
468        P: rusqlite::Params,
469    {
470        self.tx.execute(sql, params)
471    }
472
473    /// Execute one or more semicolon-separated SQL statements.
474    pub fn execute_batch(&self, sql: &str) -> rusqlite::Result<()> {
475        self.tx.execute_batch(sql)
476    }
477
478    /// Query exactly one row and map it to a host value.
479    pub fn query_row<T, P, F>(&self, sql: &str, params: P, map: F) -> rusqlite::Result<T>
480    where
481        P: rusqlite::Params,
482        F: FnOnce(&rusqlite::Row<'_>) -> rusqlite::Result<T>,
483    {
484        self.tx.query_row(sql, params, map)
485    }
486
487    /// Query any number of rows and collect their mapped host values.
488    pub fn query<T, P, F>(&self, sql: &str, params: P, map: F) -> rusqlite::Result<Vec<T>>
489    where
490        P: rusqlite::Params,
491        F: FnMut(&rusqlite::Row<'_>) -> rusqlite::Result<T>,
492    {
493        let mut statement = self.tx.prepare(sql)?;
494        let values = statement.query_map(params, map)?.collect();
495        values
496    }
497
498    pub fn stamp(&self) -> String {
499        self.stamper.stamp()
500    }
501}
502
503type WriteSql<R> =
504    Box<dyn for<'ctx, 'conn> FnOnce(SqlContext<'ctx, 'conn>) -> CovenResult<R> + Send>;
505
506pub struct WriteBatch {
507    new_blobs: Vec<NewBlob>,
508    deleted_blobs: Vec<BlobRef>,
509}
510
511impl WriteBatch {
512    fn new() -> Self {
513        Self {
514            new_blobs: Vec::new(),
515            deleted_blobs: Vec::new(),
516        }
517    }
518
519    pub fn put_blob(
520        &mut self,
521        namespace: impl Into<String>,
522        id: impl Into<String>,
523        bytes: impl Into<Vec<u8>>,
524    ) {
525        self.new_blobs.push(NewBlob {
526            namespace: namespace.into(),
527            id: id.into(),
528            bytes: bytes.into(),
529        });
530    }
531
532    pub fn delete_blob(&mut self, blob: BlobRef) {
533        self.deleted_blobs.push(blob);
534    }
535}
536
537struct NewBlob {
538    namespace: String,
539    id: String,
540    bytes: Vec<u8>,
541}
542
543#[derive(Clone)]
544pub(crate) struct StagedBlob {
545    pub namespace: String,
546    pub id: String,
547    pub staged: PathBuf,
548    pub final_path: PathBuf,
549}
550
551impl CovenHandle {
552    pub async fn sql<F, R>(&self, f: F) -> CovenResult<WriteReceipt<R>>
553    where
554        F: for<'ctx, 'conn> FnOnce(SqlContext<'ctx, 'conn>) -> CovenResult<R> + Send + 'static,
555        R: Send + 'static,
556    {
557        let stamper = self.stamper();
558        let tables = self.db().synced_tables().to_vec();
559        let gates = self.db().gates();
560        let blob_decls = self.db().blob_decls();
561        let routing_encryption = gates
562            .has_scoped_graph()
563            .then(|| self.routing_encryption())
564            .transpose()?;
565        let blob_staging = crate::sync::store::HostWriteBlobStaging::new(
566            tokio::runtime::Handle::current(),
567            self.blob_storage().await.map_err(|error| error.to_string()),
568            self.store_dir(),
569        );
570        let write_id = self.db().new_write_id();
571        let outcome = self
572            .db()
573            .call(move |conn| {
574                Ok(
575                    crate::sync::store::StoreDatabase::run_store_write_transaction_on(
576                        conn,
577                        &tables,
578                        &gates,
579                        &blob_decls,
580                        routing_encryption.as_ref(),
581                        Some(&blob_staging),
582                        write_id,
583                        |tx| f(SqlContext::new(tx, stamper)),
584                    ),
585                )
586            })
587            .await
588            .map_err(CovenError::from)?;
589        outcome
590    }
591
592    /// Run a pure read against this handle's read-only companion connection and
593    /// await the result.
594    ///
595    /// Unlike [`sql`](Self::sql), this attaches no changeset session and opens no
596    /// journaled transaction: a read pays nothing for capture it would produce
597    /// nothing for, and runs on a separate `SQLITE_OPEN_READONLY` connection thread
598    /// (concurrent with the writer, not queued behind it). The connection is
599    /// read-only, so an `INSERT`/`UPDATE`/`DELETE`/DDL in the closure is refused by
600    /// SQLite — a read cannot silently escape the sync journal because it cannot
601    /// write at all. The closure receives the `&Connection` directly, so a host
602    /// closure written against
603    /// [`CovenReadHandle::sql_read`](crate::CovenReadHandle::sql_read) runs
604    /// unchanged here.
605    ///
606    /// Read-your-writes holds for committed writes: a `sql_read` issued after an
607    /// awaited [`sql`](Self::sql) or [`write`](Self::write) sees that data (the WAL
608    /// reader reads the last committed state). It may not see another task's write
609    /// that has not yet committed.
610    pub async fn sql_read<F, R>(&self, f: F) -> CovenResult<R>
611    where
612        F: FnOnce(&rusqlite::Connection) -> CovenResult<R> + Send + 'static,
613        R: Send + 'static,
614    {
615        let outcome = self
616            .read_db()
617            .call(move |conn| Ok(f(conn)))
618            .await
619            .map_err(CovenError::from)?;
620        outcome
621    }
622
623    pub async fn write<F, S, R>(&self, f: F, sql: S) -> CovenResult<WriteReceipt<R>>
624    where
625        F: FnOnce(&mut WriteBatch) -> CovenResult<()> + Send + 'static,
626        S: for<'ctx, 'conn> FnOnce(SqlContext<'ctx, 'conn>) -> CovenResult<R> + Send + 'static,
627        R: Send + 'static,
628    {
629        let mut batch = WriteBatch::new();
630        f(&mut batch)?;
631        let sql: WriteSql<R> = Box::new(sql);
632        let staged = self.stage_blobs(batch.new_blobs).await?;
633        let staged_paths = staged
634            .iter()
635            .map(|blob| blob.staged.clone())
636            .collect::<Vec<_>>();
637        let tables = self.db().synced_tables().to_vec();
638        let db = self.db().clone();
639        let stamper = self.stamper();
640        let gates = self.db().gates();
641        let blob_decls = self.db().blob_decls();
642        let routing_encryption = gates
643            .has_scoped_graph()
644            .then(|| self.routing_encryption())
645            .transpose()?;
646        let blob_staging = crate::sync::store::HostWriteBlobStaging::new(
647            tokio::runtime::Handle::current(),
648            self.blob_storage().await.map_err(|error| error.to_string()),
649            self.store_dir(),
650        );
651        let write_id = self.db().new_write_id();
652        let deleted = batch.deleted_blobs;
653        let store_dir = self.store_dir();
654        let outcome = match db
655            .call(move |conn| {
656                Ok(run_write_batch_on_connection(
657                    conn,
658                    stamper,
659                    store_dir,
660                    staged,
661                    deleted,
662                    tables,
663                    gates,
664                    blob_decls,
665                    routing_encryption,
666                    blob_staging,
667                    write_id,
668                    sql,
669                ))
670            })
671            .await
672        {
673            Ok(outcome) => outcome,
674            Err(error) => {
675                remove_staged_paths(&staged_paths).await;
676                return Err(CovenError::from(error));
677            }
678        };
679        match outcome {
680            Ok(receipt) => {
681                if let Err(error) =
682                    crate::blob::local_cleanup::drain(self.db(), &self.store_dir()).await
683                {
684                    warn!(
685                        error = %error,
686                        "failed to drain local blob cleanup intents after write commit"
687                    );
688                }
689                Ok(receipt)
690            }
691            Err(error) => {
692                remove_staged_paths(&staged_paths).await;
693                Err(error)
694            }
695        }
696    }
697
698    async fn stage_blobs(&self, blobs: Vec<NewBlob>) -> CovenResult<Vec<StagedBlob>> {
699        let mut staged = Vec::new();
700        for blob in blobs {
701            let final_path = self
702                .store_dir()
703                .local_blob_path(&blob.namespace, &blob.id)?;
704            let staged_path = local_stage_temp_path(&final_path)?;
705            if let Err(e) = crate::local_blob::write_atomic(&staged_path, &blob.bytes).await {
706                remove_staged_files(&staged).await;
707                return Err(CovenError::Blob(e));
708            }
709            staged.push(StagedBlob {
710                namespace: blob.namespace,
711                id: blob.id,
712                staged: staged_path,
713                final_path,
714            });
715        }
716        Ok(staged)
717    }
718}
719
720fn local_stage_temp_path(final_path: &Path) -> CovenResult<PathBuf> {
721    let file_name = final_path
722        .file_name()
723        .and_then(|name| name.to_str())
724        .ok_or_else(|| {
725            CovenError::MalformedPath(format!(
726                "local blob path has no file name: {}",
727                final_path.display()
728            ))
729        })?;
730    Ok(final_path.with_file_name(format!(
731        "{file_name}{LOCAL_STAGE_MARKER}{}",
732        uuid::Uuid::new_v4()
733    )))
734}
735
736/// Remove temp debris a crashed prior process left in the blob folders, before the
737/// handle serves any read. Two temp conventions:
738///
739/// - Every atomic write (`write_atomic`/`copy_atomic`) in any blob folder — the local
740///   store, `cache/`, `pinned/` — leaves a `.tmp.<uuid>` sibling until its
741///   fsync-then-rename commits it. A crash orphans that temp. Only those older than
742///   this open (`process_start`) are crash leftovers; a temp a concurrent write is
743///   mid-producing is newer and is left in place so the sweep can never delete a live
744///   write's rename target.
745/// - `storage/local/` additionally holds write staging (`.coven-stage-<uuid>`). Open
746///   holds the exclusive store lock, so every staging temp here is a crash leftover
747///   — all are removed, no age check.
748fn remove_orphaned_local_blob_temps(
749    store_dir: &crate::store_dir::StoreDir,
750    process_start: std::time::SystemTime,
751) -> CovenResult<()> {
752    let storage = store_dir.storage_dir();
753    remove_orphaned_temps_in_dir(&storage.join("local"), &is_local_stage_temp, None)?;
754    for folder in ["local", "cache", "pinned"] {
755        remove_orphaned_temps_in_dir(
756            &storage.join(folder),
757            &crate::local_blob::is_temp_blob_path,
758            Some(process_start),
759        )?;
760    }
761    Ok(())
762}
763
764/// Recursively remove temp files under `dir` that `is_temp` recognizes. With
765/// `older_than = Some(cutoff)` a temp is removed only when its modification time is
766/// before `cutoff` — a temp at or after it is a live write, left untouched; `None`
767/// removes every recognized temp regardless of age.
768fn remove_orphaned_temps_in_dir(
769    dir: &Path,
770    is_temp: &dyn Fn(&Path) -> bool,
771    older_than: Option<std::time::SystemTime>,
772) -> CovenResult<()> {
773    let entries = match std::fs::read_dir(dir) {
774        Ok(entries) => entries,
775        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
776            debug!(
777                path = %dir.display(),
778                "blob directory absent during orphaned temp cleanup"
779            );
780            return Ok(());
781        }
782        Err(error) => return Err(CovenError::Io(error)),
783    };
784    for entry in entries {
785        let entry = entry?;
786        let path = entry.path();
787        let file_type = entry.file_type()?;
788        if file_type.is_dir() {
789            remove_orphaned_temps_in_dir(&path, is_temp, older_than)?;
790        } else if file_type.is_file() && is_temp(&path) {
791            if let Some(cutoff) = older_than {
792                let modified = entry.metadata()?.modified()?;
793                if modified >= cutoff {
794                    debug!(
795                        path = %path.display(),
796                        "leaving fresh blob temp created at or after process start"
797                    );
798                    continue;
799                }
800            }
801            remove_file_if_present(&path)?;
802        } else if file_type.is_file() && path.file_name().and_then(|name| name.to_str()).is_none() {
803            debug!(
804                path = %path.display(),
805                "skipping blob path with non-utf8 file name during orphaned temp cleanup"
806            );
807        }
808    }
809    Ok(())
810}
811
812fn is_local_stage_temp(path: &Path) -> bool {
813    path.file_name()
814        .and_then(|name| name.to_str())
815        .is_some_and(|name| name.contains(LOCAL_STAGE_MARKER))
816}
817
818async fn remove_staged_files(staged: &[StagedBlob]) {
819    let paths = staged
820        .iter()
821        .map(|blob| blob.staged.clone())
822        .collect::<Vec<_>>();
823    remove_staged_paths(&paths).await;
824}
825
826async fn remove_staged_paths(paths: &[PathBuf]) {
827    for path in paths {
828        remove_staged_path(path).await;
829    }
830}
831
832async fn remove_staged_path(path: &Path) {
833    if let Err(error) = crate::local_blob::remove_file(path).await {
834        warn!(
835            path = %path.display(),
836            error = %error,
837            "failed to remove staged local blob"
838        );
839    }
840}
841
842fn remove_file_if_present(path: &Path) -> CovenResult<()> {
843    match std::fs::remove_file(path) {
844        Ok(()) => Ok(()),
845        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
846            debug!(
847                path = %path.display(),
848                "file already absent during local blob cleanup"
849            );
850            Ok(())
851        }
852        Err(error) => Err(CovenError::Io(error)),
853    }
854}
855
856fn sync_parent_dir(path: &Path) -> CovenResult<()> {
857    let parent = path.parent().ok_or_else(|| {
858        CovenError::MalformedPath(format!("path has no parent directory: {}", path.display()))
859    })?;
860    std::fs::File::open(parent)?.sync_all()?;
861    Ok(())
862}
863
864fn run_write_batch_on_connection<R>(
865    conn: &Connection,
866    stamper: UpdatedAtStamper,
867    store_dir: crate::store_dir::StoreDir,
868    staged: Vec<StagedBlob>,
869    deleted: Vec<BlobRef>,
870    tables: Vec<SyncedTable>,
871    gates: Arc<crate::sync::gate::Gates>,
872    decls: Arc<crate::blob::decl::BlobDecls>,
873    routing_encryption: Option<crate::encryption::EncryptionService>,
874    blob_staging: crate::sync::store::HostWriteBlobStaging,
875    write_id: WriteId,
876    sql: WriteSql<R>,
877) -> CovenResult<WriteReceipt<R>> {
878    let mut moved = Vec::new();
879    let result = crate::sync::store::StoreDatabase::run_store_write_transaction_on(
880        conn,
881        &tables,
882        &gates,
883        &decls,
884        routing_encryption.as_ref(),
885        Some(&blob_staging),
886        write_id,
887        |tx| -> CovenResult<R> {
888            let cleanup_intents = deleted
889                .iter()
890                .map(|blob| {
891                    decls
892                        .row_for_blob_in_namespace(tx, &blob.namespace, &blob.id)
893                        .map_err(|error| CovenError::Blob(error.to_string()))
894                        .map(|row| match row {
895                            Some((table, row_id)) => {
896                                crate::blob::local_cleanup::LocalBlobCleanupIntent::for_row(
897                                    &blob.namespace,
898                                    &blob.id,
899                                    table,
900                                    row_id,
901                                )
902                            }
903                            None => crate::blob::local_cleanup::LocalBlobCleanupIntent::local(
904                                &blob.namespace,
905                                &blob.id,
906                            ),
907                        })
908                })
909                .collect::<CovenResult<Vec<_>>>()?;
910            for blob in &staged {
911                match decls.row_for_blob_in_namespace(tx, &blob.namespace, &blob.id) {
912                    Ok(Some(_)) => {
913                        return Err(CovenError::BlobAlreadyReferenced {
914                            namespace: blob.namespace.clone(),
915                            id: blob.id.clone(),
916                        });
917                    }
918                    Ok(None) => {}
919                    Err(e) => return Err(CovenError::Blob(e.to_string())),
920                }
921                let leased = tx
922                    .query_row(
923                        "SELECT EXISTS(\
924                             SELECT 1 FROM store_write_blob_leases \
925                             WHERE namespace = ?1 AND blob_id = ?2\
926                         )",
927                        (&blob.namespace, &blob.id),
928                        |row| row.get::<_, bool>(0),
929                    )
930                    .map_err(CovenError::from)?;
931                if leased {
932                    return Err(CovenError::BlobOwnedByPendingWrite {
933                        namespace: blob.namespace.clone(),
934                        id: blob.id.clone(),
935                    });
936                }
937                if let Some(parent) = blob.final_path.parent() {
938                    std::fs::create_dir_all(parent).map_err(|e| {
939                        CovenError::Blob(format!(
940                            "create local blob parent {}: {e}",
941                            parent.display()
942                        ))
943                    })?;
944                }
945                std::fs::rename(&blob.staged, &blob.final_path).map_err(|e| {
946                    CovenError::Blob(format!(
947                        "install staged blob {} -> {}: {e}",
948                        blob.staged.display(),
949                        blob.final_path.display()
950                    ))
951                })?;
952                moved.push(blob.clone());
953                sync_parent_dir(&blob.final_path).map_err(|e| {
954                    CovenError::Blob(format!(
955                        "sync local blob parent after installing {}: {e}",
956                        blob.final_path.display()
957                    ))
958                })?;
959            }
960
961            match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
962                sql(SqlContext::new(tx, stamper))
963            })) {
964                Ok(Ok(value)) => {
965                    for (blob, intent) in deleted.iter().zip(&cleanup_intents) {
966                        let _ = store_dir.local_blob_path(&blob.namespace, &blob.id)?;
967                        if crate::blob::local_cleanup::logical_blob_is_referenced_on(
968                            tx,
969                            &decls,
970                            &blob.namespace,
971                            &blob.id,
972                        )? {
973                            return Err(CovenError::BlobStillReferenced {
974                                namespace: blob.namespace.clone(),
975                                id: blob.id.clone(),
976                            });
977                        }
978                        crate::blob::local_cleanup::record_obsolete_copy_intents_on(
979                            tx, &decls, intent,
980                        )?;
981                    }
982                    Ok(value)
983                }
984                Ok(Err(error)) => Err(error),
985                Err(_) => Err(CovenError::WriteClosurePanicked),
986            }
987        },
988    );
989    match result {
990        Ok(value) => Ok(value),
991        Err(error) => rollback_write_batch(error, moved),
992    }
993}
994
995fn rollback_write_batch<R>(error: CovenError, moved: Vec<StagedBlob>) -> CovenResult<R> {
996    for blob in moved.iter().rev() {
997        if let Err(e) = std::fs::remove_file(&blob.final_path) {
998            warn!(
999                namespace = %blob.namespace,
1000                blob_id = %blob.id,
1001                path = %blob.final_path.display(),
1002                error = %e,
1003                "failed to remove installed local blob after write rollback"
1004            );
1005        }
1006    }
1007    Err(error)
1008}
1009
1010#[cfg(test)]
1011mod tests {
1012    use super::*;
1013
1014    use crate::blob::{BlobScope, CacheFill, Provenance};
1015    use crate::config::Config;
1016    use crate::keys::test_keyring;
1017    use crate::storage::cloud::test_utils::InMemoryCloudHome;
1018    use crate::storage::cloud::{
1019        BlobBody, BoxPartSink, CloudAccessOutcome, CloudAccessState, CloudHome, CloudHomeError,
1020        ExactSlotStorage, ObjectSlot, UploadProgress,
1021    };
1022    use crate::store_dir::StoreDir;
1023    use crate::sync::cloud_storage::CloudCipher;
1024    use crate::sync::session::BlobDecl;
1025    use crate::sync::test_helpers::{pull_into, query_text, row_exists, TestStore};
1026    use async_trait::async_trait;
1027    use rusqlite::params;
1028    use std::sync::atomic::{AtomicBool, Ordering};
1029    use std::time::Duration;
1030    use tokio::sync::mpsc;
1031    use tokio::sync::Notify;
1032
1033    fn config(dir: StoreDir) -> Config {
1034        Config::with_defaults(
1035            "lib-test".to_string(),
1036            "device-test".to_string(),
1037            dir,
1038            "Test".to_string(),
1039        )
1040    }
1041
1042    fn media_files_decl() -> BlobDecl {
1043        BlobDecl::new(
1044            "media-files",
1045            Provenance::HostProvided,
1046            CacheFill::CacheLazy,
1047        )
1048        .with_id_column("blob_id")
1049    }
1050
1051    fn files_table() -> SyncedTable {
1052        SyncedTable::new("files", coven_core::RowIdentity::SharedKey)
1053            .carries_blob(media_files_decl())
1054    }
1055
1056    fn remote_root_files_table() -> SyncedTable {
1057        SyncedTable::new("files", coven_core::RowIdentity::SharedKey)
1058            .remote_root()
1059            .carries_blob(media_files_decl())
1060    }
1061
1062    fn scoped_files_table() -> SyncedTable {
1063        SyncedTable::new("files", coven_core::RowIdentity::SharedKey)
1064            .scoped_by("audience")
1065            .carries_blob(media_files_decl())
1066    }
1067
1068    fn files_migration() -> Migration {
1069        Migration::sql(
1070            1,
1071            "test-schema",
1072            "CREATE TABLE files (
1073                id TEXT PRIMARY KEY,
1074                blob_id TEXT,
1075                size INTEGER NOT NULL,
1076                hash TEXT,
1077                _updated_at TEXT NOT NULL
1078            ) STRICT;",
1079        )
1080    }
1081
1082    fn scoped_files_migration() -> Migration {
1083        Migration::sql(
1084            1,
1085            "scoped-files",
1086            "CREATE TABLE files (
1087                id TEXT PRIMARY KEY,
1088                blob_id TEXT,
1089                size INTEGER NOT NULL,
1090                hash TEXT,
1091                audience TEXT,
1092                _updated_at TEXT NOT NULL
1093            ) STRICT;",
1094        )
1095    }
1096
1097    fn gated_roots_table() -> SyncedTable {
1098        SyncedTable::new("roots", coven_core::RowIdentity::SharedKey).gated_by("shared")
1099    }
1100
1101    fn gated_roots_migration() -> Migration {
1102        Migration::sql(
1103            1,
1104            "gated-roots",
1105            "CREATE TABLE roots (
1106                id TEXT PRIMARY KEY,
1107                title TEXT NOT NULL,
1108                shared INTEGER NOT NULL,
1109                _updated_at TEXT NOT NULL
1110            ) STRICT;",
1111        )
1112    }
1113
1114    fn open_gated_roots_handle() -> (tempfile::TempDir, CovenHandle) {
1115        let tmp = tempfile::tempdir().expect("temp dir");
1116        let handle = Coven::builder(config(StoreDir::new(tmp.path())))
1117            .synced_tables(vec![gated_roots_table()])
1118            .migrations(vec![gated_roots_migration()])
1119            .open()
1120            .expect("open gated handle");
1121        (tmp, handle)
1122    }
1123
1124    fn open_gated_roots_at(dir: StoreDir) -> CovenResult<CovenHandle> {
1125        Coven::builder(config(dir))
1126            .synced_tables(vec![gated_roots_table()])
1127            .migrations(vec![gated_roots_migration()])
1128            .open()
1129    }
1130
1131    fn precreate_database(dir: &StoreDir, sql: &str) {
1132        let conn = rusqlite::Connection::open(dir.db_path()).expect("precreate store database");
1133        conn.execute_batch(sql)
1134            .expect("seed pre-existing database state");
1135    }
1136
1137    #[test]
1138    fn precreated_empty_sqlite_file_initializes_coven_metadata() {
1139        let tmp = tempfile::tempdir().expect("temp dir");
1140        let dir = StoreDir::new(tmp.path());
1141        precreate_database(&dir, "");
1142
1143        open_gated_roots_at(dir).expect("initialize Coven in an empty SQLite database");
1144    }
1145
1146    #[test]
1147    fn existing_host_tables_without_a_coven_marker_initialize_coven_metadata() {
1148        let tmp = tempfile::tempdir().expect("temp dir");
1149        let dir = StoreDir::new(tmp.path());
1150        precreate_database(
1151            &dir,
1152            "CREATE TABLE roots (
1153                 id TEXT PRIMARY KEY,
1154                 title TEXT NOT NULL,
1155                 shared INTEGER NOT NULL,
1156                 _updated_at TEXT NOT NULL
1157             ) STRICT;
1158             PRAGMA user_version = 1;",
1159        );
1160
1161        open_gated_roots_at(dir).expect("initialize Coven beside an existing host schema");
1162    }
1163
1164    #[test]
1165    fn interrupted_coven_schema_without_a_marker_is_rejected() {
1166        let tmp = tempfile::tempdir().expect("temp dir");
1167        let dir = StoreDir::new(tmp.path());
1168        precreate_database(
1169            &dir,
1170            "CREATE TABLE protocol_state (
1171                 key TEXT PRIMARY KEY,
1172                 value TEXT NOT NULL
1173             ) STRICT;",
1174        );
1175
1176        let error = match open_gated_roots_at(dir) {
1177            Ok(_) => panic!("partial Coven schema has no valid initialization commit"),
1178            Err(error) => error,
1179        };
1180
1181        assert!(matches!(
1182            error,
1183            CovenError::Database(DbError::Message(reason))
1184                if reason.contains("without the required initialization marker")
1185        ));
1186    }
1187
1188    #[tokio::test]
1189    async fn host_sql_cannot_discover_or_mutate_the_gate_baseline() {
1190        let (_tmp, handle) = open_gated_roots_handle();
1191
1192        let discovery = handle
1193            .sql(|sql| {
1194                sql.query("PRAGMA database_list", [], |row| row.get::<_, String>(1))
1195                    .map_err(CovenError::from)
1196            })
1197            .await;
1198        assert!(
1199            discovery.is_err(),
1200            "host SQL must not enumerate coven's attached gate baseline",
1201        );
1202
1203        let mutation = handle
1204            .sql(|sql| {
1205                sql.execute(
1206                    "INSERT INTO coven_gate_empty.roots \
1207                     (id, title, shared, _updated_at) \
1208                     VALUES ('root-1', 'Private', 1, '0000000002000-0000-device-test')",
1209                    [],
1210                )?;
1211                Ok(())
1212            })
1213            .await;
1214        assert!(
1215            mutation.is_err(),
1216            "host SQL must not address coven's attached gate baseline",
1217        );
1218
1219        handle
1220            .sql(|sql| {
1221                sql.execute_batch(
1222                    "CREATE TABLE host_local (id TEXT PRIMARY KEY, value TEXT) STRICT; \
1223                     INSERT INTO host_local VALUES ('local-1', 'kept'); \
1224                     INSERT INTO roots (id, title, shared, _updated_at) \
1225                     VALUES ('root-1', 'Private', 0, \
1226                             '0000000001000-0000-device-test');",
1227                )?;
1228                Ok(())
1229            })
1230            .await
1231            .expect("arbitrary host-schema SQL remains available");
1232        let published = handle
1233            .sql(|sql| {
1234                sql.execute(
1235                    "UPDATE roots SET shared = 1, \
1236                     _updated_at = '0000000002000-0000-device-test' \
1237                     WHERE id = 'root-1'",
1238                    [],
1239                )?;
1240                Ok(())
1241            })
1242            .await
1243            .expect("flip root visible");
1244        let write_id = published.write_id.clone();
1245        let changeset = handle
1246            .db()
1247            .call(move |conn| {
1248                conn.query_row(
1249                    "SELECT changeset FROM store_writes WHERE write_id = ?1",
1250                    [write_id.as_str()],
1251                    |row| row.get::<_, Vec<u8>>(0),
1252                )
1253                .map_err(crate::database::DbError::from)
1254            })
1255            .await
1256            .expect("load gated changeset");
1257        let rows = coven_core::changeset::walk(&changeset).expect("walk gated changeset");
1258        assert_eq!(rows.len(), 1);
1259        assert_eq!(rows[0].op, coven_core::changeset::ChangeOp::Insert);
1260        assert_eq!(rows[0].pk(), Some("root-1"));
1261        assert_eq!(rows[0].col(1), Some("Private"));
1262        assert_eq!(rows[0].col(2), Some("1"));
1263        assert_eq!(rows[0].col(3), Some("0000000002000-0000-device-test"));
1264    }
1265
1266    fn open_files_handle() -> (tempfile::TempDir, CovenHandle) {
1267        let tmp = tempfile::tempdir().expect("temp dir");
1268        let dir = StoreDir::new(tmp.path());
1269        let handle = Coven::builder(config(dir))
1270            .synced_tables(vec![files_table()])
1271            .migrations(vec![files_migration()])
1272            .open()
1273            .expect("open handle");
1274        (tmp, handle)
1275    }
1276
1277    #[tokio::test]
1278    async fn second_open_of_one_store_is_refused_until_the_first_handle_drops() {
1279        let tmp = tempfile::tempdir().expect("temp dir");
1280        let dir = StoreDir::new(tmp.path());
1281        let first = Coven::builder(config(dir.clone()))
1282            .synced_tables(vec![files_table()])
1283            .migrations(vec![files_migration()])
1284            .open()
1285            .expect("first open succeeds");
1286        let clone = first.clone();
1287
1288        let second = Coven::builder(config(dir.clone()))
1289            .synced_tables(vec![files_table()])
1290            .migrations(vec![files_migration()])
1291            .open();
1292        assert!(matches!(
1293            second,
1294            Err(CovenError::AlreadyOpen { store_dir }) if store_dir == tmp.path()
1295        ));
1296
1297        drop(first);
1298
1299        let still_locked = Coven::builder(config(dir.clone()))
1300            .synced_tables(vec![files_table()])
1301            .migrations(vec![files_migration()])
1302            .open();
1303        assert!(matches!(
1304            still_locked,
1305            Err(CovenError::AlreadyOpen { store_dir }) if store_dir == tmp.path()
1306        ));
1307
1308        drop(clone);
1309
1310        Coven::builder(config(dir))
1311            .synced_tables(vec![files_table()])
1312            .migrations(vec![files_migration()])
1313            .open()
1314            .expect("open succeeds after the first handle drops");
1315    }
1316
1317    #[tokio::test]
1318    async fn a_zero_or_negative_blob_tombstone_grace_is_refused_at_open() {
1319        for grace in [chrono::Duration::zero(), chrono::Duration::seconds(-1)] {
1320            let tmp = tempfile::tempdir().expect("temp dir");
1321            let dir = StoreDir::new(tmp.path());
1322            let result = Coven::builder(config(dir))
1323                .synced_tables(vec![files_table()])
1324                .migrations(vec![files_migration()])
1325                .blob_tombstone_grace(grace)
1326                .open();
1327            assert!(
1328                matches!(result, Err(CovenError::InvalidBlobTombstoneGrace)),
1329                "grace {grace:?} must be refused at open",
1330            );
1331        }
1332    }
1333
1334    #[tokio::test]
1335    async fn a_positive_blob_tombstone_grace_opens() {
1336        let tmp = tempfile::tempdir().expect("temp dir");
1337        let dir = StoreDir::new(tmp.path());
1338        Coven::builder(config(dir))
1339            .synced_tables(vec![files_table()])
1340            .migrations(vec![files_migration()])
1341            .blob_tombstone_grace(chrono::Duration::hours(1))
1342            .open()
1343            .expect("a positive grace opens");
1344    }
1345
1346    fn open_remote_root_files_handle() -> (tempfile::TempDir, CovenHandle) {
1347        let tmp = tempfile::tempdir().expect("temp dir");
1348        let dir = StoreDir::new(tmp.path());
1349        let handle = Coven::builder(config(dir))
1350            .synced_tables(vec![remote_root_files_table()])
1351            .migrations(vec![files_migration()])
1352            .open()
1353            .expect("open handle");
1354        (tmp, handle)
1355    }
1356
1357    fn open_files_handle_in(dir: StoreDir) -> CovenHandle {
1358        try_open(&dir).expect("open handle")
1359    }
1360
1361    async fn run_test_cycle(storage: &TestStore, handle: &CovenHandle) {
1362        storage
1363            .publish_pending(handle.db(), &handle.store_dir())
1364            .await
1365            .expect("publish pending Store write");
1366    }
1367
1368    async fn merge_test_storage(
1369        handle: &CovenHandle,
1370        keypair: &crate::keys::UserKeypair,
1371    ) -> TestStore {
1372        TestStore::create(handle.db(), "lib-test", keypair.clone())
1373            .await
1374            .expect("create exact test Store")
1375    }
1376
1377    async fn publish_current_writes(handle: &CovenHandle) {
1378        let keypair = crate::keys::UserKeypair::generate();
1379        let storage = merge_test_storage(handle, &keypair).await;
1380        run_test_cycle(&storage, handle).await;
1381    }
1382
1383    #[tokio::test]
1384    async fn write_survives_reopen_before_sync_cycle() {
1385        let tmp = tempfile::tempdir().expect("temp dir");
1386        let dir = StoreDir::new(tmp.path());
1387        let handle = open_files_handle_in(dir.clone());
1388        handle
1389            .sql(|sql| {
1390                sql.execute(
1391                    "INSERT INTO files (id, blob_id, size, _updated_at) \
1392                     VALUES ('file-before-reopen', NULL, 0, ?1)",
1393                    [sql.stamp()],
1394                )?;
1395                Ok(())
1396            })
1397            .await
1398            .expect("write before reopen");
1399        drop(handle);
1400
1401        let reopened = open_files_handle_in(dir);
1402        let keypair = crate::keys::UserKeypair::generate();
1403        let storage = merge_test_storage(&reopened, &keypair).await;
1404        run_test_cycle(&storage, &reopened).await;
1405
1406        let (_peer_tmp, peer) = open_files_handle();
1407        pull_into(peer.db(), &storage, &peer.store_dir()).await;
1408        assert_eq!(
1409            query_text(
1410                peer.db(),
1411                "SELECT id FROM files WHERE id = 'file-before-reopen'"
1412            )
1413            .await,
1414            "file-before-reopen",
1415        );
1416    }
1417
1418    #[tokio::test]
1419    async fn separate_host_transactions_publish_as_separate_store_commits_after_restart() {
1420        let tmp = tempfile::tempdir().expect("temp dir");
1421        let dir = StoreDir::new(tmp.path());
1422        let handle = open_files_handle_in(dir.clone());
1423        let mut write_ids = Vec::new();
1424        for id in ["file-pending-a", "file-pending-b"] {
1425            let receipt = handle
1426                .sql(move |sql| {
1427                    sql.execute(
1428                        "INSERT INTO files (id, blob_id, size, _updated_at) VALUES (?1, NULL, 0, ?2)",
1429                        (id, sql.stamp()),
1430                    )?;
1431                    Ok(())
1432                })
1433                .await
1434                .expect("write before reopen");
1435            assert_eq!(receipt.status, coven_core::WriteStatus::Pending);
1436            write_ids.push(receipt.write_id);
1437        }
1438        drop(handle);
1439
1440        let reopened = open_files_handle_in(dir);
1441        let pending = reopened.pending_writes().await.expect("pending writes");
1442        assert_eq!(pending.len(), 2);
1443        assert_eq!(pending[0].write_id, write_ids[0]);
1444        assert_eq!(pending[1].write_id, write_ids[1]);
1445        let mut first_status = reopened
1446            .subscribe_write_status(&write_ids[0])
1447            .await
1448            .expect("subscribe after restart");
1449        assert_eq!(*first_status.borrow(), coven_core::WriteStatus::Pending);
1450        let keypair = crate::keys::UserKeypair::generate();
1451        let storage = merge_test_storage(&reopened, &keypair).await;
1452        run_test_cycle(&storage, &reopened).await;
1453
1454        first_status.changed().await.expect("published status");
1455        let first_sequence = match &*first_status.borrow() {
1456            coven_core::WriteStatus::Published(position) => position.commit().coord.sequence(),
1457            status => panic!("first host transaction is not published: {status:?}"),
1458        };
1459        assert_eq!(
1460            reopened
1461                .write_status(&write_ids[1])
1462                .await
1463                .expect("second status after first publication"),
1464            coven_core::WriteStatus::Pending,
1465        );
1466        run_test_cycle(&storage, &reopened).await;
1467        let second_sequence = match reopened
1468            .write_status(&write_ids[1])
1469            .await
1470            .expect("second status")
1471        {
1472            coven_core::WriteStatus::Published(position) => position.commit().coord.sequence(),
1473            status => panic!("second host transaction is not published: {status:?}"),
1474        };
1475        assert_eq!(second_sequence, first_sequence + 1);
1476        assert!(reopened
1477            .pending_writes()
1478            .await
1479            .expect("published writes are not pending")
1480            .is_empty());
1481
1482        let (_peer_tmp, peer) = open_files_handle();
1483        pull_into(peer.db(), &storage, &peer.store_dir()).await;
1484        assert!(row_exists(peer.db(), "SELECT 1 FROM files WHERE id = 'file-pending-a'").await);
1485        assert!(row_exists(peer.db(), "SELECT 1 FROM files WHERE id = 'file-pending-b'").await);
1486    }
1487
1488    #[tokio::test]
1489    async fn device_local_transaction_is_local_only_and_never_pending() {
1490        let (_tmp, handle) = open_files_handle();
1491        let receipt = handle
1492            .sql(|sql| {
1493                sql.execute_batch(
1494                    "CREATE TABLE local_notes (id TEXT PRIMARY KEY, body TEXT) STRICT;
1495                     INSERT INTO local_notes VALUES ('local-1', 'private');",
1496                )?;
1497                Ok("saved")
1498            })
1499            .await
1500            .expect("local transaction");
1501
1502        assert_eq!(receipt.value, "saved");
1503        assert_eq!(receipt.status, coven_core::WriteStatus::LocalOnly);
1504        assert_eq!(
1505            handle
1506                .write_status(&receipt.write_id)
1507                .await
1508                .expect("durable local status"),
1509            coven_core::WriteStatus::LocalOnly
1510        );
1511        assert!(handle
1512            .pending_writes()
1513            .await
1514            .expect("pending writes")
1515            .is_empty());
1516        let local_write_id = receipt.write_id.clone();
1517        let lease_count = handle
1518            .db()
1519            .call(move |conn| {
1520                conn.query_row(
1521                    "SELECT COUNT(*) FROM store_write_blob_leases WHERE write_id = ?1",
1522                    [local_write_id.as_str()],
1523                    |row| row.get::<_, i64>(0),
1524                )
1525                .map_err(crate::database::DbError::from)
1526            })
1527            .await
1528            .expect("count local-only blob leases");
1529        assert_eq!(lease_count, 0);
1530
1531        let keypair = crate::keys::UserKeypair::generate();
1532        let storage = merge_test_storage(&handle, &keypair).await;
1533        run_test_cycle(&storage, &handle).await;
1534        assert_eq!(
1535            handle
1536                .write_status(&receipt.write_id)
1537                .await
1538                .expect("local status after sync"),
1539            coven_core::WriteStatus::LocalOnly
1540        );
1541        assert!(handle
1542            .pending_writes()
1543            .await
1544            .expect("pending writes after sync")
1545            .is_empty());
1546    }
1547
1548    #[tokio::test]
1549    async fn mixed_transaction_tracks_and_publishes_only_shared_rows() {
1550        let (_tmp, handle) = open_files_handle();
1551        let receipt = handle
1552            .sql(|sql| {
1553                sql.execute_batch(
1554                    "CREATE TABLE local_notes (id TEXT PRIMARY KEY, body TEXT) STRICT;
1555                     INSERT INTO local_notes VALUES ('local-1', 'private');",
1556                )?;
1557                sql.execute(
1558                    "INSERT INTO files (id, blob_id, size, _updated_at)
1559                     VALUES ('shared-1', NULL, 0, ?1)",
1560                    [sql.stamp()],
1561                )?;
1562                Ok(())
1563            })
1564            .await
1565            .expect("mixed transaction");
1566
1567        assert_eq!(receipt.status, coven_core::WriteStatus::Pending);
1568        let pending = handle.pending_writes().await.expect("pending mixed write");
1569        assert_eq!(pending.len(), 1);
1570        assert_eq!(pending[0].write_id, receipt.write_id);
1571        assert_eq!(
1572            pending[0].affected_rows,
1573            vec![coven_core::AffectedRow {
1574                table: "files".to_string(),
1575                primary_key: "shared-1".to_string(),
1576            }]
1577        );
1578
1579        let keypair = crate::keys::UserKeypair::generate();
1580        let storage = merge_test_storage(&handle, &keypair).await;
1581        run_test_cycle(&storage, &handle).await;
1582        assert!(matches!(
1583            handle
1584                .write_status(&receipt.write_id)
1585                .await
1586                .expect("published mixed write"),
1587            coven_core::WriteStatus::Published(_)
1588        ));
1589
1590        let (_peer_tmp, peer) = open_files_handle();
1591        pull_into(peer.db(), &storage, &peer.store_dir()).await;
1592        assert!(row_exists(peer.db(), "SELECT 1 FROM files WHERE id = 'shared-1'").await);
1593        assert!(
1594            !row_exists(
1595                peer.db(),
1596                "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'local_notes'"
1597            )
1598            .await
1599        );
1600        assert!(
1601            row_exists(
1602                handle.db(),
1603                "SELECT 1 FROM local_notes WHERE id = 'local-1'"
1604            )
1605            .await
1606        );
1607    }
1608
1609    #[tokio::test]
1610    async fn delete_survives_reopen_before_sync_cycle() {
1611        let tmp = tempfile::tempdir().expect("temp dir");
1612        let dir = StoreDir::new(tmp.path());
1613        let handle = open_files_handle_in(dir.clone());
1614        handle
1615            .sql(|sql| {
1616                sql.execute(
1617                    "INSERT INTO files (id, blob_id, size, _updated_at) \
1618                     VALUES ('file-delete-reopen', NULL, 0, ?1)",
1619                    [sql.stamp()],
1620                )?;
1621                Ok(())
1622            })
1623            .await
1624            .expect("insert before first cycle");
1625        let keypair = crate::keys::UserKeypair::generate();
1626        let storage = merge_test_storage(&handle, &keypair).await;
1627        run_test_cycle(&storage, &handle).await;
1628
1629        let (_peer_tmp, peer) = open_files_handle();
1630        pull_into(peer.db(), &storage, &peer.store_dir()).await;
1631        assert!(
1632            row_exists(
1633                peer.db(),
1634                "SELECT 1 FROM files WHERE id = 'file-delete-reopen'"
1635            )
1636            .await,
1637            "the peer receives the insert before the delete",
1638        );
1639
1640        handle
1641            .sql(|sql| {
1642                sql.execute("DELETE FROM files WHERE id = 'file-delete-reopen'", [])?;
1643                Ok(())
1644            })
1645            .await
1646            .expect("delete before reopen");
1647        drop(handle);
1648
1649        let reopened = open_files_handle_in(dir);
1650        run_test_cycle(&storage, &reopened).await;
1651
1652        pull_into(peer.db(), &storage, &peer.store_dir()).await;
1653        assert!(
1654            !row_exists(
1655                peer.db(),
1656                "SELECT 1 FROM files WHERE id = 'file-delete-reopen'"
1657            )
1658            .await,
1659            "the delete changeset reaches the peer after reopening",
1660        );
1661    }
1662
1663    #[tokio::test]
1664    async fn pending_write_drains_only_after_changeset_push() {
1665        let (_tmp, handle) = open_files_handle();
1666        let receipt = handle
1667            .sql(|sql| {
1668                sql.execute(
1669                    "INSERT INTO files (id, blob_id, size, _updated_at) \
1670                     VALUES ('file-retry-publish', NULL, 0, ?1)",
1671                    [sql.stamp()],
1672                )?;
1673                Ok(())
1674            })
1675            .await
1676            .expect("write before failed push");
1677        let keypair = crate::keys::UserKeypair::generate();
1678        let storage = merge_test_storage(&handle, &keypair).await;
1679        storage.home.fail_exact_create_before_call(1);
1680
1681        let first = storage
1682            .publish_pending(handle.db(), &handle.store_dir())
1683            .await;
1684        assert!(
1685            first
1686                .as_ref()
1687                .is_err_and(|error| error.contains("forced failure before exact create")),
1688            "the first cycle must report the append failure while preserving its outbox: {first:?}",
1689        );
1690        assert_eq!(
1691            handle
1692                .write_status(&receipt.write_id)
1693                .await
1694                .expect("write status after failed append"),
1695            coven_core::WriteStatus::Publishing,
1696        );
1697
1698        run_test_cycle(&storage, &handle).await;
1699        assert!(
1700            matches!(
1701                handle
1702                    .write_status(&receipt.write_id)
1703                    .await
1704                    .expect("write status after retry"),
1705                coven_core::WriteStatus::Published(_)
1706            ),
1707            "the pending write is published as an immutable Store commit after retry",
1708        );
1709    }
1710
1711    async fn write_raw_file(path: &std::path::Path, bytes: &[u8]) {
1712        crate::local_blob::write_atomic(path, bytes)
1713            .await
1714            .expect("write file");
1715    }
1716
1717    async fn cleanup_intent_count(handle: &CovenHandle, namespace: &str, id: &str) -> i64 {
1718        let namespace = namespace.to_string();
1719        let id = id.to_string();
1720        handle
1721            .sql(move |sql| {
1722                sql.query_row(
1723                    "SELECT count(*) FROM local_cleanup_intents \
1724                         WHERE namespace = ?1 AND blob_id = ?2",
1725                    params![namespace, id],
1726                    |row| row.get(0),
1727                )
1728                .map_err(CovenError::from)
1729            })
1730            .await
1731            .expect("count cleanup intents")
1732            .value
1733    }
1734
1735    #[tokio::test]
1736    async fn builder_open_runs_coven_and_host_migrations() {
1737        let (_tmp, handle) = open_files_handle();
1738        let has_coven_table: i64 = handle
1739            .sql(|sql| {
1740                sql.query_row(
1741                    "SELECT count(*) FROM sqlite_master WHERE type = 'table' AND name = 'protocol_state'",
1742                    [],
1743                    |row| row.get(0),
1744                ).map_err(CovenError::from)
1745            })
1746            .await
1747            .expect("query coven table")
1748            .value;
1749        let has_host_table: i64 = handle
1750            .sql(|sql| {
1751                sql.query_row(
1752                    "SELECT count(*) FROM sqlite_master WHERE type = 'table' AND name = 'files'",
1753                    [],
1754                    |row| row.get(0),
1755                )
1756                .map_err(CovenError::from)
1757            })
1758            .await
1759            .expect("query host table")
1760            .value;
1761        assert_eq!(has_coven_table, 1);
1762        assert_eq!(has_host_table, 1);
1763    }
1764
1765    #[tokio::test]
1766    async fn open_of_a_too_new_db_yields_the_matchable_migration_variant() {
1767        let tmp = tempfile::tempdir().expect("temp dir");
1768        let dir = StoreDir::new(tmp.path());
1769
1770        // Open with a two-step ladder so the db lands at synced-schema version 2.
1771        let ahead = Coven::builder(config(dir.clone()))
1772            .synced_tables(vec![files_table()])
1773            .migrations(vec![
1774                files_migration(),
1775                Migration::sql(2, "add-extra", "CREATE TABLE extra (id TEXT PRIMARY KEY)"),
1776            ])
1777            .open()
1778            .expect("open at version 2");
1779        drop(ahead);
1780
1781        // Reopen with only the first step: an older binary meeting a db a newer one
1782        // already migrated. The remedy is "update the app", so the host must be able
1783        // to match the specific variant rather than string-scrape a DbError.
1784        let reopened = Coven::builder(config(dir))
1785            .synced_tables(vec![files_table()])
1786            .migrations(vec![files_migration()])
1787            .open();
1788        assert!(matches!(
1789            reopened,
1790            Err(CovenError::Migration(MigrationError::SchemaTooNew {
1791                current: 2,
1792                supported: 1
1793            }))
1794        ));
1795    }
1796
1797    #[tokio::test]
1798    async fn sql_reads_writes_and_stamps() {
1799        let (_tmp, handle) = open_files_handle();
1800        let id = "file-sql".to_string();
1801        handle
1802            .sql(move |sql| {
1803                sql.execute(
1804                    "INSERT INTO files (id, blob_id, size, _updated_at) VALUES (?1, NULL, 0, ?2)",
1805                    params![id, sql.stamp()],
1806                )?;
1807                Ok(())
1808            })
1809            .await
1810            .expect("insert through sql");
1811        let count: i64 = handle
1812            .sql(|sql| {
1813                sql.query_row("SELECT count(*) FROM files", [], |row| row.get(0))
1814                    .map_err(CovenError::from)
1815            })
1816            .await
1817            .expect("count rows")
1818            .value;
1819        assert_eq!(count, 1);
1820    }
1821
1822    #[tokio::test]
1823    async fn sql_surfaces_sqlite_constraint_typed() {
1824        let (_tmp, handle) = open_files_handle();
1825        handle
1826            .sql(|sql| {
1827                sql.execute(
1828                    "INSERT INTO files (id, blob_id, size, _updated_at) VALUES (?1, NULL, 0, ?2)",
1829                    params!["duplicate-id", sql.stamp()],
1830                )?;
1831                Ok(())
1832            })
1833            .await
1834            .expect("seed row");
1835
1836        let result: CovenResult<WriteReceipt<()>> = handle
1837            .sql(|sql| {
1838                sql.execute(
1839                    "INSERT INTO files (id, blob_id, size, _updated_at) VALUES (?1, NULL, 0, ?2)",
1840                    params!["duplicate-id", sql.stamp()],
1841                )?;
1842                Ok(())
1843            })
1844            .await;
1845
1846        assert!(matches!(result, Err(CovenError::Sqlite(_))));
1847    }
1848
1849    #[tokio::test]
1850    async fn sql_read_sees_a_committed_write() {
1851        let (_tmp, handle) = open_files_handle();
1852        handle
1853            .sql(|sql| {
1854                sql.execute(
1855                    "INSERT INTO files (id, blob_id, size, _updated_at) \
1856                     VALUES ('file-read-your-write', NULL, 0, ?1)",
1857                    [sql.stamp()],
1858                )?;
1859                Ok(())
1860            })
1861            .await
1862            .expect("insert through sql");
1863
1864        // The read runs on the separate read-only connection; it must observe the
1865        // just-committed write (WAL read-your-writes for committed data).
1866        let id: String = handle
1867            .sql_read(|conn| {
1868                conn.query_row(
1869                    "SELECT id FROM files WHERE id = 'file-read-your-write'",
1870                    [],
1871                    |row| row.get(0),
1872                )
1873                .map_err(CovenError::from)
1874            })
1875            .await
1876            .expect("read the committed row back through sql_read");
1877        assert_eq!(id, "file-read-your-write");
1878    }
1879
1880    #[tokio::test]
1881    async fn sql_read_cannot_write() {
1882        let (_tmp, handle) = open_files_handle();
1883        let result: CovenResult<()> = handle
1884            .sql_read(|conn| {
1885                conn.execute(
1886                    "INSERT INTO files (id, blob_id, size, _updated_at) \
1887                     VALUES ('file-read-only', NULL, 0, '0')",
1888                    [],
1889                )
1890                .map_err(CovenError::from)?;
1891                Ok(())
1892            })
1893            .await;
1894        // The companion connection is SQLITE_OPEN_READONLY, so the write is refused
1895        // at the SQLite layer with the readonly code — a read cannot escape the sync
1896        // journal because it cannot write at all.
1897        match result {
1898            Err(CovenError::Sqlite(rusqlite::Error::SqliteFailure(err, _))) => {
1899                assert_eq!(err.code, rusqlite::ErrorCode::ReadOnly);
1900            }
1901            other => panic!("expected a readonly sqlite failure, got {other:?}"),
1902        }
1903
1904        let count: i64 = handle
1905            .sql_read(|conn| {
1906                conn.query_row(
1907                    "SELECT count(*) FROM files WHERE id = 'file-read-only'",
1908                    [],
1909                    |row| row.get(0),
1910                )
1911                .map_err(CovenError::from)
1912            })
1913            .await
1914            .expect("count rows through sql_read");
1915        assert_eq!(count, 0, "the refused write left no row behind");
1916    }
1917
1918    #[tokio::test]
1919    async fn open_on_a_fresh_store_serves_sql_read() {
1920        // A fresh (empty) directory: `open` runs the writer's migrations, then opens
1921        // the read connection against the schema they created. A `sql_read` over the
1922        // host table then succeeds rather than failing on a missing table — proof the
1923        // read connection opened after, not before, the schema exists.
1924        let tmp = tempfile::tempdir().expect("temp dir");
1925        let dir = StoreDir::new(tmp.path());
1926        let handle = Coven::builder(config(dir))
1927            .synced_tables(vec![files_table()])
1928            .migrations(vec![files_migration()])
1929            .open()
1930            .expect("open on an empty directory");
1931        let count: i64 = handle
1932            .sql_read(|conn| {
1933                conn.query_row("SELECT count(*) FROM files", [], |row| row.get(0))
1934                    .map_err(CovenError::from)
1935            })
1936            .await
1937            .expect("sql_read on a fresh store");
1938        assert_eq!(count, 0);
1939    }
1940
1941    #[tokio::test]
1942    async fn open_removes_orphaned_local_blob_temps() {
1943        let tmp = tempfile::tempdir().expect("temp dir");
1944        let dir = StoreDir::new(tmp.path());
1945        let final_path = dir
1946            .local_blob_path("media-files", "tempaaaa")
1947            .expect("local path");
1948        let temp = local_stage_temp_path(&final_path).expect("stage temp path");
1949        write_raw_file(&temp, b"interrupted write").await;
1950
1951        let _handle = Coven::builder(config(dir.clone()))
1952            .synced_tables(vec![files_table()])
1953            .migrations(vec![files_migration()])
1954            .open()
1955            .expect("open handle");
1956
1957        assert!(
1958            !temp.exists(),
1959            "open removes orphaned local blob staging temps"
1960        );
1961        assert!(
1962            !dir.local_blob_path("media-files", "tempaaaa")
1963                .expect("local path")
1964                .exists(),
1965            "the interrupted blob has no committed final file"
1966        );
1967    }
1968
1969    #[tokio::test]
1970    async fn write_inserts_row_and_host_provided_blob() {
1971        let (_tmp, handle) = open_files_handle();
1972        let bytes = b"piece-bytes".to_vec();
1973        let hash = crate::blob::content_hash(&bytes);
1974        handle
1975            .write(
1976                {
1977                    let bytes = bytes.clone();
1978                    move |w| {
1979                        w.put_blob("media-files", "blobaaaa", bytes);
1980                        Ok(())
1981                    }
1982                },
1983                move |sql| {
1984                    sql.execute(
1985                        "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
1986                         VALUES (?1, ?2, ?3, ?4, ?5)",
1987                        params!["file-1", "blobaaaa", bytes.len() as i64, hash, sql.stamp()],
1988                    )?;
1989                    Ok(())
1990                },
1991            )
1992            .await
1993            .expect("write row and blob");
1994        let path = handle
1995            .store_dir()
1996            .local_blob_path("media-files", "blobaaaa")
1997            .expect("local path");
1998        assert_eq!(
1999            std::fs::read(path).expect("read local blob"),
2000            b"piece-bytes"
2001        );
2002    }
2003
2004    #[tokio::test]
2005    async fn orphaned_final_blob_is_replaced_by_next_write() {
2006        let (_tmp, handle) = open_files_handle();
2007        let path = handle
2008            .store_dir()
2009            .local_blob_path("media-files", "orphaaaa")
2010            .expect("local path");
2011        write_raw_file(&path, b"orphaned bytes").await;
2012
2013        handle
2014            .write(
2015                |w| {
2016                    w.put_blob("media-files", "orphaaaa", b"committed bytes".to_vec());
2017                    Ok(())
2018                },
2019                |sql| {
2020                    sql.execute(
2021                        "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2022                         VALUES (?1, ?2, ?3, ?4, ?5)",
2023                        params![
2024                            "file-orphan",
2025                            "orphaaaa",
2026                            15i64,
2027                            crate::blob::content_hash(b"committed bytes"),
2028                            sql.stamp()
2029                        ],
2030                    )?;
2031                    Ok(())
2032                },
2033            )
2034            .await
2035            .expect("write replaces orphaned final blob");
2036
2037        assert_eq!(
2038            std::fs::read(path).expect("read committed blob"),
2039            b"committed bytes"
2040        );
2041    }
2042
2043    #[tokio::test]
2044    async fn put_blob_rejects_id_already_referenced_by_a_row() {
2045        let (_tmp, handle) = open_files_handle();
2046        handle
2047            .write(
2048                |w| {
2049                    w.put_blob("media-files", "dupeaaaa", b"original".to_vec());
2050                    Ok(())
2051                },
2052                |sql| {
2053                    sql.execute(
2054                        "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2055                         VALUES (?1, ?2, ?3, ?4, ?5)",
2056                        params![
2057                            "file-original",
2058                            "dupeaaaa",
2059                            8i64,
2060                            crate::blob::content_hash(b"original"),
2061                            sql.stamp()
2062                        ],
2063                    )?;
2064                    Ok(())
2065                },
2066            )
2067            .await
2068            .expect("seed original blob");
2069
2070        let result: CovenResult<WriteReceipt<()>> = handle
2071            .write(
2072                |w| {
2073                    w.put_blob("media-files", "dupeaaaa", b"replacement".to_vec());
2074                    Ok(())
2075                },
2076                |sql| {
2077                    sql.execute(
2078                        "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2079                         VALUES (?1, ?2, ?3, ?4, ?5)",
2080                        params![
2081                            "file-replacement",
2082                            "dupeaaaa",
2083                            11i64,
2084                            crate::blob::content_hash(b"replacement"),
2085                            sql.stamp()
2086                        ],
2087                    )?;
2088                    Ok(())
2089                },
2090            )
2091            .await;
2092
2093        assert!(matches!(
2094            result,
2095            Err(CovenError::BlobAlreadyReferenced { .. })
2096        ));
2097        let path = handle
2098            .store_dir()
2099            .local_blob_path("media-files", "dupeaaaa")
2100            .expect("dupe path");
2101        assert_eq!(
2102            std::fs::read(path).expect("read original blob"),
2103            b"original"
2104        );
2105        let replacement_rows: i64 = handle
2106            .sql(|sql| {
2107                sql.query_row(
2108                    "SELECT count(*) FROM files WHERE id = 'file-replacement'",
2109                    [],
2110                    |row| row.get(0),
2111                )
2112                .map_err(CovenError::from)
2113            })
2114            .await
2115            .expect("count replacement rows")
2116            .value;
2117        assert_eq!(replacement_rows, 0);
2118    }
2119
2120    #[tokio::test]
2121    async fn remote_root_host_provided_write_reads_staging_through_handle_before_upload() {
2122        let (_tmp, handle) = open_remote_root_files_handle();
2123        let expected = b"remote-root-host-provided-staging-bytes".to_vec();
2124        let bytes = expected.clone();
2125        let hash = crate::blob::content_hash(&bytes);
2126
2127        handle
2128            .write(
2129                {
2130                    let bytes = bytes.clone();
2131                    move |w| {
2132                        w.put_blob("media-files", "rrhpaaaa", bytes);
2133                        Ok(())
2134                    }
2135                },
2136                move |sql| {
2137                    sql.execute(
2138                        "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2139                         VALUES (?1, ?2, ?3, ?4, ?5)",
2140                        params![
2141                            "file-remote-root",
2142                            "rrhpaaaa",
2143                            bytes.len() as i64,
2144                            hash,
2145                            sql.stamp()
2146                        ],
2147                    )?;
2148                    Ok(())
2149                },
2150            )
2151            .await
2152            .expect("write remote-root row and host-provided blob");
2153        let blob = handle
2154            .row_blob_ref("files", "file-remote-root")
2155            .await
2156            .expect("capture remote-root blob row");
2157
2158        let whole = handle
2159            .read_blob(&blob)
2160            .await
2161            .expect("read_blob serves upload staging before sync upload");
2162        assert_eq!(
2163            whole, expected,
2164            "read_blob returns the bytes written through handle.write",
2165        );
2166
2167        let (offset, len) = (12u64, 19u64);
2168        let range = handle
2169            .open_blob_stream(&blob, offset, len)
2170            .await
2171            .expect("open_blob_stream serves upload staging before sync upload");
2172        assert_eq!(
2173            range,
2174            &expected[offset as usize..(offset + len) as usize],
2175            "open_blob_stream returns the requested slice of the staged bytes",
2176        );
2177    }
2178
2179    struct RemoteOnlyStoreBlob {
2180        _tmp: tempfile::TempDir,
2181        dir: StoreDir,
2182        handle: CovenHandle,
2183        store: TestStore,
2184        encryption: coven_core::EncryptionService,
2185        destination_circle: crate::CircleId,
2186        source_object: ObjectSlot,
2187    }
2188
2189    async fn remote_only_store_blob() -> RemoteOnlyStoreBlob {
2190        let tmp = tempfile::tempdir().expect("temp dir");
2191        let dir = StoreDir::new(tmp.path());
2192        let signer = crate::keys::UserKeypair::generate();
2193        let encryption = coven_core::EncryptionService::from_key([42; 32]);
2194        let handle = Coven::builder(config(dir.clone()))
2195            .synced_tables(vec![scoped_files_table()])
2196            .migrations(vec![scoped_files_migration()])
2197            .key_custody(crate::KeyCustody::InMemory(encryption.clone().into()))
2198            .identity_custody(crate::IdentityCustody::InMemory(signer.clone()))
2199            .open()
2200            .expect("open scoped blob store");
2201        let store = TestStore::create(handle.db(), "lib-test", signer)
2202            .await
2203            .expect("create exact test Store");
2204        let authority =
2205            rusqlite::Connection::open(dir.db_path()).expect("open Circle authority database");
2206        let (destination_circle, _) =
2207            coven_core::sync::test_helpers::install_test_active_circle(&authority, "blob-circle");
2208        drop(authority);
2209
2210        let bytes = b"remote-only-circle-blob".to_vec();
2211        let hash = crate::blob::content_hash(&bytes);
2212        handle
2213            .write(
2214                {
2215                    let bytes = bytes.clone();
2216                    move |batch| {
2217                        batch.put_blob("media-files", "circleblob", bytes);
2218                        Ok(())
2219                    }
2220                },
2221                move |sql| {
2222                    sql.execute(
2223                        "INSERT INTO files
2224                         (id, blob_id, size, hash, audience, _updated_at)
2225                         VALUES ('circle-file', 'circleblob', ?1, ?2, ?3, ?4)",
2226                        params![
2227                            bytes.len() as i64,
2228                            hash,
2229                            Option::<String>::None,
2230                            sql.stamp()
2231                        ],
2232                    )?;
2233                    Ok(())
2234                },
2235            )
2236            .await
2237            .expect("write Store blob");
2238        store
2239            .publish_pending(handle.db(), &dir)
2240            .await
2241            .expect("publish Store blob");
2242        let source_object = handle
2243            .row_blob_ref("files", "circle-file")
2244            .await
2245            .expect("capture published Store blob")
2246            .stored()
2247            .expect("published Store blob has exact storage")
2248            .object()
2249            .slot()
2250            .clone();
2251        std::fs::remove_file(
2252            dir.local_blob_path("media-files", "circleblob")
2253                .expect("local blob path"),
2254        )
2255        .expect("remove local plaintext to leave a remote-only source");
2256
2257        RemoteOnlyStoreBlob {
2258            _tmp: tmp,
2259            dir,
2260            handle,
2261            store,
2262            encryption,
2263            destination_circle,
2264            source_object,
2265        }
2266    }
2267
2268    fn outbound_blob_spools(dir: &StoreDir) -> std::collections::BTreeSet<PathBuf> {
2269        let path = dir.storage_dir().join("outbound-blobs");
2270        match std::fs::read_dir(path) {
2271            Ok(entries) => entries
2272                .map(|entry| entry.expect("read outbound blob spool entry").path())
2273                .collect(),
2274            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
2275                std::collections::BTreeSet::new()
2276            }
2277            Err(error) => panic!("read outbound blob spools: {error}"),
2278        }
2279    }
2280
2281    async fn run_scoped_file_write(
2282        handle: &CovenHandle,
2283        blob_staging: Option<coven_core::sync::store::HostWriteBlobStaging>,
2284        sql: String,
2285    ) -> Result<WriteReceipt<()>, coven_core::DbError> {
2286        let tables = handle.db().synced_tables().to_vec();
2287        let gates = handle.db().gates();
2288        let blob_decls = handle.db().blob_decls();
2289        let routing_encryption = handle
2290            .routing_encryption()
2291            .expect("resolve routing encryption");
2292        let write_id = handle.db().new_write_id();
2293        handle
2294            .db()
2295            .call(move |conn| {
2296                coven_core::sync::store::StoreDatabase::run_store_write_transaction_on(
2297                    conn,
2298                    &tables,
2299                    &gates,
2300                    &blob_decls,
2301                    Some(&routing_encryption),
2302                    blob_staging.as_ref(),
2303                    write_id,
2304                    |tx| {
2305                        tx.execute_batch(&sql)?;
2306                        Ok::<_, coven_core::DbError>(())
2307                    },
2308                )
2309            })
2310            .await
2311    }
2312
2313    #[tokio::test]
2314    async fn audience_move_requires_remote_only_blob_before_committing_sql() {
2315        let fixture = remote_only_store_blob().await;
2316        ExactSlotStorage::delete_at(fixture.store.home.as_ref(), &fixture.source_object)
2317            .await
2318            .expect("remove the only remote source");
2319        fixture
2320            .handle
2321            .connect_sync_with_test_home(
2322                fixture.store.home.clone(),
2323                CloudCipher::Encrypted(fixture.encryption.clone()),
2324            )
2325            .await
2326            .expect("connect the exact test Store");
2327
2328        let destination_circle_value = fixture.destination_circle.to_string();
2329        let result = fixture
2330            .handle
2331            .sql(move |sql| {
2332                sql.execute(
2333                    "UPDATE files SET audience = ?1, _updated_at = ?2
2334                     WHERE id = 'circle-file'",
2335                    params![destination_circle_value, sql.stamp()],
2336                )?;
2337                Ok(())
2338            })
2339            .await;
2340
2341        assert!(
2342            result.is_err(),
2343            "a missing source blob must abort the audience move before SQLite commit",
2344        );
2345        let audience = fixture
2346            .handle
2347            .sql_read(|conn| {
2348                conn.query_row(
2349                    "SELECT audience FROM files WHERE id = 'circle-file'",
2350                    [],
2351                    |row| row.get::<_, Option<String>>(0),
2352                )
2353                .map_err(CovenError::from)
2354            })
2355            .await
2356            .expect("read rolled-back audience");
2357        assert_eq!(audience, None);
2358    }
2359
2360    #[tokio::test]
2361    async fn blob_audience_move_without_staging_rejects_and_rolls_back_sql() {
2362        let fixture = remote_only_store_blob().await;
2363        let destination_circle_value = fixture.destination_circle.to_string();
2364        let result = run_scoped_file_write(
2365            &fixture.handle,
2366            None,
2367            format!(
2368                "UPDATE files SET audience = '{destination_circle_value}',
2369                 _updated_at = '0000000009000-0000-device-test'
2370                 WHERE id = 'circle-file'"
2371            ),
2372        )
2373        .await;
2374
2375        let error = result.expect_err("a blob audience move cannot omit materialization");
2376        assert!(
2377            error
2378                .to_string()
2379                .contains("BlobMoveRequiresMaterialization"),
2380            "{error}",
2381        );
2382        let audience = fixture
2383            .handle
2384            .sql_read(|conn| {
2385                conn.query_row(
2386                    "SELECT audience FROM files WHERE id = 'circle-file'",
2387                    [],
2388                    |row| row.get::<_, Option<String>>(0),
2389                )
2390                .map_err(CovenError::from)
2391            })
2392            .await
2393            .expect("read rolled-back audience");
2394        assert_eq!(audience, None);
2395    }
2396
2397    #[tokio::test]
2398    async fn blob_storage_adapter_error_only_blocks_a_move_that_needs_storage() {
2399        let fixture = remote_only_store_blob().await;
2400        let adapter_error = || {
2401            coven_core::sync::store::HostWriteBlobStaging::new(
2402                tokio::runtime::Handle::current(),
2403                Err("injected blob storage adapter construction failure".to_string()),
2404                fixture.dir.clone(),
2405            )
2406        };
2407
2408        run_scoped_file_write(
2409            &fixture.handle,
2410            Some(adapter_error()),
2411            "UPDATE files SET _updated_at = '0000000009000-0000-device-test'
2412             WHERE id = 'circle-file'"
2413                .to_string(),
2414        )
2415        .await
2416        .expect("an unused adapter error does not block ordinary SQL");
2417
2418        let destination_circle_value = fixture.destination_circle.to_string();
2419        let error = run_scoped_file_write(
2420            &fixture.handle,
2421            Some(adapter_error()),
2422            format!(
2423                "UPDATE files SET audience = '{destination_circle_value}',
2424                 _updated_at = '0000000010000-0000-device-test'
2425                 WHERE id = 'circle-file'"
2426            ),
2427        )
2428        .await
2429        .expect_err("a remote-only move must surface its adapter error");
2430        assert!(
2431            error
2432                .to_string()
2433                .contains("injected blob storage adapter construction failure"),
2434            "{error}",
2435        );
2436        let audience = fixture
2437            .handle
2438            .sql_read(|conn| {
2439                conn.query_row(
2440                    "SELECT audience FROM files WHERE id = 'circle-file'",
2441                    [],
2442                    |row| row.get::<_, Option<String>>(0),
2443                )
2444                .map_err(CovenError::from)
2445            })
2446            .await
2447            .expect("read audience after adapter failure");
2448        assert_eq!(audience, None);
2449    }
2450
2451    #[tokio::test]
2452    async fn journal_failure_removes_only_the_audience_move_spool_and_rolls_back_sql() {
2453        let fixture = remote_only_store_blob().await;
2454        fixture
2455            .handle
2456            .connect_sync_with_test_home(
2457                fixture.store.home.clone(),
2458                CloudCipher::Encrypted(fixture.encryption.clone()),
2459            )
2460            .await
2461            .expect("connect the exact test Store");
2462        let before = outbound_blob_spools(&fixture.dir);
2463        rusqlite::Connection::open(fixture.dir.db_path())
2464            .expect("open database for journal fault")
2465            .execute_batch(
2466                "CREATE TRIGGER fail_audience_move_journal
2467                 BEFORE INSERT ON store_writes
2468                 BEGIN
2469                   SELECT RAISE(ABORT, 'injected Store write journal failure');
2470                 END;",
2471            )
2472            .expect("install Store write journal fault");
2473
2474        let destination_circle_value = fixture.destination_circle.to_string();
2475        let result = fixture
2476            .handle
2477            .sql(move |sql| {
2478                sql.execute(
2479                    "UPDATE files SET audience = ?1, _updated_at = ?2
2480                     WHERE id = 'circle-file'",
2481                    params![destination_circle_value, sql.stamp()],
2482                )?;
2483                Ok(())
2484            })
2485            .await;
2486
2487        assert!(result.is_err(), "the injected journal failure must surface");
2488        assert_eq!(
2489            outbound_blob_spools(&fixture.dir),
2490            before,
2491            "rollback removes the destination spool created by this attempt",
2492        );
2493        let audience = fixture
2494            .handle
2495            .sql_read(|conn| {
2496                conn.query_row(
2497                    "SELECT audience FROM files WHERE id = 'circle-file'",
2498                    [],
2499                    |row| row.get::<_, Option<String>>(0),
2500                )
2501                .map_err(CovenError::from)
2502            })
2503            .await
2504            .expect("read rolled-back audience");
2505        assert_eq!(audience, None);
2506    }
2507
2508    #[tokio::test]
2509    async fn local_audience_move_rolls_back_its_file_and_reuses_an_exact_leftover() {
2510        let fixture = remote_only_store_blob().await;
2511        fixture
2512            .handle
2513            .connect_sync_with_test_home(
2514                fixture.store.home.clone(),
2515                CloudCipher::Encrypted(fixture.encryption.clone()),
2516            )
2517            .await
2518            .expect("connect the exact test Store");
2519        let destination = fixture
2520            .dir
2521            .local_blob_path("media-files", "circleblob")
2522            .expect("resolve Local destination");
2523        let sibling = destination
2524            .parent()
2525            .expect("Local destination has a parent")
2526            .join("unrelated");
2527        std::fs::create_dir_all(sibling.parent().expect("sibling has a parent"))
2528            .expect("create Local blob directory");
2529        std::fs::write(&sibling, b"unrelated").expect("write unrelated Local file");
2530        rusqlite::Connection::open(fixture.dir.db_path())
2531            .expect("open database for Local journal fault")
2532            .execute_batch(
2533                "CREATE TRIGGER fail_local_audience_move_journal
2534                 BEFORE INSERT ON store_writes
2535                 BEGIN
2536                   SELECT RAISE(ABORT, 'injected Local Store write journal failure');
2537                 END;",
2538            )
2539            .expect("install Local Store write journal fault");
2540
2541        let result = fixture
2542            .handle
2543            .sql(|sql| {
2544                sql.execute(
2545                    "UPDATE files SET audience = 'local', _updated_at = ?1
2546                     WHERE id = 'circle-file'",
2547                    [sql.stamp()],
2548                )?;
2549                Ok(())
2550            })
2551            .await;
2552        assert!(result.is_err(), "the injected journal failure must surface");
2553        assert!(
2554            !destination.exists(),
2555            "rollback removes the Local file created by this attempt",
2556        );
2557        assert_eq!(
2558            std::fs::read(&sibling).expect("read unrelated Local file"),
2559            b"unrelated",
2560        );
2561        let audience = fixture
2562            .handle
2563            .sql_read(|conn| {
2564                conn.query_row(
2565                    "SELECT audience FROM files WHERE id = 'circle-file'",
2566                    [],
2567                    |row| row.get::<_, Option<String>>(0),
2568                )
2569                .map_err(CovenError::from)
2570            })
2571            .await
2572            .expect("read rolled-back Local audience");
2573        assert_eq!(audience, None);
2574        assert!(matches!(
2575            fixture
2576                .handle
2577                .row_blob_ref("files", "circle-file")
2578                .await
2579                .expect("load blob after failed Local move")
2580                .authority(),
2581            coven_core::blob::RowBlobAuthority::Remote(_)
2582        ));
2583
2584        rusqlite::Connection::open(fixture.dir.db_path())
2585            .expect("open database to remove Local journal fault")
2586            .execute_batch("DROP TRIGGER fail_local_audience_move_journal")
2587            .expect("remove Local Store write journal fault");
2588        std::fs::write(&destination, b"remote-only-circle-blob")
2589            .expect("model an exact file left by failed cleanup");
2590        fixture
2591            .handle
2592            .sql(|sql| {
2593                sql.execute(
2594                    "UPDATE files SET audience = 'local', _updated_at = ?1
2595                     WHERE id = 'circle-file'",
2596                    [sql.stamp()],
2597                )?;
2598                Ok(())
2599            })
2600            .await
2601            .expect("retry accepts the exact already-materialized Local file");
2602        assert_eq!(
2603            std::fs::read(&destination).expect("read retried Local destination"),
2604            b"remote-only-circle-blob",
2605        );
2606        assert_eq!(
2607            std::fs::read(&sibling).expect("read unrelated file after retry"),
2608            b"unrelated",
2609        );
2610    }
2611
2612    #[tokio::test]
2613    async fn audience_move_publishes_from_precommit_spool_after_source_disappears() {
2614        let fixture = remote_only_store_blob().await;
2615        fixture
2616            .handle
2617            .connect_sync_with_test_home(
2618                fixture.store.home.clone(),
2619                CloudCipher::Encrypted(fixture.encryption.clone()),
2620            )
2621            .await
2622            .expect("connect the exact test Store");
2623
2624        // The fabricated destination Circle names its fixed owner in its roster;
2625        // that identity must be an active Store member or the Circle would be
2626        // rotation-required and reject new content.
2627        fixture
2628            .handle
2629            .invite_member(
2630                &crate::keys::public_key_hex(
2631                    &coven_core::sync::test_helpers::test_circle_owner_keypair(),
2632                ),
2633                None,
2634                crate::MemberRole::Member,
2635            )
2636            .await
2637            .expect("register the fabricated Circle roster owner as a Store member");
2638
2639        let destination_circle_value = fixture.destination_circle.to_string();
2640        let receipt = fixture
2641            .handle
2642            .sql(move |sql| {
2643                sql.execute(
2644                    "UPDATE files SET audience = ?1, _updated_at = ?2
2645                     WHERE id = 'circle-file'",
2646                    params![destination_circle_value, sql.stamp()],
2647                )?;
2648                Ok(())
2649            })
2650            .await
2651            .expect("commit audience move after staging its destination blob");
2652        let write_id = receipt.write_id.to_string();
2653        let blob_facts = fixture
2654            .handle
2655            .sql_read(move |conn| {
2656                conn.query_row(
2657                    "SELECT blob_facts FROM store_writes WHERE write_id = ?1",
2658                    [write_id],
2659                    |row| row.get::<_, String>(0),
2660                )
2661                .map_err(CovenError::from)
2662            })
2663            .await
2664            .expect("read durable move blob facts");
2665        let blob_facts: serde_json::Value =
2666            serde_json::from_str(&blob_facts).expect("decode move blob facts");
2667        let spool_path = blob_facts["blobs"][0]["audience_move"]["remote"]["spool_path"]
2668            .as_str()
2669            .expect("move fact records its exact destination spool");
2670        assert!(
2671            std::path::Path::new(spool_path).is_file(),
2672            "the destination spool is durable before the SQL write returns",
2673        );
2674
2675        ExactSlotStorage::delete_at(fixture.store.home.as_ref(), &fixture.source_object)
2676            .await
2677            .expect("remove source after the move commits");
2678        fixture
2679            .store
2680            .publish_pending(fixture.handle.db(), &fixture.dir)
2681            .await
2682            .expect("publish the move from its durable destination spool");
2683        assert!(
2684            !std::path::Path::new(spool_path).exists(),
2685            "prepared-object completion retires the durable precommit spool",
2686        );
2687        assert!(
2688            !fixture
2689                .store
2690                .publish_pending(fixture.handle.db(), &fixture.dir)
2691                .await
2692                .expect("retry completed move publication"),
2693            "a completed move has nothing left to publish",
2694        );
2695        let published = fixture
2696            .handle
2697            .row_blob_ref("files", "circle-file")
2698            .await
2699            .expect("capture published destination Circle blob");
2700        assert_eq!(
2701            published.authority().audience(),
2702            crate::Audience::Circle(fixture.destination_circle),
2703        );
2704    }
2705
2706    #[tokio::test]
2707    async fn public_materialization_survives_store_reopen_without_a_cloud_connection() {
2708        let local = tokio::task::LocalSet::new();
2709        local
2710            .run_until(async {
2711                tokio::task::spawn_local(
2712                    run_public_materialization_survives_store_reopen_without_a_cloud_connection(),
2713                )
2714                .await
2715                .expect("public materialization task");
2716            })
2717            .await;
2718    }
2719
2720    async fn run_public_materialization_survives_store_reopen_without_a_cloud_connection() {
2721        let tmp = tempfile::tempdir().expect("temp dir");
2722        let dir = StoreDir::new(tmp.path());
2723        let signer = crate::keys::UserKeypair::generate();
2724        let keyring =
2725            coven_core::MasterKeyring::from(coven_core::EncryptionService::from_key([42; 32]));
2726        let open = || {
2727            Coven::builder(config(dir.clone()))
2728                .synced_tables(vec![remote_root_files_table()])
2729                .migrations(vec![files_migration()])
2730                .key_custody(crate::KeyCustody::InMemory(keyring.clone()))
2731                .identity_custody(crate::IdentityCustody::InMemory(signer.clone()))
2732                .open()
2733                .expect("open remote-root store")
2734        };
2735        let handle = open();
2736        let store = TestStore::create(handle.db(), "lib-test", signer.clone())
2737            .await
2738            .expect("create exact test Store");
2739        handle
2740            .connect_sync_with_test_home(
2741                store.home.clone(),
2742                CloudCipher::Encrypted(coven_core::EncryptionService::from_key([42; 32])),
2743            )
2744            .await
2745            .expect("connect exact test Store");
2746
2747        let expected = b"public materialized blob".to_vec();
2748        let bytes = expected.clone();
2749        let hash = crate::blob::content_hash(&bytes);
2750        let receipt = handle
2751            .write(
2752                {
2753                    let bytes = bytes.clone();
2754                    move |batch| {
2755                        batch.put_blob("media-files", "materialized-blob", bytes);
2756                        Ok(())
2757                    }
2758                },
2759                move |sql| {
2760                    sql.execute(
2761                        "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2762                         VALUES (?1, ?2, ?3, ?4, ?5)",
2763                        params![
2764                            "materialized-row",
2765                            "materialized-blob",
2766                            bytes.len() as i64,
2767                            hash,
2768                            sql.stamp()
2769                        ],
2770                    )?;
2771                    Ok(())
2772                },
2773            )
2774            .await
2775            .expect("write remote-root blob");
2776        let mut status = handle
2777            .subscribe_write_status(&receipt.write_id)
2778            .await
2779            .expect("subscribe to materialized write");
2780        handle.sync_now();
2781        tokio::time::timeout(Duration::from_secs(20), async {
2782            loop {
2783                let current = status.borrow().clone();
2784                match current {
2785                    coven_core::WriteStatus::Published(_) => break,
2786                    coven_core::WriteStatus::Pending | coven_core::WriteStatus::Publishing => {
2787                        status.changed().await.expect("write status remains open")
2788                    }
2789                    other => panic!("materialized write did not publish: {other:?}"),
2790                }
2791            }
2792        })
2793        .await
2794        .expect("materialized write publishes");
2795        let reference = handle
2796            .row_blob_ref("files", "materialized-row")
2797            .await
2798            .expect("capture published row blob");
2799        handle
2800            .materialize_row_blob(&reference)
2801            .await
2802            .expect("materialize through the public handle");
2803        let locator_hash = reference
2804            .stored()
2805            .expect("published row has exact storage")
2806            .locator()
2807            .locator_hash();
2808        let cached = dir
2809            .cache_blob_path("media-files", locator_hash)
2810            .expect("exact cache path");
2811        assert_eq!(std::fs::read(&cached).expect("read exact cache"), expected);
2812
2813        handle.disconnect_sync();
2814        drop(handle);
2815        let reopened = open();
2816        let reopened_reference = reopened
2817            .row_blob_ref("files", "materialized-row")
2818            .await
2819            .expect("capture reopened row blob");
2820        reopened
2821            .materialize_row_blob(&reopened_reference)
2822            .await
2823            .expect("reopen verifies the materialized cache without cloud storage");
2824    }
2825
2826    #[tokio::test]
2827    async fn sql_failure_removes_staged_blob() {
2828        let (_tmp, handle) = open_files_handle();
2829        let err = handle
2830            .write(
2831                |w| {
2832                    w.put_blob("media-files", "blobbbbb", b"staged".to_vec());
2833                    Ok(())
2834                },
2835                |_sql| Err::<(), CovenError>(CovenError::Blob("sql failed".to_string())),
2836            )
2837            .await
2838            .expect_err("write fails");
2839        assert!(err.to_string().contains("sql failed"));
2840        let path = handle
2841            .store_dir()
2842            .local_blob_path("media-files", "blobbbbb")
2843            .expect("local path");
2844        assert!(!path.exists());
2845    }
2846
2847    #[tokio::test]
2848    async fn blob_stage_failure_does_not_run_sql() {
2849        let (_tmp, handle) = open_files_handle();
2850        let result: CovenResult<WriteReceipt<()>> = handle
2851            .write(
2852                |w| {
2853                    w.put_blob("media-files", "..", b"bad".to_vec());
2854                    Ok(())
2855                },
2856                |sql| {
2857                    sql.execute(
2858                        "INSERT INTO files (id, blob_id, size, _updated_at) \
2859                         VALUES ('should-not-exist', NULL, 0, ?1)",
2860                        [sql.stamp()],
2861                    )?;
2862                    Ok(())
2863                },
2864            )
2865            .await;
2866        // A blob id that can't form a safe path is its own typed error, not the
2867        // generic blob catch-all.
2868        assert!(matches!(result, Err(CovenError::UnsafeBlobPath(_))));
2869        let count: i64 = handle
2870            .sql(|sql| {
2871                sql.query_row("SELECT count(*) FROM files", [], |row| row.get(0))
2872                    .map_err(CovenError::from)
2873            })
2874            .await
2875            .expect("count rows")
2876            .value;
2877        assert_eq!(count, 0);
2878    }
2879
2880    #[tokio::test]
2881    async fn replacement_deletes_old_blob_after_sql_drops_reference() {
2882        let (_tmp, handle) = open_files_handle();
2883        crate::blob::local_files::store(&handle.store_dir(), "media-files", "oldaaaa", b"old")
2884            .await
2885            .expect("store old");
2886        handle
2887            .sql(|sql| {
2888                sql.execute(
2889                    "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2890                     VALUES (?1, ?2, 3, ?3, ?4)",
2891                    params![
2892                        "file-1",
2893                        "oldaaaa",
2894                        crate::blob::content_hash(b"old"),
2895                        sql.stamp()
2896                    ],
2897                )?;
2898                Ok(())
2899            })
2900            .await
2901            .expect("seed row");
2902        publish_current_writes(&handle).await;
2903        crate::blob::local_files::store(&handle.store_dir(), "media-files", "oldaaaa", b"old")
2904            .await
2905            .expect("restore published blob locally");
2906        let old_ref = BlobRef {
2907            namespace: "media-files".to_string(),
2908            id: "oldaaaa".to_string(),
2909            scope: BlobScope::Master,
2910            cloud_path: None,
2911            provenance: Provenance::HostProvided,
2912            fill: CacheFill::CacheLazy,
2913        };
2914        handle
2915            .write(
2916                move |w| {
2917                    w.put_blob("media-files", "newaaaa", b"new".to_vec());
2918                    w.delete_blob(old_ref);
2919                    Ok(())
2920                },
2921                move |sql| {
2922                    sql.execute(
2923                        "UPDATE files SET blob_id = ?1, size = 3, hash = ?2, \
2924                         _updated_at = ?3 WHERE id = 'file-1'",
2925                        params!["newaaaa", crate::blob::content_hash(b"new"), sql.stamp()],
2926                    )?;
2927                    Ok(())
2928                },
2929            )
2930            .await
2931            .expect("replace blob");
2932        assert!(!handle
2933            .store_dir()
2934            .local_blob_path("media-files", "oldaaaa")
2935            .expect("old path")
2936            .exists());
2937        assert!(handle
2938            .store_dir()
2939            .local_blob_path("media-files", "newaaaa")
2940            .expect("new path")
2941            .exists());
2942    }
2943
2944    async fn queue_replacement_before_sync(
2945        handle: &CovenHandle,
2946    ) -> (WriteId, WriteId, std::path::PathBuf) {
2947        let first = handle
2948            .write(
2949                |batch| {
2950                    batch.put_blob("media-files", "ownedaaa", b"first".to_vec());
2951                    Ok(())
2952                },
2953                |sql| {
2954                    sql.execute(
2955                        "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2956                         VALUES ('owned-file', 'ownedaaa', 5, ?1, ?2)",
2957                        params![crate::blob::content_hash(b"first"), sql.stamp()],
2958                    )?;
2959                    Ok(())
2960                },
2961            )
2962            .await
2963            .expect("queue first blob write");
2964        let first_path = handle
2965            .store_dir()
2966            .local_blob_path("media-files", "ownedaaa")
2967            .expect("first blob path");
2968        let first_blob = BlobRef {
2969            namespace: "media-files".to_string(),
2970            id: "ownedaaa".to_string(),
2971            scope: BlobScope::Master,
2972            cloud_path: None,
2973            provenance: Provenance::HostProvided,
2974            fill: CacheFill::CacheLazy,
2975        };
2976        let second = handle
2977            .write(
2978                move |batch| {
2979                    batch.put_blob("media-files", "ownedbbb", b"second".to_vec());
2980                    batch.delete_blob(first_blob);
2981                    Ok(())
2982                },
2983                |sql| {
2984                    sql.execute(
2985                        "UPDATE files SET blob_id = 'ownedbbb', size = 6, hash = ?1, \
2986                         _updated_at = ?2 \
2987                         WHERE id = 'owned-file'",
2988                        params![crate::blob::content_hash(b"second"), sql.stamp()],
2989                    )?;
2990                    Ok(())
2991                },
2992            )
2993            .await
2994            .expect("queue replacement write");
2995        (first.write_id, second.write_id, first_path)
2996    }
2997
2998    async fn assert_replacement_publishes_in_order(
2999        handle: &CovenHandle,
3000        first_write: &WriteId,
3001        second_write: &WriteId,
3002        first_path: &std::path::Path,
3003    ) {
3004        let keypair = crate::keys::UserKeypair::generate();
3005        let storage = merge_test_storage(handle, &keypair).await;
3006        run_test_cycle(&storage, handle).await;
3007
3008        let first_sequence = match handle
3009            .write_status(first_write)
3010            .await
3011            .expect("first status")
3012        {
3013            coven_core::WriteStatus::Published(position) => position.commit().coord.sequence(),
3014            status => panic!("first replacement write is not published: {status:?}"),
3015        };
3016        assert_eq!(
3017            handle
3018                .write_status(second_write)
3019                .await
3020                .expect("second status after first publication"),
3021            coven_core::WriteStatus::Pending,
3022            "one Store publication consumes one pending host transaction",
3023        );
3024
3025        run_test_cycle(&storage, handle).await;
3026        let second_sequence = match handle
3027            .write_status(second_write)
3028            .await
3029            .expect("second status")
3030        {
3031            coven_core::WriteStatus::Published(position) => position.commit().coord.sequence(),
3032            status => panic!("second replacement write is not published: {status:?}"),
3033        };
3034        assert_eq!(
3035            second_sequence,
3036            first_sequence + 1,
3037            "replacement writes publish in their host transaction order"
3038        );
3039        let blob_objects = storage
3040            .home
3041            .keys()
3042            .into_iter()
3043            .filter(|key| key.starts_with("media-files/opaque/"))
3044            .count();
3045        assert_eq!(blob_objects, 2, "both row versions upload their blob bytes");
3046        assert!(
3047            !first_path.exists(),
3048            "the first write releases its local bytes after publication"
3049        );
3050    }
3051
3052    #[tokio::test]
3053    async fn pending_write_owns_blob_bytes_until_its_publication() {
3054        let local = tokio::task::LocalSet::new();
3055        local
3056            .run_until(async {
3057                tokio::task::spawn_local(run_pending_write_owns_blob_bytes_until_its_publication())
3058                    .await
3059                    .expect("pending-write blob ownership test task");
3060            })
3061            .await;
3062    }
3063
3064    async fn run_pending_write_owns_blob_bytes_until_its_publication() {
3065        let (_tmp, handle) = open_files_handle();
3066        let (first_write, second_write, first_path) = queue_replacement_before_sync(&handle).await;
3067
3068        assert_eq!(
3069            std::fs::read(&first_path).expect("first write still owns its bytes"),
3070            b"first"
3071        );
3072        let overwrite: CovenResult<WriteReceipt<()>> = handle
3073            .write(
3074                |batch| {
3075                    batch.put_blob("media-files", "ownedaaa", b"overwritten".to_vec());
3076                    Ok(())
3077                },
3078                |sql| {
3079                    sql.execute(
3080                        "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
3081                         VALUES ('overwrite', 'ownedaaa', 11, ?1, ?2)",
3082                        params![crate::blob::content_hash(b"overwritten"), sql.stamp()],
3083                    )?;
3084                    Ok(())
3085                },
3086            )
3087            .await;
3088        assert!(matches!(
3089            overwrite,
3090            Err(CovenError::BlobOwnedByPendingWrite { .. })
3091        ));
3092        assert_eq!(
3093            std::fs::read(&first_path).expect("lease prevents overwrite"),
3094            b"first"
3095        );
3096        assert_replacement_publishes_in_order(&handle, &first_write, &second_write, &first_path)
3097            .await;
3098    }
3099
3100    #[tokio::test]
3101    async fn pending_write_blob_ownership_survives_restart() {
3102        let local = tokio::task::LocalSet::new();
3103        local
3104            .run_until(async {
3105                tokio::task::spawn_local(run_pending_write_blob_ownership_survives_restart())
3106                    .await
3107                    .expect("reopened pending-write blob ownership test task");
3108            })
3109            .await;
3110    }
3111
3112    async fn run_pending_write_blob_ownership_survives_restart() {
3113        let tmp = tempfile::tempdir().expect("temp dir");
3114        let dir = StoreDir::new(tmp.path());
3115        let handle = open_files_handle_in(dir.clone());
3116        let (first_write, second_write, first_path) = queue_replacement_before_sync(&handle).await;
3117        drop(handle);
3118
3119        let reopened = open_files_handle_in(dir);
3120        assert_eq!(
3121            std::fs::read(&first_path).expect("reopened first write still owns its bytes"),
3122            b"first"
3123        );
3124        assert_replacement_publishes_in_order(&reopened, &first_write, &second_write, &first_path)
3125            .await;
3126    }
3127
3128    #[tokio::test]
3129    async fn author_delete_drops_all_local_blob_copies() {
3130        let (_tmp, handle) = open_files_handle();
3131        crate::blob::local_files::store(&handle.store_dir(), "media-files", "oldcccc", b"old")
3132            .await
3133            .expect("store old");
3134        handle
3135            .sql(|sql| {
3136                sql.execute(
3137                    "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
3138                     VALUES (?1, ?2, 3, ?3, ?4)",
3139                    params![
3140                        "file-1",
3141                        "oldcccc",
3142                        crate::blob::content_hash(b"old"),
3143                        sql.stamp()
3144                    ],
3145                )?;
3146                Ok(())
3147            })
3148            .await
3149            .expect("seed row");
3150        publish_current_writes(&handle).await;
3151        let published = handle
3152            .row_blob_ref("files", "file-1")
3153            .await
3154            .expect("capture exact published blob");
3155        let locator_hash = published
3156            .stored()
3157            .expect("published row has exact storage")
3158            .locator()
3159            .locator_hash();
3160        let pinned = handle
3161            .store_dir()
3162            .pinned_blob_path("media-files", locator_hash)
3163            .expect("pinned path");
3164        let cached = handle
3165            .store_dir()
3166            .cache_blob_path("media-files", locator_hash)
3167            .expect("cache path");
3168        crate::blob::local_files::store(&handle.store_dir(), "media-files", "oldcccc", b"old")
3169            .await
3170            .expect("restore published blob locally");
3171        write_raw_file(&pinned, b"old").await;
3172        write_raw_file(&cached, b"old").await;
3173        let old_ref = BlobRef {
3174            namespace: "media-files".to_string(),
3175            id: "oldcccc".to_string(),
3176            scope: BlobScope::Master,
3177            cloud_path: None,
3178            provenance: Provenance::HostProvided,
3179            fill: CacheFill::CacheLazy,
3180        };
3181
3182        handle
3183            .write(
3184                move |w| {
3185                    w.delete_blob(old_ref);
3186                    Ok(())
3187                },
3188                move |sql| {
3189                    sql.execute(
3190                        "UPDATE files SET blob_id = NULL, size = 0, hash = NULL, _updated_at = ?1 \
3191                         WHERE id = 'file-1'",
3192                        [sql.stamp()],
3193                    )?;
3194                    Ok(())
3195                },
3196            )
3197            .await
3198            .expect("delete blob");
3199
3200        assert!(!handle
3201            .store_dir()
3202            .local_blob_path("media-files", "oldcccc")
3203            .expect("local path")
3204            .exists());
3205        assert!(!pinned.exists(), "pinned copy is removed");
3206        assert!(!cached.exists(), "cache copy is removed");
3207    }
3208
3209    #[tokio::test]
3210    async fn failed_local_blob_cleanup_keeps_intent_for_later_drain() {
3211        let (_tmp, handle) = open_files_handle();
3212        crate::blob::local_files::store(&handle.store_dir(), "media-files", "oldddddd", b"old")
3213            .await
3214            .expect("store old");
3215        handle
3216            .sql(|sql| {
3217                sql.execute(
3218                    "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
3219                     VALUES (?1, ?2, 3, ?3, ?4)",
3220                    params![
3221                        "file-1",
3222                        "oldddddd",
3223                        crate::blob::content_hash(b"old"),
3224                        sql.stamp()
3225                    ],
3226                )?;
3227                Ok(())
3228            })
3229            .await
3230            .expect("seed row");
3231        publish_current_writes(&handle).await;
3232        let published = handle
3233            .row_blob_ref("files", "file-1")
3234            .await
3235            .expect("capture exact published blob");
3236        let locator_hash = published
3237            .stored()
3238            .expect("published row has exact storage")
3239            .locator()
3240            .locator_hash();
3241        let pinned = handle
3242            .store_dir()
3243            .pinned_blob_path("media-files", locator_hash)
3244            .expect("pinned path");
3245        crate::blob::local_files::store(&handle.store_dir(), "media-files", "oldddddd", b"old")
3246            .await
3247            .expect("restore published blob locally");
3248        std::fs::create_dir_all(&pinned).expect("create pinned blocker");
3249        let old_ref = BlobRef {
3250            namespace: "media-files".to_string(),
3251            id: "oldddddd".to_string(),
3252            scope: BlobScope::Master,
3253            cloud_path: None,
3254            provenance: Provenance::HostProvided,
3255            fill: CacheFill::CacheLazy,
3256        };
3257
3258        handle
3259            .write(
3260                move |w| {
3261                    w.delete_blob(old_ref);
3262                    Ok(())
3263                },
3264                |sql| {
3265                    sql.execute(
3266                        "UPDATE files SET blob_id = NULL, size = 0, hash = NULL, _updated_at = ?1 \
3267                         WHERE id = 'file-1'",
3268                        [sql.stamp()],
3269                    )?;
3270                    Ok(())
3271                },
3272            )
3273            .await
3274            .expect("row delete commits despite cleanup failure");
3275
3276        assert_eq!(
3277            cleanup_intent_count(&handle, "media-files", "oldddddd").await,
3278            2
3279        );
3280        assert!(handle
3281            .store_dir()
3282            .local_blob_path("media-files", "oldddddd")
3283            .expect("local path")
3284            .exists());
3285
3286        std::fs::remove_dir_all(&pinned).expect("remove pinned blocker");
3287        handle
3288            .write(
3289                |_| Ok(()),
3290                |sql| {
3291                    sql.execute(
3292                        "INSERT INTO files (id, blob_id, size, _updated_at) \
3293                         VALUES (?1, NULL, 0, ?2)",
3294                        params!["drain-trigger", sql.stamp()],
3295                    )?;
3296                    Ok(())
3297                },
3298            )
3299            .await
3300            .expect("later committed write drains pending cleanup");
3301
3302        assert_eq!(
3303            cleanup_intent_count(&handle, "media-files", "oldddddd").await,
3304            0
3305        );
3306        assert!(!handle
3307            .store_dir()
3308            .local_blob_path("media-files", "oldddddd")
3309            .expect("local path")
3310            .exists());
3311    }
3312
3313    #[tokio::test]
3314    async fn write_drain_separates_live_local_source_from_deleted_exact_cache() {
3315        let (_tmp, handle) = open_files_handle();
3316        let blob_id = "shared01";
3317        handle
3318            .sql(move |sql| {
3319                let hash = crate::blob::content_hash(b"live");
3320                for id in ["remote-deletes", "still-live"] {
3321                    sql.execute(
3322                        "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
3323                         VALUES (?1, 'shared01', 4, ?2, \
3324                                 '0000000001000-0000-dev-remote')",
3325                        params![id, &hash],
3326                    )?;
3327                }
3328                Ok(())
3329            })
3330            .await
3331            .expect("seed two rows sharing the blob");
3332
3333        let local = handle
3334            .store_dir()
3335            .local_blob_path("media-files", blob_id)
3336            .expect("local path");
3337        write_raw_file(&local, b"live").await;
3338
3339        let (_source_tmp, source) = open_files_handle();
3340        let storage = Arc::new(
3341            TestStore::create(
3342                source.db(),
3343                "lib-test",
3344                crate::keys::UserKeypair::generate(),
3345            )
3346            .await
3347            .expect("create remote exact test Store"),
3348        );
3349        source
3350            .write(
3351                |batch| {
3352                    batch.put_blob("media-files", "shared01", b"live".to_vec());
3353                    Ok(())
3354                },
3355                |sql| {
3356                    sql.execute(
3357                        "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
3358                         VALUES ('remote-deletes', 'shared01', 4, ?1, ?2)",
3359                        params![crate::blob::content_hash(b"live"), sql.stamp()],
3360                    )?;
3361                    Ok(())
3362                },
3363            )
3364            .await
3365            .expect("insert remote row");
3366        storage
3367            .publish_pending(source.db(), &source.store_dir())
3368            .await
3369            .expect("publish remote insert");
3370        let remote_reference = source
3371            .row_blob_ref("files", "remote-deletes")
3372            .await
3373            .expect("capture exact remote blob");
3374        let locator_hash = remote_reference
3375            .stored()
3376            .expect("published row has exact storage")
3377            .locator()
3378            .locator_hash();
3379        let pinned = handle
3380            .store_dir()
3381            .pinned_blob_path("media-files", locator_hash)
3382            .expect("pinned path");
3383        let cached = handle
3384            .store_dir()
3385            .cache_blob_path("media-files", locator_hash)
3386            .expect("cache path");
3387        write_raw_file(&pinned, b"live").await;
3388        write_raw_file(&cached, b"live").await;
3389        let delete = source
3390            .sql(|sql| {
3391                sql.execute("DELETE FROM files WHERE id = 'remote-deletes'", [])?;
3392                Ok(())
3393            })
3394            .await
3395            .expect("delete remote row");
3396        storage
3397            .publish_pending(source.db(), &source.store_dir())
3398            .await
3399            .expect("publish remote delete");
3400
3401        assert!(matches!(
3402            source
3403                .write_status(&delete.write_id)
3404                .await
3405                .expect("remote delete status"),
3406            coven_core::WriteStatus::Published(_)
3407        ));
3408        let (device_id, sequence) = source
3409            .db()
3410            .call(|conn| {
3411                conn.query_row(
3412                    "SELECT device_id, seq FROM materialized_commits ORDER BY seq DESC LIMIT 1",
3413                    [],
3414                    |row| Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)),
3415                )
3416                .map_err(coven_core::database::DbError::from)
3417            })
3418            .await
3419            .expect("load remote delete stream coordinate");
3420        let sequence = u64::try_from(sequence).expect("remote delete sequence is non-negative");
3421
3422        let (commit_reached, resume_pull) = handle.db().arm_test_pause(
3423            coven_core::database::DatabaseTestPoint::PullAfterRemoteCommit {
3424                device_id,
3425                seq: sequence,
3426            },
3427        );
3428        let pull_handle = handle.clone();
3429        let pull_storage = storage.clone();
3430        let pull = tokio::spawn(async move {
3431            pull_into(
3432                pull_handle.db(),
3433                pull_storage.as_ref(),
3434                &pull_handle.store_dir(),
3435            )
3436            .await
3437        });
3438
3439        commit_reached.notified().await;
3440        handle
3441            .write(
3442                |_| Ok(()),
3443                |sql| {
3444                    sql.execute(
3445                        "INSERT INTO files (id, blob_id, size, _updated_at) \
3446                         VALUES ('drain-trigger', NULL, 0, ?1)",
3447                        [sql.stamp()],
3448                    )?;
3449                    Ok(())
3450                },
3451            )
3452            .await
3453            .expect("write drains cleanup queue");
3454        resume_pull.notify_one();
3455        pull.await.expect("pull task");
3456
3457        assert!(
3458            !row_exists(
3459                handle.db(),
3460                "SELECT 1 FROM files WHERE id = 'remote-deletes'"
3461            )
3462            .await
3463        );
3464        assert!(row_exists(handle.db(), "SELECT 1 FROM files WHERE id = 'still-live'").await);
3465        assert!(local.exists(), "the live blob's local copy survives");
3466        assert!(
3467            !pinned.exists(),
3468            "the deleted locator's pinned copy is removed"
3469        );
3470        assert!(
3471            !cached.exists(),
3472            "the deleted locator's cache copy is removed"
3473        );
3474        assert_eq!(
3475            cleanup_intent_count(&handle, "media-files", blob_id).await,
3476            0,
3477        );
3478    }
3479
3480    #[tokio::test]
3481    async fn replacement_is_rejected_while_sql_still_references_old_blob() {
3482        let (_tmp, handle) = open_files_handle();
3483        crate::blob::local_files::store(&handle.store_dir(), "media-files", "oldbbbb", b"old")
3484            .await
3485            .expect("store old");
3486        handle
3487            .sql(|sql| {
3488                sql.execute(
3489                    "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
3490                     VALUES (?1, ?2, 3, ?3, ?4)",
3491                    params![
3492                        "file-1",
3493                        "oldbbbb",
3494                        crate::blob::content_hash(b"old"),
3495                        sql.stamp()
3496                    ],
3497                )?;
3498                Ok(())
3499            })
3500            .await
3501            .expect("seed row");
3502        let old_ref = BlobRef {
3503            namespace: "media-files".to_string(),
3504            id: "oldbbbb".to_string(),
3505            scope: BlobScope::Master,
3506            cloud_path: None,
3507            provenance: Provenance::HostProvided,
3508            fill: CacheFill::CacheLazy,
3509        };
3510        let result: CovenResult<WriteReceipt<()>> = handle
3511            .write(
3512                move |w| {
3513                    w.put_blob("media-files", "newbbbb", b"new".to_vec());
3514                    w.delete_blob(old_ref);
3515                    Ok(())
3516                },
3517                move |sql| {
3518                    sql.execute(
3519                        "UPDATE files SET _updated_at = ?1 WHERE id = 'file-1'",
3520                        [sql.stamp()],
3521                    )?;
3522                    Ok(())
3523                },
3524            )
3525            .await;
3526        assert!(matches!(
3527            result,
3528            Err(CovenError::BlobStillReferenced { .. })
3529        ));
3530        assert!(handle
3531            .store_dir()
3532            .local_blob_path("media-files", "oldbbbb")
3533            .expect("old path")
3534            .exists());
3535        assert!(!handle
3536            .store_dir()
3537            .local_blob_path("media-files", "newbbbb")
3538            .expect("new path")
3539            .exists());
3540    }
3541
3542    #[tokio::test]
3543    async fn sql_panic_removes_moved_blob() {
3544        let (_tmp, handle) = open_files_handle();
3545        let result: CovenResult<WriteReceipt<()>> = handle
3546            .write(
3547                |w| {
3548                    w.put_blob("media-files", "panicccc", b"new".to_vec());
3549                    Ok(())
3550                },
3551                |_sql| panic!("boom"),
3552            )
3553            .await;
3554        assert!(matches!(result, Err(CovenError::WriteClosurePanicked)));
3555        assert!(!handle
3556            .store_dir()
3557            .local_blob_path("media-files", "panicccc")
3558            .expect("panic path")
3559            .exists());
3560    }
3561
3562    #[tokio::test]
3563    async fn concurrent_duplicate_blob_write_does_not_delete_committed_blob() {
3564        let (_tmp, handle) = open_files_handle();
3565        let winner = handle.clone();
3566        let loser = handle.clone();
3567
3568        let write_winner = tokio::spawn(async move {
3569            winner
3570                .write(
3571                    move |w| {
3572                        w.put_blob("media-files", "raceblob", b"committed".to_vec());
3573                        Ok(())
3574                    },
3575                    move |sql| {
3576                        sql.execute(
3577                            "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
3578                             VALUES (?1, ?2, ?3, ?4, ?5)",
3579                            params![
3580                                "winner",
3581                                "raceblob",
3582                                9i64,
3583                                crate::blob::content_hash(b"committed"),
3584                                sql.stamp()
3585                            ],
3586                        )?;
3587                        Ok(())
3588                    },
3589                )
3590                .await
3591        });
3592
3593        let write_loser = tokio::spawn(async move {
3594            loser
3595                .write(
3596                    move |w| {
3597                        w.put_blob("media-files", "raceblob", b"rolled-back".to_vec());
3598                        Ok(())
3599                    },
3600                    |_sql| Err::<(), CovenError>(CovenError::Blob("force rollback".to_string())),
3601                )
3602                .await
3603        });
3604
3605        let winner_result = write_winner.await.expect("winner task");
3606        let loser_result = write_loser.await.expect("loser task");
3607        assert!(winner_result.is_ok() || loser_result.is_ok());
3608        assert!(winner_result.is_err() || loser_result.is_err());
3609
3610        let path = handle
3611            .store_dir()
3612            .local_blob_path("media-files", "raceblob")
3613            .expect("race path");
3614        assert_eq!(std::fs::read(path).expect("read race blob"), b"committed");
3615        let rows: i64 = handle
3616            .sql(|sql| {
3617                sql.query_row(
3618                    "SELECT count(*) FROM files WHERE id = 'winner'",
3619                    [],
3620                    |row| row.get(0),
3621                )
3622                .map_err(CovenError::from)
3623            })
3624            .await
3625            .expect("count winner row")
3626            .value;
3627        assert_eq!(rows, 1);
3628    }
3629
3630    /// A [`CloudHome`] that blocks the sync loop inside a cycle on demand. It
3631    /// delegates every call to an inner [`InMemoryCloudHome`], but once `armed`,
3632    /// the first cloud operation of a cycle signals `entered` and parks on
3633    /// `release` until the test wakes it — holding the loop mid-cycle so the test
3634    /// can observe the store-directory lock while a cycle is in flight.
3635    ///
3636    /// Arming happens only after `connect_sync_with_test_home` returns, so the
3637    /// connect-time bootstrap runs unblocked and only a loop cycle is gated.
3638    struct GateCloudHome {
3639        inner: InMemoryCloudHome,
3640        armed: Arc<AtomicBool>,
3641        gated: AtomicBool,
3642        entered: mpsc::UnboundedSender<()>,
3643        release: Arc<Notify>,
3644    }
3645
3646    impl GateCloudHome {
3647        async fn gate(&self) {
3648            if self.armed.load(Ordering::Acquire) && !self.gated.swap(true, Ordering::AcqRel) {
3649                self.entered
3650                    .send(())
3651                    .expect("test observes the loop entering a cycle");
3652                self.release.notified().await;
3653            }
3654        }
3655    }
3656
3657    #[async_trait]
3658    impl CloudHome for GateCloudHome {
3659        fn exact_slot_storage(self: Arc<Self>) -> Option<Arc<dyn ExactSlotStorage>> {
3660            Some(self)
3661        }
3662
3663        async fn put_object(&self, key: &str, data: Vec<u8>) -> Result<(), CloudHomeError> {
3664            self.gate().await;
3665            self.inner.put_object(key, data).await
3666        }
3667
3668        async fn open_multipart<'a>(
3669            &'a self,
3670            key: &str,
3671            total_len: u64,
3672        ) -> Result<BoxPartSink<'a>, CloudHomeError> {
3673            self.gate().await;
3674            self.inner.open_multipart(key, total_len).await
3675        }
3676
3677        fn multipart_threshold(&self) -> u64 {
3678            self.inner.multipart_threshold()
3679        }
3680
3681        async fn read(&self, key: &str) -> Result<Vec<u8>, CloudHomeError> {
3682            self.gate().await;
3683            self.inner.read(key).await
3684        }
3685
3686        async fn read_range(
3687            &self,
3688            key: &str,
3689            start: u64,
3690            end: u64,
3691        ) -> Result<Vec<u8>, CloudHomeError> {
3692            self.gate().await;
3693            self.inner.read_range(key, start, end).await
3694        }
3695
3696        async fn list(&self, prefix: &str) -> Result<Vec<String>, CloudHomeError> {
3697            self.gate().await;
3698            self.inner.list(prefix).await
3699        }
3700
3701        async fn delete(&self, key: &str) -> Result<(), CloudHomeError> {
3702            self.gate().await;
3703            self.inner.delete(key).await
3704        }
3705
3706        async fn exists(&self, key: &str) -> Result<bool, CloudHomeError> {
3707            self.gate().await;
3708            self.inner.exists(key).await
3709        }
3710
3711        async fn set_access(
3712            &self,
3713            desired: CloudAccessState,
3714        ) -> Result<CloudAccessOutcome, CloudHomeError> {
3715            self.gate().await;
3716            self.inner.set_access(desired).await
3717        }
3718    }
3719
3720    #[async_trait]
3721    impl ExactSlotStorage for GateCloudHome {
3722        async fn provider_binding(
3723            &self,
3724        ) -> Result<crate::sync::storage::ResolvedProviderBinding, CloudHomeError> {
3725            self.gate().await;
3726            ExactSlotStorage::provider_binding(&self.inner).await
3727        }
3728
3729        async fn allocate_slot(&self, key: &str) -> Result<ObjectSlot, CloudHomeError> {
3730            self.gate().await;
3731            ExactSlotStorage::allocate_slot(&self.inner, key).await
3732        }
3733
3734        async fn create_at(
3735            &self,
3736            slot: &ObjectSlot,
3737            body: BlobBody,
3738            progress: &UploadProgress<'_>,
3739        ) -> Result<(), CloudHomeError> {
3740            self.gate().await;
3741            ExactSlotStorage::create_at(&self.inner, slot, body, progress).await
3742        }
3743
3744        async fn read_at(&self, slot: &ObjectSlot) -> Result<Vec<u8>, CloudHomeError> {
3745            self.gate().await;
3746            ExactSlotStorage::read_at(&self.inner, slot).await
3747        }
3748
3749        async fn read_range_at(
3750            &self,
3751            slot: &ObjectSlot,
3752            start: u64,
3753            end: u64,
3754        ) -> Result<Vec<u8>, CloudHomeError> {
3755            self.gate().await;
3756            ExactSlotStorage::read_range_at(&self.inner, slot, start, end).await
3757        }
3758
3759        async fn read_at_to_file(
3760            &self,
3761            slot: &ObjectSlot,
3762            destination: &std::path::Path,
3763        ) -> Result<(), crate::storage::cloud::CloudFileReadError> {
3764            self.gate().await;
3765            ExactSlotStorage::read_at_to_file(&self.inner, slot, destination).await
3766        }
3767
3768        async fn delete_at(&self, slot: &ObjectSlot) -> Result<(), CloudHomeError> {
3769            self.gate().await;
3770            ExactSlotStorage::delete_at(&self.inner, slot).await
3771        }
3772    }
3773
3774    fn try_open(dir: &StoreDir) -> CovenResult<CovenHandle> {
3775        Coven::builder(config(dir.clone()))
3776            .synced_tables(vec![files_table()])
3777            .migrations(vec![files_migration()])
3778            .open()
3779    }
3780
3781    /// Reopen `dir`, retrying while a still-running sync loop holds the lock.
3782    /// Retries only the lock refusal; any other open error fails immediately.
3783    /// Fails the calling test if the lock is not released within the budget.
3784    async fn open_when_lock_released(dir: &StoreDir) -> CovenHandle {
3785        for _ in 0..100 {
3786            match try_open(dir) {
3787                Ok(handle) => return handle,
3788                Err(CovenError::AlreadyOpen { .. }) => {
3789                    tokio::time::sleep(Duration::from_millis(100)).await;
3790                }
3791                Err(other) => panic!("reopen failed for a reason other than the lock: {other}"),
3792            }
3793        }
3794        panic!("store lock never released: reopen kept failing");
3795    }
3796
3797    #[tokio::test]
3798    async fn lock_is_held_until_the_sync_loop_exits_its_cycle() {
3799        test_keyring::install();
3800
3801        let tmp = tempfile::tempdir().expect("temp dir");
3802        let dir = StoreDir::new(tmp.path());
3803        // A store id distinct from every other test's — `try_open`/
3804        // `open_when_lock_released` below reopen the same directory (the
3805        // lock is path-scoped, not store-id-scoped) but this test's own
3806        // identity establishment must not collide with a concurrently
3807        // running test that also establishes one under the shared default id.
3808        let handle = Coven::builder(Config::with_defaults(
3809            "lock-held-until-cycle-exits".to_string(),
3810            "device-test".to_string(),
3811            dir.clone(),
3812            "Test".to_string(),
3813        ))
3814        .synced_tables(vec![files_table()])
3815        .migrations(vec![files_migration()])
3816        .open()
3817        .expect("open handle");
3818        handle
3819            .initialize_identity()
3820            .expect("establish this store's identity before connecting");
3821
3822        let armed = Arc::new(AtomicBool::new(false));
3823        let (entered_tx, mut entered_rx) = mpsc::unbounded_channel();
3824        let release = Arc::new(Notify::new());
3825        let home = Arc::new(GateCloudHome {
3826            inner: InMemoryCloudHome::new(),
3827            armed: armed.clone(),
3828            gated: AtomicBool::new(false),
3829            entered: entered_tx,
3830            release: release.clone(),
3831        });
3832        let connect_handle = handle.clone();
3833        tokio::spawn(async move {
3834            connect_handle
3835                .connect_sync_with_test_home(home, CloudCipher::Plaintext)
3836                .await
3837        })
3838        .await
3839        .expect("gating test connection task completes")
3840        .expect("connect over the gating test home");
3841
3842        // Bootstrap is done; gate the loop's next cycle and wait until it parks
3843        // mid-cycle inside the cloud home.
3844        armed.store(true, Ordering::Release);
3845        tokio::time::timeout(Duration::from_secs(30), entered_rx.recv())
3846            .await
3847            .expect("sync loop reached a cycle within the timeout")
3848            .expect("gating home signalled the cycle entry");
3849
3850        // Drop the last handle while the loop is still mid-cycle. The loop owns a
3851        // guard clone, so the lock must stay held: a concurrent open is refused.
3852        drop(handle);
3853        assert!(
3854            matches!(
3855                try_open(&dir),
3856                Err(CovenError::AlreadyOpen { store_dir }) if store_dir == tmp.path()
3857            ),
3858            "a concurrent open must be refused while the sync loop is mid-cycle",
3859        );
3860
3861        // Release the cycle; the loop finishes, sees its channels closed, exits,
3862        // and drops the last guard clone — the lock frees and a reopen succeeds.
3863        release.notify_one();
3864        let _reopened = open_when_lock_released(&dir).await;
3865    }
3866
3867    #[tokio::test]
3868    async fn normal_shutdown_releases_the_lock_for_reopen() {
3869        test_keyring::install();
3870
3871        let tmp = tempfile::tempdir().expect("temp dir");
3872        let dir = StoreDir::new(tmp.path());
3873        // A store id distinct from every other test's — see the identical
3874        // note in `lock_is_held_until_the_sync_loop_exits_its_cycle`.
3875        let handle = Coven::builder(Config::with_defaults(
3876            "normal-shutdown-releases-lock".to_string(),
3877            "device-test".to_string(),
3878            dir.clone(),
3879            "Test".to_string(),
3880        ))
3881        .synced_tables(vec![files_table()])
3882        .migrations(vec![files_migration()])
3883        .open()
3884        .expect("open handle");
3885        handle
3886            .initialize_identity()
3887            .expect("establish this store's identity before connecting");
3888        let connect_handle = handle.clone();
3889        tokio::spawn(async move {
3890            connect_handle
3891                .connect_sync_with_test_home(
3892                    Arc::new(InMemoryCloudHome::new()),
3893                    CloudCipher::Plaintext,
3894                )
3895                .await
3896        })
3897        .await
3898        .expect("shutdown test connection task completes")
3899        .expect("connect over the test home");
3900
3901        drop(handle);
3902
3903        let _reopened = open_when_lock_released(&dir).await;
3904    }
3905
3906    /// The open-time orphan sweep clears crash-left atomic-write temps (`.tmp.<uuid>`)
3907    /// under every blob folder — `cache/`, `pinned/`, and the local store — that
3908    /// predate this open, while leaving a temp a concurrent write is mid-producing
3909    /// (newer than process start) and every committed blob untouched. Local-store
3910    /// staging temps (`.coven-stage-<uuid>`) are still swept regardless of age.
3911    #[test]
3912    fn open_time_sweep_clears_stale_blob_temps_but_keeps_fresh_ones() {
3913        let tmp = tempfile::tempdir().expect("temp dir");
3914        let ld = StoreDir::new(tmp.path());
3915
3916        let process_start = std::time::SystemTime::now();
3917        let stale = process_start - Duration::from_secs(3600);
3918        let fresh = process_start + Duration::from_secs(3600);
3919
3920        let write_with_mtime = |path: &Path, mtime: std::time::SystemTime| {
3921            std::fs::create_dir_all(path.parent().unwrap()).expect("create dir");
3922            std::fs::write(path, b"x").expect("write file");
3923            std::fs::OpenOptions::new()
3924                .write(true)
3925                .open(path)
3926                .expect("open to set mtime")
3927                .set_modified(mtime)
3928                .expect("set mtime");
3929        };
3930        let temp_name = |marker: &str| format!("{marker}{}", uuid::Uuid::new_v4());
3931
3932        let cache_shard = ld
3933            .storage_dir()
3934            .join("cache")
3935            .join("release_files")
3936            .join("ab")
3937            .join("cd");
3938        let pinned_shard = ld
3939            .storage_dir()
3940            .join("pinned")
3941            .join("photos")
3942            .join("ef")
3943            .join("gh");
3944        let local_ns = ld.storage_dir().join("local").join("audio");
3945
3946        let stale_cache_temp = cache_shard.join(temp_name(crate::local_blob::TEMP_BLOB_PREFIX));
3947        let fresh_cache_temp = cache_shard.join(temp_name(crate::local_blob::TEMP_BLOB_PREFIX));
3948        // A committed blob with an OLD mtime: not a temp, so age is irrelevant — it must
3949        // survive, proving the sweep classifies by the temp prefix, not by age.
3950        let committed_cache = cache_shard.join("blob0aaa");
3951        let stale_pinned_temp = pinned_shard.join(temp_name(crate::local_blob::TEMP_BLOB_PREFIX));
3952        let fresh_pinned_temp = pinned_shard.join(temp_name(crate::local_blob::TEMP_BLOB_PREFIX));
3953        // The local store's own `write_atomic` leaves `.tmp.<uuid>` temps too; a crash
3954        // orphans them the same way. A committed local blob (old mtime) must survive.
3955        let stale_local_temp = local_ns.join(temp_name(crate::local_blob::TEMP_BLOB_PREFIX));
3956        let fresh_local_temp = local_ns.join(temp_name(crate::local_blob::TEMP_BLOB_PREFIX));
3957        let committed_local = local_ns.join("blob0bbb");
3958        let stale_local_stage = local_ns.join(temp_name(&format!("blob{LOCAL_STAGE_MARKER}")));
3959
3960        write_with_mtime(&stale_cache_temp, stale);
3961        write_with_mtime(&fresh_cache_temp, fresh);
3962        write_with_mtime(&committed_cache, stale);
3963        write_with_mtime(&stale_pinned_temp, stale);
3964        write_with_mtime(&fresh_pinned_temp, fresh);
3965        write_with_mtime(&stale_local_temp, stale);
3966        write_with_mtime(&fresh_local_temp, fresh);
3967        write_with_mtime(&committed_local, stale);
3968        write_with_mtime(&stale_local_stage, stale);
3969
3970        remove_orphaned_local_blob_temps(&ld, process_start).expect("open-time orphan sweep");
3971
3972        assert!(
3973            !stale_cache_temp.exists(),
3974            "a cache temp older than process start is a crash leftover, removed",
3975        );
3976        assert!(
3977            !stale_pinned_temp.exists(),
3978            "a pinned temp older than process start is a crash leftover, removed",
3979        );
3980        assert!(
3981            !stale_local_temp.exists(),
3982            "a local-store atomic-write temp older than process start is a crash leftover, removed",
3983        );
3984        assert!(
3985            fresh_cache_temp.exists(),
3986            "a cache temp newer than process start is a live write, left alone",
3987        );
3988        assert!(
3989            fresh_pinned_temp.exists(),
3990            "a pinned temp newer than process start is a live write, left alone",
3991        );
3992        assert!(
3993            fresh_local_temp.exists(),
3994            "a local-store temp newer than process start is a live write, left alone",
3995        );
3996        assert!(
3997            committed_cache.exists(),
3998            "a committed cache blob is never a temp — kept regardless of its mtime",
3999        );
4000        assert!(
4001            committed_local.exists(),
4002            "a committed local-store blob is never a temp — kept regardless of its mtime",
4003        );
4004        assert!(
4005            !stale_local_stage.exists(),
4006            "local-store staging temps are still swept at open",
4007        );
4008    }
4009
4010    // ========================================================================
4011    // Read-only opens (CovenReadHandle)
4012    // ========================================================================
4013
4014    fn try_open_read_only(dir: &StoreDir) -> CovenResult<crate::read_handle::CovenReadHandle> {
4015        Coven::builder(config(dir.clone()))
4016            .synced_tables(vec![files_table()])
4017            .migrations(vec![files_migration()])
4018            .open_read_only()
4019    }
4020
4021    async fn read_files_count(handle: &crate::read_handle::CovenReadHandle) -> i64 {
4022        handle
4023            .sql_read(|conn| {
4024                conn.query_row("SELECT count(*) FROM files", [], |row| row.get(0))
4025                    .map_err(CovenError::from)
4026            })
4027            .await
4028            .expect("count files through the read handle")
4029    }
4030
4031    /// The requirement's failing baseline made to pass: a second FULL open is still
4032    /// refused while the first holds the store, but a read-only open succeeds
4033    /// against the same held store — it takes no writer lock.
4034    #[tokio::test]
4035    async fn read_only_open_succeeds_while_a_full_open_holds_the_store() {
4036        let tmp = tempfile::tempdir().expect("temp dir");
4037        let dir = StoreDir::new(tmp.path());
4038        let writer = open_files_handle_in(dir.clone());
4039
4040        // A second full open is refused (the invariant the lock protects).
4041        assert!(matches!(
4042            try_open(&dir),
4043            Err(CovenError::AlreadyOpen { store_dir }) if store_dir == tmp.path()
4044        ));
4045
4046        // A read-only open succeeds against the very same held store.
4047        let reader = try_open_read_only(&dir).expect("read-only open succeeds under a full open");
4048        assert_eq!(read_files_count(&reader).await, 0);
4049
4050        drop(writer);
4051    }
4052
4053    /// Multiple read-only opens coexist with each other and with the writer.
4054    #[tokio::test]
4055    async fn multiple_read_only_opens_coexist_with_a_writer() {
4056        let tmp = tempfile::tempdir().expect("temp dir");
4057        let dir = StoreDir::new(tmp.path());
4058        let writer = open_files_handle_in(dir.clone());
4059
4060        let reader_a = try_open_read_only(&dir).expect("first read-only open");
4061        let reader_b = try_open_read_only(&dir).expect("second read-only open");
4062
4063        assert_eq!(read_files_count(&reader_a).await, 0);
4064        assert_eq!(read_files_count(&reader_b).await, 0);
4065        drop(writer);
4066    }
4067
4068    /// A read-only handle sees rows the writer committed — both before the reader
4069    /// opened and after (WAL cross-connection visibility on the one db file).
4070    #[tokio::test]
4071    async fn read_only_open_sees_committed_writer_data() {
4072        let tmp = tempfile::tempdir().expect("temp dir");
4073        let dir = StoreDir::new(tmp.path());
4074        let writer = open_files_handle_in(dir.clone());
4075
4076        writer
4077            .sql(|sql| {
4078                sql.execute(
4079                    "INSERT INTO files (id, blob_id, size, _updated_at) \
4080                     VALUES ('row-before-reader', NULL, 0, ?1)",
4081                    [sql.stamp()],
4082                )?;
4083                Ok(())
4084            })
4085            .await
4086            .expect("writer inserts before the reader opens");
4087
4088        let reader = try_open_read_only(&dir).expect("read-only open");
4089        assert_eq!(
4090            read_files_count(&reader).await,
4091            1,
4092            "the reader sees the row committed before it opened",
4093        );
4094
4095        // A commit after the reader is open is visible on its next read: coven runs
4096        // each read as its own transaction, so it never pins an old WAL snapshot.
4097        writer
4098            .sql(|sql| {
4099                sql.execute(
4100                    "INSERT INTO files (id, blob_id, size, _updated_at) \
4101                     VALUES ('row-after-reader', NULL, 0, ?1)",
4102                    [sql.stamp()],
4103                )?;
4104                Ok(())
4105            })
4106            .await
4107            .expect("writer inserts after the reader opened");
4108        assert_eq!(
4109            read_files_count(&reader).await,
4110            2,
4111            "the reader sees a row the writer committed after the reader opened",
4112        );
4113        drop(writer);
4114    }
4115
4116    /// A read-only handle has no write API at all — writes are absent by
4117    /// construction. The one place a raw statement could be run is the
4118    /// `sql_read` closure's `&Connection`; because the connection is
4119    /// `SQLITE_OPEN_READONLY`, SQLite refuses the write there rather than
4120    /// mutating the store.
4121    #[tokio::test]
4122    async fn read_only_handle_connection_refuses_writes() {
4123        let tmp = tempfile::tempdir().expect("temp dir");
4124        let dir = StoreDir::new(tmp.path());
4125        let _writer = open_files_handle_in(dir.clone());
4126        let reader = try_open_read_only(&dir).expect("read-only open");
4127
4128        let result: CovenResult<()> = reader
4129            .sql_read(|conn| {
4130                conn.execute(
4131                    "INSERT INTO files (id, blob_id, size, _updated_at) \
4132                     VALUES ('should-not-write', NULL, 0, 'x')",
4133                    [],
4134                )?;
4135                Ok(())
4136            })
4137            .await;
4138        assert!(
4139            matches!(result, Err(CovenError::Sqlite(_))),
4140            "a write through the read-only connection is refused by SQLite: {result:?}",
4141        );
4142    }
4143
4144    /// A read-only open runs no migration ladder, but refuses a db a newer binary
4145    /// migrated past what this binary knows — the writer's `SchemaTooNew` policy.
4146    #[tokio::test]
4147    async fn read_only_open_refuses_a_too_new_schema() {
4148        let tmp = tempfile::tempdir().expect("temp dir");
4149        let dir = StoreDir::new(tmp.path());
4150
4151        // A newer binary migrates the db to synced-schema version 2.
4152        let ahead = Coven::builder(config(dir.clone()))
4153            .synced_tables(vec![files_table()])
4154            .migrations(vec![
4155                files_migration(),
4156                Migration::sql(2, "add-extra", "CREATE TABLE extra (id TEXT PRIMARY KEY)"),
4157            ])
4158            .open()
4159            .expect("open at version 2");
4160        drop(ahead);
4161
4162        // An older binary opens the same db read-only with only the version-1 ladder:
4163        // it cannot understand the schema, so it refuses with the matchable variant.
4164        let reopened = Coven::builder(config(dir))
4165            .synced_tables(vec![files_table()])
4166            .migrations(vec![files_migration()])
4167            .open_read_only();
4168        assert!(matches!(
4169            reopened,
4170            Err(CovenError::Migration(MigrationError::SchemaTooNew {
4171                current: 2,
4172                supported: 1
4173            }))
4174        ));
4175    }
4176
4177    /// End-to-end blob read through the read handle: the writer stores a
4178    /// host-provided blob on a remote-root table (its bytes land in the local store
4179    /// as upload staging); the read-only handle resolves the blob's locality and
4180    /// serves those bytes — no cloud, no writer lock.
4181    #[tokio::test]
4182    async fn read_only_handle_reads_a_host_provided_blob() {
4183        let tmp = tempfile::tempdir().expect("temp dir");
4184        let dir = StoreDir::new(tmp.path());
4185        let writer = Coven::builder(config(dir.clone()))
4186            .synced_tables(vec![remote_root_files_table()])
4187            .migrations(vec![files_migration()])
4188            .open()
4189            .expect("open writer");
4190
4191        let bytes = b"read-only-handle-serves-these-blob-bytes".to_vec();
4192        let hash = crate::blob::content_hash(&bytes);
4193        writer
4194            .write(
4195                {
4196                    let bytes = bytes.clone();
4197                    move |w| {
4198                        w.put_blob("media-files", "roblob01", bytes);
4199                        Ok(())
4200                    }
4201                },
4202                {
4203                    let len = bytes.len() as i64;
4204                    move |sql| {
4205                        sql.execute(
4206                            "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
4207                             VALUES (?1, ?2, ?3, ?4, ?5)",
4208                            params!["file-ro", "roblob01", len, hash, sql.stamp()],
4209                        )?;
4210                        Ok(())
4211                    }
4212                },
4213            )
4214            .await
4215            .expect("writer stores the row and blob");
4216
4217        let reader = Coven::builder(config(dir))
4218            .synced_tables(vec![remote_root_files_table()])
4219            .migrations(vec![files_migration()])
4220            .open_read_only()
4221            .expect("read-only open");
4222
4223        let blob = reader
4224            .row_blob_ref("files", "file-ro")
4225            .await
4226            .expect("capture read-only blob row");
4227        let read = reader
4228            .read_blob(&blob)
4229            .await
4230            .expect("read blob via read handle");
4231        assert_eq!(
4232            read, bytes,
4233            "the read handle serves the blob the writer stored"
4234        );
4235
4236        // A ranged read through the same handle serves the requested slice.
4237        let (offset, len) = (5u64, 10u64);
4238        let range = reader
4239            .open_blob_stream(&blob, offset, len)
4240            .await
4241            .expect("ranged read via read handle");
4242        assert_eq!(range, &bytes[offset as usize..(offset + len) as usize]);
4243        drop(writer);
4244    }
4245
4246    /// Concurrent same-path cache populate by two producers (the writer's and the
4247    /// reader's fetch both writing the same blob into `cache/`) is safe: the atomic
4248    /// temp-then-rename write means the destination is always one producer's whole
4249    /// file, never a torn interleave. This is the primitive requirement 6 rests on.
4250    #[tokio::test]
4251    async fn concurrent_same_blob_cache_writes_never_tear() {
4252        let tmp = tempfile::tempdir().expect("temp dir");
4253        let dir = StoreDir::new(tmp.path());
4254        let dest = dir
4255            .cache_blob_path(
4256                "media-files",
4257                coven_core::sync::store_commit::ObjectHash::digest(b"raceblob"),
4258            )
4259            .expect("cache path");
4260        if let Some(parent) = dest.parent() {
4261            std::fs::create_dir_all(parent).expect("create cache shard dir");
4262        }
4263
4264        // Two full-length payloads that differ everywhere, so any interleave would be
4265        // detectable as a mixed file. In production both producers write the same
4266        // blob's bytes; the distinct payloads only make tearing observable.
4267        let a = vec![b'A'; 64 * 1024];
4268        let b = vec![b'B'; 64 * 1024];
4269        let (ra, rb) = tokio::join!(
4270            crate::local_blob::write_atomic(&dest, &a),
4271            crate::local_blob::write_atomic(&dest, &b),
4272        );
4273        ra.expect("first concurrent cache write");
4274        rb.expect("second concurrent cache write");
4275
4276        let final_bytes = crate::local_blob::read(&dest)
4277            .await
4278            .expect("read cache file");
4279        assert!(
4280            final_bytes == a || final_bytes == b,
4281            "the cache file is exactly one producer's whole payload, never a torn mix",
4282        );
4283    }
4284}