1use std::num::NonZeroUsize;
5use std::path::{Path, PathBuf};
6use std::sync::Arc;
7
8use rusqlite::Connection;
9use tracing::{debug, warn};
10
11use crate::blob::local_files::LocalBlobError;
12use crate::blob::{BlobRef, BlobTransitionObserver};
13use crate::clock::{ClockRef, SystemClock};
14use crate::config::{Config, HomeStorage};
15use crate::custody::KeyCustody;
16use crate::database::{Database, DbError, OpenError};
17use crate::handle::CovenHandle;
18use crate::identity_custody::IdentityCustody;
19use crate::keys::StoreKeys;
20use crate::migration::{Migration, MigrationError};
21use crate::store_dir::PathTokenError;
22use crate::sync::hlc::UpdatedAtStamper;
23use crate::sync::session::SyncedTable;
24use crate::sync::sync_manager::ConfigProvider;
25use coven_core::{WriteId, WriteReceipt};
26
27pub type CovenResult<T> = Result<T, CovenError>;
28
29const LOCAL_STAGE_MARKER: &str = ".coven-stage-";
30
31#[derive(Debug, thiserror::Error)]
32pub enum CovenError {
33 #[error("database error: {0}")]
34 Database(#[from] DbError),
35 #[error("migration error: {0}")]
36 Migration(MigrationError),
37 #[error("sqlite error: {0}")]
38 Sqlite(#[from] rusqlite::Error),
39 #[error("blob error: {0}")]
40 Blob(String),
41 #[error("unsafe blob path: {0}")]
42 UnsafeBlobPath(#[from] PathTokenError),
43 #[error("malformed local path: {0}")]
44 MalformedPath(String),
45 #[error("the write SQL closure panicked")]
46 WriteClosurePanicked,
47 #[error("synced_tables must be set before opening a coven store")]
48 MissingSyncedTables,
49 #[error("migrations must be set before opening a coven store")]
50 MissingMigrations,
51 #[error("candidate resolution failed: {0}")]
52 CandidateResolution(String),
53 #[error("blob_tombstone_grace must be a positive duration")]
54 InvalidBlobTombstoneGrace,
55 #[error("browsable cloud storage cannot be used with scoped table {table:?}")]
56 BrowsableStorageWithScopedTable { table: String },
57 #[error("blob {namespace}/{id} is still referenced by a row after the write")]
58 BlobStillReferenced { namespace: String, id: String },
59 #[error("blob {namespace}/{id} is already referenced by a row")]
60 BlobAlreadyReferenced { namespace: String, id: String },
61 #[error("blob {namespace}/{id} is owned by an unpublished write")]
62 BlobOwnedByPendingWrite { namespace: String, id: String },
63 #[error("store is already open: {}", store_dir.display())]
64 AlreadyOpen { store_dir: PathBuf },
65 #[error("I/O error: {0}")]
66 Io(#[from] std::io::Error),
67}
68
69impl From<OpenError> for CovenError {
70 fn from(value: OpenError) -> Self {
71 match value {
72 OpenError::Migration(e) => CovenError::Migration(e),
73 OpenError::Db(e) => CovenError::Database(e),
74 }
75 }
76}
77
78impl From<LocalBlobError> for CovenError {
79 fn from(value: LocalBlobError) -> Self {
80 CovenError::Blob(value.to_string())
81 }
82}
83
84#[derive(Clone)]
85pub struct CovenConfig(ConfigProvider);
86
87impl CovenConfig {
88 fn current(&self) -> Config {
89 (self.0)()
90 }
91
92 fn provider(&self) -> ConfigProvider {
93 self.0.clone()
94 }
95}
96
97impl From<Config> for CovenConfig {
98 fn from(value: Config) -> Self {
99 let config = value;
100 Self(Arc::new(move || config.clone()))
101 }
102}
103
104impl<F> From<F> for CovenConfig
105where
106 F: Fn() -> Config + Send + Sync + 'static,
107{
108 fn from(value: F) -> Self {
109 Self(Arc::new(value))
110 }
111}
112
113pub struct Coven;
114
115impl Coven {
116 pub fn builder(config: impl Into<CovenConfig>) -> CovenBuilder {
117 let config = config.into();
118 let current = config.current();
119 CovenBuilder {
120 config,
121 synced_tables: None,
122 migrations: None,
123 blob_tombstone_grace: crate::blob::delete::BLOB_TOMBSTONE_GRACE,
124 max_concurrent_uploads: NonZeroUsize::MIN,
125 max_concurrent_downloads: NonZeroUsize::MIN,
126 clock: Arc::new(SystemClock),
127 key_service: StoreKeys::new(current.store_id),
128 key_custody: KeyCustody::Keyring,
129 identity_custody: IdentityCustody::Keyring,
130 cloudkit_ops: None,
131 observer: None,
132 }
133 }
134}
135
136pub struct CovenBuilder {
137 config: CovenConfig,
138 synced_tables: Option<Vec<SyncedTable>>,
139 migrations: Option<Vec<Migration>>,
140 blob_tombstone_grace: chrono::Duration,
141 max_concurrent_uploads: NonZeroUsize,
142 max_concurrent_downloads: NonZeroUsize,
143 clock: ClockRef,
144 key_service: StoreKeys,
145 key_custody: KeyCustody,
146 identity_custody: IdentityCustody,
147 cloudkit_ops: Option<Arc<dyn crate::storage::cloud::cloudkit::CloudKitOps>>,
148 observer: Option<Arc<dyn BlobTransitionObserver>>,
149}
150
151pub(crate) struct StoreOpenGuard {
171 _file: std::fs::File,
172}
173
174impl StoreOpenGuard {
175 #[cfg(test)]
177 pub(crate) fn acquire_for_test(store_dir: &crate::store_dir::StoreDir) -> std::sync::Arc<Self> {
178 std::sync::Arc::new(Self::acquire(store_dir).expect("acquire store open guard"))
179 }
180
181 pub(crate) fn acquire(store_dir: &crate::store_dir::StoreDir) -> CovenResult<Self> {
182 let db_path = store_dir.db_path();
183 let Some(dir) = db_path.parent() else {
184 return Err(CovenError::MalformedPath(format!(
185 "store database path has no parent: {}",
186 db_path.display()
187 )));
188 };
189 std::fs::create_dir_all(dir)?;
190 let file = std::fs::OpenOptions::new()
191 .read(true)
192 .write(true)
193 .create(true)
194 .truncate(false)
195 .open(dir.join(".coven-lock"))?;
196 match file.try_lock() {
197 Ok(()) => Ok(Self { _file: file }),
198 Err(std::fs::TryLockError::WouldBlock) => Err(CovenError::AlreadyOpen {
199 store_dir: dir.to_path_buf(),
200 }),
201 Err(std::fs::TryLockError::Error(error)) => Err(CovenError::Io(error)),
202 }
203 }
204}
205
206impl CovenBuilder {
207 pub fn synced_tables(mut self, tables: Vec<SyncedTable>) -> Self {
208 self.synced_tables = Some(tables);
209 self
210 }
211
212 pub fn blob_tombstone_grace(mut self, grace: chrono::Duration) -> Self {
218 self.blob_tombstone_grace = grace;
219 self
220 }
221
222 pub fn max_concurrent_uploads(mut self, n: NonZeroUsize) -> Self {
226 self.max_concurrent_uploads = n;
227 self
228 }
229
230 pub fn max_concurrent_downloads(mut self, n: NonZeroUsize) -> Self {
234 self.max_concurrent_downloads = n;
235 self
236 }
237
238 pub fn migrations(mut self, migrations: Vec<Migration>) -> Self {
242 self.migrations = Some(migrations);
243 self
244 }
245
246 pub fn clock(mut self, clock: ClockRef) -> Self {
247 self.clock = clock;
248 self
249 }
250
251 pub fn key_service(mut self, key_service: StoreKeys) -> Self {
252 self.key_service = key_service;
253 self
254 }
255
256 pub fn key_custody(mut self, custody: KeyCustody) -> Self {
262 self.key_custody = custody;
263 self
264 }
265
266 pub fn identity_custody(mut self, custody: IdentityCustody) -> Self {
274 self.identity_custody = custody;
275 self
276 }
277
278 pub fn cloudkit_ops(
279 mut self,
280 ops: Arc<dyn crate::storage::cloud::cloudkit::CloudKitOps>,
281 ) -> Self {
282 self.cloudkit_ops = Some(ops);
283 self
284 }
285
286 pub fn apply_cloudkit_ops(
287 mut self,
288 ops: Option<Arc<dyn crate::storage::cloud::cloudkit::CloudKitOps>>,
289 ) -> Self {
290 self.cloudkit_ops = ops;
291 self
292 }
293
294 pub fn observer(mut self, observer: Arc<dyn BlobTransitionObserver>) -> Self {
295 self.observer = Some(observer);
296 self
297 }
298
299 pub fn open(self) -> CovenResult<CovenHandle> {
310 let config = self.config.current();
311 let tables = self.synced_tables.ok_or(CovenError::MissingSyncedTables)?;
312 let migrations = self.migrations.ok_or(CovenError::MissingMigrations)?;
313 validate_storage_scope(&config, &tables)?;
314 if self.blob_tombstone_grace <= chrono::Duration::zero() {
315 return Err(CovenError::InvalidBlobTombstoneGrace);
316 }
317 let db_path = config.store_dir.db_path();
318 let provider = self.config.provider();
319 let store_dir = config.store_dir.clone();
320 let transfer_limits = crate::blob::TransferLimits {
321 uploads: self.max_concurrent_uploads,
322 downloads: self.max_concurrent_downloads,
323 };
324 let open_guard = Arc::new(StoreOpenGuard::acquire(&store_dir)?);
325 remove_orphaned_local_blob_temps(&store_dir, std::time::SystemTime::now())?;
326 let (db, stamper) = Database::open(
327 &db_path,
328 tables.clone(),
329 self.blob_tombstone_grace,
330 transfer_limits,
331 config.device_id.clone(),
332 &migrations,
333 )?;
334 let read_db = Database::open_read_only(
341 &db_path,
342 tables,
343 self.blob_tombstone_grace,
344 transfer_limits,
345 config.device_id.clone(),
346 &migrations,
347 )?;
348 let key_custody = self.key_custody.resolve(&config.store_id, &store_dir);
349 let identity_custody = self.identity_custody.resolve(&config.store_id, &store_dir);
350 Ok(CovenHandle::new(
351 db,
352 read_db,
353 stamper,
354 store_dir,
355 provider,
356 self.key_service,
357 key_custody,
358 identity_custody,
359 self.clock,
360 self.cloudkit_ops,
361 self.observer,
362 open_guard,
363 ))
364 }
365
366 pub fn open_read_only(self) -> CovenResult<crate::read_handle::CovenReadHandle> {
386 let config = self.config.current();
387 let tables = self.synced_tables.ok_or(CovenError::MissingSyncedTables)?;
388 let migrations = self.migrations.ok_or(CovenError::MissingMigrations)?;
389 validate_storage_scope(&config, &tables)?;
390 let db_path = config.store_dir.db_path();
391 let provider = self.config.provider();
392 let store_dir = config.store_dir.clone();
393 let db = Database::open_read_only(
397 &db_path,
398 tables,
399 self.blob_tombstone_grace,
400 crate::blob::TransferLimits {
401 uploads: self.max_concurrent_uploads,
402 downloads: self.max_concurrent_downloads,
403 },
404 config.device_id.clone(),
405 &migrations,
406 )?;
407 let key_custody = self.key_custody.resolve(&config.store_id, &store_dir);
408 let identity_custody = self.identity_custody.resolve(&config.store_id, &store_dir);
409 Ok(crate::read_handle::CovenReadHandle::new(
410 db,
411 store_dir,
412 provider,
413 self.key_service,
414 key_custody,
415 identity_custody,
416 self.clock,
417 self.cloudkit_ops,
418 ))
419 }
420}
421
422fn validate_storage_scope(config: &Config, tables: &[SyncedTable]) -> CovenResult<()> {
423 if config.cloud_home.storage == HomeStorage::Browsable {
424 if let Some(table) = tables
425 .iter()
426 .find(|table| table.audience_column().is_some())
427 {
428 return Err(CovenError::BrowsableStorageWithScopedTable {
429 table: table.name().to_string(),
430 });
431 }
432 }
433 Ok(())
434}
435
436pub struct SqlContext<'ctx, 'conn> {
456 tx: &'ctx rusqlite::Transaction<'conn>,
457 stamper: UpdatedAtStamper,
458}
459
460impl<'ctx, 'conn> SqlContext<'ctx, 'conn> {
461 pub(crate) fn new(tx: &'ctx rusqlite::Transaction<'conn>, stamper: UpdatedAtStamper) -> Self {
462 Self { tx, stamper }
463 }
464
465 pub fn execute<P>(&self, sql: &str, params: P) -> rusqlite::Result<usize>
467 where
468 P: rusqlite::Params,
469 {
470 self.tx.execute(sql, params)
471 }
472
473 pub fn execute_batch(&self, sql: &str) -> rusqlite::Result<()> {
475 self.tx.execute_batch(sql)
476 }
477
478 pub fn query_row<T, P, F>(&self, sql: &str, params: P, map: F) -> rusqlite::Result<T>
480 where
481 P: rusqlite::Params,
482 F: FnOnce(&rusqlite::Row<'_>) -> rusqlite::Result<T>,
483 {
484 self.tx.query_row(sql, params, map)
485 }
486
487 pub fn query<T, P, F>(&self, sql: &str, params: P, map: F) -> rusqlite::Result<Vec<T>>
489 where
490 P: rusqlite::Params,
491 F: FnMut(&rusqlite::Row<'_>) -> rusqlite::Result<T>,
492 {
493 let mut statement = self.tx.prepare(sql)?;
494 let values = statement.query_map(params, map)?.collect();
495 values
496 }
497
498 pub fn stamp(&self) -> String {
499 self.stamper.stamp()
500 }
501}
502
503type WriteSql<R> =
504 Box<dyn for<'ctx, 'conn> FnOnce(SqlContext<'ctx, 'conn>) -> CovenResult<R> + Send>;
505
506pub struct WriteBatch {
507 new_blobs: Vec<NewBlob>,
508 deleted_blobs: Vec<BlobRef>,
509}
510
511impl WriteBatch {
512 fn new() -> Self {
513 Self {
514 new_blobs: Vec::new(),
515 deleted_blobs: Vec::new(),
516 }
517 }
518
519 pub fn put_blob(
520 &mut self,
521 namespace: impl Into<String>,
522 id: impl Into<String>,
523 bytes: impl Into<Vec<u8>>,
524 ) {
525 self.new_blobs.push(NewBlob {
526 namespace: namespace.into(),
527 id: id.into(),
528 bytes: bytes.into(),
529 });
530 }
531
532 pub fn delete_blob(&mut self, blob: BlobRef) {
533 self.deleted_blobs.push(blob);
534 }
535}
536
537struct NewBlob {
538 namespace: String,
539 id: String,
540 bytes: Vec<u8>,
541}
542
543#[derive(Clone)]
544pub(crate) struct StagedBlob {
545 pub namespace: String,
546 pub id: String,
547 pub staged: PathBuf,
548 pub final_path: PathBuf,
549}
550
551impl CovenHandle {
552 pub async fn sql<F, R>(&self, f: F) -> CovenResult<WriteReceipt<R>>
553 where
554 F: for<'ctx, 'conn> FnOnce(SqlContext<'ctx, 'conn>) -> CovenResult<R> + Send + 'static,
555 R: Send + 'static,
556 {
557 let stamper = self.stamper();
558 let tables = self.db().synced_tables().to_vec();
559 let gates = self.db().gates();
560 let blob_decls = self.db().blob_decls();
561 let routing_encryption = gates
562 .has_scoped_graph()
563 .then(|| self.routing_encryption())
564 .transpose()?;
565 let blob_staging = crate::sync::store::HostWriteBlobStaging::new(
566 tokio::runtime::Handle::current(),
567 self.blob_storage().await.map_err(|error| error.to_string()),
568 self.store_dir(),
569 );
570 let write_id = self.db().new_write_id();
571 let outcome = self
572 .db()
573 .call(move |conn| {
574 Ok(
575 crate::sync::store::StoreDatabase::run_store_write_transaction_on(
576 conn,
577 &tables,
578 &gates,
579 &blob_decls,
580 routing_encryption.as_ref(),
581 Some(&blob_staging),
582 write_id,
583 |tx| f(SqlContext::new(tx, stamper)),
584 ),
585 )
586 })
587 .await
588 .map_err(CovenError::from)?;
589 outcome
590 }
591
592 pub async fn sql_read<F, R>(&self, f: F) -> CovenResult<R>
611 where
612 F: FnOnce(&rusqlite::Connection) -> CovenResult<R> + Send + 'static,
613 R: Send + 'static,
614 {
615 let outcome = self
616 .read_db()
617 .call(move |conn| Ok(f(conn)))
618 .await
619 .map_err(CovenError::from)?;
620 outcome
621 }
622
623 pub async fn write<F, S, R>(&self, f: F, sql: S) -> CovenResult<WriteReceipt<R>>
624 where
625 F: FnOnce(&mut WriteBatch) -> CovenResult<()> + Send + 'static,
626 S: for<'ctx, 'conn> FnOnce(SqlContext<'ctx, 'conn>) -> CovenResult<R> + Send + 'static,
627 R: Send + 'static,
628 {
629 let mut batch = WriteBatch::new();
630 f(&mut batch)?;
631 let sql: WriteSql<R> = Box::new(sql);
632 let staged = self.stage_blobs(batch.new_blobs).await?;
633 let staged_paths = staged
634 .iter()
635 .map(|blob| blob.staged.clone())
636 .collect::<Vec<_>>();
637 let tables = self.db().synced_tables().to_vec();
638 let db = self.db().clone();
639 let stamper = self.stamper();
640 let gates = self.db().gates();
641 let blob_decls = self.db().blob_decls();
642 let routing_encryption = gates
643 .has_scoped_graph()
644 .then(|| self.routing_encryption())
645 .transpose()?;
646 let blob_staging = crate::sync::store::HostWriteBlobStaging::new(
647 tokio::runtime::Handle::current(),
648 self.blob_storage().await.map_err(|error| error.to_string()),
649 self.store_dir(),
650 );
651 let write_id = self.db().new_write_id();
652 let deleted = batch.deleted_blobs;
653 let store_dir = self.store_dir();
654 let outcome = match db
655 .call(move |conn| {
656 Ok(run_write_batch_on_connection(
657 conn,
658 stamper,
659 store_dir,
660 staged,
661 deleted,
662 tables,
663 gates,
664 blob_decls,
665 routing_encryption,
666 blob_staging,
667 write_id,
668 sql,
669 ))
670 })
671 .await
672 {
673 Ok(outcome) => outcome,
674 Err(error) => {
675 remove_staged_paths(&staged_paths).await;
676 return Err(CovenError::from(error));
677 }
678 };
679 match outcome {
680 Ok(receipt) => {
681 if let Err(error) =
682 crate::blob::local_cleanup::drain(self.db(), &self.store_dir()).await
683 {
684 warn!(
685 error = %error,
686 "failed to drain local blob cleanup intents after write commit"
687 );
688 }
689 Ok(receipt)
690 }
691 Err(error) => {
692 remove_staged_paths(&staged_paths).await;
693 Err(error)
694 }
695 }
696 }
697
698 async fn stage_blobs(&self, blobs: Vec<NewBlob>) -> CovenResult<Vec<StagedBlob>> {
699 let mut staged = Vec::new();
700 for blob in blobs {
701 let final_path = self
702 .store_dir()
703 .local_blob_path(&blob.namespace, &blob.id)?;
704 let staged_path = local_stage_temp_path(&final_path)?;
705 if let Err(e) = crate::local_blob::write_atomic(&staged_path, &blob.bytes).await {
706 remove_staged_files(&staged).await;
707 return Err(CovenError::Blob(e));
708 }
709 staged.push(StagedBlob {
710 namespace: blob.namespace,
711 id: blob.id,
712 staged: staged_path,
713 final_path,
714 });
715 }
716 Ok(staged)
717 }
718}
719
720fn local_stage_temp_path(final_path: &Path) -> CovenResult<PathBuf> {
721 let file_name = final_path
722 .file_name()
723 .and_then(|name| name.to_str())
724 .ok_or_else(|| {
725 CovenError::MalformedPath(format!(
726 "local blob path has no file name: {}",
727 final_path.display()
728 ))
729 })?;
730 Ok(final_path.with_file_name(format!(
731 "{file_name}{LOCAL_STAGE_MARKER}{}",
732 uuid::Uuid::new_v4()
733 )))
734}
735
736fn remove_orphaned_local_blob_temps(
749 store_dir: &crate::store_dir::StoreDir,
750 process_start: std::time::SystemTime,
751) -> CovenResult<()> {
752 let storage = store_dir.storage_dir();
753 remove_orphaned_temps_in_dir(&storage.join("local"), &is_local_stage_temp, None)?;
754 for folder in ["local", "cache", "pinned"] {
755 remove_orphaned_temps_in_dir(
756 &storage.join(folder),
757 &crate::local_blob::is_temp_blob_path,
758 Some(process_start),
759 )?;
760 }
761 Ok(())
762}
763
764fn remove_orphaned_temps_in_dir(
769 dir: &Path,
770 is_temp: &dyn Fn(&Path) -> bool,
771 older_than: Option<std::time::SystemTime>,
772) -> CovenResult<()> {
773 let entries = match std::fs::read_dir(dir) {
774 Ok(entries) => entries,
775 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
776 debug!(
777 path = %dir.display(),
778 "blob directory absent during orphaned temp cleanup"
779 );
780 return Ok(());
781 }
782 Err(error) => return Err(CovenError::Io(error)),
783 };
784 for entry in entries {
785 let entry = entry?;
786 let path = entry.path();
787 let file_type = entry.file_type()?;
788 if file_type.is_dir() {
789 remove_orphaned_temps_in_dir(&path, is_temp, older_than)?;
790 } else if file_type.is_file() && is_temp(&path) {
791 if let Some(cutoff) = older_than {
792 let modified = entry.metadata()?.modified()?;
793 if modified >= cutoff {
794 debug!(
795 path = %path.display(),
796 "leaving fresh blob temp created at or after process start"
797 );
798 continue;
799 }
800 }
801 remove_file_if_present(&path)?;
802 } else if file_type.is_file() && path.file_name().and_then(|name| name.to_str()).is_none() {
803 debug!(
804 path = %path.display(),
805 "skipping blob path with non-utf8 file name during orphaned temp cleanup"
806 );
807 }
808 }
809 Ok(())
810}
811
812fn is_local_stage_temp(path: &Path) -> bool {
813 path.file_name()
814 .and_then(|name| name.to_str())
815 .is_some_and(|name| name.contains(LOCAL_STAGE_MARKER))
816}
817
818async fn remove_staged_files(staged: &[StagedBlob]) {
819 let paths = staged
820 .iter()
821 .map(|blob| blob.staged.clone())
822 .collect::<Vec<_>>();
823 remove_staged_paths(&paths).await;
824}
825
826async fn remove_staged_paths(paths: &[PathBuf]) {
827 for path in paths {
828 remove_staged_path(path).await;
829 }
830}
831
832async fn remove_staged_path(path: &Path) {
833 if let Err(error) = crate::local_blob::remove_file(path).await {
834 warn!(
835 path = %path.display(),
836 error = %error,
837 "failed to remove staged local blob"
838 );
839 }
840}
841
842fn remove_file_if_present(path: &Path) -> CovenResult<()> {
843 match std::fs::remove_file(path) {
844 Ok(()) => Ok(()),
845 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
846 debug!(
847 path = %path.display(),
848 "file already absent during local blob cleanup"
849 );
850 Ok(())
851 }
852 Err(error) => Err(CovenError::Io(error)),
853 }
854}
855
856fn sync_parent_dir(path: &Path) -> CovenResult<()> {
857 let parent = path.parent().ok_or_else(|| {
858 CovenError::MalformedPath(format!("path has no parent directory: {}", path.display()))
859 })?;
860 std::fs::File::open(parent)?.sync_all()?;
861 Ok(())
862}
863
864fn run_write_batch_on_connection<R>(
865 conn: &Connection,
866 stamper: UpdatedAtStamper,
867 store_dir: crate::store_dir::StoreDir,
868 staged: Vec<StagedBlob>,
869 deleted: Vec<BlobRef>,
870 tables: Vec<SyncedTable>,
871 gates: Arc<crate::sync::gate::Gates>,
872 decls: Arc<crate::blob::decl::BlobDecls>,
873 routing_encryption: Option<crate::encryption::EncryptionService>,
874 blob_staging: crate::sync::store::HostWriteBlobStaging,
875 write_id: WriteId,
876 sql: WriteSql<R>,
877) -> CovenResult<WriteReceipt<R>> {
878 let mut moved = Vec::new();
879 let result = crate::sync::store::StoreDatabase::run_store_write_transaction_on(
880 conn,
881 &tables,
882 &gates,
883 &decls,
884 routing_encryption.as_ref(),
885 Some(&blob_staging),
886 write_id,
887 |tx| -> CovenResult<R> {
888 let cleanup_intents = deleted
889 .iter()
890 .map(|blob| {
891 decls
892 .row_for_blob_in_namespace(tx, &blob.namespace, &blob.id)
893 .map_err(|error| CovenError::Blob(error.to_string()))
894 .map(|row| match row {
895 Some((table, row_id)) => {
896 crate::blob::local_cleanup::LocalBlobCleanupIntent::for_row(
897 &blob.namespace,
898 &blob.id,
899 table,
900 row_id,
901 )
902 }
903 None => crate::blob::local_cleanup::LocalBlobCleanupIntent::local(
904 &blob.namespace,
905 &blob.id,
906 ),
907 })
908 })
909 .collect::<CovenResult<Vec<_>>>()?;
910 for blob in &staged {
911 match decls.row_for_blob_in_namespace(tx, &blob.namespace, &blob.id) {
912 Ok(Some(_)) => {
913 return Err(CovenError::BlobAlreadyReferenced {
914 namespace: blob.namespace.clone(),
915 id: blob.id.clone(),
916 });
917 }
918 Ok(None) => {}
919 Err(e) => return Err(CovenError::Blob(e.to_string())),
920 }
921 let leased = tx
922 .query_row(
923 "SELECT EXISTS(\
924 SELECT 1 FROM store_write_blob_leases \
925 WHERE namespace = ?1 AND blob_id = ?2\
926 )",
927 (&blob.namespace, &blob.id),
928 |row| row.get::<_, bool>(0),
929 )
930 .map_err(CovenError::from)?;
931 if leased {
932 return Err(CovenError::BlobOwnedByPendingWrite {
933 namespace: blob.namespace.clone(),
934 id: blob.id.clone(),
935 });
936 }
937 if let Some(parent) = blob.final_path.parent() {
938 std::fs::create_dir_all(parent).map_err(|e| {
939 CovenError::Blob(format!(
940 "create local blob parent {}: {e}",
941 parent.display()
942 ))
943 })?;
944 }
945 std::fs::rename(&blob.staged, &blob.final_path).map_err(|e| {
946 CovenError::Blob(format!(
947 "install staged blob {} -> {}: {e}",
948 blob.staged.display(),
949 blob.final_path.display()
950 ))
951 })?;
952 moved.push(blob.clone());
953 sync_parent_dir(&blob.final_path).map_err(|e| {
954 CovenError::Blob(format!(
955 "sync local blob parent after installing {}: {e}",
956 blob.final_path.display()
957 ))
958 })?;
959 }
960
961 match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
962 sql(SqlContext::new(tx, stamper))
963 })) {
964 Ok(Ok(value)) => {
965 for (blob, intent) in deleted.iter().zip(&cleanup_intents) {
966 let _ = store_dir.local_blob_path(&blob.namespace, &blob.id)?;
967 if crate::blob::local_cleanup::logical_blob_is_referenced_on(
968 tx,
969 &decls,
970 &blob.namespace,
971 &blob.id,
972 )? {
973 return Err(CovenError::BlobStillReferenced {
974 namespace: blob.namespace.clone(),
975 id: blob.id.clone(),
976 });
977 }
978 crate::blob::local_cleanup::record_obsolete_copy_intents_on(
979 tx, &decls, intent,
980 )?;
981 }
982 Ok(value)
983 }
984 Ok(Err(error)) => Err(error),
985 Err(_) => Err(CovenError::WriteClosurePanicked),
986 }
987 },
988 );
989 match result {
990 Ok(value) => Ok(value),
991 Err(error) => rollback_write_batch(error, moved),
992 }
993}
994
995fn rollback_write_batch<R>(error: CovenError, moved: Vec<StagedBlob>) -> CovenResult<R> {
996 for blob in moved.iter().rev() {
997 if let Err(e) = std::fs::remove_file(&blob.final_path) {
998 warn!(
999 namespace = %blob.namespace,
1000 blob_id = %blob.id,
1001 path = %blob.final_path.display(),
1002 error = %e,
1003 "failed to remove installed local blob after write rollback"
1004 );
1005 }
1006 }
1007 Err(error)
1008}
1009
1010#[cfg(test)]
1011mod tests {
1012 use super::*;
1013
1014 use crate::blob::{BlobScope, CacheFill, Provenance};
1015 use crate::config::Config;
1016 use crate::keys::test_keyring;
1017 use crate::storage::cloud::test_utils::InMemoryCloudHome;
1018 use crate::storage::cloud::{
1019 BlobBody, BoxPartSink, CloudAccessOutcome, CloudAccessState, CloudHome, CloudHomeError,
1020 ExactSlotStorage, ObjectSlot, UploadProgress,
1021 };
1022 use crate::store_dir::StoreDir;
1023 use crate::sync::cloud_storage::CloudCipher;
1024 use crate::sync::session::BlobDecl;
1025 use crate::sync::test_helpers::{pull_into, query_text, row_exists, TestStore};
1026 use async_trait::async_trait;
1027 use rusqlite::params;
1028 use std::sync::atomic::{AtomicBool, Ordering};
1029 use std::time::Duration;
1030 use tokio::sync::mpsc;
1031 use tokio::sync::Notify;
1032
1033 fn config(dir: StoreDir) -> Config {
1034 Config::with_defaults(
1035 "lib-test".to_string(),
1036 "device-test".to_string(),
1037 dir,
1038 "Test".to_string(),
1039 )
1040 }
1041
1042 fn media_files_decl() -> BlobDecl {
1043 BlobDecl::new(
1044 "media-files",
1045 Provenance::HostProvided,
1046 CacheFill::CacheLazy,
1047 )
1048 .with_id_column("blob_id")
1049 }
1050
1051 fn files_table() -> SyncedTable {
1052 SyncedTable::new("files", coven_core::RowIdentity::SharedKey)
1053 .carries_blob(media_files_decl())
1054 }
1055
1056 fn remote_root_files_table() -> SyncedTable {
1057 SyncedTable::new("files", coven_core::RowIdentity::SharedKey)
1058 .remote_root()
1059 .carries_blob(media_files_decl())
1060 }
1061
1062 fn scoped_files_table() -> SyncedTable {
1063 SyncedTable::new("files", coven_core::RowIdentity::SharedKey)
1064 .scoped_by("audience")
1065 .carries_blob(media_files_decl())
1066 }
1067
1068 fn files_migration() -> Migration {
1069 Migration::sql(
1070 1,
1071 "test-schema",
1072 "CREATE TABLE files (
1073 id TEXT PRIMARY KEY,
1074 blob_id TEXT,
1075 size INTEGER NOT NULL,
1076 hash TEXT,
1077 _updated_at TEXT NOT NULL
1078 ) STRICT;",
1079 )
1080 }
1081
1082 fn scoped_files_migration() -> Migration {
1083 Migration::sql(
1084 1,
1085 "scoped-files",
1086 "CREATE TABLE files (
1087 id TEXT PRIMARY KEY,
1088 blob_id TEXT,
1089 size INTEGER NOT NULL,
1090 hash TEXT,
1091 audience TEXT,
1092 _updated_at TEXT NOT NULL
1093 ) STRICT;",
1094 )
1095 }
1096
1097 fn gated_roots_table() -> SyncedTable {
1098 SyncedTable::new("roots", coven_core::RowIdentity::SharedKey).gated_by("shared")
1099 }
1100
1101 fn gated_roots_migration() -> Migration {
1102 Migration::sql(
1103 1,
1104 "gated-roots",
1105 "CREATE TABLE roots (
1106 id TEXT PRIMARY KEY,
1107 title TEXT NOT NULL,
1108 shared INTEGER NOT NULL,
1109 _updated_at TEXT NOT NULL
1110 ) STRICT;",
1111 )
1112 }
1113
1114 fn open_gated_roots_handle() -> (tempfile::TempDir, CovenHandle) {
1115 let tmp = tempfile::tempdir().expect("temp dir");
1116 let handle = Coven::builder(config(StoreDir::new(tmp.path())))
1117 .synced_tables(vec![gated_roots_table()])
1118 .migrations(vec![gated_roots_migration()])
1119 .open()
1120 .expect("open gated handle");
1121 (tmp, handle)
1122 }
1123
1124 fn open_gated_roots_at(dir: StoreDir) -> CovenResult<CovenHandle> {
1125 Coven::builder(config(dir))
1126 .synced_tables(vec![gated_roots_table()])
1127 .migrations(vec![gated_roots_migration()])
1128 .open()
1129 }
1130
1131 fn precreate_database(dir: &StoreDir, sql: &str) {
1132 let conn = rusqlite::Connection::open(dir.db_path()).expect("precreate store database");
1133 conn.execute_batch(sql)
1134 .expect("seed pre-existing database state");
1135 }
1136
1137 #[test]
1138 fn precreated_empty_sqlite_file_initializes_coven_metadata() {
1139 let tmp = tempfile::tempdir().expect("temp dir");
1140 let dir = StoreDir::new(tmp.path());
1141 precreate_database(&dir, "");
1142
1143 open_gated_roots_at(dir).expect("initialize Coven in an empty SQLite database");
1144 }
1145
1146 #[test]
1147 fn existing_host_tables_without_a_coven_marker_initialize_coven_metadata() {
1148 let tmp = tempfile::tempdir().expect("temp dir");
1149 let dir = StoreDir::new(tmp.path());
1150 precreate_database(
1151 &dir,
1152 "CREATE TABLE roots (
1153 id TEXT PRIMARY KEY,
1154 title TEXT NOT NULL,
1155 shared INTEGER NOT NULL,
1156 _updated_at TEXT NOT NULL
1157 ) STRICT;
1158 PRAGMA user_version = 1;",
1159 );
1160
1161 open_gated_roots_at(dir).expect("initialize Coven beside an existing host schema");
1162 }
1163
1164 #[test]
1165 fn interrupted_coven_schema_without_a_marker_is_rejected() {
1166 let tmp = tempfile::tempdir().expect("temp dir");
1167 let dir = StoreDir::new(tmp.path());
1168 precreate_database(
1169 &dir,
1170 "CREATE TABLE protocol_state (
1171 key TEXT PRIMARY KEY,
1172 value TEXT NOT NULL
1173 ) STRICT;",
1174 );
1175
1176 let error = match open_gated_roots_at(dir) {
1177 Ok(_) => panic!("partial Coven schema has no valid initialization commit"),
1178 Err(error) => error,
1179 };
1180
1181 assert!(matches!(
1182 error,
1183 CovenError::Database(DbError::Message(reason))
1184 if reason.contains("without the required initialization marker")
1185 ));
1186 }
1187
1188 #[tokio::test]
1189 async fn host_sql_cannot_discover_or_mutate_the_gate_baseline() {
1190 let (_tmp, handle) = open_gated_roots_handle();
1191
1192 let discovery = handle
1193 .sql(|sql| {
1194 sql.query("PRAGMA database_list", [], |row| row.get::<_, String>(1))
1195 .map_err(CovenError::from)
1196 })
1197 .await;
1198 assert!(
1199 discovery.is_err(),
1200 "host SQL must not enumerate coven's attached gate baseline",
1201 );
1202
1203 let mutation = handle
1204 .sql(|sql| {
1205 sql.execute(
1206 "INSERT INTO coven_gate_empty.roots \
1207 (id, title, shared, _updated_at) \
1208 VALUES ('root-1', 'Private', 1, '0000000002000-0000-device-test')",
1209 [],
1210 )?;
1211 Ok(())
1212 })
1213 .await;
1214 assert!(
1215 mutation.is_err(),
1216 "host SQL must not address coven's attached gate baseline",
1217 );
1218
1219 handle
1220 .sql(|sql| {
1221 sql.execute_batch(
1222 "CREATE TABLE host_local (id TEXT PRIMARY KEY, value TEXT) STRICT; \
1223 INSERT INTO host_local VALUES ('local-1', 'kept'); \
1224 INSERT INTO roots (id, title, shared, _updated_at) \
1225 VALUES ('root-1', 'Private', 0, \
1226 '0000000001000-0000-device-test');",
1227 )?;
1228 Ok(())
1229 })
1230 .await
1231 .expect("arbitrary host-schema SQL remains available");
1232 let published = handle
1233 .sql(|sql| {
1234 sql.execute(
1235 "UPDATE roots SET shared = 1, \
1236 _updated_at = '0000000002000-0000-device-test' \
1237 WHERE id = 'root-1'",
1238 [],
1239 )?;
1240 Ok(())
1241 })
1242 .await
1243 .expect("flip root visible");
1244 let write_id = published.write_id.clone();
1245 let changeset = handle
1246 .db()
1247 .call(move |conn| {
1248 conn.query_row(
1249 "SELECT changeset FROM store_writes WHERE write_id = ?1",
1250 [write_id.as_str()],
1251 |row| row.get::<_, Vec<u8>>(0),
1252 )
1253 .map_err(crate::database::DbError::from)
1254 })
1255 .await
1256 .expect("load gated changeset");
1257 let rows = coven_core::changeset::walk(&changeset).expect("walk gated changeset");
1258 assert_eq!(rows.len(), 1);
1259 assert_eq!(rows[0].op, coven_core::changeset::ChangeOp::Insert);
1260 assert_eq!(rows[0].pk(), Some("root-1"));
1261 assert_eq!(rows[0].col(1), Some("Private"));
1262 assert_eq!(rows[0].col(2), Some("1"));
1263 assert_eq!(rows[0].col(3), Some("0000000002000-0000-device-test"));
1264 }
1265
1266 fn open_files_handle() -> (tempfile::TempDir, CovenHandle) {
1267 let tmp = tempfile::tempdir().expect("temp dir");
1268 let dir = StoreDir::new(tmp.path());
1269 let handle = Coven::builder(config(dir))
1270 .synced_tables(vec![files_table()])
1271 .migrations(vec![files_migration()])
1272 .open()
1273 .expect("open handle");
1274 (tmp, handle)
1275 }
1276
1277 #[tokio::test]
1278 async fn second_open_of_one_store_is_refused_until_the_first_handle_drops() {
1279 let tmp = tempfile::tempdir().expect("temp dir");
1280 let dir = StoreDir::new(tmp.path());
1281 let first = Coven::builder(config(dir.clone()))
1282 .synced_tables(vec![files_table()])
1283 .migrations(vec![files_migration()])
1284 .open()
1285 .expect("first open succeeds");
1286 let clone = first.clone();
1287
1288 let second = Coven::builder(config(dir.clone()))
1289 .synced_tables(vec![files_table()])
1290 .migrations(vec![files_migration()])
1291 .open();
1292 assert!(matches!(
1293 second,
1294 Err(CovenError::AlreadyOpen { store_dir }) if store_dir == tmp.path()
1295 ));
1296
1297 drop(first);
1298
1299 let still_locked = Coven::builder(config(dir.clone()))
1300 .synced_tables(vec![files_table()])
1301 .migrations(vec![files_migration()])
1302 .open();
1303 assert!(matches!(
1304 still_locked,
1305 Err(CovenError::AlreadyOpen { store_dir }) if store_dir == tmp.path()
1306 ));
1307
1308 drop(clone);
1309
1310 Coven::builder(config(dir))
1311 .synced_tables(vec![files_table()])
1312 .migrations(vec![files_migration()])
1313 .open()
1314 .expect("open succeeds after the first handle drops");
1315 }
1316
1317 #[tokio::test]
1318 async fn a_zero_or_negative_blob_tombstone_grace_is_refused_at_open() {
1319 for grace in [chrono::Duration::zero(), chrono::Duration::seconds(-1)] {
1320 let tmp = tempfile::tempdir().expect("temp dir");
1321 let dir = StoreDir::new(tmp.path());
1322 let result = Coven::builder(config(dir))
1323 .synced_tables(vec![files_table()])
1324 .migrations(vec![files_migration()])
1325 .blob_tombstone_grace(grace)
1326 .open();
1327 assert!(
1328 matches!(result, Err(CovenError::InvalidBlobTombstoneGrace)),
1329 "grace {grace:?} must be refused at open",
1330 );
1331 }
1332 }
1333
1334 #[tokio::test]
1335 async fn a_positive_blob_tombstone_grace_opens() {
1336 let tmp = tempfile::tempdir().expect("temp dir");
1337 let dir = StoreDir::new(tmp.path());
1338 Coven::builder(config(dir))
1339 .synced_tables(vec![files_table()])
1340 .migrations(vec![files_migration()])
1341 .blob_tombstone_grace(chrono::Duration::hours(1))
1342 .open()
1343 .expect("a positive grace opens");
1344 }
1345
1346 fn open_remote_root_files_handle() -> (tempfile::TempDir, CovenHandle) {
1347 let tmp = tempfile::tempdir().expect("temp dir");
1348 let dir = StoreDir::new(tmp.path());
1349 let handle = Coven::builder(config(dir))
1350 .synced_tables(vec![remote_root_files_table()])
1351 .migrations(vec![files_migration()])
1352 .open()
1353 .expect("open handle");
1354 (tmp, handle)
1355 }
1356
1357 fn open_files_handle_in(dir: StoreDir) -> CovenHandle {
1358 try_open(&dir).expect("open handle")
1359 }
1360
1361 async fn run_test_cycle(storage: &TestStore, handle: &CovenHandle) {
1362 storage
1363 .publish_pending(handle.db(), &handle.store_dir())
1364 .await
1365 .expect("publish pending Store write");
1366 }
1367
1368 async fn merge_test_storage(
1369 handle: &CovenHandle,
1370 keypair: &crate::keys::UserKeypair,
1371 ) -> TestStore {
1372 TestStore::create(handle.db(), "lib-test", keypair.clone())
1373 .await
1374 .expect("create exact test Store")
1375 }
1376
1377 async fn publish_current_writes(handle: &CovenHandle) {
1378 let keypair = crate::keys::UserKeypair::generate();
1379 let storage = merge_test_storage(handle, &keypair).await;
1380 run_test_cycle(&storage, handle).await;
1381 }
1382
1383 #[tokio::test]
1384 async fn write_survives_reopen_before_sync_cycle() {
1385 let tmp = tempfile::tempdir().expect("temp dir");
1386 let dir = StoreDir::new(tmp.path());
1387 let handle = open_files_handle_in(dir.clone());
1388 handle
1389 .sql(|sql| {
1390 sql.execute(
1391 "INSERT INTO files (id, blob_id, size, _updated_at) \
1392 VALUES ('file-before-reopen', NULL, 0, ?1)",
1393 [sql.stamp()],
1394 )?;
1395 Ok(())
1396 })
1397 .await
1398 .expect("write before reopen");
1399 drop(handle);
1400
1401 let reopened = open_files_handle_in(dir);
1402 let keypair = crate::keys::UserKeypair::generate();
1403 let storage = merge_test_storage(&reopened, &keypair).await;
1404 run_test_cycle(&storage, &reopened).await;
1405
1406 let (_peer_tmp, peer) = open_files_handle();
1407 pull_into(peer.db(), &storage, &peer.store_dir()).await;
1408 assert_eq!(
1409 query_text(
1410 peer.db(),
1411 "SELECT id FROM files WHERE id = 'file-before-reopen'"
1412 )
1413 .await,
1414 "file-before-reopen",
1415 );
1416 }
1417
1418 #[tokio::test]
1419 async fn separate_host_transactions_publish_as_separate_store_commits_after_restart() {
1420 let tmp = tempfile::tempdir().expect("temp dir");
1421 let dir = StoreDir::new(tmp.path());
1422 let handle = open_files_handle_in(dir.clone());
1423 let mut write_ids = Vec::new();
1424 for id in ["file-pending-a", "file-pending-b"] {
1425 let receipt = handle
1426 .sql(move |sql| {
1427 sql.execute(
1428 "INSERT INTO files (id, blob_id, size, _updated_at) VALUES (?1, NULL, 0, ?2)",
1429 (id, sql.stamp()),
1430 )?;
1431 Ok(())
1432 })
1433 .await
1434 .expect("write before reopen");
1435 assert_eq!(receipt.status, coven_core::WriteStatus::Pending);
1436 write_ids.push(receipt.write_id);
1437 }
1438 drop(handle);
1439
1440 let reopened = open_files_handle_in(dir);
1441 let pending = reopened.pending_writes().await.expect("pending writes");
1442 assert_eq!(pending.len(), 2);
1443 assert_eq!(pending[0].write_id, write_ids[0]);
1444 assert_eq!(pending[1].write_id, write_ids[1]);
1445 let mut first_status = reopened
1446 .subscribe_write_status(&write_ids[0])
1447 .await
1448 .expect("subscribe after restart");
1449 assert_eq!(*first_status.borrow(), coven_core::WriteStatus::Pending);
1450 let keypair = crate::keys::UserKeypair::generate();
1451 let storage = merge_test_storage(&reopened, &keypair).await;
1452 run_test_cycle(&storage, &reopened).await;
1453
1454 first_status.changed().await.expect("published status");
1455 let first_sequence = match &*first_status.borrow() {
1456 coven_core::WriteStatus::Published(position) => position.commit().coord.sequence(),
1457 status => panic!("first host transaction is not published: {status:?}"),
1458 };
1459 assert_eq!(
1460 reopened
1461 .write_status(&write_ids[1])
1462 .await
1463 .expect("second status after first publication"),
1464 coven_core::WriteStatus::Pending,
1465 );
1466 run_test_cycle(&storage, &reopened).await;
1467 let second_sequence = match reopened
1468 .write_status(&write_ids[1])
1469 .await
1470 .expect("second status")
1471 {
1472 coven_core::WriteStatus::Published(position) => position.commit().coord.sequence(),
1473 status => panic!("second host transaction is not published: {status:?}"),
1474 };
1475 assert_eq!(second_sequence, first_sequence + 1);
1476 assert!(reopened
1477 .pending_writes()
1478 .await
1479 .expect("published writes are not pending")
1480 .is_empty());
1481
1482 let (_peer_tmp, peer) = open_files_handle();
1483 pull_into(peer.db(), &storage, &peer.store_dir()).await;
1484 assert!(row_exists(peer.db(), "SELECT 1 FROM files WHERE id = 'file-pending-a'").await);
1485 assert!(row_exists(peer.db(), "SELECT 1 FROM files WHERE id = 'file-pending-b'").await);
1486 }
1487
1488 #[tokio::test]
1489 async fn device_local_transaction_is_local_only_and_never_pending() {
1490 let (_tmp, handle) = open_files_handle();
1491 let receipt = handle
1492 .sql(|sql| {
1493 sql.execute_batch(
1494 "CREATE TABLE local_notes (id TEXT PRIMARY KEY, body TEXT) STRICT;
1495 INSERT INTO local_notes VALUES ('local-1', 'private');",
1496 )?;
1497 Ok("saved")
1498 })
1499 .await
1500 .expect("local transaction");
1501
1502 assert_eq!(receipt.value, "saved");
1503 assert_eq!(receipt.status, coven_core::WriteStatus::LocalOnly);
1504 assert_eq!(
1505 handle
1506 .write_status(&receipt.write_id)
1507 .await
1508 .expect("durable local status"),
1509 coven_core::WriteStatus::LocalOnly
1510 );
1511 assert!(handle
1512 .pending_writes()
1513 .await
1514 .expect("pending writes")
1515 .is_empty());
1516 let local_write_id = receipt.write_id.clone();
1517 let lease_count = handle
1518 .db()
1519 .call(move |conn| {
1520 conn.query_row(
1521 "SELECT COUNT(*) FROM store_write_blob_leases WHERE write_id = ?1",
1522 [local_write_id.as_str()],
1523 |row| row.get::<_, i64>(0),
1524 )
1525 .map_err(crate::database::DbError::from)
1526 })
1527 .await
1528 .expect("count local-only blob leases");
1529 assert_eq!(lease_count, 0);
1530
1531 let keypair = crate::keys::UserKeypair::generate();
1532 let storage = merge_test_storage(&handle, &keypair).await;
1533 run_test_cycle(&storage, &handle).await;
1534 assert_eq!(
1535 handle
1536 .write_status(&receipt.write_id)
1537 .await
1538 .expect("local status after sync"),
1539 coven_core::WriteStatus::LocalOnly
1540 );
1541 assert!(handle
1542 .pending_writes()
1543 .await
1544 .expect("pending writes after sync")
1545 .is_empty());
1546 }
1547
1548 #[tokio::test]
1549 async fn mixed_transaction_tracks_and_publishes_only_shared_rows() {
1550 let (_tmp, handle) = open_files_handle();
1551 let receipt = handle
1552 .sql(|sql| {
1553 sql.execute_batch(
1554 "CREATE TABLE local_notes (id TEXT PRIMARY KEY, body TEXT) STRICT;
1555 INSERT INTO local_notes VALUES ('local-1', 'private');",
1556 )?;
1557 sql.execute(
1558 "INSERT INTO files (id, blob_id, size, _updated_at)
1559 VALUES ('shared-1', NULL, 0, ?1)",
1560 [sql.stamp()],
1561 )?;
1562 Ok(())
1563 })
1564 .await
1565 .expect("mixed transaction");
1566
1567 assert_eq!(receipt.status, coven_core::WriteStatus::Pending);
1568 let pending = handle.pending_writes().await.expect("pending mixed write");
1569 assert_eq!(pending.len(), 1);
1570 assert_eq!(pending[0].write_id, receipt.write_id);
1571 assert_eq!(
1572 pending[0].affected_rows,
1573 vec![coven_core::AffectedRow {
1574 table: "files".to_string(),
1575 primary_key: "shared-1".to_string(),
1576 }]
1577 );
1578
1579 let keypair = crate::keys::UserKeypair::generate();
1580 let storage = merge_test_storage(&handle, &keypair).await;
1581 run_test_cycle(&storage, &handle).await;
1582 assert!(matches!(
1583 handle
1584 .write_status(&receipt.write_id)
1585 .await
1586 .expect("published mixed write"),
1587 coven_core::WriteStatus::Published(_)
1588 ));
1589
1590 let (_peer_tmp, peer) = open_files_handle();
1591 pull_into(peer.db(), &storage, &peer.store_dir()).await;
1592 assert!(row_exists(peer.db(), "SELECT 1 FROM files WHERE id = 'shared-1'").await);
1593 assert!(
1594 !row_exists(
1595 peer.db(),
1596 "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'local_notes'"
1597 )
1598 .await
1599 );
1600 assert!(
1601 row_exists(
1602 handle.db(),
1603 "SELECT 1 FROM local_notes WHERE id = 'local-1'"
1604 )
1605 .await
1606 );
1607 }
1608
1609 #[tokio::test]
1610 async fn delete_survives_reopen_before_sync_cycle() {
1611 let tmp = tempfile::tempdir().expect("temp dir");
1612 let dir = StoreDir::new(tmp.path());
1613 let handle = open_files_handle_in(dir.clone());
1614 handle
1615 .sql(|sql| {
1616 sql.execute(
1617 "INSERT INTO files (id, blob_id, size, _updated_at) \
1618 VALUES ('file-delete-reopen', NULL, 0, ?1)",
1619 [sql.stamp()],
1620 )?;
1621 Ok(())
1622 })
1623 .await
1624 .expect("insert before first cycle");
1625 let keypair = crate::keys::UserKeypair::generate();
1626 let storage = merge_test_storage(&handle, &keypair).await;
1627 run_test_cycle(&storage, &handle).await;
1628
1629 let (_peer_tmp, peer) = open_files_handle();
1630 pull_into(peer.db(), &storage, &peer.store_dir()).await;
1631 assert!(
1632 row_exists(
1633 peer.db(),
1634 "SELECT 1 FROM files WHERE id = 'file-delete-reopen'"
1635 )
1636 .await,
1637 "the peer receives the insert before the delete",
1638 );
1639
1640 handle
1641 .sql(|sql| {
1642 sql.execute("DELETE FROM files WHERE id = 'file-delete-reopen'", [])?;
1643 Ok(())
1644 })
1645 .await
1646 .expect("delete before reopen");
1647 drop(handle);
1648
1649 let reopened = open_files_handle_in(dir);
1650 run_test_cycle(&storage, &reopened).await;
1651
1652 pull_into(peer.db(), &storage, &peer.store_dir()).await;
1653 assert!(
1654 !row_exists(
1655 peer.db(),
1656 "SELECT 1 FROM files WHERE id = 'file-delete-reopen'"
1657 )
1658 .await,
1659 "the delete changeset reaches the peer after reopening",
1660 );
1661 }
1662
1663 #[tokio::test]
1664 async fn pending_write_drains_only_after_changeset_push() {
1665 let (_tmp, handle) = open_files_handle();
1666 let receipt = handle
1667 .sql(|sql| {
1668 sql.execute(
1669 "INSERT INTO files (id, blob_id, size, _updated_at) \
1670 VALUES ('file-retry-publish', NULL, 0, ?1)",
1671 [sql.stamp()],
1672 )?;
1673 Ok(())
1674 })
1675 .await
1676 .expect("write before failed push");
1677 let keypair = crate::keys::UserKeypair::generate();
1678 let storage = merge_test_storage(&handle, &keypair).await;
1679 storage.home.fail_exact_create_before_call(1);
1680
1681 let first = storage
1682 .publish_pending(handle.db(), &handle.store_dir())
1683 .await;
1684 assert!(
1685 first
1686 .as_ref()
1687 .is_err_and(|error| error.contains("forced failure before exact create")),
1688 "the first cycle must report the append failure while preserving its outbox: {first:?}",
1689 );
1690 assert_eq!(
1691 handle
1692 .write_status(&receipt.write_id)
1693 .await
1694 .expect("write status after failed append"),
1695 coven_core::WriteStatus::Publishing,
1696 );
1697
1698 run_test_cycle(&storage, &handle).await;
1699 assert!(
1700 matches!(
1701 handle
1702 .write_status(&receipt.write_id)
1703 .await
1704 .expect("write status after retry"),
1705 coven_core::WriteStatus::Published(_)
1706 ),
1707 "the pending write is published as an immutable Store commit after retry",
1708 );
1709 }
1710
1711 async fn write_raw_file(path: &std::path::Path, bytes: &[u8]) {
1712 crate::local_blob::write_atomic(path, bytes)
1713 .await
1714 .expect("write file");
1715 }
1716
1717 async fn cleanup_intent_count(handle: &CovenHandle, namespace: &str, id: &str) -> i64 {
1718 let namespace = namespace.to_string();
1719 let id = id.to_string();
1720 handle
1721 .sql(move |sql| {
1722 sql.query_row(
1723 "SELECT count(*) FROM local_cleanup_intents \
1724 WHERE namespace = ?1 AND blob_id = ?2",
1725 params![namespace, id],
1726 |row| row.get(0),
1727 )
1728 .map_err(CovenError::from)
1729 })
1730 .await
1731 .expect("count cleanup intents")
1732 .value
1733 }
1734
1735 #[tokio::test]
1736 async fn builder_open_runs_coven_and_host_migrations() {
1737 let (_tmp, handle) = open_files_handle();
1738 let has_coven_table: i64 = handle
1739 .sql(|sql| {
1740 sql.query_row(
1741 "SELECT count(*) FROM sqlite_master WHERE type = 'table' AND name = 'protocol_state'",
1742 [],
1743 |row| row.get(0),
1744 ).map_err(CovenError::from)
1745 })
1746 .await
1747 .expect("query coven table")
1748 .value;
1749 let has_host_table: i64 = handle
1750 .sql(|sql| {
1751 sql.query_row(
1752 "SELECT count(*) FROM sqlite_master WHERE type = 'table' AND name = 'files'",
1753 [],
1754 |row| row.get(0),
1755 )
1756 .map_err(CovenError::from)
1757 })
1758 .await
1759 .expect("query host table")
1760 .value;
1761 assert_eq!(has_coven_table, 1);
1762 assert_eq!(has_host_table, 1);
1763 }
1764
1765 #[tokio::test]
1766 async fn open_of_a_too_new_db_yields_the_matchable_migration_variant() {
1767 let tmp = tempfile::tempdir().expect("temp dir");
1768 let dir = StoreDir::new(tmp.path());
1769
1770 let ahead = Coven::builder(config(dir.clone()))
1772 .synced_tables(vec![files_table()])
1773 .migrations(vec![
1774 files_migration(),
1775 Migration::sql(2, "add-extra", "CREATE TABLE extra (id TEXT PRIMARY KEY)"),
1776 ])
1777 .open()
1778 .expect("open at version 2");
1779 drop(ahead);
1780
1781 let reopened = Coven::builder(config(dir))
1785 .synced_tables(vec![files_table()])
1786 .migrations(vec![files_migration()])
1787 .open();
1788 assert!(matches!(
1789 reopened,
1790 Err(CovenError::Migration(MigrationError::SchemaTooNew {
1791 current: 2,
1792 supported: 1
1793 }))
1794 ));
1795 }
1796
1797 #[tokio::test]
1798 async fn sql_reads_writes_and_stamps() {
1799 let (_tmp, handle) = open_files_handle();
1800 let id = "file-sql".to_string();
1801 handle
1802 .sql(move |sql| {
1803 sql.execute(
1804 "INSERT INTO files (id, blob_id, size, _updated_at) VALUES (?1, NULL, 0, ?2)",
1805 params![id, sql.stamp()],
1806 )?;
1807 Ok(())
1808 })
1809 .await
1810 .expect("insert through sql");
1811 let count: i64 = handle
1812 .sql(|sql| {
1813 sql.query_row("SELECT count(*) FROM files", [], |row| row.get(0))
1814 .map_err(CovenError::from)
1815 })
1816 .await
1817 .expect("count rows")
1818 .value;
1819 assert_eq!(count, 1);
1820 }
1821
1822 #[tokio::test]
1823 async fn sql_surfaces_sqlite_constraint_typed() {
1824 let (_tmp, handle) = open_files_handle();
1825 handle
1826 .sql(|sql| {
1827 sql.execute(
1828 "INSERT INTO files (id, blob_id, size, _updated_at) VALUES (?1, NULL, 0, ?2)",
1829 params!["duplicate-id", sql.stamp()],
1830 )?;
1831 Ok(())
1832 })
1833 .await
1834 .expect("seed row");
1835
1836 let result: CovenResult<WriteReceipt<()>> = handle
1837 .sql(|sql| {
1838 sql.execute(
1839 "INSERT INTO files (id, blob_id, size, _updated_at) VALUES (?1, NULL, 0, ?2)",
1840 params!["duplicate-id", sql.stamp()],
1841 )?;
1842 Ok(())
1843 })
1844 .await;
1845
1846 assert!(matches!(result, Err(CovenError::Sqlite(_))));
1847 }
1848
1849 #[tokio::test]
1850 async fn sql_read_sees_a_committed_write() {
1851 let (_tmp, handle) = open_files_handle();
1852 handle
1853 .sql(|sql| {
1854 sql.execute(
1855 "INSERT INTO files (id, blob_id, size, _updated_at) \
1856 VALUES ('file-read-your-write', NULL, 0, ?1)",
1857 [sql.stamp()],
1858 )?;
1859 Ok(())
1860 })
1861 .await
1862 .expect("insert through sql");
1863
1864 let id: String = handle
1867 .sql_read(|conn| {
1868 conn.query_row(
1869 "SELECT id FROM files WHERE id = 'file-read-your-write'",
1870 [],
1871 |row| row.get(0),
1872 )
1873 .map_err(CovenError::from)
1874 })
1875 .await
1876 .expect("read the committed row back through sql_read");
1877 assert_eq!(id, "file-read-your-write");
1878 }
1879
1880 #[tokio::test]
1881 async fn sql_read_cannot_write() {
1882 let (_tmp, handle) = open_files_handle();
1883 let result: CovenResult<()> = handle
1884 .sql_read(|conn| {
1885 conn.execute(
1886 "INSERT INTO files (id, blob_id, size, _updated_at) \
1887 VALUES ('file-read-only', NULL, 0, '0')",
1888 [],
1889 )
1890 .map_err(CovenError::from)?;
1891 Ok(())
1892 })
1893 .await;
1894 match result {
1898 Err(CovenError::Sqlite(rusqlite::Error::SqliteFailure(err, _))) => {
1899 assert_eq!(err.code, rusqlite::ErrorCode::ReadOnly);
1900 }
1901 other => panic!("expected a readonly sqlite failure, got {other:?}"),
1902 }
1903
1904 let count: i64 = handle
1905 .sql_read(|conn| {
1906 conn.query_row(
1907 "SELECT count(*) FROM files WHERE id = 'file-read-only'",
1908 [],
1909 |row| row.get(0),
1910 )
1911 .map_err(CovenError::from)
1912 })
1913 .await
1914 .expect("count rows through sql_read");
1915 assert_eq!(count, 0, "the refused write left no row behind");
1916 }
1917
1918 #[tokio::test]
1919 async fn open_on_a_fresh_store_serves_sql_read() {
1920 let tmp = tempfile::tempdir().expect("temp dir");
1925 let dir = StoreDir::new(tmp.path());
1926 let handle = Coven::builder(config(dir))
1927 .synced_tables(vec![files_table()])
1928 .migrations(vec![files_migration()])
1929 .open()
1930 .expect("open on an empty directory");
1931 let count: i64 = handle
1932 .sql_read(|conn| {
1933 conn.query_row("SELECT count(*) FROM files", [], |row| row.get(0))
1934 .map_err(CovenError::from)
1935 })
1936 .await
1937 .expect("sql_read on a fresh store");
1938 assert_eq!(count, 0);
1939 }
1940
1941 #[tokio::test]
1942 async fn open_removes_orphaned_local_blob_temps() {
1943 let tmp = tempfile::tempdir().expect("temp dir");
1944 let dir = StoreDir::new(tmp.path());
1945 let final_path = dir
1946 .local_blob_path("media-files", "tempaaaa")
1947 .expect("local path");
1948 let temp = local_stage_temp_path(&final_path).expect("stage temp path");
1949 write_raw_file(&temp, b"interrupted write").await;
1950
1951 let _handle = Coven::builder(config(dir.clone()))
1952 .synced_tables(vec![files_table()])
1953 .migrations(vec![files_migration()])
1954 .open()
1955 .expect("open handle");
1956
1957 assert!(
1958 !temp.exists(),
1959 "open removes orphaned local blob staging temps"
1960 );
1961 assert!(
1962 !dir.local_blob_path("media-files", "tempaaaa")
1963 .expect("local path")
1964 .exists(),
1965 "the interrupted blob has no committed final file"
1966 );
1967 }
1968
1969 #[tokio::test]
1970 async fn write_inserts_row_and_host_provided_blob() {
1971 let (_tmp, handle) = open_files_handle();
1972 let bytes = b"piece-bytes".to_vec();
1973 let hash = crate::blob::content_hash(&bytes);
1974 handle
1975 .write(
1976 {
1977 let bytes = bytes.clone();
1978 move |w| {
1979 w.put_blob("media-files", "blobaaaa", bytes);
1980 Ok(())
1981 }
1982 },
1983 move |sql| {
1984 sql.execute(
1985 "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
1986 VALUES (?1, ?2, ?3, ?4, ?5)",
1987 params!["file-1", "blobaaaa", bytes.len() as i64, hash, sql.stamp()],
1988 )?;
1989 Ok(())
1990 },
1991 )
1992 .await
1993 .expect("write row and blob");
1994 let path = handle
1995 .store_dir()
1996 .local_blob_path("media-files", "blobaaaa")
1997 .expect("local path");
1998 assert_eq!(
1999 std::fs::read(path).expect("read local blob"),
2000 b"piece-bytes"
2001 );
2002 }
2003
2004 #[tokio::test]
2005 async fn orphaned_final_blob_is_replaced_by_next_write() {
2006 let (_tmp, handle) = open_files_handle();
2007 let path = handle
2008 .store_dir()
2009 .local_blob_path("media-files", "orphaaaa")
2010 .expect("local path");
2011 write_raw_file(&path, b"orphaned bytes").await;
2012
2013 handle
2014 .write(
2015 |w| {
2016 w.put_blob("media-files", "orphaaaa", b"committed bytes".to_vec());
2017 Ok(())
2018 },
2019 |sql| {
2020 sql.execute(
2021 "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2022 VALUES (?1, ?2, ?3, ?4, ?5)",
2023 params![
2024 "file-orphan",
2025 "orphaaaa",
2026 15i64,
2027 crate::blob::content_hash(b"committed bytes"),
2028 sql.stamp()
2029 ],
2030 )?;
2031 Ok(())
2032 },
2033 )
2034 .await
2035 .expect("write replaces orphaned final blob");
2036
2037 assert_eq!(
2038 std::fs::read(path).expect("read committed blob"),
2039 b"committed bytes"
2040 );
2041 }
2042
2043 #[tokio::test]
2044 async fn put_blob_rejects_id_already_referenced_by_a_row() {
2045 let (_tmp, handle) = open_files_handle();
2046 handle
2047 .write(
2048 |w| {
2049 w.put_blob("media-files", "dupeaaaa", b"original".to_vec());
2050 Ok(())
2051 },
2052 |sql| {
2053 sql.execute(
2054 "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2055 VALUES (?1, ?2, ?3, ?4, ?5)",
2056 params![
2057 "file-original",
2058 "dupeaaaa",
2059 8i64,
2060 crate::blob::content_hash(b"original"),
2061 sql.stamp()
2062 ],
2063 )?;
2064 Ok(())
2065 },
2066 )
2067 .await
2068 .expect("seed original blob");
2069
2070 let result: CovenResult<WriteReceipt<()>> = handle
2071 .write(
2072 |w| {
2073 w.put_blob("media-files", "dupeaaaa", b"replacement".to_vec());
2074 Ok(())
2075 },
2076 |sql| {
2077 sql.execute(
2078 "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2079 VALUES (?1, ?2, ?3, ?4, ?5)",
2080 params![
2081 "file-replacement",
2082 "dupeaaaa",
2083 11i64,
2084 crate::blob::content_hash(b"replacement"),
2085 sql.stamp()
2086 ],
2087 )?;
2088 Ok(())
2089 },
2090 )
2091 .await;
2092
2093 assert!(matches!(
2094 result,
2095 Err(CovenError::BlobAlreadyReferenced { .. })
2096 ));
2097 let path = handle
2098 .store_dir()
2099 .local_blob_path("media-files", "dupeaaaa")
2100 .expect("dupe path");
2101 assert_eq!(
2102 std::fs::read(path).expect("read original blob"),
2103 b"original"
2104 );
2105 let replacement_rows: i64 = handle
2106 .sql(|sql| {
2107 sql.query_row(
2108 "SELECT count(*) FROM files WHERE id = 'file-replacement'",
2109 [],
2110 |row| row.get(0),
2111 )
2112 .map_err(CovenError::from)
2113 })
2114 .await
2115 .expect("count replacement rows")
2116 .value;
2117 assert_eq!(replacement_rows, 0);
2118 }
2119
2120 #[tokio::test]
2121 async fn remote_root_host_provided_write_reads_staging_through_handle_before_upload() {
2122 let (_tmp, handle) = open_remote_root_files_handle();
2123 let expected = b"remote-root-host-provided-staging-bytes".to_vec();
2124 let bytes = expected.clone();
2125 let hash = crate::blob::content_hash(&bytes);
2126
2127 handle
2128 .write(
2129 {
2130 let bytes = bytes.clone();
2131 move |w| {
2132 w.put_blob("media-files", "rrhpaaaa", bytes);
2133 Ok(())
2134 }
2135 },
2136 move |sql| {
2137 sql.execute(
2138 "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2139 VALUES (?1, ?2, ?3, ?4, ?5)",
2140 params![
2141 "file-remote-root",
2142 "rrhpaaaa",
2143 bytes.len() as i64,
2144 hash,
2145 sql.stamp()
2146 ],
2147 )?;
2148 Ok(())
2149 },
2150 )
2151 .await
2152 .expect("write remote-root row and host-provided blob");
2153 let blob = handle
2154 .row_blob_ref("files", "file-remote-root")
2155 .await
2156 .expect("capture remote-root blob row");
2157
2158 let whole = handle
2159 .read_blob(&blob)
2160 .await
2161 .expect("read_blob serves upload staging before sync upload");
2162 assert_eq!(
2163 whole, expected,
2164 "read_blob returns the bytes written through handle.write",
2165 );
2166
2167 let (offset, len) = (12u64, 19u64);
2168 let range = handle
2169 .open_blob_stream(&blob, offset, len)
2170 .await
2171 .expect("open_blob_stream serves upload staging before sync upload");
2172 assert_eq!(
2173 range,
2174 &expected[offset as usize..(offset + len) as usize],
2175 "open_blob_stream returns the requested slice of the staged bytes",
2176 );
2177 }
2178
2179 struct RemoteOnlyStoreBlob {
2180 _tmp: tempfile::TempDir,
2181 dir: StoreDir,
2182 handle: CovenHandle,
2183 store: TestStore,
2184 encryption: coven_core::EncryptionService,
2185 destination_circle: crate::CircleId,
2186 source_object: ObjectSlot,
2187 }
2188
2189 async fn remote_only_store_blob() -> RemoteOnlyStoreBlob {
2190 let tmp = tempfile::tempdir().expect("temp dir");
2191 let dir = StoreDir::new(tmp.path());
2192 let signer = crate::keys::UserKeypair::generate();
2193 let encryption = coven_core::EncryptionService::from_key([42; 32]);
2194 let handle = Coven::builder(config(dir.clone()))
2195 .synced_tables(vec![scoped_files_table()])
2196 .migrations(vec![scoped_files_migration()])
2197 .key_custody(crate::KeyCustody::InMemory(encryption.clone().into()))
2198 .identity_custody(crate::IdentityCustody::InMemory(signer.clone()))
2199 .open()
2200 .expect("open scoped blob store");
2201 let store = TestStore::create(handle.db(), "lib-test", signer)
2202 .await
2203 .expect("create exact test Store");
2204 let authority =
2205 rusqlite::Connection::open(dir.db_path()).expect("open Circle authority database");
2206 let (destination_circle, _) =
2207 coven_core::sync::test_helpers::install_test_active_circle(&authority, "blob-circle");
2208 drop(authority);
2209
2210 let bytes = b"remote-only-circle-blob".to_vec();
2211 let hash = crate::blob::content_hash(&bytes);
2212 handle
2213 .write(
2214 {
2215 let bytes = bytes.clone();
2216 move |batch| {
2217 batch.put_blob("media-files", "circleblob", bytes);
2218 Ok(())
2219 }
2220 },
2221 move |sql| {
2222 sql.execute(
2223 "INSERT INTO files
2224 (id, blob_id, size, hash, audience, _updated_at)
2225 VALUES ('circle-file', 'circleblob', ?1, ?2, ?3, ?4)",
2226 params![
2227 bytes.len() as i64,
2228 hash,
2229 Option::<String>::None,
2230 sql.stamp()
2231 ],
2232 )?;
2233 Ok(())
2234 },
2235 )
2236 .await
2237 .expect("write Store blob");
2238 store
2239 .publish_pending(handle.db(), &dir)
2240 .await
2241 .expect("publish Store blob");
2242 let source_object = handle
2243 .row_blob_ref("files", "circle-file")
2244 .await
2245 .expect("capture published Store blob")
2246 .stored()
2247 .expect("published Store blob has exact storage")
2248 .object()
2249 .slot()
2250 .clone();
2251 std::fs::remove_file(
2252 dir.local_blob_path("media-files", "circleblob")
2253 .expect("local blob path"),
2254 )
2255 .expect("remove local plaintext to leave a remote-only source");
2256
2257 RemoteOnlyStoreBlob {
2258 _tmp: tmp,
2259 dir,
2260 handle,
2261 store,
2262 encryption,
2263 destination_circle,
2264 source_object,
2265 }
2266 }
2267
2268 fn outbound_blob_spools(dir: &StoreDir) -> std::collections::BTreeSet<PathBuf> {
2269 let path = dir.storage_dir().join("outbound-blobs");
2270 match std::fs::read_dir(path) {
2271 Ok(entries) => entries
2272 .map(|entry| entry.expect("read outbound blob spool entry").path())
2273 .collect(),
2274 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
2275 std::collections::BTreeSet::new()
2276 }
2277 Err(error) => panic!("read outbound blob spools: {error}"),
2278 }
2279 }
2280
2281 async fn run_scoped_file_write(
2282 handle: &CovenHandle,
2283 blob_staging: Option<coven_core::sync::store::HostWriteBlobStaging>,
2284 sql: String,
2285 ) -> Result<WriteReceipt<()>, coven_core::DbError> {
2286 let tables = handle.db().synced_tables().to_vec();
2287 let gates = handle.db().gates();
2288 let blob_decls = handle.db().blob_decls();
2289 let routing_encryption = handle
2290 .routing_encryption()
2291 .expect("resolve routing encryption");
2292 let write_id = handle.db().new_write_id();
2293 handle
2294 .db()
2295 .call(move |conn| {
2296 coven_core::sync::store::StoreDatabase::run_store_write_transaction_on(
2297 conn,
2298 &tables,
2299 &gates,
2300 &blob_decls,
2301 Some(&routing_encryption),
2302 blob_staging.as_ref(),
2303 write_id,
2304 |tx| {
2305 tx.execute_batch(&sql)?;
2306 Ok::<_, coven_core::DbError>(())
2307 },
2308 )
2309 })
2310 .await
2311 }
2312
2313 #[tokio::test]
2314 async fn audience_move_requires_remote_only_blob_before_committing_sql() {
2315 let fixture = remote_only_store_blob().await;
2316 ExactSlotStorage::delete_at(fixture.store.home.as_ref(), &fixture.source_object)
2317 .await
2318 .expect("remove the only remote source");
2319 fixture
2320 .handle
2321 .connect_sync_with_test_home(
2322 fixture.store.home.clone(),
2323 CloudCipher::Encrypted(fixture.encryption.clone()),
2324 )
2325 .await
2326 .expect("connect the exact test Store");
2327
2328 let destination_circle_value = fixture.destination_circle.to_string();
2329 let result = fixture
2330 .handle
2331 .sql(move |sql| {
2332 sql.execute(
2333 "UPDATE files SET audience = ?1, _updated_at = ?2
2334 WHERE id = 'circle-file'",
2335 params![destination_circle_value, sql.stamp()],
2336 )?;
2337 Ok(())
2338 })
2339 .await;
2340
2341 assert!(
2342 result.is_err(),
2343 "a missing source blob must abort the audience move before SQLite commit",
2344 );
2345 let audience = fixture
2346 .handle
2347 .sql_read(|conn| {
2348 conn.query_row(
2349 "SELECT audience FROM files WHERE id = 'circle-file'",
2350 [],
2351 |row| row.get::<_, Option<String>>(0),
2352 )
2353 .map_err(CovenError::from)
2354 })
2355 .await
2356 .expect("read rolled-back audience");
2357 assert_eq!(audience, None);
2358 }
2359
2360 #[tokio::test]
2361 async fn blob_audience_move_without_staging_rejects_and_rolls_back_sql() {
2362 let fixture = remote_only_store_blob().await;
2363 let destination_circle_value = fixture.destination_circle.to_string();
2364 let result = run_scoped_file_write(
2365 &fixture.handle,
2366 None,
2367 format!(
2368 "UPDATE files SET audience = '{destination_circle_value}',
2369 _updated_at = '0000000009000-0000-device-test'
2370 WHERE id = 'circle-file'"
2371 ),
2372 )
2373 .await;
2374
2375 let error = result.expect_err("a blob audience move cannot omit materialization");
2376 assert!(
2377 error
2378 .to_string()
2379 .contains("BlobMoveRequiresMaterialization"),
2380 "{error}",
2381 );
2382 let audience = fixture
2383 .handle
2384 .sql_read(|conn| {
2385 conn.query_row(
2386 "SELECT audience FROM files WHERE id = 'circle-file'",
2387 [],
2388 |row| row.get::<_, Option<String>>(0),
2389 )
2390 .map_err(CovenError::from)
2391 })
2392 .await
2393 .expect("read rolled-back audience");
2394 assert_eq!(audience, None);
2395 }
2396
2397 #[tokio::test]
2398 async fn blob_storage_adapter_error_only_blocks_a_move_that_needs_storage() {
2399 let fixture = remote_only_store_blob().await;
2400 let adapter_error = || {
2401 coven_core::sync::store::HostWriteBlobStaging::new(
2402 tokio::runtime::Handle::current(),
2403 Err("injected blob storage adapter construction failure".to_string()),
2404 fixture.dir.clone(),
2405 )
2406 };
2407
2408 run_scoped_file_write(
2409 &fixture.handle,
2410 Some(adapter_error()),
2411 "UPDATE files SET _updated_at = '0000000009000-0000-device-test'
2412 WHERE id = 'circle-file'"
2413 .to_string(),
2414 )
2415 .await
2416 .expect("an unused adapter error does not block ordinary SQL");
2417
2418 let destination_circle_value = fixture.destination_circle.to_string();
2419 let error = run_scoped_file_write(
2420 &fixture.handle,
2421 Some(adapter_error()),
2422 format!(
2423 "UPDATE files SET audience = '{destination_circle_value}',
2424 _updated_at = '0000000010000-0000-device-test'
2425 WHERE id = 'circle-file'"
2426 ),
2427 )
2428 .await
2429 .expect_err("a remote-only move must surface its adapter error");
2430 assert!(
2431 error
2432 .to_string()
2433 .contains("injected blob storage adapter construction failure"),
2434 "{error}",
2435 );
2436 let audience = fixture
2437 .handle
2438 .sql_read(|conn| {
2439 conn.query_row(
2440 "SELECT audience FROM files WHERE id = 'circle-file'",
2441 [],
2442 |row| row.get::<_, Option<String>>(0),
2443 )
2444 .map_err(CovenError::from)
2445 })
2446 .await
2447 .expect("read audience after adapter failure");
2448 assert_eq!(audience, None);
2449 }
2450
2451 #[tokio::test]
2452 async fn journal_failure_removes_only_the_audience_move_spool_and_rolls_back_sql() {
2453 let fixture = remote_only_store_blob().await;
2454 fixture
2455 .handle
2456 .connect_sync_with_test_home(
2457 fixture.store.home.clone(),
2458 CloudCipher::Encrypted(fixture.encryption.clone()),
2459 )
2460 .await
2461 .expect("connect the exact test Store");
2462 let before = outbound_blob_spools(&fixture.dir);
2463 rusqlite::Connection::open(fixture.dir.db_path())
2464 .expect("open database for journal fault")
2465 .execute_batch(
2466 "CREATE TRIGGER fail_audience_move_journal
2467 BEFORE INSERT ON store_writes
2468 BEGIN
2469 SELECT RAISE(ABORT, 'injected Store write journal failure');
2470 END;",
2471 )
2472 .expect("install Store write journal fault");
2473
2474 let destination_circle_value = fixture.destination_circle.to_string();
2475 let result = fixture
2476 .handle
2477 .sql(move |sql| {
2478 sql.execute(
2479 "UPDATE files SET audience = ?1, _updated_at = ?2
2480 WHERE id = 'circle-file'",
2481 params![destination_circle_value, sql.stamp()],
2482 )?;
2483 Ok(())
2484 })
2485 .await;
2486
2487 assert!(result.is_err(), "the injected journal failure must surface");
2488 assert_eq!(
2489 outbound_blob_spools(&fixture.dir),
2490 before,
2491 "rollback removes the destination spool created by this attempt",
2492 );
2493 let audience = fixture
2494 .handle
2495 .sql_read(|conn| {
2496 conn.query_row(
2497 "SELECT audience FROM files WHERE id = 'circle-file'",
2498 [],
2499 |row| row.get::<_, Option<String>>(0),
2500 )
2501 .map_err(CovenError::from)
2502 })
2503 .await
2504 .expect("read rolled-back audience");
2505 assert_eq!(audience, None);
2506 }
2507
2508 #[tokio::test]
2509 async fn local_audience_move_rolls_back_its_file_and_reuses_an_exact_leftover() {
2510 let fixture = remote_only_store_blob().await;
2511 fixture
2512 .handle
2513 .connect_sync_with_test_home(
2514 fixture.store.home.clone(),
2515 CloudCipher::Encrypted(fixture.encryption.clone()),
2516 )
2517 .await
2518 .expect("connect the exact test Store");
2519 let destination = fixture
2520 .dir
2521 .local_blob_path("media-files", "circleblob")
2522 .expect("resolve Local destination");
2523 let sibling = destination
2524 .parent()
2525 .expect("Local destination has a parent")
2526 .join("unrelated");
2527 std::fs::create_dir_all(sibling.parent().expect("sibling has a parent"))
2528 .expect("create Local blob directory");
2529 std::fs::write(&sibling, b"unrelated").expect("write unrelated Local file");
2530 rusqlite::Connection::open(fixture.dir.db_path())
2531 .expect("open database for Local journal fault")
2532 .execute_batch(
2533 "CREATE TRIGGER fail_local_audience_move_journal
2534 BEFORE INSERT ON store_writes
2535 BEGIN
2536 SELECT RAISE(ABORT, 'injected Local Store write journal failure');
2537 END;",
2538 )
2539 .expect("install Local Store write journal fault");
2540
2541 let result = fixture
2542 .handle
2543 .sql(|sql| {
2544 sql.execute(
2545 "UPDATE files SET audience = 'local', _updated_at = ?1
2546 WHERE id = 'circle-file'",
2547 [sql.stamp()],
2548 )?;
2549 Ok(())
2550 })
2551 .await;
2552 assert!(result.is_err(), "the injected journal failure must surface");
2553 assert!(
2554 !destination.exists(),
2555 "rollback removes the Local file created by this attempt",
2556 );
2557 assert_eq!(
2558 std::fs::read(&sibling).expect("read unrelated Local file"),
2559 b"unrelated",
2560 );
2561 let audience = fixture
2562 .handle
2563 .sql_read(|conn| {
2564 conn.query_row(
2565 "SELECT audience FROM files WHERE id = 'circle-file'",
2566 [],
2567 |row| row.get::<_, Option<String>>(0),
2568 )
2569 .map_err(CovenError::from)
2570 })
2571 .await
2572 .expect("read rolled-back Local audience");
2573 assert_eq!(audience, None);
2574 assert!(matches!(
2575 fixture
2576 .handle
2577 .row_blob_ref("files", "circle-file")
2578 .await
2579 .expect("load blob after failed Local move")
2580 .authority(),
2581 coven_core::blob::RowBlobAuthority::Remote(_)
2582 ));
2583
2584 rusqlite::Connection::open(fixture.dir.db_path())
2585 .expect("open database to remove Local journal fault")
2586 .execute_batch("DROP TRIGGER fail_local_audience_move_journal")
2587 .expect("remove Local Store write journal fault");
2588 std::fs::write(&destination, b"remote-only-circle-blob")
2589 .expect("model an exact file left by failed cleanup");
2590 fixture
2591 .handle
2592 .sql(|sql| {
2593 sql.execute(
2594 "UPDATE files SET audience = 'local', _updated_at = ?1
2595 WHERE id = 'circle-file'",
2596 [sql.stamp()],
2597 )?;
2598 Ok(())
2599 })
2600 .await
2601 .expect("retry accepts the exact already-materialized Local file");
2602 assert_eq!(
2603 std::fs::read(&destination).expect("read retried Local destination"),
2604 b"remote-only-circle-blob",
2605 );
2606 assert_eq!(
2607 std::fs::read(&sibling).expect("read unrelated file after retry"),
2608 b"unrelated",
2609 );
2610 }
2611
2612 #[tokio::test]
2613 async fn audience_move_publishes_from_precommit_spool_after_source_disappears() {
2614 let fixture = remote_only_store_blob().await;
2615 fixture
2616 .handle
2617 .connect_sync_with_test_home(
2618 fixture.store.home.clone(),
2619 CloudCipher::Encrypted(fixture.encryption.clone()),
2620 )
2621 .await
2622 .expect("connect the exact test Store");
2623
2624 fixture
2628 .handle
2629 .invite_member(
2630 &crate::keys::public_key_hex(
2631 &coven_core::sync::test_helpers::test_circle_owner_keypair(),
2632 ),
2633 None,
2634 crate::MemberRole::Member,
2635 )
2636 .await
2637 .expect("register the fabricated Circle roster owner as a Store member");
2638
2639 let destination_circle_value = fixture.destination_circle.to_string();
2640 let receipt = fixture
2641 .handle
2642 .sql(move |sql| {
2643 sql.execute(
2644 "UPDATE files SET audience = ?1, _updated_at = ?2
2645 WHERE id = 'circle-file'",
2646 params![destination_circle_value, sql.stamp()],
2647 )?;
2648 Ok(())
2649 })
2650 .await
2651 .expect("commit audience move after staging its destination blob");
2652 let write_id = receipt.write_id.to_string();
2653 let blob_facts = fixture
2654 .handle
2655 .sql_read(move |conn| {
2656 conn.query_row(
2657 "SELECT blob_facts FROM store_writes WHERE write_id = ?1",
2658 [write_id],
2659 |row| row.get::<_, String>(0),
2660 )
2661 .map_err(CovenError::from)
2662 })
2663 .await
2664 .expect("read durable move blob facts");
2665 let blob_facts: serde_json::Value =
2666 serde_json::from_str(&blob_facts).expect("decode move blob facts");
2667 let spool_path = blob_facts["blobs"][0]["audience_move"]["remote"]["spool_path"]
2668 .as_str()
2669 .expect("move fact records its exact destination spool");
2670 assert!(
2671 std::path::Path::new(spool_path).is_file(),
2672 "the destination spool is durable before the SQL write returns",
2673 );
2674
2675 ExactSlotStorage::delete_at(fixture.store.home.as_ref(), &fixture.source_object)
2676 .await
2677 .expect("remove source after the move commits");
2678 fixture
2679 .store
2680 .publish_pending(fixture.handle.db(), &fixture.dir)
2681 .await
2682 .expect("publish the move from its durable destination spool");
2683 assert!(
2684 !std::path::Path::new(spool_path).exists(),
2685 "prepared-object completion retires the durable precommit spool",
2686 );
2687 assert!(
2688 !fixture
2689 .store
2690 .publish_pending(fixture.handle.db(), &fixture.dir)
2691 .await
2692 .expect("retry completed move publication"),
2693 "a completed move has nothing left to publish",
2694 );
2695 let published = fixture
2696 .handle
2697 .row_blob_ref("files", "circle-file")
2698 .await
2699 .expect("capture published destination Circle blob");
2700 assert_eq!(
2701 published.authority().audience(),
2702 crate::Audience::Circle(fixture.destination_circle),
2703 );
2704 }
2705
2706 #[tokio::test]
2707 async fn public_materialization_survives_store_reopen_without_a_cloud_connection() {
2708 let local = tokio::task::LocalSet::new();
2709 local
2710 .run_until(async {
2711 tokio::task::spawn_local(
2712 run_public_materialization_survives_store_reopen_without_a_cloud_connection(),
2713 )
2714 .await
2715 .expect("public materialization task");
2716 })
2717 .await;
2718 }
2719
2720 async fn run_public_materialization_survives_store_reopen_without_a_cloud_connection() {
2721 let tmp = tempfile::tempdir().expect("temp dir");
2722 let dir = StoreDir::new(tmp.path());
2723 let signer = crate::keys::UserKeypair::generate();
2724 let keyring =
2725 coven_core::MasterKeyring::from(coven_core::EncryptionService::from_key([42; 32]));
2726 let open = || {
2727 Coven::builder(config(dir.clone()))
2728 .synced_tables(vec![remote_root_files_table()])
2729 .migrations(vec![files_migration()])
2730 .key_custody(crate::KeyCustody::InMemory(keyring.clone()))
2731 .identity_custody(crate::IdentityCustody::InMemory(signer.clone()))
2732 .open()
2733 .expect("open remote-root store")
2734 };
2735 let handle = open();
2736 let store = TestStore::create(handle.db(), "lib-test", signer.clone())
2737 .await
2738 .expect("create exact test Store");
2739 handle
2740 .connect_sync_with_test_home(
2741 store.home.clone(),
2742 CloudCipher::Encrypted(coven_core::EncryptionService::from_key([42; 32])),
2743 )
2744 .await
2745 .expect("connect exact test Store");
2746
2747 let expected = b"public materialized blob".to_vec();
2748 let bytes = expected.clone();
2749 let hash = crate::blob::content_hash(&bytes);
2750 let receipt = handle
2751 .write(
2752 {
2753 let bytes = bytes.clone();
2754 move |batch| {
2755 batch.put_blob("media-files", "materialized-blob", bytes);
2756 Ok(())
2757 }
2758 },
2759 move |sql| {
2760 sql.execute(
2761 "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2762 VALUES (?1, ?2, ?3, ?4, ?5)",
2763 params![
2764 "materialized-row",
2765 "materialized-blob",
2766 bytes.len() as i64,
2767 hash,
2768 sql.stamp()
2769 ],
2770 )?;
2771 Ok(())
2772 },
2773 )
2774 .await
2775 .expect("write remote-root blob");
2776 let mut status = handle
2777 .subscribe_write_status(&receipt.write_id)
2778 .await
2779 .expect("subscribe to materialized write");
2780 handle.sync_now();
2781 tokio::time::timeout(Duration::from_secs(20), async {
2782 loop {
2783 let current = status.borrow().clone();
2784 match current {
2785 coven_core::WriteStatus::Published(_) => break,
2786 coven_core::WriteStatus::Pending | coven_core::WriteStatus::Publishing => {
2787 status.changed().await.expect("write status remains open")
2788 }
2789 other => panic!("materialized write did not publish: {other:?}"),
2790 }
2791 }
2792 })
2793 .await
2794 .expect("materialized write publishes");
2795 let reference = handle
2796 .row_blob_ref("files", "materialized-row")
2797 .await
2798 .expect("capture published row blob");
2799 handle
2800 .materialize_row_blob(&reference)
2801 .await
2802 .expect("materialize through the public handle");
2803 let locator_hash = reference
2804 .stored()
2805 .expect("published row has exact storage")
2806 .locator()
2807 .locator_hash();
2808 let cached = dir
2809 .cache_blob_path("media-files", locator_hash)
2810 .expect("exact cache path");
2811 assert_eq!(std::fs::read(&cached).expect("read exact cache"), expected);
2812
2813 handle.disconnect_sync();
2814 drop(handle);
2815 let reopened = open();
2816 let reopened_reference = reopened
2817 .row_blob_ref("files", "materialized-row")
2818 .await
2819 .expect("capture reopened row blob");
2820 reopened
2821 .materialize_row_blob(&reopened_reference)
2822 .await
2823 .expect("reopen verifies the materialized cache without cloud storage");
2824 }
2825
2826 #[tokio::test]
2827 async fn sql_failure_removes_staged_blob() {
2828 let (_tmp, handle) = open_files_handle();
2829 let err = handle
2830 .write(
2831 |w| {
2832 w.put_blob("media-files", "blobbbbb", b"staged".to_vec());
2833 Ok(())
2834 },
2835 |_sql| Err::<(), CovenError>(CovenError::Blob("sql failed".to_string())),
2836 )
2837 .await
2838 .expect_err("write fails");
2839 assert!(err.to_string().contains("sql failed"));
2840 let path = handle
2841 .store_dir()
2842 .local_blob_path("media-files", "blobbbbb")
2843 .expect("local path");
2844 assert!(!path.exists());
2845 }
2846
2847 #[tokio::test]
2848 async fn blob_stage_failure_does_not_run_sql() {
2849 let (_tmp, handle) = open_files_handle();
2850 let result: CovenResult<WriteReceipt<()>> = handle
2851 .write(
2852 |w| {
2853 w.put_blob("media-files", "..", b"bad".to_vec());
2854 Ok(())
2855 },
2856 |sql| {
2857 sql.execute(
2858 "INSERT INTO files (id, blob_id, size, _updated_at) \
2859 VALUES ('should-not-exist', NULL, 0, ?1)",
2860 [sql.stamp()],
2861 )?;
2862 Ok(())
2863 },
2864 )
2865 .await;
2866 assert!(matches!(result, Err(CovenError::UnsafeBlobPath(_))));
2869 let count: i64 = handle
2870 .sql(|sql| {
2871 sql.query_row("SELECT count(*) FROM files", [], |row| row.get(0))
2872 .map_err(CovenError::from)
2873 })
2874 .await
2875 .expect("count rows")
2876 .value;
2877 assert_eq!(count, 0);
2878 }
2879
2880 #[tokio::test]
2881 async fn replacement_deletes_old_blob_after_sql_drops_reference() {
2882 let (_tmp, handle) = open_files_handle();
2883 crate::blob::local_files::store(&handle.store_dir(), "media-files", "oldaaaa", b"old")
2884 .await
2885 .expect("store old");
2886 handle
2887 .sql(|sql| {
2888 sql.execute(
2889 "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2890 VALUES (?1, ?2, 3, ?3, ?4)",
2891 params![
2892 "file-1",
2893 "oldaaaa",
2894 crate::blob::content_hash(b"old"),
2895 sql.stamp()
2896 ],
2897 )?;
2898 Ok(())
2899 })
2900 .await
2901 .expect("seed row");
2902 publish_current_writes(&handle).await;
2903 crate::blob::local_files::store(&handle.store_dir(), "media-files", "oldaaaa", b"old")
2904 .await
2905 .expect("restore published blob locally");
2906 let old_ref = BlobRef {
2907 namespace: "media-files".to_string(),
2908 id: "oldaaaa".to_string(),
2909 scope: BlobScope::Master,
2910 cloud_path: None,
2911 provenance: Provenance::HostProvided,
2912 fill: CacheFill::CacheLazy,
2913 };
2914 handle
2915 .write(
2916 move |w| {
2917 w.put_blob("media-files", "newaaaa", b"new".to_vec());
2918 w.delete_blob(old_ref);
2919 Ok(())
2920 },
2921 move |sql| {
2922 sql.execute(
2923 "UPDATE files SET blob_id = ?1, size = 3, hash = ?2, \
2924 _updated_at = ?3 WHERE id = 'file-1'",
2925 params!["newaaaa", crate::blob::content_hash(b"new"), sql.stamp()],
2926 )?;
2927 Ok(())
2928 },
2929 )
2930 .await
2931 .expect("replace blob");
2932 assert!(!handle
2933 .store_dir()
2934 .local_blob_path("media-files", "oldaaaa")
2935 .expect("old path")
2936 .exists());
2937 assert!(handle
2938 .store_dir()
2939 .local_blob_path("media-files", "newaaaa")
2940 .expect("new path")
2941 .exists());
2942 }
2943
2944 async fn queue_replacement_before_sync(
2945 handle: &CovenHandle,
2946 ) -> (WriteId, WriteId, std::path::PathBuf) {
2947 let first = handle
2948 .write(
2949 |batch| {
2950 batch.put_blob("media-files", "ownedaaa", b"first".to_vec());
2951 Ok(())
2952 },
2953 |sql| {
2954 sql.execute(
2955 "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2956 VALUES ('owned-file', 'ownedaaa', 5, ?1, ?2)",
2957 params![crate::blob::content_hash(b"first"), sql.stamp()],
2958 )?;
2959 Ok(())
2960 },
2961 )
2962 .await
2963 .expect("queue first blob write");
2964 let first_path = handle
2965 .store_dir()
2966 .local_blob_path("media-files", "ownedaaa")
2967 .expect("first blob path");
2968 let first_blob = BlobRef {
2969 namespace: "media-files".to_string(),
2970 id: "ownedaaa".to_string(),
2971 scope: BlobScope::Master,
2972 cloud_path: None,
2973 provenance: Provenance::HostProvided,
2974 fill: CacheFill::CacheLazy,
2975 };
2976 let second = handle
2977 .write(
2978 move |batch| {
2979 batch.put_blob("media-files", "ownedbbb", b"second".to_vec());
2980 batch.delete_blob(first_blob);
2981 Ok(())
2982 },
2983 |sql| {
2984 sql.execute(
2985 "UPDATE files SET blob_id = 'ownedbbb', size = 6, hash = ?1, \
2986 _updated_at = ?2 \
2987 WHERE id = 'owned-file'",
2988 params![crate::blob::content_hash(b"second"), sql.stamp()],
2989 )?;
2990 Ok(())
2991 },
2992 )
2993 .await
2994 .expect("queue replacement write");
2995 (first.write_id, second.write_id, first_path)
2996 }
2997
2998 async fn assert_replacement_publishes_in_order(
2999 handle: &CovenHandle,
3000 first_write: &WriteId,
3001 second_write: &WriteId,
3002 first_path: &std::path::Path,
3003 ) {
3004 let keypair = crate::keys::UserKeypair::generate();
3005 let storage = merge_test_storage(handle, &keypair).await;
3006 run_test_cycle(&storage, handle).await;
3007
3008 let first_sequence = match handle
3009 .write_status(first_write)
3010 .await
3011 .expect("first status")
3012 {
3013 coven_core::WriteStatus::Published(position) => position.commit().coord.sequence(),
3014 status => panic!("first replacement write is not published: {status:?}"),
3015 };
3016 assert_eq!(
3017 handle
3018 .write_status(second_write)
3019 .await
3020 .expect("second status after first publication"),
3021 coven_core::WriteStatus::Pending,
3022 "one Store publication consumes one pending host transaction",
3023 );
3024
3025 run_test_cycle(&storage, handle).await;
3026 let second_sequence = match handle
3027 .write_status(second_write)
3028 .await
3029 .expect("second status")
3030 {
3031 coven_core::WriteStatus::Published(position) => position.commit().coord.sequence(),
3032 status => panic!("second replacement write is not published: {status:?}"),
3033 };
3034 assert_eq!(
3035 second_sequence,
3036 first_sequence + 1,
3037 "replacement writes publish in their host transaction order"
3038 );
3039 let blob_objects = storage
3040 .home
3041 .keys()
3042 .into_iter()
3043 .filter(|key| key.starts_with("media-files/opaque/"))
3044 .count();
3045 assert_eq!(blob_objects, 2, "both row versions upload their blob bytes");
3046 assert!(
3047 !first_path.exists(),
3048 "the first write releases its local bytes after publication"
3049 );
3050 }
3051
3052 #[tokio::test]
3053 async fn pending_write_owns_blob_bytes_until_its_publication() {
3054 let local = tokio::task::LocalSet::new();
3055 local
3056 .run_until(async {
3057 tokio::task::spawn_local(run_pending_write_owns_blob_bytes_until_its_publication())
3058 .await
3059 .expect("pending-write blob ownership test task");
3060 })
3061 .await;
3062 }
3063
3064 async fn run_pending_write_owns_blob_bytes_until_its_publication() {
3065 let (_tmp, handle) = open_files_handle();
3066 let (first_write, second_write, first_path) = queue_replacement_before_sync(&handle).await;
3067
3068 assert_eq!(
3069 std::fs::read(&first_path).expect("first write still owns its bytes"),
3070 b"first"
3071 );
3072 let overwrite: CovenResult<WriteReceipt<()>> = handle
3073 .write(
3074 |batch| {
3075 batch.put_blob("media-files", "ownedaaa", b"overwritten".to_vec());
3076 Ok(())
3077 },
3078 |sql| {
3079 sql.execute(
3080 "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
3081 VALUES ('overwrite', 'ownedaaa', 11, ?1, ?2)",
3082 params![crate::blob::content_hash(b"overwritten"), sql.stamp()],
3083 )?;
3084 Ok(())
3085 },
3086 )
3087 .await;
3088 assert!(matches!(
3089 overwrite,
3090 Err(CovenError::BlobOwnedByPendingWrite { .. })
3091 ));
3092 assert_eq!(
3093 std::fs::read(&first_path).expect("lease prevents overwrite"),
3094 b"first"
3095 );
3096 assert_replacement_publishes_in_order(&handle, &first_write, &second_write, &first_path)
3097 .await;
3098 }
3099
3100 #[tokio::test]
3101 async fn pending_write_blob_ownership_survives_restart() {
3102 let local = tokio::task::LocalSet::new();
3103 local
3104 .run_until(async {
3105 tokio::task::spawn_local(run_pending_write_blob_ownership_survives_restart())
3106 .await
3107 .expect("reopened pending-write blob ownership test task");
3108 })
3109 .await;
3110 }
3111
3112 async fn run_pending_write_blob_ownership_survives_restart() {
3113 let tmp = tempfile::tempdir().expect("temp dir");
3114 let dir = StoreDir::new(tmp.path());
3115 let handle = open_files_handle_in(dir.clone());
3116 let (first_write, second_write, first_path) = queue_replacement_before_sync(&handle).await;
3117 drop(handle);
3118
3119 let reopened = open_files_handle_in(dir);
3120 assert_eq!(
3121 std::fs::read(&first_path).expect("reopened first write still owns its bytes"),
3122 b"first"
3123 );
3124 assert_replacement_publishes_in_order(&reopened, &first_write, &second_write, &first_path)
3125 .await;
3126 }
3127
3128 #[tokio::test]
3129 async fn author_delete_drops_all_local_blob_copies() {
3130 let (_tmp, handle) = open_files_handle();
3131 crate::blob::local_files::store(&handle.store_dir(), "media-files", "oldcccc", b"old")
3132 .await
3133 .expect("store old");
3134 handle
3135 .sql(|sql| {
3136 sql.execute(
3137 "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
3138 VALUES (?1, ?2, 3, ?3, ?4)",
3139 params![
3140 "file-1",
3141 "oldcccc",
3142 crate::blob::content_hash(b"old"),
3143 sql.stamp()
3144 ],
3145 )?;
3146 Ok(())
3147 })
3148 .await
3149 .expect("seed row");
3150 publish_current_writes(&handle).await;
3151 let published = handle
3152 .row_blob_ref("files", "file-1")
3153 .await
3154 .expect("capture exact published blob");
3155 let locator_hash = published
3156 .stored()
3157 .expect("published row has exact storage")
3158 .locator()
3159 .locator_hash();
3160 let pinned = handle
3161 .store_dir()
3162 .pinned_blob_path("media-files", locator_hash)
3163 .expect("pinned path");
3164 let cached = handle
3165 .store_dir()
3166 .cache_blob_path("media-files", locator_hash)
3167 .expect("cache path");
3168 crate::blob::local_files::store(&handle.store_dir(), "media-files", "oldcccc", b"old")
3169 .await
3170 .expect("restore published blob locally");
3171 write_raw_file(&pinned, b"old").await;
3172 write_raw_file(&cached, b"old").await;
3173 let old_ref = BlobRef {
3174 namespace: "media-files".to_string(),
3175 id: "oldcccc".to_string(),
3176 scope: BlobScope::Master,
3177 cloud_path: None,
3178 provenance: Provenance::HostProvided,
3179 fill: CacheFill::CacheLazy,
3180 };
3181
3182 handle
3183 .write(
3184 move |w| {
3185 w.delete_blob(old_ref);
3186 Ok(())
3187 },
3188 move |sql| {
3189 sql.execute(
3190 "UPDATE files SET blob_id = NULL, size = 0, hash = NULL, _updated_at = ?1 \
3191 WHERE id = 'file-1'",
3192 [sql.stamp()],
3193 )?;
3194 Ok(())
3195 },
3196 )
3197 .await
3198 .expect("delete blob");
3199
3200 assert!(!handle
3201 .store_dir()
3202 .local_blob_path("media-files", "oldcccc")
3203 .expect("local path")
3204 .exists());
3205 assert!(!pinned.exists(), "pinned copy is removed");
3206 assert!(!cached.exists(), "cache copy is removed");
3207 }
3208
3209 #[tokio::test]
3210 async fn failed_local_blob_cleanup_keeps_intent_for_later_drain() {
3211 let (_tmp, handle) = open_files_handle();
3212 crate::blob::local_files::store(&handle.store_dir(), "media-files", "oldddddd", b"old")
3213 .await
3214 .expect("store old");
3215 handle
3216 .sql(|sql| {
3217 sql.execute(
3218 "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
3219 VALUES (?1, ?2, 3, ?3, ?4)",
3220 params![
3221 "file-1",
3222 "oldddddd",
3223 crate::blob::content_hash(b"old"),
3224 sql.stamp()
3225 ],
3226 )?;
3227 Ok(())
3228 })
3229 .await
3230 .expect("seed row");
3231 publish_current_writes(&handle).await;
3232 let published = handle
3233 .row_blob_ref("files", "file-1")
3234 .await
3235 .expect("capture exact published blob");
3236 let locator_hash = published
3237 .stored()
3238 .expect("published row has exact storage")
3239 .locator()
3240 .locator_hash();
3241 let pinned = handle
3242 .store_dir()
3243 .pinned_blob_path("media-files", locator_hash)
3244 .expect("pinned path");
3245 crate::blob::local_files::store(&handle.store_dir(), "media-files", "oldddddd", b"old")
3246 .await
3247 .expect("restore published blob locally");
3248 std::fs::create_dir_all(&pinned).expect("create pinned blocker");
3249 let old_ref = BlobRef {
3250 namespace: "media-files".to_string(),
3251 id: "oldddddd".to_string(),
3252 scope: BlobScope::Master,
3253 cloud_path: None,
3254 provenance: Provenance::HostProvided,
3255 fill: CacheFill::CacheLazy,
3256 };
3257
3258 handle
3259 .write(
3260 move |w| {
3261 w.delete_blob(old_ref);
3262 Ok(())
3263 },
3264 |sql| {
3265 sql.execute(
3266 "UPDATE files SET blob_id = NULL, size = 0, hash = NULL, _updated_at = ?1 \
3267 WHERE id = 'file-1'",
3268 [sql.stamp()],
3269 )?;
3270 Ok(())
3271 },
3272 )
3273 .await
3274 .expect("row delete commits despite cleanup failure");
3275
3276 assert_eq!(
3277 cleanup_intent_count(&handle, "media-files", "oldddddd").await,
3278 2
3279 );
3280 assert!(handle
3281 .store_dir()
3282 .local_blob_path("media-files", "oldddddd")
3283 .expect("local path")
3284 .exists());
3285
3286 std::fs::remove_dir_all(&pinned).expect("remove pinned blocker");
3287 handle
3288 .write(
3289 |_| Ok(()),
3290 |sql| {
3291 sql.execute(
3292 "INSERT INTO files (id, blob_id, size, _updated_at) \
3293 VALUES (?1, NULL, 0, ?2)",
3294 params!["drain-trigger", sql.stamp()],
3295 )?;
3296 Ok(())
3297 },
3298 )
3299 .await
3300 .expect("later committed write drains pending cleanup");
3301
3302 assert_eq!(
3303 cleanup_intent_count(&handle, "media-files", "oldddddd").await,
3304 0
3305 );
3306 assert!(!handle
3307 .store_dir()
3308 .local_blob_path("media-files", "oldddddd")
3309 .expect("local path")
3310 .exists());
3311 }
3312
3313 #[tokio::test]
3314 async fn write_drain_separates_live_local_source_from_deleted_exact_cache() {
3315 let (_tmp, handle) = open_files_handle();
3316 let blob_id = "shared01";
3317 handle
3318 .sql(move |sql| {
3319 let hash = crate::blob::content_hash(b"live");
3320 for id in ["remote-deletes", "still-live"] {
3321 sql.execute(
3322 "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
3323 VALUES (?1, 'shared01', 4, ?2, \
3324 '0000000001000-0000-dev-remote')",
3325 params![id, &hash],
3326 )?;
3327 }
3328 Ok(())
3329 })
3330 .await
3331 .expect("seed two rows sharing the blob");
3332
3333 let local = handle
3334 .store_dir()
3335 .local_blob_path("media-files", blob_id)
3336 .expect("local path");
3337 write_raw_file(&local, b"live").await;
3338
3339 let (_source_tmp, source) = open_files_handle();
3340 let storage = Arc::new(
3341 TestStore::create(
3342 source.db(),
3343 "lib-test",
3344 crate::keys::UserKeypair::generate(),
3345 )
3346 .await
3347 .expect("create remote exact test Store"),
3348 );
3349 source
3350 .write(
3351 |batch| {
3352 batch.put_blob("media-files", "shared01", b"live".to_vec());
3353 Ok(())
3354 },
3355 |sql| {
3356 sql.execute(
3357 "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
3358 VALUES ('remote-deletes', 'shared01', 4, ?1, ?2)",
3359 params![crate::blob::content_hash(b"live"), sql.stamp()],
3360 )?;
3361 Ok(())
3362 },
3363 )
3364 .await
3365 .expect("insert remote row");
3366 storage
3367 .publish_pending(source.db(), &source.store_dir())
3368 .await
3369 .expect("publish remote insert");
3370 let remote_reference = source
3371 .row_blob_ref("files", "remote-deletes")
3372 .await
3373 .expect("capture exact remote blob");
3374 let locator_hash = remote_reference
3375 .stored()
3376 .expect("published row has exact storage")
3377 .locator()
3378 .locator_hash();
3379 let pinned = handle
3380 .store_dir()
3381 .pinned_blob_path("media-files", locator_hash)
3382 .expect("pinned path");
3383 let cached = handle
3384 .store_dir()
3385 .cache_blob_path("media-files", locator_hash)
3386 .expect("cache path");
3387 write_raw_file(&pinned, b"live").await;
3388 write_raw_file(&cached, b"live").await;
3389 let delete = source
3390 .sql(|sql| {
3391 sql.execute("DELETE FROM files WHERE id = 'remote-deletes'", [])?;
3392 Ok(())
3393 })
3394 .await
3395 .expect("delete remote row");
3396 storage
3397 .publish_pending(source.db(), &source.store_dir())
3398 .await
3399 .expect("publish remote delete");
3400
3401 assert!(matches!(
3402 source
3403 .write_status(&delete.write_id)
3404 .await
3405 .expect("remote delete status"),
3406 coven_core::WriteStatus::Published(_)
3407 ));
3408 let (device_id, sequence) = source
3409 .db()
3410 .call(|conn| {
3411 conn.query_row(
3412 "SELECT device_id, seq FROM materialized_commits ORDER BY seq DESC LIMIT 1",
3413 [],
3414 |row| Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)),
3415 )
3416 .map_err(coven_core::database::DbError::from)
3417 })
3418 .await
3419 .expect("load remote delete stream coordinate");
3420 let sequence = u64::try_from(sequence).expect("remote delete sequence is non-negative");
3421
3422 let (commit_reached, resume_pull) = handle.db().arm_test_pause(
3423 coven_core::database::DatabaseTestPoint::PullAfterRemoteCommit {
3424 device_id,
3425 seq: sequence,
3426 },
3427 );
3428 let pull_handle = handle.clone();
3429 let pull_storage = storage.clone();
3430 let pull = tokio::spawn(async move {
3431 pull_into(
3432 pull_handle.db(),
3433 pull_storage.as_ref(),
3434 &pull_handle.store_dir(),
3435 )
3436 .await
3437 });
3438
3439 commit_reached.notified().await;
3440 handle
3441 .write(
3442 |_| Ok(()),
3443 |sql| {
3444 sql.execute(
3445 "INSERT INTO files (id, blob_id, size, _updated_at) \
3446 VALUES ('drain-trigger', NULL, 0, ?1)",
3447 [sql.stamp()],
3448 )?;
3449 Ok(())
3450 },
3451 )
3452 .await
3453 .expect("write drains cleanup queue");
3454 resume_pull.notify_one();
3455 pull.await.expect("pull task");
3456
3457 assert!(
3458 !row_exists(
3459 handle.db(),
3460 "SELECT 1 FROM files WHERE id = 'remote-deletes'"
3461 )
3462 .await
3463 );
3464 assert!(row_exists(handle.db(), "SELECT 1 FROM files WHERE id = 'still-live'").await);
3465 assert!(local.exists(), "the live blob's local copy survives");
3466 assert!(
3467 !pinned.exists(),
3468 "the deleted locator's pinned copy is removed"
3469 );
3470 assert!(
3471 !cached.exists(),
3472 "the deleted locator's cache copy is removed"
3473 );
3474 assert_eq!(
3475 cleanup_intent_count(&handle, "media-files", blob_id).await,
3476 0,
3477 );
3478 }
3479
3480 #[tokio::test]
3481 async fn replacement_is_rejected_while_sql_still_references_old_blob() {
3482 let (_tmp, handle) = open_files_handle();
3483 crate::blob::local_files::store(&handle.store_dir(), "media-files", "oldbbbb", b"old")
3484 .await
3485 .expect("store old");
3486 handle
3487 .sql(|sql| {
3488 sql.execute(
3489 "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
3490 VALUES (?1, ?2, 3, ?3, ?4)",
3491 params![
3492 "file-1",
3493 "oldbbbb",
3494 crate::blob::content_hash(b"old"),
3495 sql.stamp()
3496 ],
3497 )?;
3498 Ok(())
3499 })
3500 .await
3501 .expect("seed row");
3502 let old_ref = BlobRef {
3503 namespace: "media-files".to_string(),
3504 id: "oldbbbb".to_string(),
3505 scope: BlobScope::Master,
3506 cloud_path: None,
3507 provenance: Provenance::HostProvided,
3508 fill: CacheFill::CacheLazy,
3509 };
3510 let result: CovenResult<WriteReceipt<()>> = handle
3511 .write(
3512 move |w| {
3513 w.put_blob("media-files", "newbbbb", b"new".to_vec());
3514 w.delete_blob(old_ref);
3515 Ok(())
3516 },
3517 move |sql| {
3518 sql.execute(
3519 "UPDATE files SET _updated_at = ?1 WHERE id = 'file-1'",
3520 [sql.stamp()],
3521 )?;
3522 Ok(())
3523 },
3524 )
3525 .await;
3526 assert!(matches!(
3527 result,
3528 Err(CovenError::BlobStillReferenced { .. })
3529 ));
3530 assert!(handle
3531 .store_dir()
3532 .local_blob_path("media-files", "oldbbbb")
3533 .expect("old path")
3534 .exists());
3535 assert!(!handle
3536 .store_dir()
3537 .local_blob_path("media-files", "newbbbb")
3538 .expect("new path")
3539 .exists());
3540 }
3541
3542 #[tokio::test]
3543 async fn sql_panic_removes_moved_blob() {
3544 let (_tmp, handle) = open_files_handle();
3545 let result: CovenResult<WriteReceipt<()>> = handle
3546 .write(
3547 |w| {
3548 w.put_blob("media-files", "panicccc", b"new".to_vec());
3549 Ok(())
3550 },
3551 |_sql| panic!("boom"),
3552 )
3553 .await;
3554 assert!(matches!(result, Err(CovenError::WriteClosurePanicked)));
3555 assert!(!handle
3556 .store_dir()
3557 .local_blob_path("media-files", "panicccc")
3558 .expect("panic path")
3559 .exists());
3560 }
3561
3562 #[tokio::test]
3563 async fn concurrent_duplicate_blob_write_does_not_delete_committed_blob() {
3564 let (_tmp, handle) = open_files_handle();
3565 let winner = handle.clone();
3566 let loser = handle.clone();
3567
3568 let write_winner = tokio::spawn(async move {
3569 winner
3570 .write(
3571 move |w| {
3572 w.put_blob("media-files", "raceblob", b"committed".to_vec());
3573 Ok(())
3574 },
3575 move |sql| {
3576 sql.execute(
3577 "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
3578 VALUES (?1, ?2, ?3, ?4, ?5)",
3579 params![
3580 "winner",
3581 "raceblob",
3582 9i64,
3583 crate::blob::content_hash(b"committed"),
3584 sql.stamp()
3585 ],
3586 )?;
3587 Ok(())
3588 },
3589 )
3590 .await
3591 });
3592
3593 let write_loser = tokio::spawn(async move {
3594 loser
3595 .write(
3596 move |w| {
3597 w.put_blob("media-files", "raceblob", b"rolled-back".to_vec());
3598 Ok(())
3599 },
3600 |_sql| Err::<(), CovenError>(CovenError::Blob("force rollback".to_string())),
3601 )
3602 .await
3603 });
3604
3605 let winner_result = write_winner.await.expect("winner task");
3606 let loser_result = write_loser.await.expect("loser task");
3607 assert!(winner_result.is_ok() || loser_result.is_ok());
3608 assert!(winner_result.is_err() || loser_result.is_err());
3609
3610 let path = handle
3611 .store_dir()
3612 .local_blob_path("media-files", "raceblob")
3613 .expect("race path");
3614 assert_eq!(std::fs::read(path).expect("read race blob"), b"committed");
3615 let rows: i64 = handle
3616 .sql(|sql| {
3617 sql.query_row(
3618 "SELECT count(*) FROM files WHERE id = 'winner'",
3619 [],
3620 |row| row.get(0),
3621 )
3622 .map_err(CovenError::from)
3623 })
3624 .await
3625 .expect("count winner row")
3626 .value;
3627 assert_eq!(rows, 1);
3628 }
3629
3630 struct GateCloudHome {
3639 inner: InMemoryCloudHome,
3640 armed: Arc<AtomicBool>,
3641 gated: AtomicBool,
3642 entered: mpsc::UnboundedSender<()>,
3643 release: Arc<Notify>,
3644 }
3645
3646 impl GateCloudHome {
3647 async fn gate(&self) {
3648 if self.armed.load(Ordering::Acquire) && !self.gated.swap(true, Ordering::AcqRel) {
3649 self.entered
3650 .send(())
3651 .expect("test observes the loop entering a cycle");
3652 self.release.notified().await;
3653 }
3654 }
3655 }
3656
3657 #[async_trait]
3658 impl CloudHome for GateCloudHome {
3659 fn exact_slot_storage(self: Arc<Self>) -> Option<Arc<dyn ExactSlotStorage>> {
3660 Some(self)
3661 }
3662
3663 async fn put_object(&self, key: &str, data: Vec<u8>) -> Result<(), CloudHomeError> {
3664 self.gate().await;
3665 self.inner.put_object(key, data).await
3666 }
3667
3668 async fn open_multipart<'a>(
3669 &'a self,
3670 key: &str,
3671 total_len: u64,
3672 ) -> Result<BoxPartSink<'a>, CloudHomeError> {
3673 self.gate().await;
3674 self.inner.open_multipart(key, total_len).await
3675 }
3676
3677 fn multipart_threshold(&self) -> u64 {
3678 self.inner.multipart_threshold()
3679 }
3680
3681 async fn read(&self, key: &str) -> Result<Vec<u8>, CloudHomeError> {
3682 self.gate().await;
3683 self.inner.read(key).await
3684 }
3685
3686 async fn read_range(
3687 &self,
3688 key: &str,
3689 start: u64,
3690 end: u64,
3691 ) -> Result<Vec<u8>, CloudHomeError> {
3692 self.gate().await;
3693 self.inner.read_range(key, start, end).await
3694 }
3695
3696 async fn list(&self, prefix: &str) -> Result<Vec<String>, CloudHomeError> {
3697 self.gate().await;
3698 self.inner.list(prefix).await
3699 }
3700
3701 async fn delete(&self, key: &str) -> Result<(), CloudHomeError> {
3702 self.gate().await;
3703 self.inner.delete(key).await
3704 }
3705
3706 async fn exists(&self, key: &str) -> Result<bool, CloudHomeError> {
3707 self.gate().await;
3708 self.inner.exists(key).await
3709 }
3710
3711 async fn set_access(
3712 &self,
3713 desired: CloudAccessState,
3714 ) -> Result<CloudAccessOutcome, CloudHomeError> {
3715 self.gate().await;
3716 self.inner.set_access(desired).await
3717 }
3718 }
3719
3720 #[async_trait]
3721 impl ExactSlotStorage for GateCloudHome {
3722 async fn provider_binding(
3723 &self,
3724 ) -> Result<crate::sync::storage::ResolvedProviderBinding, CloudHomeError> {
3725 self.gate().await;
3726 ExactSlotStorage::provider_binding(&self.inner).await
3727 }
3728
3729 async fn allocate_slot(&self, key: &str) -> Result<ObjectSlot, CloudHomeError> {
3730 self.gate().await;
3731 ExactSlotStorage::allocate_slot(&self.inner, key).await
3732 }
3733
3734 async fn create_at(
3735 &self,
3736 slot: &ObjectSlot,
3737 body: BlobBody,
3738 progress: &UploadProgress<'_>,
3739 ) -> Result<(), CloudHomeError> {
3740 self.gate().await;
3741 ExactSlotStorage::create_at(&self.inner, slot, body, progress).await
3742 }
3743
3744 async fn read_at(&self, slot: &ObjectSlot) -> Result<Vec<u8>, CloudHomeError> {
3745 self.gate().await;
3746 ExactSlotStorage::read_at(&self.inner, slot).await
3747 }
3748
3749 async fn read_range_at(
3750 &self,
3751 slot: &ObjectSlot,
3752 start: u64,
3753 end: u64,
3754 ) -> Result<Vec<u8>, CloudHomeError> {
3755 self.gate().await;
3756 ExactSlotStorage::read_range_at(&self.inner, slot, start, end).await
3757 }
3758
3759 async fn read_at_to_file(
3760 &self,
3761 slot: &ObjectSlot,
3762 destination: &std::path::Path,
3763 ) -> Result<(), crate::storage::cloud::CloudFileReadError> {
3764 self.gate().await;
3765 ExactSlotStorage::read_at_to_file(&self.inner, slot, destination).await
3766 }
3767
3768 async fn delete_at(&self, slot: &ObjectSlot) -> Result<(), CloudHomeError> {
3769 self.gate().await;
3770 ExactSlotStorage::delete_at(&self.inner, slot).await
3771 }
3772 }
3773
3774 fn try_open(dir: &StoreDir) -> CovenResult<CovenHandle> {
3775 Coven::builder(config(dir.clone()))
3776 .synced_tables(vec![files_table()])
3777 .migrations(vec![files_migration()])
3778 .open()
3779 }
3780
3781 async fn open_when_lock_released(dir: &StoreDir) -> CovenHandle {
3785 for _ in 0..100 {
3786 match try_open(dir) {
3787 Ok(handle) => return handle,
3788 Err(CovenError::AlreadyOpen { .. }) => {
3789 tokio::time::sleep(Duration::from_millis(100)).await;
3790 }
3791 Err(other) => panic!("reopen failed for a reason other than the lock: {other}"),
3792 }
3793 }
3794 panic!("store lock never released: reopen kept failing");
3795 }
3796
3797 #[tokio::test]
3798 async fn lock_is_held_until_the_sync_loop_exits_its_cycle() {
3799 test_keyring::install();
3800
3801 let tmp = tempfile::tempdir().expect("temp dir");
3802 let dir = StoreDir::new(tmp.path());
3803 let handle = Coven::builder(Config::with_defaults(
3809 "lock-held-until-cycle-exits".to_string(),
3810 "device-test".to_string(),
3811 dir.clone(),
3812 "Test".to_string(),
3813 ))
3814 .synced_tables(vec![files_table()])
3815 .migrations(vec![files_migration()])
3816 .open()
3817 .expect("open handle");
3818 handle
3819 .initialize_identity()
3820 .expect("establish this store's identity before connecting");
3821
3822 let armed = Arc::new(AtomicBool::new(false));
3823 let (entered_tx, mut entered_rx) = mpsc::unbounded_channel();
3824 let release = Arc::new(Notify::new());
3825 let home = Arc::new(GateCloudHome {
3826 inner: InMemoryCloudHome::new(),
3827 armed: armed.clone(),
3828 gated: AtomicBool::new(false),
3829 entered: entered_tx,
3830 release: release.clone(),
3831 });
3832 let connect_handle = handle.clone();
3833 tokio::spawn(async move {
3834 connect_handle
3835 .connect_sync_with_test_home(home, CloudCipher::Plaintext)
3836 .await
3837 })
3838 .await
3839 .expect("gating test connection task completes")
3840 .expect("connect over the gating test home");
3841
3842 armed.store(true, Ordering::Release);
3845 tokio::time::timeout(Duration::from_secs(30), entered_rx.recv())
3846 .await
3847 .expect("sync loop reached a cycle within the timeout")
3848 .expect("gating home signalled the cycle entry");
3849
3850 drop(handle);
3853 assert!(
3854 matches!(
3855 try_open(&dir),
3856 Err(CovenError::AlreadyOpen { store_dir }) if store_dir == tmp.path()
3857 ),
3858 "a concurrent open must be refused while the sync loop is mid-cycle",
3859 );
3860
3861 release.notify_one();
3864 let _reopened = open_when_lock_released(&dir).await;
3865 }
3866
3867 #[tokio::test]
3868 async fn normal_shutdown_releases_the_lock_for_reopen() {
3869 test_keyring::install();
3870
3871 let tmp = tempfile::tempdir().expect("temp dir");
3872 let dir = StoreDir::new(tmp.path());
3873 let handle = Coven::builder(Config::with_defaults(
3876 "normal-shutdown-releases-lock".to_string(),
3877 "device-test".to_string(),
3878 dir.clone(),
3879 "Test".to_string(),
3880 ))
3881 .synced_tables(vec![files_table()])
3882 .migrations(vec![files_migration()])
3883 .open()
3884 .expect("open handle");
3885 handle
3886 .initialize_identity()
3887 .expect("establish this store's identity before connecting");
3888 let connect_handle = handle.clone();
3889 tokio::spawn(async move {
3890 connect_handle
3891 .connect_sync_with_test_home(
3892 Arc::new(InMemoryCloudHome::new()),
3893 CloudCipher::Plaintext,
3894 )
3895 .await
3896 })
3897 .await
3898 .expect("shutdown test connection task completes")
3899 .expect("connect over the test home");
3900
3901 drop(handle);
3902
3903 let _reopened = open_when_lock_released(&dir).await;
3904 }
3905
3906 #[test]
3912 fn open_time_sweep_clears_stale_blob_temps_but_keeps_fresh_ones() {
3913 let tmp = tempfile::tempdir().expect("temp dir");
3914 let ld = StoreDir::new(tmp.path());
3915
3916 let process_start = std::time::SystemTime::now();
3917 let stale = process_start - Duration::from_secs(3600);
3918 let fresh = process_start + Duration::from_secs(3600);
3919
3920 let write_with_mtime = |path: &Path, mtime: std::time::SystemTime| {
3921 std::fs::create_dir_all(path.parent().unwrap()).expect("create dir");
3922 std::fs::write(path, b"x").expect("write file");
3923 std::fs::OpenOptions::new()
3924 .write(true)
3925 .open(path)
3926 .expect("open to set mtime")
3927 .set_modified(mtime)
3928 .expect("set mtime");
3929 };
3930 let temp_name = |marker: &str| format!("{marker}{}", uuid::Uuid::new_v4());
3931
3932 let cache_shard = ld
3933 .storage_dir()
3934 .join("cache")
3935 .join("release_files")
3936 .join("ab")
3937 .join("cd");
3938 let pinned_shard = ld
3939 .storage_dir()
3940 .join("pinned")
3941 .join("photos")
3942 .join("ef")
3943 .join("gh");
3944 let local_ns = ld.storage_dir().join("local").join("audio");
3945
3946 let stale_cache_temp = cache_shard.join(temp_name(crate::local_blob::TEMP_BLOB_PREFIX));
3947 let fresh_cache_temp = cache_shard.join(temp_name(crate::local_blob::TEMP_BLOB_PREFIX));
3948 let committed_cache = cache_shard.join("blob0aaa");
3951 let stale_pinned_temp = pinned_shard.join(temp_name(crate::local_blob::TEMP_BLOB_PREFIX));
3952 let fresh_pinned_temp = pinned_shard.join(temp_name(crate::local_blob::TEMP_BLOB_PREFIX));
3953 let stale_local_temp = local_ns.join(temp_name(crate::local_blob::TEMP_BLOB_PREFIX));
3956 let fresh_local_temp = local_ns.join(temp_name(crate::local_blob::TEMP_BLOB_PREFIX));
3957 let committed_local = local_ns.join("blob0bbb");
3958 let stale_local_stage = local_ns.join(temp_name(&format!("blob{LOCAL_STAGE_MARKER}")));
3959
3960 write_with_mtime(&stale_cache_temp, stale);
3961 write_with_mtime(&fresh_cache_temp, fresh);
3962 write_with_mtime(&committed_cache, stale);
3963 write_with_mtime(&stale_pinned_temp, stale);
3964 write_with_mtime(&fresh_pinned_temp, fresh);
3965 write_with_mtime(&stale_local_temp, stale);
3966 write_with_mtime(&fresh_local_temp, fresh);
3967 write_with_mtime(&committed_local, stale);
3968 write_with_mtime(&stale_local_stage, stale);
3969
3970 remove_orphaned_local_blob_temps(&ld, process_start).expect("open-time orphan sweep");
3971
3972 assert!(
3973 !stale_cache_temp.exists(),
3974 "a cache temp older than process start is a crash leftover, removed",
3975 );
3976 assert!(
3977 !stale_pinned_temp.exists(),
3978 "a pinned temp older than process start is a crash leftover, removed",
3979 );
3980 assert!(
3981 !stale_local_temp.exists(),
3982 "a local-store atomic-write temp older than process start is a crash leftover, removed",
3983 );
3984 assert!(
3985 fresh_cache_temp.exists(),
3986 "a cache temp newer than process start is a live write, left alone",
3987 );
3988 assert!(
3989 fresh_pinned_temp.exists(),
3990 "a pinned temp newer than process start is a live write, left alone",
3991 );
3992 assert!(
3993 fresh_local_temp.exists(),
3994 "a local-store temp newer than process start is a live write, left alone",
3995 );
3996 assert!(
3997 committed_cache.exists(),
3998 "a committed cache blob is never a temp — kept regardless of its mtime",
3999 );
4000 assert!(
4001 committed_local.exists(),
4002 "a committed local-store blob is never a temp — kept regardless of its mtime",
4003 );
4004 assert!(
4005 !stale_local_stage.exists(),
4006 "local-store staging temps are still swept at open",
4007 );
4008 }
4009
4010 fn try_open_read_only(dir: &StoreDir) -> CovenResult<crate::read_handle::CovenReadHandle> {
4015 Coven::builder(config(dir.clone()))
4016 .synced_tables(vec![files_table()])
4017 .migrations(vec![files_migration()])
4018 .open_read_only()
4019 }
4020
4021 async fn read_files_count(handle: &crate::read_handle::CovenReadHandle) -> i64 {
4022 handle
4023 .sql_read(|conn| {
4024 conn.query_row("SELECT count(*) FROM files", [], |row| row.get(0))
4025 .map_err(CovenError::from)
4026 })
4027 .await
4028 .expect("count files through the read handle")
4029 }
4030
4031 #[tokio::test]
4035 async fn read_only_open_succeeds_while_a_full_open_holds_the_store() {
4036 let tmp = tempfile::tempdir().expect("temp dir");
4037 let dir = StoreDir::new(tmp.path());
4038 let writer = open_files_handle_in(dir.clone());
4039
4040 assert!(matches!(
4042 try_open(&dir),
4043 Err(CovenError::AlreadyOpen { store_dir }) if store_dir == tmp.path()
4044 ));
4045
4046 let reader = try_open_read_only(&dir).expect("read-only open succeeds under a full open");
4048 assert_eq!(read_files_count(&reader).await, 0);
4049
4050 drop(writer);
4051 }
4052
4053 #[tokio::test]
4055 async fn multiple_read_only_opens_coexist_with_a_writer() {
4056 let tmp = tempfile::tempdir().expect("temp dir");
4057 let dir = StoreDir::new(tmp.path());
4058 let writer = open_files_handle_in(dir.clone());
4059
4060 let reader_a = try_open_read_only(&dir).expect("first read-only open");
4061 let reader_b = try_open_read_only(&dir).expect("second read-only open");
4062
4063 assert_eq!(read_files_count(&reader_a).await, 0);
4064 assert_eq!(read_files_count(&reader_b).await, 0);
4065 drop(writer);
4066 }
4067
4068 #[tokio::test]
4071 async fn read_only_open_sees_committed_writer_data() {
4072 let tmp = tempfile::tempdir().expect("temp dir");
4073 let dir = StoreDir::new(tmp.path());
4074 let writer = open_files_handle_in(dir.clone());
4075
4076 writer
4077 .sql(|sql| {
4078 sql.execute(
4079 "INSERT INTO files (id, blob_id, size, _updated_at) \
4080 VALUES ('row-before-reader', NULL, 0, ?1)",
4081 [sql.stamp()],
4082 )?;
4083 Ok(())
4084 })
4085 .await
4086 .expect("writer inserts before the reader opens");
4087
4088 let reader = try_open_read_only(&dir).expect("read-only open");
4089 assert_eq!(
4090 read_files_count(&reader).await,
4091 1,
4092 "the reader sees the row committed before it opened",
4093 );
4094
4095 writer
4098 .sql(|sql| {
4099 sql.execute(
4100 "INSERT INTO files (id, blob_id, size, _updated_at) \
4101 VALUES ('row-after-reader', NULL, 0, ?1)",
4102 [sql.stamp()],
4103 )?;
4104 Ok(())
4105 })
4106 .await
4107 .expect("writer inserts after the reader opened");
4108 assert_eq!(
4109 read_files_count(&reader).await,
4110 2,
4111 "the reader sees a row the writer committed after the reader opened",
4112 );
4113 drop(writer);
4114 }
4115
4116 #[tokio::test]
4122 async fn read_only_handle_connection_refuses_writes() {
4123 let tmp = tempfile::tempdir().expect("temp dir");
4124 let dir = StoreDir::new(tmp.path());
4125 let _writer = open_files_handle_in(dir.clone());
4126 let reader = try_open_read_only(&dir).expect("read-only open");
4127
4128 let result: CovenResult<()> = reader
4129 .sql_read(|conn| {
4130 conn.execute(
4131 "INSERT INTO files (id, blob_id, size, _updated_at) \
4132 VALUES ('should-not-write', NULL, 0, 'x')",
4133 [],
4134 )?;
4135 Ok(())
4136 })
4137 .await;
4138 assert!(
4139 matches!(result, Err(CovenError::Sqlite(_))),
4140 "a write through the read-only connection is refused by SQLite: {result:?}",
4141 );
4142 }
4143
4144 #[tokio::test]
4147 async fn read_only_open_refuses_a_too_new_schema() {
4148 let tmp = tempfile::tempdir().expect("temp dir");
4149 let dir = StoreDir::new(tmp.path());
4150
4151 let ahead = Coven::builder(config(dir.clone()))
4153 .synced_tables(vec![files_table()])
4154 .migrations(vec![
4155 files_migration(),
4156 Migration::sql(2, "add-extra", "CREATE TABLE extra (id TEXT PRIMARY KEY)"),
4157 ])
4158 .open()
4159 .expect("open at version 2");
4160 drop(ahead);
4161
4162 let reopened = Coven::builder(config(dir))
4165 .synced_tables(vec![files_table()])
4166 .migrations(vec![files_migration()])
4167 .open_read_only();
4168 assert!(matches!(
4169 reopened,
4170 Err(CovenError::Migration(MigrationError::SchemaTooNew {
4171 current: 2,
4172 supported: 1
4173 }))
4174 ));
4175 }
4176
4177 #[tokio::test]
4182 async fn read_only_handle_reads_a_host_provided_blob() {
4183 let tmp = tempfile::tempdir().expect("temp dir");
4184 let dir = StoreDir::new(tmp.path());
4185 let writer = Coven::builder(config(dir.clone()))
4186 .synced_tables(vec![remote_root_files_table()])
4187 .migrations(vec![files_migration()])
4188 .open()
4189 .expect("open writer");
4190
4191 let bytes = b"read-only-handle-serves-these-blob-bytes".to_vec();
4192 let hash = crate::blob::content_hash(&bytes);
4193 writer
4194 .write(
4195 {
4196 let bytes = bytes.clone();
4197 move |w| {
4198 w.put_blob("media-files", "roblob01", bytes);
4199 Ok(())
4200 }
4201 },
4202 {
4203 let len = bytes.len() as i64;
4204 move |sql| {
4205 sql.execute(
4206 "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
4207 VALUES (?1, ?2, ?3, ?4, ?5)",
4208 params!["file-ro", "roblob01", len, hash, sql.stamp()],
4209 )?;
4210 Ok(())
4211 }
4212 },
4213 )
4214 .await
4215 .expect("writer stores the row and blob");
4216
4217 let reader = Coven::builder(config(dir))
4218 .synced_tables(vec![remote_root_files_table()])
4219 .migrations(vec![files_migration()])
4220 .open_read_only()
4221 .expect("read-only open");
4222
4223 let blob = reader
4224 .row_blob_ref("files", "file-ro")
4225 .await
4226 .expect("capture read-only blob row");
4227 let read = reader
4228 .read_blob(&blob)
4229 .await
4230 .expect("read blob via read handle");
4231 assert_eq!(
4232 read, bytes,
4233 "the read handle serves the blob the writer stored"
4234 );
4235
4236 let (offset, len) = (5u64, 10u64);
4238 let range = reader
4239 .open_blob_stream(&blob, offset, len)
4240 .await
4241 .expect("ranged read via read handle");
4242 assert_eq!(range, &bytes[offset as usize..(offset + len) as usize]);
4243 drop(writer);
4244 }
4245
4246 #[tokio::test]
4251 async fn concurrent_same_blob_cache_writes_never_tear() {
4252 let tmp = tempfile::tempdir().expect("temp dir");
4253 let dir = StoreDir::new(tmp.path());
4254 let dest = dir
4255 .cache_blob_path(
4256 "media-files",
4257 coven_core::sync::store_commit::ObjectHash::digest(b"raceblob"),
4258 )
4259 .expect("cache path");
4260 if let Some(parent) = dest.parent() {
4261 std::fs::create_dir_all(parent).expect("create cache shard dir");
4262 }
4263
4264 let a = vec![b'A'; 64 * 1024];
4268 let b = vec![b'B'; 64 * 1024];
4269 let (ra, rb) = tokio::join!(
4270 crate::local_blob::write_atomic(&dest, &a),
4271 crate::local_blob::write_atomic(&dest, &b),
4272 );
4273 ra.expect("first concurrent cache write");
4274 rb.expect("second concurrent cache write");
4275
4276 let final_bytes = crate::local_blob::read(&dest)
4277 .await
4278 .expect("read cache file");
4279 assert!(
4280 final_bytes == a || final_bytes == b,
4281 "the cache file is exactly one producer's whole payload, never a torn mix",
4282 );
4283 }
4284}