1use super::{DatabaseAccess, DatabaseInner, Mode, StoreDatabase, Writer};
4use crate::internal_schema::CACHE_NAME_USED_SQL;
5use crate::sqlite::StoreSql;
6use crate::store_call::StoreCall;
7use crate::DatabaseChanges;
8use crate::{file_queue, file_ref, write_schema::WriteSchema, DbError, FileRef, FileUpload};
9use coven_crypto::SecretBytes;
10use coven_foundation::task::CarriesBug;
11use coven_foundation::{
12 files::{FileName, StoreDir},
13 id_source::{DeviceId, FileId, ObjectId},
14};
15use std::sync::Arc;
16
17#[path = "file_cache.rs"]
18pub(crate) mod cache;
19pub(crate) use cache::{
20 clear_entries as clear_cached, declare_budgets, kept as cache_kept, object_path,
21 object_path_of, publish as publish_cached, remove_entries as remove_cached,
22 set_budget as set_cache_budget, trim_entries as trim_cached,
23};
24
25pub struct FileDatabase<M: Mode = Writer> {
47 database: StoreDatabase<M>,
48}
49
50impl<M: Mode> Clone for FileDatabase<M> {
52 fn clone(&self) -> Self {
53 Self { database: self.database.clone() }
54 }
55}
56
57impl<M: Mode> FileDatabase<M> {
58 pub fn new(database: StoreDatabase<M>) -> Self {
60 Self { database }
61 }
62
63 pub fn bugs(&self) -> &coven_foundation::task::BugSinkRef {
66 self.database.bugs()
67 }
68
69 pub fn records(&self) -> Option<&FileDatabase<Writer>> {
72 M::records(self)
73 }
74
75 async fn run<T: Send + 'static, E: From<DbError> + CarriesBug + Send + 'static>(
78 &self,
79 run: impl FnOnce(&mut StoreSql, &StoreCall<'_>, &WriteSchema, &StoreDir, DeviceId) -> Result<T, E>
80 + Send
81 + 'static,
82 ) -> Result<T, E> {
83 self.database
84 .call(move |open| {
85 let access = M::access(open);
86 access.with_writer(|writer, call| {
87 run(writer, call, &access.write_schema, &access.directory, access.device)
88 })
89 })
90 .await
91 }
92
93 pub async fn file_ref(&self, file: FileId) -> Result<FileRef, DbError> {
95 self.database.file_ref(file).await
96 }
97 pub async fn open_local(
101 &self,
102 file: &FileRef,
103 place: coven_foundation::files::LeasePlace,
104 ) -> Result<crate::Located<crate::LocalFileStream>, crate::LocalFileError> {
105 self.database.open_local_file(file, place).await
106 }
107
108 pub async fn cached(&self, file: &FileRef, index: i64) -> Result<Option<Vec<u8>>, DbError> {
111 let file = file.clone();
112 self.database.call(move |open| cache::read(M::access(open), &file, index)).await
113 }
114 pub async fn repair_cached(
118 &self,
119 file: &FileRef,
120 index: i64,
121 bytes: Vec<u8>,
122 ) -> Result<(), DbError> {
123 let file = file.clone();
124 self.database.call(move |open| cache::repair(M::access(open), &file, index, &bytes)).await
125 }
126 pub async fn cache_range(
132 &self,
133 file: &FileRef,
134 first: i64,
135 chunks: i64,
136 name: FileName,
137 bytes: Vec<u8>,
138 ) -> Result<(), DbError> {
139 let file = file.clone();
140 self.database
141 .call(move |open| cache::put(M::access(open), &file, first, chunks, &name, &bytes))
142 .await
143 }
144 async fn remove_released(
148 &self,
149 chosen: crate::file_removals::Chosen,
150 ) -> Result<Vec<DbError>, DbError> {
151 self.database
152 .call(move |open| Ok::<_, DbError>(M::access(open).remove_released(chosen)))
153 .await
154 }
155 async fn removed(&self, chosen: crate::file_removals::Chosen) -> Result<(), DbError> {
156 let failures = self.remove_released(chosen).await?;
157 crate::write_failure::after_cleanup(Ok(()), failures)
158 }
159 pub async fn trim_cache(&self, namespace: &str) -> Result<(), DbError> {
161 let namespace = namespace.to_owned();
162 let chosen = self
163 .run(move |db, _, _, _, _| {
164 let paths = cache::trim(db, &namespace)?;
165 Ok::<_, DbError>(crate::file_removals::choose_cache(&paths))
166 })
167 .await?;
168 self.removed(chosen).await
169 }
170 #[cfg(any(test, feature = "test-utils"))]
174 pub async fn missing_bytes(&self, file: &FileRef) -> Result<u64, DbError> {
175 let file = file.clone();
176 self.run(move |db, _, _, _, device| {
177 Ok(file.size() - crate::cache_progress::covered(db, &file, device)?)
178 })
179 .await
180 }
181 pub async fn budget(&self, namespace: &str) -> Result<u64, DbError> {
183 let namespace = namespace.to_owned();
184 self.run(move |db, _, _, _, _| cache::budget(db, &namespace)).await
185 }
186}
187
188impl FileDatabase {
189 pub fn uploads_in_flight(&self) -> crate::InFlight {
192 self.database.uploads_in_flight()
193 }
194
195 pub fn downloads_in_flight(&self) -> crate::InFlight {
198 self.database.downloads_in_flight()
199 }
200
201 async fn read<T: Send + 'static, E: From<DbError> + CarriesBug + Send + 'static>(
204 &self,
205 read: impl FnOnce(&StoreSql, &StoreCall<'_>) -> Result<T, E> + Send + 'static,
206 ) -> Result<T, E> {
207 self.database.call(move |inner| inner.access.with_committed(read)).await
208 }
209 pub async fn local_evidence<P: crate::PendingValue>(
214 &self,
215 file: &FileRef,
216 upload_subject: impl FnOnce(&crate::FileLocation) -> P::Subject + Send + 'static,
217 move_subject: impl FnOnce(crate::MoveId) -> P::Subject + Send + 'static,
218 ) -> Result<Option<LocalEvidence<P>>, DbError>
219 where
220 P::Subject: Send,
221 {
222 let file = file.clone();
223 self.run(move |db, call, _, _, device| {
224 crate::transaction_constants::read_with_constants(db, call, || {
225 if file.place() != (crate::FilePlace::DeviceOnly { device }) {
226 return Ok(None);
227 }
228 let Some(upload) = file_queue::upload_of(db, file.id())? else {
229 return Ok(None);
230 };
231 let (location, of) = db.query_row(
232 "SELECT location,move FROM _coven_file_uploads WHERE id=?1",
233 [upload],
234 |r| Ok((r.get::<_, String>(0)?, r.get::<_, crate::MoveId>(1)?)),
235 )?;
236 let location = crate::FileLocation::decode(&location)?;
237 let record = crate::pending::find::<P>(db, &upload_subject(&location))?;
238 let moving = crate::pending::find::<P>(db, &move_subject(of))?;
239 let original = crate::user_file::read(db, file.id())?.map(|original| original.path);
240 Ok(Some(LocalEvidence { upload, location, of, original, record, moving }))
241 })
242 })
243 .await
244 }
245 pub async fn fail_move<P: crate::PendingValue>(
251 &self,
252 upload: i64,
253 record: P,
254 ) -> Result<(), DbError> {
255 self.run(move |db, _, _, _, _| {
256 crate::pending::state_transaction(db, |db| {
257 let Some(of) = record.move_of() else {
258 return Err(DbError::Invariant {
259 what: "a failed move's record names the move",
260 });
261 };
262 if crate::file_moves::of_upload(db, upload)? != of {
263 return Err(DbError::Invariant {
264 what: "a move fails through one of its own uploads",
265 });
266 }
267 if crate::file_moves::state(db, of)? != Some(crate::file_moves::MoveState::Moving) {
268 return Ok(());
269 }
270 crate::file_moves::end_unswitched(db, of, crate::file_moves::MoveState::Failed)?;
271 crate::pending::save(db, &record)
272 })
273 })
274 .await
275 }
276 pub async fn clear_original<P: crate::PendingValue>(
282 &self,
283 upload: i64,
284 subject: P::Subject,
285 checked: impl Fn(&P, &std::path::Path) -> bool + Send + 'static,
286 ) -> Result<Option<std::path::PathBuf>, DbError>
287 where
288 P::Subject: Send + 'static,
289 {
290 self.run(move |db, _, _, _, _| {
291 crate::pending::state_transaction(db, |db| {
292 let original: Option<Option<Vec<u8>>> = db
293 .query("SELECT original FROM _coven_file_uploads WHERE id=?1", [upload], |r| {
294 r.get(0)
295 })?
296 .pop();
297 let Some(Some(original)) = original else {
298 return Ok(None);
299 };
300 let path = crate::user_file::decode_path(original)?;
301 if crate::pending::find::<P>(db, &subject)?
302 .is_some_and(|record| checked(&record, &path))
303 {
304 crate::pending::clear(db, &subject)?;
305 crate::retry::clear(db, crate::RetrySubject::Upload(upload))?;
306 return Ok(Some(path));
307 }
308 Ok(None)
309 })
310 })
311 .await
312 }
313 pub async fn retire_unlinked<S: serde::Serialize + Send + 'static>(
319 &self,
320 in_flight: std::collections::BTreeSet<ObjectId>,
321 subject: impl Fn(&crate::FileLocation) -> S + Send + 'static,
322 ) -> Result<Vec<i64>, DbError> {
323 let database = &self.database;
325 let work = database
326 .call(move |inner| {
327 inner.access.with_committed(|db, _| {
328 Ok::<_, DbError>(!file_queue::retirable(db)?.is_empty())
329 })
330 })
331 .await?;
332 if !work {
333 return Ok(Vec::new());
334 }
335 database
336 .call(move |inner| {
337 inner.with_files("retire uploads", |writer, call, files| {
341 crate::local_write::local_write(
342 writer,
343 call,
344 &inner.access.write_schema,
345 inner.access.device,
346 inner.clock.now(),
347 files,
348 |files| files.retire_unlinked(&in_flight, subject),
349 )
350 })
351 })
352 .await
353 }
354 pub async fn fail_download<P: crate::PendingValue>(&self, record: P) -> Result<(), DbError> {
358 self.database.record_pending(record).await
359 }
360 pub async fn clear_download_pending<P: crate::PendingValue>(
363 &self,
364 subject: P::Subject,
365 checked: impl Fn(&P) -> bool + Send + 'static,
366 ) -> Result<(), DbError>
367 where
368 P::Subject: Send + 'static,
369 {
370 self.database.clear_checked::<P>(subject, checked).await
371 }
372 pub async fn clear_upload_pending<P: crate::PendingValue>(
377 &self,
378 object: (DeviceId, ObjectId),
379 subject: P::Subject,
380 checked: impl Fn(&P) -> bool + Send + 'static,
381 ) -> Result<Option<crate::RetrySubject>, DbError>
382 where
383 P::Subject: Send + 'static,
384 {
385 self.database.clear_pending_if::<P>(object, subject, checked).await
386 }
387 pub async fn fail_local_read<P, E>(
393 &self,
394 file: FileId,
395 record: P,
396 next: impl FnOnce(Option<crate::RetryWait>) -> Result<crate::RetryWait, E> + Send + 'static,
397 ) -> Result<Option<(crate::RetrySubject, crate::RetryWait)>, E>
398 where
399 P: crate::PendingValue,
400 E: From<DbError> + CarriesBug + Send + 'static,
401 {
402 self.run(move |db, _, _, _, _| {
403 crate::pending::state_transaction(db, |db| {
404 let Some(id) = file_queue::upload_of(db, file)? else {
405 return Ok(Ok(None));
406 };
407 let subject = crate::RetrySubject::Upload(id);
408 let wait = match next(crate::retry::find(db, subject)?) {
411 Ok(wait) => wait,
412 Err(refused) => return Ok(Err(refused)),
413 };
414 crate::pending::save(db, &record)?;
415 crate::retry::save(db, subject, &wait)?;
416 Ok(Ok(Some((subject, wait))))
417 })
418 })
419 .await?
420 }
421
422 pub async fn uploads(&self) -> Result<Vec<FileUpload>, DbError> {
424 self.read(|db, _| file_queue::read(db)).await
425 }
426 #[cfg(any(test, feature = "test-utils"))]
428 pub async fn all_uploads(&self) -> Result<Vec<FileUpload>, DbError> {
429 self.read(|db, _| file_queue::read_all(db)).await
430 }
431 pub async fn bound_uploads(&self) -> Result<std::collections::BTreeSet<i64>, DbError> {
434 self.read(|db, _| file_queue::bound(db)).await
435 }
436 pub async fn uploads_paused(&self) -> Result<bool, DbError> {
438 self.read(|db, _| file_queue::paused(db)).await
439 }
440 pub async fn writes_paused(&self) -> Result<bool, DbError> {
443 self.read(|db, _| crate::operation::writes_paused(db)).await
444 }
445 pub async fn unmetered_only(&self) -> Result<bool, DbError> {
448 self.read(|db, _| file_queue::unmetered_only(db)).await
449 }
450 pub async fn upload_work<P: crate::PendingValue>(
454 &self,
455 subject: impl Fn(&crate::FileLocation) -> P::Subject + Send + 'static,
456 ) -> Result<Vec<(FileUpload, Option<P>)>, DbError> {
457 self.run(move |db, call, _, _, _| {
458 crate::transaction_constants::read_with_constants(db, call, || {
459 file_queue::read(db)?
460 .into_iter()
461 .filter(|upload| !upload.stored)
463 .map(|upload| {
464 let record = crate::pending::find::<P>(db, &subject(&upload.location))?;
465 Ok((upload, record))
466 })
467 .collect()
468 })
469 })
470 .await
471 }
472 pub async fn abandoned_uploads<P: crate::PendingValue>(
476 &self,
477 subject: impl Fn(&crate::FileLocation) -> P::Subject + Send + 'static,
478 ) -> Result<Vec<(FileUpload, Option<P>)>, DbError> {
479 self.run(move |db, call, _, _, _| {
480 crate::transaction_constants::read_with_constants(db, call, || {
481 file_queue::abandoned(db)?
482 .into_iter()
483 .map(|upload| {
484 let record = crate::pending::find::<P>(db, &subject(&upload.location))?;
485 Ok((upload, record))
486 })
487 .collect()
488 })
489 })
490 .await
491 }
492 pub async fn forget_session(&self, id: i64) -> Result<(), DbError> {
497 self.run(move |db, _, _, _, _| {
498 db.transaction(|db| crate::file_queue::forget_session(db, id))
499 })
500 .await
501 }
502 pub async fn fail_upload<P: crate::PendingValue>(
506 &self,
507 id: i64,
508 record: P,
509 retry: Option<crate::RetryWait>,
510 ) -> Result<(), DbError> {
511 self.run(move |db, _, _, _, _| {
512 db.transaction(|db| {
513 if !db.query_row(
514 "SELECT EXISTS(SELECT 1 FROM _coven_file_uploads WHERE id=?1)",
515 [id],
516 |r| r.get::<_, bool>(0),
517 )? {
518 return Err(DbError::Invariant {
519 what: "the upload recording progress is queued",
520 });
521 }
522 crate::retry::save_failure(
523 db,
524 &record,
525 crate::RetrySubject::Upload(id),
526 retry.as_ref(),
527 )
528 })
529 })
530 .await
531 }
532 pub async fn upload_chunk_hash(
535 &self,
536 location: &crate::FileLocation,
537 index: u64,
538 ) -> Result<coven_crypto::ContentHash, DbError> {
539 let location = location.clone();
540 self.read(move |db, _| file_queue::chunk_hash(db, &location, index)).await
541 }
542 pub async fn open_upload_source(
548 &self,
549 upload: &FileUpload,
550 place: coven_foundation::files::LeasePlace,
551 ) -> Result<Option<crate::LocalFileStream>, crate::LocalFileError> {
552 let (id, object, size) = (upload.id, upload.location.object(), upload.size);
553 self.database
554 .call(move |inner| open_upload_source(&inner.access, id, object, size, place))
555 .await
556 }
557
558 #[expect(
563 clippy::map_err_ignore,
564 reason = "a byte count past SQLite's integers breaks the named invariant"
565 )]
566 pub async fn record_session(
567 &self,
568 id: i64,
569 session: SecretBytes,
570 progress: crate::UploadProgress,
571 ) -> Result<(), DbError> {
572 let count = |bytes: u64| {
573 i64::try_from(bytes)
574 .map_err(|_| DbError::Invariant { what: "an upload's size fits SQLite" })
575 };
576 let confirmed = count(progress.bytes_confirmed)?;
577 let total = count(progress.bytes_total)?;
578 self.run(move |db, _, _, _, _| {
579 db.transaction(|db| {
580 changed(crate::file_queue::record_session(
581 db,
582 id,
583 session.as_bytes(),
584 confirmed,
585 total,
586 )?)
587 })
588 })
589 .await
590 }
591 pub async fn complete_upload<S: serde::Serialize + Send + 'static>(
601 &self,
602 id: i64,
603 subject: S,
604 ) -> Result<crate::UploadCompletion, DbError> {
605 let subject = crate::pending::SubjectKey::of(&subject)?;
606 self.database.call(move |inner| complete(inner, id, subject)).await
607 }
608 pub async fn reserve_cache_file(
611 &self,
612 file: &FileRef,
613 name: FileName,
614 ) -> Result<CacheReservation, DbError> {
615 let database = &self.database;
616 let writing = BytesWriting::of(database).await;
617 let file = file.clone();
618 database
619 .call(move |inner| {
620 inner.access.with_writer(|writer, _| {
621 writer.transaction(|db| {
622 if db.query_row(
623 &format!("SELECT {CACHE_NAME_USED_SQL}"),
624 [name.as_str()],
625 |r| r.get::<_, bool>(0),
626 )? {
627 return Err(DbError::FileNameReused { name: name.as_str().to_owned() });
628 }
629 crate::file_removals::record_cached(db, name.as_str())?;
630 Ok(())
631 })?;
632 inner.staging.reserve([crate::file_removals::OwnedFile::Cached(name.clone())]);
633 Ok::<_, DbError>(())
634 })?;
635 Ok(CacheReservation {
636 reserved: inner.staging.clone(),
637 file,
638 name,
639 writing: Some(writing),
640 partial: false,
641 })
642 })
643 .await
644 }
645
646 pub fn queued_work_changes(&self) -> DatabaseChanges<DbError> {
650 self.database.observer.changes(Self::queued_work_reads(), true, DbError::from)
651 }
652
653 pub fn queued_work_commits(&self) -> DatabaseChanges<DbError> {
658 self.database.observer.changes(Self::queued_work_reads(), false, DbError::from)
659 }
660
661 fn queued_work_reads() -> sqlive::ReadSet {
663 vec![
668 sqlive::TableRead {
669 table: "_coven_write_pause".into(),
670 columns: ["paused".into()].into(),
671 scope: sqlive::RowScope::All,
672 inserted_only: false,
673 },
674 sqlive::TableRead {
675 table: "_coven_file_uploads".into(),
676 columns: ["location".into()].into(),
677 scope: sqlive::RowScope::All,
678 inserted_only: true,
679 },
680 sqlive::TableRead {
681 table: "_coven_upload_files".into(),
682 columns: ["file".into()].into(),
683 scope: sqlive::RowScope::All,
684 inserted_only: false,
685 },
686 ]
687 }
688}
689
690pub struct CacheReservation {
693 reserved: Arc<crate::reserved_names::ReservedNames>,
695 file: FileRef,
696 name: FileName,
697 writing: Option<BytesWriting>,
699 partial: bool,
701}
702impl CacheReservation {
703 pub fn name(&self) -> &FileName {
705 &self.name
706 }
707}
708struct BytesWriting {
715 _lease: tokio::sync::OwnedRwLockReadGuard<()>,
716 _store: Arc<std::sync::RwLock<Option<DatabaseInner>>>,
717}
718impl BytesWriting {
719 async fn of(database: &StoreDatabase) -> Self {
722 Self {
723 _lease: database.writer.file_tasks.clone().read_owned().await,
724 _store: database.inner.clone(),
725 }
726 }
727}
728impl Drop for CacheReservation {
729 fn drop(&mut self) {
730 let Some(_writing) = self.writing.take() else {
732 return;
733 };
734 self.reserved.release([crate::file_removals::OwnedFile::Cached(self.name.clone())]);
735 }
736}
737fn complete(
740 inner: &DatabaseInner,
741 id: i64,
742 subject: crate::pending::SubjectKey,
743) -> Result<crate::UploadCompletion, DbError> {
744 inner.with_files("complete an upload", |writer, call, files| {
745 crate::local_write::local_write(
746 writer,
747 call,
748 &inner.access.write_schema,
749 inner.access.device,
750 inner.clock.now(),
751 files,
752 |files| {
753 files.complete_upload(
754 id,
755 Box::new(move |db, _| crate::pending::clear_key(db, &subject)),
756 )
757 },
758 )
759 })
760}
761fn open_upload_source<L>(
764 access: &DatabaseAccess<L>,
765 upload: i64,
766 object: ObjectId,
767 size: u64,
768 place: coven_foundation::files::LeasePlace,
769) -> Result<Option<crate::LocalFileStream>, crate::LocalFileError> {
770 let source = access.with_writer(|db, _| file_ref::queued_source(db, upload))?;
771 source
772 .map(|source| {
773 file_ref::open_source(&access.directory, &object.to_string(), size, source, place)
774 })
775 .transpose()
776}
777
778impl StoreDatabase {
779 pub async fn settle_files(&self) -> Result<(), DbError> {
784 self.call(move |inner| {
785 let waiting = inner.access.with_committed(|db, _| {
786 Ok::<_, DbError>(db.query_row(
787 "SELECT EXISTS(SELECT 1 FROM _coven_unreferenced_files)
788 AND NOT EXISTS(SELECT 1 FROM _coven_write_pause)",
789 [],
790 |r| r.get::<_, bool>(0),
791 )?)
792 })?;
793 if !waiting {
794 return Ok(());
795 }
796 inner.with_files("settle files", |writer, call, files| {
797 crate::local_write::local_write(
798 writer,
799 call,
800 &inner.access.write_schema,
801 inner.access.device,
802 inner.clock.now(),
803 files,
804 |files| files.settle(),
805 )
806 })
807 })
808 .await
809 }
810}
811
812fn changed(count: usize) -> Result<(), DbError> {
813 if count == 1 {
814 Ok(())
815 } else {
816 Err(DbError::Invariant { what: "a queued upload's update changes exactly its row" })
817 }
818}
819
820#[derive(Debug)]
823pub struct LocalEvidence<P> {
824 pub upload: i64,
826 pub location: crate::FileLocation,
828 pub of: crate::MoveId,
830 pub original: Option<std::path::PathBuf>,
832 pub record: Option<P>,
834 pub moving: Option<P>,
836}
837
838#[path = "file_database_pins.rs"]
839mod pins;
840
841#[path = "file_database_partial_pins.rs"]
842mod partial_pins;
843pub(crate) use partial_pins::remove as remove_partial_pin;
844
845#[path = "file_database_prepared_pin.rs"]
846mod prepared_pin;
847pub use prepared_pin::PreparedPin;
848
849#[path = "file_database_copies.rs"]
850pub(crate) mod copies;
851pub use copies::{CopyReservation, FolderCopy, PreparedDeviceOnly};
852
853#[cfg(test)]
854#[path = "file_database_tests.rs"]
855pub(crate) mod tests;