1#[path = "file_cache.rs"]
4mod file_cache;
5#[path = "file_cache_retry.rs"]
6mod file_cache_retry;
7#[path = "file_device_only.rs"]
8mod file_device_only;
9#[path = "file_error.rs"]
10mod file_error;
11#[path = "file_failure.rs"]
12mod file_failure;
13#[path = "file_keep.rs"]
14mod file_keep;
15#[path = "file_read.rs"]
16mod file_read;
17#[path = "file_status.rs"]
18mod file_status;
19#[path = "file_storage.rs"]
20mod file_storage;
21#[path = "file_upload.rs"]
22mod file_upload;
23#[path = "file_upload_abandoned.rs"]
24mod file_upload_abandoned;
25#[path = "file_upload_connections.rs"]
26mod file_upload_connections;
27#[path = "file_upload_failure.rs"]
28mod file_upload_failure;
29#[path = "file_upload_pool.rs"]
30mod file_upload_pool;
31#[path = "file_upload_queue.rs"]
32mod file_upload_queue;
33#[path = "file_upload_retry.rs"]
34mod file_upload_retry;
35#[path = "file_upload_settings.rs"]
36mod file_upload_settings;
37use crate::SyncError;
38pub use coven_database::PinProgress;
39use coven_database::{DbError, FileDatabase, Mode, ReadOnly, Writer};
40use coven_foundation::locks::lock_unpoisoned;
41use coven_foundation::{
42 files::{FileBudget, StoreDir},
43 id_source::IdSourceRef,
44};
45use coven_storage::Storage;
46pub use file_device_only::{DeviceOnlyError, DeviceOnlyStep};
47pub use file_error::FileReadError;
48pub use file_read::{FileRangeStream, FileStream, ReadProgress};
49pub use file_status::FileStatus;
50pub use file_storage::ConnectionComing;
51pub(crate) use file_storage::StorageState;
52pub(crate) use file_upload_connections::SpareConnection;
53pub use file_upload_pool::DrainOutcome;
54pub(crate) use file_upload_queue::names_original;
55pub use file_upload_queue::{QueuedUpload, UploadPhase, UploadProgress, UploadQueue, UploadWait};
56pub use file_upload_settings::UploadSettings;
57use std::{
58 collections::BTreeMap,
59 sync::{
60 atomic::{AtomicBool, AtomicUsize, Ordering},
61 Arc, Mutex,
62 },
63};
64use tokio::sync::{watch, Notify};
65
66use coven_storage::TransferBudget;
67pub use coven_storage::TransferLimits;
68
69pub trait FilesMode: Mode {
73 type Uploads: Send + Sync + 'static;
75}
76impl FilesMode for Writer {
77 type Uploads = UploadSettings;
78}
79impl FilesMode for ReadOnly {
80 type Uploads = ();
81}
82
83pub struct Files<M: FilesMode = Writer> {
89 inner: Arc<FilesInner<M>>,
90 worker: Arc<FileWorker>,
91}
92impl<M: FilesMode> Clone for Files<M> {
94 fn clone(&self) -> Self {
95 Self { inner: self.inner.clone(), worker: self.worker.clone() }
96 }
97}
98type FileReadLocks = BTreeMap<
100 (String, coven_foundation::id_source::ObjectId),
101 std::sync::Weak<tokio::sync::Mutex<()>>,
102>;
103struct FilesInner<M: FilesMode = Writer> {
104 database: FileDatabase<M>,
105 directory: StoreDir,
106 storage: watch::Sender<file_storage::StorageSlot>,
110 send: Arc<crate::SequenceState>,
111 waits: Arc<crate::RetryWaits>,
112 connection_waits: Mutex<BTreeMap<coven_database::RetrySubject, u64>>,
117 ids: IdSourceRef,
118 clock: coven_foundation::clock::ClockRef,
120 uploads: M::Uploads,
123 unmetered_only: AtomicBool,
127 network: coven_foundation::network_watch::NetworkView,
129 hold_wake: Notify,
132 #[cfg(test)]
134 pub(crate) held_downloads: tokio::sync::watch::Sender<usize>,
135 #[cfg(any(test, feature = "test-utils"))]
137 reads_awaiting_connection: tokio::sync::watch::Sender<usize>,
138 #[cfg(test)]
141 pub(crate) shared_reads: tokio::sync::watch::Sender<usize>,
142 upload_wake: Arc<Notify>,
144 upload_parks: crate::Parks,
147 drain: tokio::sync::Mutex<()>,
149 app_retries: Mutex<Vec<file_upload_pool::AppRetry>>,
151 worker_ended: std::sync::OnceLock<file_upload_pool::WorkerEnd>,
155 changing: AtomicUsize,
158 cache: tokio::sync::RwLock<()>,
159 reads: Mutex<FileReadLocks>,
160 transfers: Arc<TransferBudget>,
161 files: FileBudget,
165 pins: tokio::sync::Mutex<()>,
166 closed: AtomicBool,
167 #[cfg(test)]
171 blocked_end_gate: Mutex<Option<Arc<dyn Fn() + Send + Sync>>>,
172 #[cfg(test)]
177 device_only_gate: Mutex<Option<file_device_only::DeviceOnlyGate>>,
178}
179struct FileWorker {
180 stop: watch::Sender<bool>,
181 task: tokio::sync::Mutex<Option<coven_foundation::task::OwnedTask>>,
184}
185impl Drop for FileWorker {
186 fn drop(&mut self) {
187 self.stop.send_replace(true);
188 }
189}
190impl Files<ReadOnly> {
191 pub fn read_only(
194 database: FileDatabase<ReadOnly>,
195 directory: StoreDir,
196 storage: Option<Arc<dyn Storage>>,
197 send: Arc<crate::SequenceState>,
198 waits: Arc<crate::RetryWaits>,
199 ids: IdSourceRef,
200 transfers: Arc<TransferBudget>,
201 files: FileBudget,
202 clock: coven_foundation::clock::ClockRef,
203 network: coven_foundation::network_watch::NetworkView,
204 ) -> Self {
205 let inner = Arc::new(FilesInner::new(
206 database,
207 directory,
208 storage,
209 send,
210 waits,
211 ids,
212 transfers,
213 files,
214 clock,
215 network,
216 (),
217 ));
218 let worker =
219 FileWorker { stop: watch::channel(false).0, task: tokio::sync::Mutex::new(None) };
220 Self { inner, worker: Arc::new(worker) }
221 }
222}
223impl<M: FilesMode> FilesInner<M> {
224 fn new(
225 database: FileDatabase<M>,
226 directory: StoreDir,
227 storage: Option<Arc<dyn Storage>>,
228 send: Arc<crate::SequenceState>,
229 waits: Arc<crate::RetryWaits>,
230 ids: IdSourceRef,
231 transfers: Arc<TransferBudget>,
232 files: FileBudget,
233 clock: coven_foundation::clock::ClockRef,
234 network: coven_foundation::network_watch::NetworkView,
235 uploads: M::Uploads,
236 ) -> Self {
237 Self {
238 database,
239 directory,
240 storage: watch::channel(file_storage::StorageSlot::opened(storage)).0,
241 send,
242 waits,
243 connection_waits: Mutex::new(BTreeMap::new()),
244 ids,
245 clock,
246 uploads,
247 unmetered_only: AtomicBool::new(false),
248 network,
249 hold_wake: Notify::new(),
250 #[cfg(test)]
251 held_downloads: tokio::sync::watch::channel(0).0,
252 #[cfg(any(test, feature = "test-utils"))]
253 reads_awaiting_connection: tokio::sync::watch::channel(0).0,
254 #[cfg(test)]
255 shared_reads: tokio::sync::watch::channel(0).0,
256 upload_wake: Arc::new(Notify::new()),
257 upload_parks: crate::Parks::new(),
258 drain: tokio::sync::Mutex::new(()),
259 app_retries: Mutex::new(Vec::new()),
260 worker_ended: std::sync::OnceLock::new(),
261 changing: AtomicUsize::new(0),
262 cache: tokio::sync::RwLock::new(()),
263 reads: Mutex::new(BTreeMap::new()),
264 transfers,
265 files,
266 pins: tokio::sync::Mutex::new(()),
267 closed: AtomicBool::new(false),
268 #[cfg(test)]
269 blocked_end_gate: Mutex::new(None),
270 #[cfg(test)]
271 device_only_gate: Mutex::new(None),
272 }
273 }
274}
275impl<M: FilesMode> Files<M> {
276 pub async fn close(&self) {
281 self.inner.closed.store(true, Ordering::Release);
282 self.inner.hold_wake.notify_waiters();
284 self.worker.stop.send_replace(true);
285 if let Some(task) = self.worker.task.lock().await.take() {
286 task.ended().await;
287 }
288 let _drain = self.inner.drain.lock().await;
289 let _pins = self.inner.pins.lock().await;
290 let _cache = self.inner.cache.write().await;
291 lock_unpoisoned(&self.inner.app_retries).clear();
292 self.inner.storage.send_modify(|slot| slot.close());
293 }
294}
295impl Files {
296 #[cfg(any(test, feature = "test-utils"))]
299 pub fn uploads_parked_from_now(
300 &self,
301 ) -> impl std::future::Future<Output = ()> + Send + 'static {
302 self.inner.upload_parks.parked_from_now()
303 }
304
305 pub fn new(
308 database: FileDatabase,
309 directory: StoreDir,
310 storage: Option<Arc<dyn Storage>>,
311 send: Arc<crate::SequenceState>,
312 waits: Arc<crate::RetryWaits>,
313 ids: IdSourceRef,
314 transfers: Arc<TransferBudget>,
315 files: FileBudget,
316 clock: coven_foundation::clock::ClockRef,
317 network: coven_foundation::network_watch::NetworkView,
318 ) -> Self {
319 let uploads_changes = database.queued_work_changes();
320 let (stop, _) = watch::channel(false);
321 let inner = Arc::new(FilesInner::new(
322 database,
323 directory,
324 storage,
325 send,
326 waits,
327 ids,
328 transfers,
329 files,
330 clock,
331 network,
332 UploadSettings::new(),
333 ));
334 let owner = Arc::downgrade(&inner);
338 let task = coven_foundation::task::spawn_owned(
339 "the upload worker",
340 inner.database.bugs().clone(),
341 upload_worker(Arc::downgrade(&inner), stop.subscribe(), uploads_changes),
342 |bug| async move {
343 if let Some(owner) = owner.upgrade() {
344 if let Err(error) = owner.waits.sync_halted(bug.clone()).await {
347 let error = crate::SyncError::from(error);
348 if !coven_foundation::task::report_carried(
349 owner.database.bugs().as_ref(),
350 &error,
351 ) {
352 tracing::debug!(
353 "halting sync after the upload worker's panic failed: {error}"
354 );
355 }
356 }
357 owner.worker_ended(file_upload_pool::WorkerEnd::Panicked(bug));
358 }
359 },
360 );
361 let worker = FileWorker { stop, task: tokio::sync::Mutex::new(Some(task)) };
362 Self { inner, worker: Arc::new(worker) }
363 }
364 pub fn transfer_limits(&self) -> TransferLimits {
366 self.inner.transfers.limits()
367 }
368 pub fn set_transfer_limits(&self, limits: TransferLimits) {
372 self.inner.transfers.set_limits(limits);
373 self.inner.upload_wake.notify_one();
374 }
375 pub async fn retry_uploads_now(&self) -> Result<DrainOutcome, SyncError> {
381 self.inner.ask_retry()?.answered().await
384 }
385 pub fn set_uploads_paused(
393 &self,
394 sql: &coven_database::SqlContext<'_, '_>,
395 paused: bool,
396 ) -> Result<(), DbError> {
397 self.inner.check_open()?;
398 sql.set_uploads_paused(paused)?;
399 let inner = self.inner.clone();
400 sql.after_commit(move || {
401 let change = inner.hold_change_blocking();
404 inner.set_paused(paused);
405 drop(change);
406 if !paused {
407 inner.upload_wake.notify_one();
408 }
409 });
410 Ok(())
411 }
412 pub fn set_unmetered_only(
419 &self,
420 sql: &coven_database::SqlContext<'_, '_>,
421 unmetered_only: bool,
422 ) -> Result<(), DbError> {
423 self.inner.check_open()?;
424 sql.set_unmetered_only(unmetered_only)?;
425 let inner = self.inner.clone();
426 sql.after_commit(move || {
427 let change = inner.hold_change_blocking();
428 inner.unmetered_only.store(unmetered_only, Ordering::Release);
429 drop(change);
430 if !unmetered_only {
431 inner.upload_wake.notify_one();
432 inner.hold_wake.notify_waiters();
433 }
434 });
435 Ok(())
436 }
437
438 pub async fn set_storage(
442 &self,
443 storage: Option<Arc<dyn Storage>>,
444 ready: impl std::future::Future<Output = Result<(), SyncError>>,
445 ) -> Result<(), SyncError> {
446 let _changing = ConnectionChange::start(&self.inner.changing, &self.inner.upload_wake);
447 let _drain = self.inner.drain.lock().await;
448 let _pins = self.inner.pins.lock().await;
449 let _cache = self.inner.cache.write().await;
450 self.inner.check_open()?;
451 ready.await?;
452 self.inner.storage.send_modify(|slot| slot.connect(storage));
453 Ok(())
454 }
455}
456
457struct ConnectionChange<'a> {
461 changing: &'a AtomicUsize,
462 wake: &'a Notify,
463}
464impl<'a> ConnectionChange<'a> {
465 fn start(changing: &'a AtomicUsize, wake: &'a Notify) -> Self {
466 changing.fetch_add(1, Ordering::AcqRel);
467 wake.notify_one();
468 Self { changing, wake }
469 }
470}
471impl Drop for ConnectionChange<'_> {
472 fn drop(&mut self) {
473 self.changing.fetch_sub(1, Ordering::AcqRel);
474 self.wake.notify_one();
475 }
476}
477
478async fn upload_worker(
484 owner: std::sync::Weak<FilesInner>,
485 stopping: watch::Receiver<bool>,
486 changes: coven_database::DatabaseChanges<coven_database::DbError>,
487) {
488 let ended = serve_uploads(owner.clone(), stopping, changes).await;
489 if let (Some(ended), Some(owner)) = (ended, owner.upgrade()) {
492 owner.worker_ended(file_upload_pool::WorkerEnd::of(ended));
493 }
494}
495
496async fn serve_uploads(
500 owner: std::sync::Weak<FilesInner>,
501 mut stopping: watch::Receiver<bool>,
502 mut changes: coven_database::DatabaseChanges<coven_database::DbError>,
503) -> Option<SyncError> {
504 let first = owner.upgrade()?;
505 let parks = first.upload_parks.clone();
506 let _running = parks.running();
507 let wake = first.upload_wake.clone();
508 wake.notify_one();
510 let mut sending = first.send.watch();
511 let mut syncing = first.waits.syncing();
512 let mut reports = first.waits.network_reports();
513 let mut network = first.network.clone();
515 drop(first);
516 let mut retry: crate::retry_waits::Wake = Box::pin(std::future::pending());
518 loop {
519 tokio::select! {
520 biased;
521 never = parks.polling() => match never {},
522 _ = stopping.changed() => return None,
523 result = changes.next() => if let Err(error) = result {
524 if !matches!(error, DbError::StoreClosed) {
525 tracing::error!(?error, "file commit observation failed");
526 }
527 return Some(error.into());
528 },
529 () = wake.notified() => {},
530 result = sending.changed() => if result.is_err() { return Some(DbError::StoreClosed.into()) },
533 result = syncing.changed() => if result.is_err() { return Some(DbError::StoreClosed.into()) },
535 result = reports.changed() => if result.is_err() { return Some(DbError::StoreClosed.into()) },
536 () = network.changed() => {},
537 () = &mut retry => {},
538 never = parks.parked() => match never {},
539 }
540 let owner = owner.upgrade()?;
541 let run = async {
542 match owner.drain_uploads().await {
543 Ok(_) => (),
544 Err(SyncError::NoStorage) => {
545 tracing::debug!("file uploads waiting for storage");
546 }
547 Err(SyncError::Storage(error))
548 if error.failure() == coven_storage::StorageFailure::CatchUpRequired =>
549 {
550 tracing::debug!("file uploads waiting for a completed catch-up");
551 }
552 Err(SyncError::Database(DbError::StoreClosed)) => (),
553 Err(error) => show_worker_failure(&owner, error).await?,
554 }
555 match owner.keep_pinned().await {
558 Ok(()) | Err(SyncError::Database(DbError::StoreClosed)) => Ok(()),
559 Err(error) => show_worker_failure(&owner, error).await,
560 }
561 };
562 let ran =
563 tokio::select! { biased; _ = stopping.changed() => return None, ran = run => ran };
564 if let Err(error) = ran {
565 return Some(error);
566 }
567 retry = match owner.database.bound_uploads().await {
570 Ok(bound) => {
571 owner.next_upload_retry(&bound).unwrap_or_else(|| Box::pin(std::future::pending()))
572 }
573 Err(DbError::StoreClosed) => return Some(DbError::StoreClosed.into()),
574 Err(error) => {
577 if let Err(error) = show_worker_failure(&owner, error.into()).await {
578 return Some(error);
579 }
580 Box::pin(std::future::pending())
581 }
582 };
583 }
584}
585
586async fn show_worker_failure(owner: &FilesInner, error: SyncError) -> Result<(), SyncError> {
592 if coven_foundation::task::report_carried(owner.database.bugs().as_ref(), &error) {
593 return Ok(());
594 }
595 owner.waits.show_worker_failure(error).await?;
596 Ok(())
597}
598
599impl<M: FilesMode> FilesInner<M> {
600 fn connection_due(&self, subject: coven_database::RetrySubject) -> bool {
601 let connection = self.storage.borrow().connection();
602 lock_unpoisoned(&self.connection_waits).get(&subject) != Some(&connection)
603 }
604
605 fn wait_for_connection(&self, subject: coven_database::RetrySubject) {
608 let connection = self.storage.borrow().connection();
609 lock_unpoisoned(&self.connection_waits).insert(subject, connection);
610 }
611
612 fn forget_connection_wait(&self, subject: coven_database::RetrySubject) {
613 lock_unpoisoned(&self.connection_waits).remove(&subject);
614 }
615
616 fn retain_connection_waits(&self, keep: impl Fn(&coven_database::RetrySubject) -> bool) {
618 lock_unpoisoned(&self.connection_waits).retain(|subject, _| keep(subject));
619 }
620
621 pub(crate) async fn lock_file(
624 &self,
625 file: &coven_database::FileRef,
626 location: &coven_database::FileLocation,
627 ) -> tokio::sync::OwnedMutexGuard<()> {
628 let lock = {
629 let mut reads = lock_unpoisoned(&self.reads);
630 reads.retain(|_, lock| lock.strong_count() != 0);
631 let key = (file.namespace().to_owned(), location.object());
632 match reads.get(&key).and_then(std::sync::Weak::upgrade) {
633 Some(lock) => {
634 #[cfg(test)]
635 self.shared_reads.send_modify(|shared| *shared += 1);
636 lock
637 }
638 None => {
639 let lock = Arc::new(tokio::sync::Mutex::new(()));
640 reads.insert(key, Arc::downgrade(&lock));
641 lock
642 }
643 }
644 };
645 lock.lock_owned().await
646 }
647
648 pub(crate) fn check_open(&self) -> Result<(), DbError> {
649 if self.closed.load(Ordering::Acquire) {
650 Err(DbError::StoreClosed)
651 } else {
652 Ok(())
653 }
654 }
655 pub(crate) fn metered_holds(&self, bound: bool) -> bool {
659 self.unmetered_only.load(Ordering::Acquire)
660 && self.network.current().metered
661 && held_on_metered(bound)
662 }
663 pub(crate) fn downloads_held(&self) -> bool {
665 self.metered_holds(false)
666 }
667 pub(crate) async fn until_downloads_unheld(&self) -> Result<(), DbError> {
671 let mut network = self.network.clone();
672 loop {
673 let woken = self.hold_wake.notified();
674 tokio::pin!(woken);
675 woken.as_mut().enable();
676 self.check_open()?;
677 network.read();
678 if !self.downloads_held() {
679 return Ok(());
680 }
681 #[cfg(test)]
682 let _held = Waiting::new(&self.held_downloads);
683 tokio::select! {
684 () = woken => {}
685 () = network.changed() => {}
686 }
687 }
688 }
689 pub(crate) async fn trim(&self, namespace: &str) {
690 if let Err(error) = self.database.trim_cache(namespace).await {
691 tracing::error!(?error, namespace, "cache eviction failed");
694 }
695 }
696}
697impl FilesInner {
698 pub(crate) async fn read_unmetered_only(&self) -> Result<(), DbError> {
703 let _change = self.lock_hold_change().await;
704 let unmetered_only = self.database.unmetered_only().await?;
705 self.unmetered_only.store(unmetered_only, Ordering::Release);
706 Ok(())
707 }
708}
709#[cfg(any(test, feature = "test-utils"))]
712pub(crate) struct Waiting<'a>(&'a tokio::sync::watch::Sender<usize>);
713
714#[cfg(any(test, feature = "test-utils"))]
715impl<'a> Waiting<'a> {
716 pub(crate) fn new(waiting: &'a tokio::sync::watch::Sender<usize>) -> Self {
717 waiting.send_modify(|waiting| *waiting += 1);
718 Self(waiting)
719 }
720}
721
722#[cfg(any(test, feature = "test-utils"))]
723impl Drop for Waiting<'_> {
724 fn drop(&mut self) {
725 self.0.send_modify(|waiting| *waiting -= 1);
726 }
727}
728
729#[derive(Clone, Copy, Debug, PartialEq, Eq)]
731pub(crate) enum TrailingHold {
732 Paused,
734 MemberKeysChanging,
738 Metered,
741}
742
743pub(crate) fn held_on_metered(bound: bool) -> bool {
748 !bound
749}
750
751#[cfg(test)]
752#[path = "files_tests.rs"]
753mod tests;