Skip to main content

coven_sync/
files.rs

1//! File transfers and cached reads, composed beside the database at the root.
2
3#[path = "file_cache.rs"]
4mod file_cache;
5#[path = "file_cache_retry.rs"]
6mod file_cache_retry;
7#[path = "file_device_only.rs"]
8mod file_device_only;
9#[path = "file_error.rs"]
10mod file_error;
11#[path = "file_failure.rs"]
12mod file_failure;
13#[path = "file_keep.rs"]
14mod file_keep;
15#[path = "file_read.rs"]
16mod file_read;
17#[path = "file_status.rs"]
18mod file_status;
19#[path = "file_storage.rs"]
20mod file_storage;
21#[path = "file_upload.rs"]
22mod file_upload;
23#[path = "file_upload_abandoned.rs"]
24mod file_upload_abandoned;
25#[path = "file_upload_connections.rs"]
26mod file_upload_connections;
27#[path = "file_upload_failure.rs"]
28mod file_upload_failure;
29#[path = "file_upload_pool.rs"]
30mod file_upload_pool;
31#[path = "file_upload_queue.rs"]
32mod file_upload_queue;
33#[path = "file_upload_retry.rs"]
34mod file_upload_retry;
35#[path = "file_upload_settings.rs"]
36mod file_upload_settings;
37use crate::SyncError;
38pub use coven_database::PinProgress;
39use coven_database::{DbError, FileDatabase, Mode, ReadOnly, Writer};
40use coven_foundation::locks::lock_unpoisoned;
41use coven_foundation::{
42    files::{FileBudget, StoreDir},
43    id_source::IdSourceRef,
44};
45use coven_storage::Storage;
46pub use file_device_only::{DeviceOnlyError, DeviceOnlyStep};
47pub use file_error::FileReadError;
48pub use file_read::{FileRangeStream, FileStream, ReadProgress};
49pub use file_status::FileStatus;
50pub use file_storage::ConnectionComing;
51pub(crate) use file_storage::StorageState;
52pub(crate) use file_upload_connections::SpareConnection;
53pub use file_upload_pool::DrainOutcome;
54pub(crate) use file_upload_queue::names_original;
55pub use file_upload_queue::{QueuedUpload, UploadPhase, UploadProgress, UploadQueue, UploadWait};
56pub use file_upload_settings::UploadSettings;
57use std::{
58    collections::BTreeMap,
59    sync::{
60        atomic::{AtomicBool, AtomicUsize, Ordering},
61        Arc, Mutex,
62    },
63};
64use tokio::sync::{watch, Notify};
65
66use coven_storage::TransferBudget;
67pub use coven_storage::TransferLimits;
68
69/// What a handle's mode adds to its file owner: the writer's upload
70/// settings, which the upload runs and the app's setting changes use, and
71/// nothing for a read-only handle, which runs no upload. Sealed with `Mode`.
72pub trait FilesMode: Mode {
73    /// The mode's upload state, beside what every file read uses.
74    type Uploads: Send + Sync + 'static;
75}
76impl FilesMode for Writer {
77    type Uploads = UploadSettings;
78}
79impl FilesMode for ReadOnly {
80    type Uploads = ();
81}
82
83/// Owns storage, database file operations and the device's file area
84/// (Appendix F.3), for the store's writer or a read-only handle
85/// (`Files<ReadOnly>`), which only reads files and keeps its cache. The
86/// writer's starts one commit-driven worker, for uploads and pins; no sync
87/// loop or timer is created. Call `close` before closing the database.
88pub struct Files<M: FilesMode = Writer> {
89    inner: Arc<FilesInner<M>>,
90    worker: Arc<FileWorker>,
91}
92// Written out: `derive(Clone)` would ask the mode to be `Clone`.
93impl<M: FilesMode> Clone for Files<M> {
94    fn clone(&self) -> Self {
95        Self { inner: self.inner.clone(), worker: self.worker.clone() }
96    }
97}
98/// One lock per stored object's cache entries, by namespace and object.
99type FileReadLocks = BTreeMap<
100    (String, coven_foundation::id_source::ObjectId),
101    std::sync::Weak<tokio::sync::Mutex<()>>,
102>;
103struct FilesInner<M: FilesMode = Writer> {
104    database: FileDatabase<M>,
105    directory: StoreDir,
106    /// Whether storage is connected, and if not why: the one place it is
107    /// kept, written by the connection's owner through `set_storage` and
108    /// `expect_connection`, and by `close`. Readers wait on it.
109    storage: watch::Sender<file_storage::StorageSlot>,
110    send: Arc<crate::SequenceState>,
111    waits: Arc<crate::RetryWaits>,
112    /// Subjects whose failure needs a new connection, each with the number
113    /// of the connection it failed on: it is due again once the slot's
114    /// connection number differs. Reopening creates a new connection and
115    /// permits one attempt again.
116    connection_waits: Mutex<BTreeMap<coven_database::RetrySubject, u64>>,
117    ids: IdSourceRef,
118    /// Paces the progress a long call reports.
119    clock: coven_foundation::clock::ClockRef,
120    /// The mode's upload state: the writer's copies of the upload settings,
121    /// and nothing for a read-only handle.
122    uploads: M::Uploads,
123    /// The persisted setting to move files only on unmetered networks, as
124    /// of the last upload run or app change; parts and pins' downloads
125    /// check it, with the network, between requests (§17.9).
126    unmetered_only: AtomicBool,
127    /// The device's network, as the store's watch says it is (§6.7).
128    network: coven_foundation::network_watch::NetworkView,
129    /// Wakes a pin's download waiting for the network as the setting
130    /// changes or the store closes.
131    hold_wake: Notify,
132    /// How many pins' downloads wait for the network now, for a test.
133    #[cfg(test)]
134    pub(crate) held_downloads: tokio::sync::watch::Sender<usize>,
135    /// How many reads wait for a coming connection now, for a test.
136    #[cfg(any(test, feature = "test-utils"))]
137    reads_awaiting_connection: tokio::sync::watch::Sender<usize>,
138    /// How many reads have found another read of the same stored bytes
139    /// holding their lock, for a test.
140    #[cfg(test)]
141    pub(crate) shared_reads: tokio::sync::watch::Sender<usize>,
142    /// Wakes the upload worker, or its run in progress.
143    upload_wake: Arc<Notify>,
144    /// Each time the upload worker has taken everything that can wake it
145    /// and waits.
146    upload_parks: crate::Parks,
147    /// One upload run at a time.
148    drain: tokio::sync::Mutex<()>,
149    /// App retries waiting for an upload run to serve them.
150    app_retries: Mutex<Vec<file_upload_pool::AppRetry>>,
151    /// Why the upload worker ended, other than told to stop, set under
152    /// `app_retries`' lock: no run serves a retry once it is set, so each
153    /// fails with it (E15).
154    worker_ended: std::sync::OnceLock<file_upload_pool::WorkerEnd>,
155    /// Connection changes waiting for the uploads in flight to end; while
156    /// any waits, no upload starts.
157    changing: AtomicUsize,
158    cache: tokio::sync::RwLock<()>,
159    reads: Mutex<FileReadLocks>,
160    transfers: Arc<TransferBudget>,
161    /// The installation's places for open files: coven's own work leases
162    /// from it (an upload, a device-only call), and every file stream takes a
163    /// place of the lease its caller gives (§17.3).
164    files: FileBudget,
165    pins: tokio::sync::Mutex<()>,
166    closed: AtomicBool,
167    /// Runs where an upload run that found uploads blocked, with none in
168    /// flight, is about to act on that, so a test can change the setting
169    /// between the run's check and what it does.
170    #[cfg(test)]
171    blocked_end_gate: Mutex<Option<Arc<dyn Fn() + Send + Sync>>>,
172    /// Holds a device-only call at the place a test names: where a copy's
173    /// bytes were just appended, or where a copy is renamed to its path and
174    /// not yet recorded placed. It says it got there, then waits to be
175    /// resumed.
176    #[cfg(test)]
177    device_only_gate: Mutex<Option<file_device_only::DeviceOnlyGate>>,
178}
179struct FileWorker {
180    stop: watch::Sender<bool>,
181    /// The writer's upload worker, until the owner closes; a read-only
182    /// handle runs none.
183    task: tokio::sync::Mutex<Option<coven_foundation::task::OwnedTask>>,
184}
185impl Drop for FileWorker {
186    fn drop(&mut self) {
187        self.stop.send_replace(true);
188    }
189}
190impl Files<ReadOnly> {
191    /// Compose a read-only handle's file owner, which reads files and keeps
192    /// its cache; it uploads, pins and records nothing, and runs no worker.
193    pub fn read_only(
194        database: FileDatabase<ReadOnly>,
195        directory: StoreDir,
196        storage: Option<Arc<dyn Storage>>,
197        send: Arc<crate::SequenceState>,
198        waits: Arc<crate::RetryWaits>,
199        ids: IdSourceRef,
200        transfers: Arc<TransferBudget>,
201        files: FileBudget,
202        clock: coven_foundation::clock::ClockRef,
203        network: coven_foundation::network_watch::NetworkView,
204    ) -> Self {
205        let inner = Arc::new(FilesInner::new(
206            database,
207            directory,
208            storage,
209            send,
210            waits,
211            ids,
212            transfers,
213            files,
214            clock,
215            network,
216            (),
217        ));
218        let worker =
219            FileWorker { stop: watch::channel(false).0, task: tokio::sync::Mutex::new(None) };
220        Self { inner, worker: Arc::new(worker) }
221    }
222}
223impl<M: FilesMode> FilesInner<M> {
224    fn new(
225        database: FileDatabase<M>,
226        directory: StoreDir,
227        storage: Option<Arc<dyn Storage>>,
228        send: Arc<crate::SequenceState>,
229        waits: Arc<crate::RetryWaits>,
230        ids: IdSourceRef,
231        transfers: Arc<TransferBudget>,
232        files: FileBudget,
233        clock: coven_foundation::clock::ClockRef,
234        network: coven_foundation::network_watch::NetworkView,
235        uploads: M::Uploads,
236    ) -> Self {
237        Self {
238            database,
239            directory,
240            storage: watch::channel(file_storage::StorageSlot::opened(storage)).0,
241            send,
242            waits,
243            connection_waits: Mutex::new(BTreeMap::new()),
244            ids,
245            clock,
246            uploads,
247            unmetered_only: AtomicBool::new(false),
248            network,
249            hold_wake: Notify::new(),
250            #[cfg(test)]
251            held_downloads: tokio::sync::watch::channel(0).0,
252            #[cfg(any(test, feature = "test-utils"))]
253            reads_awaiting_connection: tokio::sync::watch::channel(0).0,
254            #[cfg(test)]
255            shared_reads: tokio::sync::watch::channel(0).0,
256            upload_wake: Arc::new(Notify::new()),
257            upload_parks: crate::Parks::new(),
258            drain: tokio::sync::Mutex::new(()),
259            app_retries: Mutex::new(Vec::new()),
260            worker_ended: std::sync::OnceLock::new(),
261            changing: AtomicUsize::new(0),
262            cache: tokio::sync::RwLock::new(()),
263            reads: Mutex::new(BTreeMap::new()),
264            transfers,
265            files,
266            pins: tokio::sync::Mutex::new(()),
267            closed: AtomicBool::new(false),
268            #[cfg(test)]
269            blocked_end_gate: Mutex::new(None),
270            #[cfg(test)]
271            device_only_gate: Mutex::new(None),
272        }
273    }
274}
275impl<M: FilesMode> Files<M> {
276    /// Stop background work, wait for in-flight database and transfer work,
277    /// then release the storage client, while clones of this owner live on.
278    /// An app retry still waiting fails as the store closes, as does a read
279    /// of a file stream still open.
280    pub async fn close(&self) {
281        self.inner.closed.store(true, Ordering::Release);
282        // A pin's download waiting for the network fails as the store closes.
283        self.inner.hold_wake.notify_waiters();
284        self.worker.stop.send_replace(true);
285        if let Some(task) = self.worker.task.lock().await.take() {
286            task.ended().await;
287        }
288        let _drain = self.inner.drain.lock().await;
289        let _pins = self.inner.pins.lock().await;
290        let _cache = self.inner.cache.write().await;
291        lock_unpoisoned(&self.inner.app_retries).clear();
292        self.inner.storage.send_modify(|slot| slot.close());
293    }
294}
295impl Files {
296    /// Each time the upload worker has taken every change, wake and due
297    /// retry that was ready, and waits for the next.
298    #[cfg(any(test, feature = "test-utils"))]
299    pub fn uploads_parked_from_now(
300        &self,
301    ) -> impl std::future::Future<Output = ()> + Send + 'static {
302        self.inner.upload_parks.parked_from_now()
303    }
304
305    /// Compose the writer's file owner and start its upload worker.
306    /// Storage may be absent while uploads wait.
307    pub fn new(
308        database: FileDatabase,
309        directory: StoreDir,
310        storage: Option<Arc<dyn Storage>>,
311        send: Arc<crate::SequenceState>,
312        waits: Arc<crate::RetryWaits>,
313        ids: IdSourceRef,
314        transfers: Arc<TransferBudget>,
315        files: FileBudget,
316        clock: coven_foundation::clock::ClockRef,
317        network: coven_foundation::network_watch::NetworkView,
318    ) -> Self {
319        let uploads_changes = database.queued_work_changes();
320        let (stop, _) = watch::channel(false);
321        let inner = Arc::new(FilesInner::new(
322            database,
323            directory,
324            storage,
325            send,
326            waits,
327            ids,
328            transfers,
329            files,
330            clock,
331            network,
332            UploadSettings::new(),
333        ));
334        // A panic in the worker goes to the store's bug sink, halts sync
335        // until the store reopens, as the sync loop's own panic does, and
336        // fails the app retries it would have served (E15).
337        let owner = Arc::downgrade(&inner);
338        let task = coven_foundation::task::spawn_owned(
339            "the upload worker",
340            inner.database.bugs().clone(),
341            upload_worker(Arc::downgrade(&inner), stop.subscribe(), uploads_changes),
342            |bug| async move {
343                if let Some(owner) = owner.upgrade() {
344                    // A failed write has failed the handle, which reports
345                    // it, or its bug goes to the sink.
346                    if let Err(error) = owner.waits.sync_halted(bug.clone()).await {
347                        let error = crate::SyncError::from(error);
348                        if !coven_foundation::task::report_carried(
349                            owner.database.bugs().as_ref(),
350                            &error,
351                        ) {
352                            tracing::debug!(
353                                "halting sync after the upload worker's panic failed: {error}"
354                            );
355                        }
356                    }
357                    owner.worker_ended(file_upload_pool::WorkerEnd::Panicked(bug));
358                }
359            },
360        );
361        let worker = FileWorker { stop, task: tokio::sync::Mutex::new(Some(task)) };
362        Self { inner, worker: Arc::new(worker) }
363    }
364    /// The limits shared by this handle's transfer consumers.
365    pub fn transfer_limits(&self) -> TransferLimits {
366        self.inner.transfers.limits()
367    }
368    /// Change upload and download admission. The next upload to start sees
369    /// the new upload limit; uploads in flight keep running. Active response
370    /// bodies retain their leases until they finish or are dropped.
371    pub fn set_transfer_limits(&self, limits: TransferLimits) {
372        self.inner.transfers.set_limits(limits);
373        self.inner.upload_wake.notify_one();
374    }
375    /// Optionally retry recoverable uploads early: the upload worker attempts
376    /// every upload that can retry, whatever its wait, and this returns once
377    /// each attempt has ended. Provider cooldowns and failures that never
378    /// clear still hold (§17.9), and while uploads are paused only bound ones
379    /// are tried; while sync is stopped none is (§17.4).
380    pub async fn retry_uploads_now(&self) -> Result<DrainOutcome, SyncError> {
381        // The worker stopping with the store drops the retry unanswered; a
382        // worker that ended otherwise answers it with why.
383        self.inner.ask_retry()?.answered().await
384    }
385    /// Persist the pause setting, as a step of `sql`'s write; a running
386    /// trailing upload stops between confirmed parts, preserving its
387    /// provider session (E7). Bound uploads run on (§17.4). Once the write
388    /// commits, and before it returns, the copy parts check takes the new
389    /// setting, after any upload run that read the old one has acted on it:
390    /// so no retry asked after the write returns is answered from the
391    /// setting it replaced.
392    pub fn set_uploads_paused(
393        &self,
394        sql: &coven_database::SqlContext<'_, '_>,
395        paused: bool,
396    ) -> Result<(), DbError> {
397        self.inner.check_open()?;
398        sql.set_uploads_paused(paused)?;
399        let inner = self.inner.clone();
400        sql.after_commit(move || {
401            // The write's effects run on its blocking thread, where waiting
402            // for the run is allowed.
403            let change = inner.hold_change_blocking();
404            inner.set_paused(paused);
405            drop(change);
406            if !paused {
407                inner.upload_wake.notify_one();
408            }
409        });
410        Ok(())
411    }
412    /// Persist the setting to move files only on unmetered networks, as a
413    /// step of `sql`'s write (§17.9): while the network is metered, a running
414    /// trailing upload stops between confirmed parts, as a pause stops it,
415    /// and pins' downloads wait before their next request to storage. Bound
416    /// uploads run on. Once the write commits, and before it returns, the
417    /// uploads and downloads take the new setting, as they take a pause.
418    pub fn set_unmetered_only(
419        &self,
420        sql: &coven_database::SqlContext<'_, '_>,
421        unmetered_only: bool,
422    ) -> Result<(), DbError> {
423        self.inner.check_open()?;
424        sql.set_unmetered_only(unmetered_only)?;
425        let inner = self.inner.clone();
426        sql.after_commit(move || {
427            let change = inner.hold_change_blocking();
428            inner.unmetered_only.store(unmetered_only, Ordering::Release);
429            drop(change);
430            if !unmetered_only {
431                inner.upload_wake.notify_one();
432                inner.hold_wake.notify_waiters();
433            }
434        });
435        Ok(())
436    }
437
438    /// Finish transfers before verifying and committing a connection change.
439    /// No upload starts meanwhile. A failed setup leaves the current
440    /// connection installed.
441    pub async fn set_storage(
442        &self,
443        storage: Option<Arc<dyn Storage>>,
444        ready: impl std::future::Future<Output = Result<(), SyncError>>,
445    ) -> Result<(), SyncError> {
446        let _changing = ConnectionChange::start(&self.inner.changing, &self.inner.upload_wake);
447        let _drain = self.inner.drain.lock().await;
448        let _pins = self.inner.pins.lock().await;
449        let _cache = self.inner.cache.write().await;
450        self.inner.check_open()?;
451        ready.await?;
452        self.inner.storage.send_modify(|slot| slot.connect(storage));
453        Ok(())
454    }
455}
456
457/// A connection change waiting for, then holding, the upload runs. While it
458/// lasts no upload starts, so the run in progress ends once its attempts do;
459/// as it ends, also when its caller is cancelled, the upload worker runs again.
460struct ConnectionChange<'a> {
461    changing: &'a AtomicUsize,
462    wake: &'a Notify,
463}
464impl<'a> ConnectionChange<'a> {
465    fn start(changing: &'a AtomicUsize, wake: &'a Notify) -> Self {
466        changing.fetch_add(1, Ordering::AcqRel);
467        wake.notify_one();
468        Self { changing, wake }
469    }
470}
471impl Drop for ConnectionChange<'_> {
472    fn drop(&mut self) {
473        self.changing.fetch_sub(1, Ordering::AcqRel);
474        self.wake.notify_one();
475    }
476}
477
478/// The upload worker: runs uploads whenever queued work, a changed
479/// prerequisite or a due retry may let one start, while sync is started
480/// (§17.4, §17.9). A run keeps the
481/// upload limit's number in flight until none can start. After each run it
482/// makes the pins coven keeps that may be made (§17.8).
483async fn upload_worker(
484    owner: std::sync::Weak<FilesInner>,
485    stopping: watch::Receiver<bool>,
486    changes: coven_database::DatabaseChanges<coven_database::DbError>,
487) {
488    let ended = serve_uploads(owner.clone(), stopping, changes).await;
489    // No run serves an app retry once the worker has ended, so each waiting
490    // one, and each asked after, hears why.
491    if let (Some(ended), Some(owner)) = (ended, owner.upgrade()) {
492        owner.worker_ended(file_upload_pool::WorkerEnd::of(ended));
493    }
494}
495
496/// The upload worker's runs, until it is told to stop or its owner is gone,
497/// which end it with `None`, or until what it watches fails, which ends it
498/// with that failure.
499async fn serve_uploads(
500    owner: std::sync::Weak<FilesInner>,
501    mut stopping: watch::Receiver<bool>,
502    mut changes: coven_database::DatabaseChanges<coven_database::DbError>,
503) -> Option<SyncError> {
504    let first = owner.upgrade()?;
505    let parks = first.upload_parks.clone();
506    let _running = parks.running();
507    let wake = first.upload_wake.clone();
508    // Pins coven keeps resume as the store opens (§17.8).
509    wake.notify_one();
510    let mut sending = first.send.watch();
511    let mut syncing = first.waits.syncing();
512    let mut reports = first.waits.network_reports();
513    // The network turning unmetered lets held uploads and pins go (§17.9).
514    let mut network = first.network.clone();
515    drop(first);
516    // The earliest due upload retry, armed after each run (§6.7).
517    let mut retry: crate::retry_waits::Wake = Box::pin(std::future::pending());
518    loop {
519        tokio::select! {
520            biased;
521            never = parks.polling() => match never {},
522            _ = stopping.changed() => return None,
523            result = changes.next() => if let Err(error) = result {
524                if !matches!(error, DbError::StoreClosed) {
525                    tracing::error!(?error, "file commit observation failed");
526                }
527                return Some(error.into());
528            },
529            () = wake.notified() => {},
530            // The store's sequence state, retry waits and network go only
531            // with the store.
532            result = sending.changed() => if result.is_err() { return Some(DbError::StoreClosed.into()) },
533            // Starting sync lets uploads run (§17.4).
534            result = syncing.changed() => if result.is_err() { return Some(DbError::StoreClosed.into()) },
535            result = reports.changed() => if result.is_err() { return Some(DbError::StoreClosed.into()) },
536            () = network.changed() => {},
537            () = &mut retry => {},
538            never = parks.parked() => match never {},
539        }
540        let owner = owner.upgrade()?;
541        let run = async {
542            match owner.drain_uploads().await {
543                Ok(_) => (),
544                Err(SyncError::NoStorage) => {
545                    tracing::debug!("file uploads waiting for storage");
546                }
547                Err(SyncError::Storage(error))
548                    if error.failure() == coven_storage::StorageFailure::CatchUpRequired =>
549                {
550                    tracing::debug!("file uploads waiting for a completed catch-up");
551                }
552                Err(SyncError::Database(DbError::StoreClosed)) => (),
553                Err(error) => show_worker_failure(&owner, error).await?,
554            }
555            // The switches a run made can ask for pins; a paused queue or no
556            // storage holds no pin made from coven's copy.
557            match owner.keep_pinned().await {
558                Ok(()) | Err(SyncError::Database(DbError::StoreClosed)) => Ok(()),
559                Err(error) => show_worker_failure(&owner, error).await,
560            }
561        };
562        let ran =
563            tokio::select! { biased; _ = stopping.changed() => return None, ran = run => ran };
564        if let Err(error) = ran {
565            return Some(error);
566        }
567        // A paused queue still runs its bound uploads, whose retries the
568        // worker must wake for (§17.4).
569        retry = match owner.database.bound_uploads().await {
570            Ok(bound) => {
571                owner.next_upload_retry(&bound).unwrap_or_else(|| Box::pin(std::future::pending()))
572            }
573            Err(DbError::StoreClosed) => return Some(DbError::StoreClosed.into()),
574            // Commits, network reports and wakes still run the worker; no
575            // retry wait wakes it until a later run arms one.
576            Err(error) => {
577                if let Err(error) = show_worker_failure(&owner, error.into()).await {
578                    return Some(error);
579                }
580                Box::pin(std::future::pending())
581            }
582        };
583    }
584}
585
586/// A failure of the upload worker's own work, which no caller hears: a bug
587/// goes to the store's bug sink, and any other failure is written as how
588/// the last pass ended, so the sync state says the queue is failing (§19.1)
589/// rather than only the log. A failed write of that has failed the store's
590/// handle, and ends the worker.
591async fn show_worker_failure(owner: &FilesInner, error: SyncError) -> Result<(), SyncError> {
592    if coven_foundation::task::report_carried(owner.database.bugs().as_ref(), &error) {
593        return Ok(());
594    }
595    owner.waits.show_worker_failure(error).await?;
596    Ok(())
597}
598
599impl<M: FilesMode> FilesInner<M> {
600    fn connection_due(&self, subject: coven_database::RetrySubject) -> bool {
601        let connection = self.storage.borrow().connection();
602        lock_unpoisoned(&self.connection_waits).get(&subject) != Some(&connection)
603    }
604
605    /// The subject's failure needs a new connection: it is due again when
606    /// the connection changes.
607    fn wait_for_connection(&self, subject: coven_database::RetrySubject) {
608        let connection = self.storage.borrow().connection();
609        lock_unpoisoned(&self.connection_waits).insert(subject, connection);
610    }
611
612    fn forget_connection_wait(&self, subject: coven_database::RetrySubject) {
613        lock_unpoisoned(&self.connection_waits).remove(&subject);
614    }
615
616    /// Keep only the connection waits `keep` accepts.
617    fn retain_connection_waits(&self, keep: impl Fn(&coven_database::RetrySubject) -> bool) {
618        lock_unpoisoned(&self.connection_waits).retain(|subject, _| keep(subject));
619    }
620
621    /// Hold the lock on one stored object's cache entries while reading
622    /// them, fetching and caching chunks. Files over the same bytes share it.
623    pub(crate) async fn lock_file(
624        &self,
625        file: &coven_database::FileRef,
626        location: &coven_database::FileLocation,
627    ) -> tokio::sync::OwnedMutexGuard<()> {
628        let lock = {
629            let mut reads = lock_unpoisoned(&self.reads);
630            reads.retain(|_, lock| lock.strong_count() != 0);
631            let key = (file.namespace().to_owned(), location.object());
632            match reads.get(&key).and_then(std::sync::Weak::upgrade) {
633                Some(lock) => {
634                    #[cfg(test)]
635                    self.shared_reads.send_modify(|shared| *shared += 1);
636                    lock
637                }
638                None => {
639                    let lock = Arc::new(tokio::sync::Mutex::new(()));
640                    reads.insert(key, Arc::downgrade(&lock));
641                    lock
642                }
643            }
644        };
645        lock.lock_owned().await
646    }
647
648    pub(crate) fn check_open(&self) -> Result<(), DbError> {
649        if self.closed.load(Ordering::Acquire) {
650            Err(DbError::StoreClosed)
651        } else {
652            Ok(())
653        }
654    }
655    /// Whether the setting to move files only on unmetered networks holds a
656    /// transfer now: it is on, the network is metered, and it holds this
657    /// transfer, which for an upload depends on whether it is bound.
658    pub(crate) fn metered_holds(&self, bound: bool) -> bool {
659        self.unmetered_only.load(Ordering::Acquire)
660            && self.network.current().metered
661            && held_on_metered(bound)
662    }
663    /// Whether a pin's download waits for the network now (§17.8).
664    pub(crate) fn downloads_held(&self) -> bool {
665        self.metered_holds(false)
666    }
667    /// Wait until no metered network holds a pin's download: the network
668    /// turns unmetered, or the setting goes off. Fails once the store
669    /// closes.
670    pub(crate) async fn until_downloads_unheld(&self) -> Result<(), DbError> {
671        let mut network = self.network.clone();
672        loop {
673            let woken = self.hold_wake.notified();
674            tokio::pin!(woken);
675            woken.as_mut().enable();
676            self.check_open()?;
677            network.read();
678            if !self.downloads_held() {
679                return Ok(());
680            }
681            #[cfg(test)]
682            let _held = Waiting::new(&self.held_downloads);
683            tokio::select! {
684                () = woken => {}
685                () = network.changed() => {}
686            }
687        }
688    }
689    pub(crate) async fn trim(&self, namespace: &str) {
690        if let Err(error) = self.database.trim_cache(namespace).await {
691            // Space reclamation is ancillary to a successful read (§17.8),
692            // but its failure must remain visible in the structured log.
693            tracing::error!(?error, namespace, "cache eviction failed");
694        }
695    }
696}
697impl FilesInner {
698    /// Take the persisted setting to move files only on unmetered networks
699    /// as this owner's copy, as an upload run does for both settings: a pin
700    /// prepared as the store opens holds as the setting says, before any
701    /// upload run has read it.
702    pub(crate) async fn read_unmetered_only(&self) -> Result<(), DbError> {
703        let _change = self.lock_hold_change().await;
704        let unmetered_only = self.database.unmetered_only().await?;
705        self.unmetered_only.store(unmetered_only, Ordering::Release);
706        Ok(())
707    }
708}
709/// A wait counted in a test's count of such waits while it waits: a pin's
710/// download waiting for the network, or a read for a coming connection.
711#[cfg(any(test, feature = "test-utils"))]
712pub(crate) struct Waiting<'a>(&'a tokio::sync::watch::Sender<usize>);
713
714#[cfg(any(test, feature = "test-utils"))]
715impl<'a> Waiting<'a> {
716    pub(crate) fn new(waiting: &'a tokio::sync::watch::Sender<usize>) -> Self {
717        waiting.send_modify(|waiting| *waiting += 1);
718        Self(waiting)
719    }
720}
721
722#[cfg(any(test, feature = "test-utils"))]
723impl Drop for Waiting<'_> {
724    fn drop(&mut self) {
725        self.0.send_modify(|waiting| *waiting -= 1);
726    }
727}
728
729/// What holds trailing uploads (§17.4, §17.9).
730#[derive(Clone, Copy, Debug, PartialEq, Eq)]
731pub(crate) enum TrailingHold {
732    /// The app paused uploads.
733    Paused,
734    /// A self-replacement pauses this device's writes, so a trailing
735    /// upload's completion would have to wait for it (§12.3). A bound
736    /// upload's completion goes through: the replacement waits for it.
737    MemberKeysChanging,
738    /// The network is metered and the app moves files only on unmetered
739    /// networks.
740    Metered,
741}
742
743/// Whether the setting to move files only on unmetered networks holds an
744/// upload on a metered network: the trailing ones, as the pause holds them,
745/// since every later write waits for a bound one (§17.4). The Lean model's
746/// `Upload.heldOnMetered`; the views' `coven_uploads.metered` says the same.
747pub(crate) fn held_on_metered(bound: bool) -> bool {
748    !bound
749}
750
751#[cfg(test)]
752#[path = "files_tests.rs"]
753mod tests;