Skip to main content

coven_sync/
file_upload_pool.rs

1//! The upload worker's runs (§17.9, E7). While uploads can run, a run keeps
2//! as many attempts in flight as the upload limit allows, in each of two
3//! lanes: bound uploads, which writes wait for, and trailing ones (§17.4).
4//! As each one ends, the next eligible upload of its lane starts, files
5//! queued meanwhile included; no upload waits for the others to finish, and
6//! no bound upload for a trailing one. Pausing holds the trailing lane only.
7//! An app's early retry joins the run in progress.
8
9use crate::{
10    files::file_failure::upload_subject, files::FilesInner, files::StorageState,
11    files::TrailingHold, retry_waits::Ending, NetworkOwner, PendingRecord, Retry, SyncError,
12};
13use coven_database::{DbError, FileUpload, RetrySubject};
14use coven_foundation::locks::lock_unpoisoned;
15use coven_storage::StorageFailure;
16use futures_util::{future::BoxFuture, stream::FuturesUnordered, StreamExt};
17use std::{
18    collections::{BTreeMap, BTreeSet, VecDeque},
19    sync::atomic::Ordering,
20    time::Duration,
21};
22use tokio::sync::oneshot;
23
24/// Result of an app retry (`retry_uploads_now`), or of one upload run.
25/// Individual failures stay in `pending()`.
26#[derive(Debug, PartialEq, Eq)]
27pub enum DrainOutcome {
28    /// Every upload the retry covered was attempted; this many stored a
29    /// file.
30    Drained {
31        /// Uploads whose bytes storage confirmed for files waiting for them.
32        /// One whose move failed or was cancelled while it was sent stored no
33        /// file, and doesn't count (§17.3).
34        uploaded: usize,
35    },
36    /// No waiting upload exists.
37    QueueEmpty,
38    /// Uploads were paused: no trailing upload was tried. Each bound upload
39    /// the retry covered was, since the pause holds none (§17.4).
40    Paused,
41    /// The network is metered and the app moves files only on unmetered
42    /// networks: no trailing upload was tried. Each bound upload the retry
43    /// covered was, since the setting holds none (§17.9).
44    Metered,
45    /// Sync is stopped: no upload was tried (§17.4).
46    NotSyncing,
47}
48
49/// An app's early retry, waiting for its answer. A run takes it when it
50/// next reads the queue: the retry then covers every upload that can ever
51/// retry, each eligible whatever its wait, and is answered once each covered
52/// upload's attempt has ended.
53pub(super) struct AppRetry {
54    /// The queue ids it covers, fixed when a run takes it; each leaves as its
55    /// attempt ends. `None` until taken.
56    covers: Option<BTreeSet<i64>>,
57    /// How many covered uploads were stored.
58    uploaded: usize,
59    /// Its place among the retries asked, which says whether a run's reads
60    /// of the queue and the holds came after it.
61    asked: u64,
62    reply: oneshot::Sender<Result<DrainOutcome, SyncError>>,
63}
64
65/// The answer to an app's early retry: how its covered uploads' attempts
66/// ended, or nothing if the worker stopped with the store first.
67pub(super) struct UploadRetryAnswer(oneshot::Receiver<Result<DrainOutcome, SyncError>>);
68
69impl UploadRetryAnswer {
70    /// Wait for the answer; the store closed if the worker stopped first.
71    pub(super) async fn answered(self) -> Result<DrainOutcome, SyncError> {
72        self.0.await.unwrap_or_else(|_| Err(DbError::StoreClosed.into()))
73    }
74
75    /// The answer if the run already gave it; `None` when it did not.
76    #[cfg(test)]
77    pub(super) fn try_answered(&mut self) -> Option<Result<DrainOutcome, SyncError>> {
78        self.0.try_recv().ok()
79    }
80}
81
82/// Why the upload worker ended, which each app retry from then on hears.
83#[derive(Debug)]
84pub(super) enum WorkerEnd {
85    /// It panicked with this bug, which the store's bug sink heard: each
86    /// retry hears it as `SyncError::WorkerFailed` (E15).
87    Panicked(coven_foundation::task::Bug),
88    /// The store closed under it.
89    Closed,
90    /// What it watches failed, as when the handle fails, with one shared
91    /// cause.
92    Failed(crate::LocalFailure, std::sync::Arc<dyn std::error::Error + Send + Sync>),
93}
94
95impl WorkerEnd {
96    /// The end of a worker that stopped on `error`, the failure of what it
97    /// watches.
98    pub(super) fn of(error: SyncError) -> Self {
99        match error {
100            SyncError::Database(DbError::StoreClosed) => Self::Closed,
101            error => {
102                let (failure, source) = error.into_shared();
103                Self::Failed(failure, source)
104            }
105        }
106    }
107
108    fn error(&self) -> SyncError {
109        match self {
110            Self::Panicked(bug) => SyncError::WorkerFailed(bug.clone()),
111            Self::Closed => DbError::StoreClosed.into(),
112            Self::Failed(failure, source) => {
113                SyncError::Failed { failure: *failure, source: source.clone() }
114            }
115        }
116    }
117}
118
119/// The lane an upload runs in (§17.4): a bound upload's write and every
120/// later one wait for it, so neither the pause nor trailing uploads hold it.
121#[derive(Clone, Copy, Debug, PartialEq, Eq)]
122pub(super) enum Lane {
123    Bound,
124    Trailing,
125}
126
127impl Lane {
128    pub(super) fn of(upload: &FileUpload) -> Self {
129        match upload.holds {
130            Some(_) => Self::Bound,
131            None => Self::Trailing,
132        }
133    }
134    pub(super) fn index(self) -> usize {
135        match self {
136            Self::Bound => 0,
137            Self::Trailing => 1,
138        }
139    }
140}
141
142/// Why no upload may start now.
143#[derive(Clone, Copy, Debug)]
144enum Blocked {
145    Closed,
146    /// Sync is stopped, which stops both lanes (§17.4).
147    NotSyncing,
148    NoStorage,
149    /// The pause or a metered network holds trailing uploads, and no bound
150    /// upload waits.
151    Held(TrailingHold),
152    /// Files wait for the sequence to catch up (§5.4).
153    CatchUp,
154    /// The provider asked every consumer to wait this long (§6.7).
155    Cooling(Duration),
156    /// A connection change waits for the uploads in flight to end.
157    Changing,
158}
159
160impl Blocked {
161    /// What an app retry learns; `None` keeps it for the run after a
162    /// connection change, which serves it on the new connection.
163    fn retry_answer(self) -> Option<Result<DrainOutcome, SyncError>> {
164        Some(match self {
165            Self::Closed => Err(DbError::StoreClosed.into()),
166            Self::NotSyncing => Ok(DrainOutcome::NotSyncing),
167            Self::NoStorage => Err(SyncError::NoStorage),
168            Self::Held(TrailingHold::Paused) => Ok(DrainOutcome::Paused),
169            Self::Held(TrailingHold::Metered) => Ok(DrainOutcome::Metered),
170            // As for an app write (§12.3).
171            Self::Held(TrailingHold::MemberKeysChanging) => Err(DbError::MemberKeysChanging.into()),
172            Self::CatchUp => Err(StorageFailure::CatchUpRequired.into()),
173            // An app retry overrides ordinary backoff, never a provider cooldown.
174            Self::Cooling(remaining) => Err(StorageFailure::RateLimited
175                .with_source(coven_storage::CoolingDown { remaining })
176                .into()),
177            Self::Changing => return None,
178        })
179    }
180
181    /// What the run itself returns. Automatic work waits out a cooldown, a
182    /// connection change and a self-replacement's write pause without
183    /// failing.
184    fn run_outcome(self, uploaded: usize) -> Result<DrainOutcome, SyncError> {
185        match self {
186            Self::Cooling(_) | Self::Changing | Self::Held(TrailingHold::MemberKeysChanging) => {
187                Ok(DrainOutcome::Drained { uploaded })
188            }
189            blocked => blocked.retry_answer().unwrap_or(Ok(DrainOutcome::Drained { uploaded })),
190        }
191    }
192}
193
194type Attempt<'a> = BoxFuture<'a, (i64, Result<super::file_upload::Attempted, SyncError>)>;
195
196/// One run's scheduling state.
197#[derive(Default)]
198struct Run<'a> {
199    /// Attempts in flight, each yielding its queue id and result.
200    attempts: FuturesUnordered<Attempt<'a>>,
201    /// The network reports seen as each attempt in flight started.
202    started: BTreeMap<i64, u64>,
203    /// The lane of each attempt in flight.
204    lanes: BTreeMap<i64, Lane>,
205    /// The bound uploads of the last read of the queue, whose retries wake
206    /// the run while trailing uploads are held.
207    bound: BTreeSet<i64>,
208    /// A commit since that read may have queued a bound upload.
209    committed: bool,
210    /// Eligible uploads from the last read of the queue, oldest first, not
211    /// started yet.
212    ready: VecDeque<FileUpload>,
213    /// Something may have made more uploads eligible since that read.
214    stale: bool,
215    /// Uploads whose attempt failed after a report arrived while it ran,
216    /// with which waits those reports end; they end the wait each failed
217    /// into.
218    crossed: BTreeMap<RetrySubject, Ending>,
219    /// Files this run stored.
220    uploaded: usize,
221}
222
223impl Run<'_> {
224    fn in_flight(&self, lane: Lane) -> usize {
225        self.lanes.values().filter(|of| **of == lane).count()
226    }
227
228    /// The connections `lane` uses: one for each attempt in flight, and
229    /// those its attempts took beside their own for parts sent together.
230    fn used(&self, lane: Lane, owner: &FilesInner) -> usize {
231        self.in_flight(lane) + owner.connections_beside(lane)
232    }
233
234    /// Whether an upload of some lane a hold leaves open could start: a
235    /// lane with room, and for the bound lane, a bound upload the last read
236    /// found or a commit since may have queued.
237    fn room(&self, limit: usize, held: bool, owner: &FilesInner) -> bool {
238        (self.used(Lane::Bound, owner) < limit && (self.committed || !self.bound.is_empty()))
239            || (!held && self.used(Lane::Trailing, owner) < limit)
240    }
241}
242
243impl FilesInner {
244    /// Queue an app retry for the upload worker, and wake it. A worker that
245    /// panicked serves none: the retry fails with its bug, as
246    /// `SyncError::WorkerFailed`.
247    pub(super) fn ask_retry(&self) -> Result<UploadRetryAnswer, SyncError> {
248        let (reply, answer) = oneshot::channel();
249        {
250            // Closing sets the flag before clearing this list under its lock,
251            // and the worker's panic sets its bug before answering it, so a
252            // retry queued here is either cleared, answered or refused.
253            let mut retries = lock_unpoisoned(&self.app_retries);
254            self.check_open()?;
255            if let Some(ended) = self.worker_ended.get() {
256                return Err(ended.error());
257            }
258            let asked = self.retry_asked();
259            retries.push(AppRetry { covers: None, uploaded: 0, asked, reply });
260        }
261        self.upload_wake.notify_one();
262        Ok(UploadRetryAnswer(answer))
263    }
264
265    /// The upload worker ended, on a panic or on the failure of what it
266    /// watches: no run will serve an app retry, so each waiting one is
267    /// answered with why, as each asked from now on is (E15).
268    pub(super) fn worker_ended(&self, end: WorkerEnd) {
269        let mut retries = lock_unpoisoned(&self.app_retries);
270        let ended = self.worker_ended.get_or_init(|| end);
271        for retry in retries.drain(..) {
272            #[expect(
273                clippy::let_underscore_must_use,
274                reason = "a retry whose caller stopped waiting needs no answer"
275            )]
276            let _ = retry.reply.send(Err(ended.error()));
277        }
278    }
279
280    /// Answer every queued app retry with this; `None` keeps them all, as
281    /// they are, for the next run.
282    fn answer_retries(&self, answer: impl Fn() -> Option<Result<DrainOutcome, SyncError>>) {
283        let mut retries = lock_unpoisoned(&self.app_retries);
284        if answer().is_none() {
285            return;
286        }
287        for retry in retries.drain(..) {
288            if let Some(result) = answer() {
289                // The caller may have stopped waiting; nothing is lost.
290                #[expect(
291                    clippy::let_underscore_must_use,
292                    reason = "a retry whose caller stopped waiting needs no answer"
293                )]
294                let _ = retry.reply.send(result);
295            }
296        }
297    }
298
299    /// A test's run of the upload worker that begins with an app retry of its
300    /// own, whose answer it returns.
301    #[cfg(test)]
302    pub(crate) async fn drain_uploads_retrying(&self) -> Result<DrainOutcome, SyncError> {
303        let mut own = self.ask_retry()?;
304        let run = self.drain_uploads().await;
305        // Unanswered, its covered uploads were not all attempted: the run
306        // failed, or a connection change stopped it.
307        own.try_answered().unwrap_or(run)
308    }
309
310    /// One run of the upload worker: start uploads while they can run,
311    /// keeping the upload limit's number in flight, and end once none is in
312    /// flight and none can start. A run that fails on a bug returns it to
313    /// the worker, which reports it; each waiting app retry hears it as
314    /// `WorkerFailed` (E15).
315    pub(crate) async fn drain_uploads(&self) -> Result<DrainOutcome, SyncError> {
316        use coven_foundation::task::CarriesBug;
317        let _run = self.drain.lock().await;
318        let result = self.serve_uploads().await;
319        match result {
320            Ok(outcome) => Ok(outcome),
321            Err(error) if lock_unpoisoned(&self.app_retries).is_empty() => Err(error),
322            Err(error) => {
323                if let Some(bug) = error.bug().cloned() {
324                    self.answer_retries(|| Some(Err(SyncError::WorkerFailed(bug.clone()))));
325                    return Err(error);
326                }
327                // Every waiting app retry learns the failure that ended the run.
328                let (failure, source) = error.into_shared();
329                self.answer_retries(|| {
330                    Some(Err(SyncError::Failed { failure, source: source.clone() }))
331                });
332                Err(SyncError::Failed { failure, source })
333            }
334        }
335    }
336
337    /// Why no upload may start now, from the owner's current state.
338    fn blocked(&self) -> Option<Blocked> {
339        if self.check_open().is_err() {
340            Some(Blocked::Closed)
341        } else if !self.waits.started() {
342            Some(Blocked::NotSyncing)
343        } else if !matches!(self.storage_state(), StorageState::Connected(_)) {
344            Some(Blocked::NoStorage)
345        } else if self.changing.load(Ordering::Acquire) > 0 {
346            Some(Blocked::Changing)
347        } else if !self.send.files_allowed() {
348            Some(Blocked::CatchUp)
349        } else {
350            self.waits.cooldown().map(Blocked::Cooling)
351        }
352    }
353
354    fn end_blocked(&self, blocked: Blocked, uploaded: usize) -> Result<DrainOutcome, SyncError> {
355        self.answer_retries(|| blocked.retry_answer());
356        blocked.run_outcome(uploaded)
357    }
358
359    /// End a run that a hold on trailing uploads stops. Each app retry a
360    /// read of the queue covered is answered with the hold: a trailing upload
361    /// it covers was not tried, and each bound one was, or none waited. One
362    /// asked since could cover a bound upload queued after that read, which
363    /// the hold doesn't stop, or come after the hold ended, so it waits for
364    /// the next run, which reads the queue and the hold again.
365    fn end_held(
366        &self,
367        hold: TrailingHold,
368        uploaded: usize,
369        covered: impl Fn(&AppRetry) -> bool,
370    ) -> Result<DrainOutcome, SyncError> {
371        let blocked = Blocked::Held(hold);
372        let mut retries = lock_unpoisoned(&self.app_retries);
373        let (answered, asked_since): (Vec<_>, Vec<_>) =
374            retries.drain(..).partition(|retry| covered(retry));
375        *retries = asked_since;
376        let waiting = !retries.is_empty();
377        drop(retries);
378        for retry in answered {
379            if let Some(answer) = blocked.retry_answer() {
380                #[expect(
381                    clippy::let_underscore_must_use,
382                    reason = "a retry whose caller stopped waiting needs no answer"
383                )]
384                let _ = retry.reply.send(answer);
385            }
386        }
387        if waiting {
388            self.upload_wake.notify_one();
389        }
390        blocked.run_outcome(uploaded)
391    }
392
393    #[cfg(test)]
394    fn pass_blocked_end_gate(&self) {
395        let gate = lock_unpoisoned(&self.blocked_end_gate).clone();
396        if let Some(gate) = gate {
397            gate();
398        }
399    }
400
401    async fn serve_uploads(&self) -> Result<DrainOutcome, SyncError> {
402        // Observe before reading the queue, so nothing that happens while it
403        // is read is missed. Only commits count: the read covers the state
404        // before it, and a report of that state would say a bound upload may
405        // have been queued since the read, so the run would read the queue
406        // again while no slot is free and take an app retry there.
407        let mut changes = self.database.queued_work_commits();
408        let mut sending = self.send.watch();
409        let mut reports = self.waits.network_reports();
410        let mut rearmed = self.waits.rearmed();
411        let mut syncing = self.waits.syncing();
412        let mut network = self.network.clone();
413        if let Some(blocked @ Blocked::Closed) = self.blocked() {
414            return self.end_blocked(blocked, 0);
415        }
416        // Uploads no file waits for any more go first, paused, stopped or
417        // with no storage too; an attempt in flight retires its own as it
418        // ends (§17.9). One holding a provider session has it aborted
419        // first, which needs storage and sync started.
420        self.end_abandoned_sessions().await?;
421        for id in self
422            .database
423            .retire_unlinked(self.database.uploads_in_flight().current(), upload_subject)
424            .await?
425        {
426            self.waits.forget(RetrySubject::Upload(id));
427            self.forget_connection_wait(RetrySubject::Upload(id));
428        }
429        if let Some(blocked @ (Blocked::NotSyncing | Blocked::NoStorage)) = self.blocked() {
430            return self.end_blocked(blocked, 0);
431        }
432        // The persisted settings govern; parts check these copies between
433        // sends. A change of either waits until this run has acted on what
434        // it read, so the run cannot answer a retry asked after a resume
435        // with the pause.
436        let change = self.lock_hold_change().await;
437        // The app retries asked before the reads below, which cover them.
438        let read_after = self.retries_asked();
439        let paused = self.database.uploads_paused().await?;
440        self.set_paused(paused);
441        let unmetered_only = self.database.unmetered_only().await?;
442        self.unmetered_only.store(unmetered_only, Ordering::Release);
443        #[cfg(test)]
444        {
445            let gate = self.take_settings_read_gate();
446            if let Some((read, resume)) = gate {
447                read.send(()).expect("the test waits for the read");
448                resume.await.expect("the test resumes the run");
449            }
450        }
451        self.read_writes_paused().await?;
452        let work = self.database.upload_work::<PendingRecord>(upload_subject).await?;
453        // The pause, a self-replacement's write pause and a metered network
454        // hold trailing uploads only (§12.3, §17.4, §17.9).
455        if let Some(hold) = self.trailing_hold() {
456            if work.iter().all(|(item, _)| Lane::of(item) == Lane::Trailing) {
457                #[cfg(test)]
458                self.pass_blocked_end_gate();
459                return self.end_held(hold, 0, |retry| retry.asked < read_after);
460            }
461        }
462        drop(change);
463        if work.is_empty() {
464            self.answer_retries(|| Some(Ok(DrainOutcome::QueueEmpty)));
465            return Ok(DrainOutcome::QueueEmpty);
466        }
467        let mut run = Run::default();
468        let mut work = Some(work);
469        loop {
470            if let Some(blocked) = self.blocked() {
471                if run.attempts.is_empty() {
472                    #[cfg(test)]
473                    self.pass_blocked_end_gate();
474                    return self.end_blocked(blocked, run.uploaded);
475                }
476            } else {
477                self.act_on_reports(&mut run).await;
478                let limit = self.transfers.limits().uploads.get();
479                if work.is_some() || (run.stale && run.room(limit, self.trailing_held(), self)) {
480                    // The first read went stale if a report ended waits since.
481                    let work = match work.take() {
482                        Some(work) if !run.stale => work,
483                        _ => {
484                            // A self-replacement's pause starts and ends
485                            // with a commit, which made the read stale.
486                            self.read_writes_paused().await?;
487                            self.database.upload_work::<PendingRecord>(upload_subject).await?
488                        }
489                    };
490                    self.read_ready(&mut run, work);
491                }
492                self.start_ready(&mut run, limit);
493                if run.attempts.is_empty() && run.ready.is_empty() {
494                    // Held, the run ends once nothing it read is left to
495                    // start and no bound upload could have been queued since.
496                    let held = self.trailing_held();
497                    if held && (!run.stale || !run.room(limit, held, self)) {
498                        #[cfg(test)]
499                        self.pass_blocked_end_gate();
500                        // As with the read above, a setting's change waits
501                        // while the run acts on the hold it saw; after a
502                        // resume since the check, the run serves uploads
503                        // again.
504                        let _change = self.lock_hold_change().await;
505                        self.read_writes_paused().await?;
506                        if let Some(hold) = self.trailing_hold() {
507                            return self
508                                .end_held(hold, run.uploaded, |retry| retry.covers.is_some());
509                        }
510                        run.stale = true;
511                        continue;
512                    }
513                    if !run.stale
514                        && lock_unpoisoned(&self.app_retries)
515                            .iter()
516                            .all(|retry| retry.covers.is_some())
517                    {
518                        // Nothing a taken retry covers is left to start.
519                        for retry in lock_unpoisoned(&self.app_retries).drain(..) {
520                            #[expect(
521                                clippy::let_underscore_must_use,
522                                reason = "a retry whose caller stopped waiting needs no answer"
523                            )]
524                            let _ = retry
525                                .reply
526                                .send(Ok(DrainOutcome::Drained { uploaded: retry.uploaded }));
527                        }
528                        return Ok(DrainOutcome::Drained { uploaded: run.uploaded });
529                    }
530                    run.stale = true;
531                    continue;
532                }
533            }
534            // A due retry matters only while an upload could start; it
535            // would wake at once for a wait an attempt in flight serves.
536            let open = self.blocked().is_none()
537                && run.room(self.transfers.limits().uploads.get(), self.trailing_held(), self);
538            let retry = if open { self.next_upload_retry(&run.bound) } else { None };
539            tokio::select! {
540                Some((id, result)) = run.attempts.next(), if !run.attempts.is_empty() => {
541                    self.finished(&mut run, id, result?);
542                }
543                () = self.upload_wake.notified() => run.stale = true,
544                // A connection a part took beside its attempt's ended: an
545                // upload waiting for one may start.
546                () = self.connection_freed() => {}
547                change = changes.next() => {
548                    change?;
549                    run.stale = true;
550                    run.committed = true;
551                }
552                _ = sending.changed() => run.stale = true,
553                _ = syncing.changed() => run.stale = true,
554                _ = reports.changed() => {}
555                () = network.changed() => run.stale = true,
556                _ = rearmed.changed() => {}
557                () = async {
558                    match retry {
559                        Some(retry) => retry.await,
560                        None => std::future::pending().await,
561                    }
562                } => run.stale = true,
563            }
564        }
565    }
566
567    /// The run's own work failed with `error`, which no caller hears: a bug
568    /// it carries goes to the sink, and any other failure is logged (E15).
569    fn failed_on_its_own(&self, what: &'static str, error: &SyncError) {
570        if !coven_foundation::task::report_carried(self.database.bugs().as_ref(), error) {
571            tracing::error!(?error, "{what}");
572        }
573    }
574
575    /// Act on reports for upload waits, as each plan does (§6.7).
576    /// A failure keeps every wait and the report, for the next plan.
577    async fn act_on_reports(&self, run: &mut Run<'_>) {
578        if !run.crossed.is_empty() {
579            let crossed = std::mem::take(&mut run.crossed);
580            match self.waits.end_waits_after_report(NetworkOwner::Uploads, &crossed).await {
581                Ok(()) => run.stale = true,
582                Err(error) => {
583                    self.failed_on_its_own("ending reported upload waits failed", &error);
584                    run.crossed = crossed;
585                }
586            }
587        }
588        match self.waits.end_reported_waits(NetworkOwner::Uploads).await {
589            Ok(true) => run.stale = true,
590            Ok(false) => {}
591            Err(error) => self.failed_on_its_own("ending reported upload waits failed", &error),
592        }
593    }
594
595    /// Take the queue just read as the run's next uploads: each one not in
596    /// flight that may retry and whose wait is over, or that an app retry
597    /// covers. App retries not yet taken cover every upload that may retry.
598    /// While trailing uploads are held, only bound ones are taken.
599    fn read_ready(&self, run: &mut Run<'_>, work: Vec<(FileUpload, Option<PendingRecord>)>) {
600        run.stale = false;
601        run.committed = false;
602        run.bound = work
603            .iter()
604            .filter(|(item, _)| Lane::of(item) == Lane::Bound)
605            .map(|(item, _)| item.id)
606            .collect();
607        let held = self.trailing_held();
608        let retryable: BTreeSet<i64> = work
609            .iter()
610            .filter(|(_, record)| {
611                record.as_ref().is_none_or(|record| record.retry() != Retry::Never)
612            })
613            .map(|(item, _)| item.id)
614            .collect();
615        let forced: BTreeSet<i64> = {
616            let mut retries = lock_unpoisoned(&self.app_retries);
617            // A retry whose caller stopped waiting is dropped.
618            retries.retain(|retry| !retry.reply.is_closed());
619            for retry in retries.iter_mut() {
620                retry.covers.get_or_insert_with(|| retryable.clone());
621            }
622            retries.iter().flat_map(|retry| retry.covers.iter().flatten().copied()).collect()
623        };
624        self.answer_covered();
625        run.ready.clear();
626        for (item, record) in work {
627            let subject = RetrySubject::Upload(item.id);
628            if !retryable.contains(&item.id) || (held && Lane::of(&item) == Lane::Trailing) {
629                continue;
630            }
631            // An upload in flight, or one an app retry starts whatever its
632            // wait, was evaluated: its ended wait wakes nothing.
633            if run.started.contains_key(&item.id) {
634                self.waits.seen(subject);
635            } else if forced.contains(&item.id) {
636                self.waits.seen(subject);
637                run.ready.push_back(item);
638            } else if self.retry_due(&item, record.as_ref()) {
639                run.ready.push_back(item);
640            }
641        }
642    }
643
644    /// Start ready uploads, oldest first, until each lane has the limit's
645    /// number in flight. A trailing upload taken before a hold began waits
646    /// for the run's next read once the hold ends.
647    fn start_ready<'a>(&'a self, run: &mut Run<'a>, limit: usize) {
648        let held = self.trailing_held();
649        let mut waiting = VecDeque::new();
650        while let Some(item) = run.ready.pop_front() {
651            let lane = Lane::of(&item);
652            if lane == Lane::Trailing && held {
653                continue;
654            }
655            if run.used(lane, self) >= limit {
656                waiting.push_back(item);
657                continue;
658            }
659            let id = item.id;
660            run.lanes.insert(id, lane);
661            self.set_lane_attempts(lane, run.in_flight(lane));
662            run.started.insert(id, self.waits.network_reported());
663            // A resumed session starts where the provider confirmed it.
664            let progress = coven_database::TransferProgress {
665                bytes_transferred: item.progress.map_or(0, |progress| progress.bytes_confirmed),
666                bytes_total: item.bytes_total,
667            };
668            let sending = self.database.uploads_in_flight().start(
669                item.location.uploader(),
670                item.location.object(),
671                item.files.clone(),
672                progress,
673            );
674            run.attempts.push(Box::pin(async move {
675                let attempted = self.attempt_upload(item, sending.meter()).await;
676                sending.end(match &attempted {
677                    Ok(attempted) => attempted.transfer_end(),
678                    // The attempt failed, and so did recording its failure.
679                    Err(_) => coven_database::TransferEnd::Failed,
680                });
681                (id, attempted)
682            }));
683        }
684        // An upload waiting for a connection takes the next one freed,
685        // before a part beside an attempt does.
686        for lane in [Lane::Bound, Lane::Trailing] {
687            let waits = waiting.iter().any(|item| Lane::of(item) == lane);
688            self.set_lane_waiting(lane, waits);
689        }
690        run.ready = waiting;
691    }
692
693    /// One attempt ended: its slot is free, and the app retries covering it
694    /// count it. Only an upload that stored a file counts as uploaded; one
695    /// that retired because no file waited for it any more stored none.
696    fn finished(&self, run: &mut Run<'_>, id: i64, attempted: super::file_upload::Attempted) {
697        use super::file_upload::Attempted;
698        if let Some(lane) = run.lanes.remove(&id) {
699            self.set_lane_attempts(lane, run.in_flight(lane));
700        }
701        let started = run.started.remove(&id);
702        let stored = attempted == Attempted::Stored;
703        match attempted {
704            Attempted::Stored => run.uploaded += 1,
705            Attempted::Retired => {}
706            Attempted::Failed | Attempted::Held => {
707                if let Some(ending) = started.and_then(|seen| self.waits.reports_since(seen)) {
708                    run.crossed.insert(RetrySubject::Upload(id), ending);
709                }
710            }
711        }
712        for retry in lock_unpoisoned(&self.app_retries).iter_mut() {
713            if let Some(covers) = &mut retry.covers {
714                if covers.remove(&id) && stored {
715                    retry.uploaded += 1;
716                }
717            }
718        }
719        self.answer_covered();
720    }
721
722    /// Answer each taken app retry whose covered uploads were all attempted.
723    fn answer_covered(&self) {
724        let mut retries = lock_unpoisoned(&self.app_retries);
725        let mut kept = Vec::new();
726        for retry in retries.drain(..) {
727            match &retry.covers {
728                Some(covers) if covers.is_empty() => {
729                    #[expect(
730                        clippy::let_underscore_must_use,
731                        reason = "a retry whose caller stopped waiting needs no answer"
732                    )]
733                    let _ =
734                        retry.reply.send(Ok(DrainOutcome::Drained { uploaded: retry.uploaded }));
735                }
736                _ => kept.push(retry),
737            }
738        }
739        *retries = kept;
740    }
741}