Skip to main content

coven_sync/
sync_loop.rs

1//! Event-driven sequence passes with durable backoff and retained app wakes.
2use crate::{
3    LastPass, PassFailure, RetryOutcome, RetryWaits, SequenceStatus, SequenceSync, SyncError,
4    SyncFailure,
5};
6use coven_database::{DbError, RetrySubject};
7use std::{sync::Arc, time::Duration};
8use tokio::sync::{mpsc, oneshot};
9
10#[derive(Clone)]
11/// Control of one serialized pass loop. Commands survive caller cancellation.
12/// It publishes its activity to the owner's `SequenceState`.
13pub struct SyncLoop {
14    commands: mpsc::UnboundedSender<Command>,
15    /// Where the person's retry requests wait for this loop's passes.
16    waits: Arc<RetryWaits>,
17    /// Why the loop ended, once it has: what a command to it fails with
18    /// from then on (E15).
19    ended: Arc<std::sync::OnceLock<LoopEnd>>,
20    /// Each time the loop has taken everything that can wake it and waits.
21    #[cfg(any(test, feature = "test-utils"))]
22    parks: crate::Parks,
23    /// How many pauses have been asked of the loop.
24    #[cfg(any(test, feature = "test-utils"))]
25    pauses: Arc<tokio::sync::watch::Sender<usize>>,
26}
27/// Why a sync loop ended (E15).
28#[derive(Clone)]
29enum LoopEnd {
30    /// It panicked, or its own work hit a bug, which the sink heard.
31    Bug(coven_foundation::task::Bug),
32    /// Its own work failed otherwise, as a write does once it has failed
33    /// the handle.
34    Failed { failure: crate::LocalFailure, source: Arc<dyn std::error::Error + Send + Sync> },
35    /// Its store closed.
36    StoreClosed,
37    /// Its owner closed it as it released the connection, or dropped every
38    /// handle to it.
39    Released,
40}
41
42impl LoopEnd {
43    /// The loop ends on `error` from its own work, whose bug, if any, has
44    /// gone to the sink; with it, how the last pass shows `error`.
45    fn on(error: SyncError) -> (Self, SyncFailure) {
46        use coven_foundation::task::CarriesBug;
47        if let Some(bug) = error.bug().cloned() {
48            return (Self::Bug(bug), error.into());
49        }
50        let failure = error.local_failure();
51        let shown = SyncFailure::from(error);
52        let source: Arc<dyn std::error::Error + Send + Sync> = match &shown {
53            SyncFailure::Other(source) => source.clone(),
54            SyncFailure::Storage(source) => source.clone(),
55            shown => Arc::new(shown.clone()),
56        };
57        (Self::Failed { failure, source }, shown)
58    }
59
60    /// What a command to the ended loop fails with.
61    fn error(&self) -> SyncError {
62        match self {
63            Self::Bug(bug) => SyncError::WorkerFailed(bug.clone()),
64            Self::Failed { failure, source } => {
65                SyncError::Failed { failure: *failure, source: source.clone() }
66            }
67            Self::StoreClosed => DbError::StoreClosed.into(),
68            Self::Released => SyncError::NoStorage,
69        }
70    }
71}
72
73enum Command {
74    Start,
75    Now,
76    Stop,
77    Pause(oneshot::Sender<()>),
78    Resume,
79    Close(oneshot::Sender<Result<(), SyncError>>),
80}
81#[expect(
82    clippy::map_err_ignore,
83    reason = "a command fails only when the loop has ended, and `gone` says why"
84)]
85impl SyncLoop {
86    /// Compose a stopped loop for an open, connected sequence owner.
87    /// `codes` completes member-key replacement and installs delivered S3 keys
88    /// on this device after each pass (§11.5, §14.3).
89    /// A panic in the loop goes to the store's bug sink, and ends the loop
90    /// as a failure: the last pass is recorded failed with the bug, sync
91    /// stops, and the retries waiting, those the panicking pass took among
92    /// them, are answered with it as `SyncError::WorkerFailed` (E15).
93    pub fn new(sync: Arc<SequenceSync>, codes: crate::RestoreCodes) -> Self {
94        let (commands, receiver) = mpsc::unbounded_channel();
95        let waits = sync.waits.clone();
96        // Weak, so the connection's owners go when the loop lets them go.
97        let owner = Arc::downgrade(&sync);
98        let ended = Arc::new(std::sync::OnceLock::new());
99        let unwound = ended.clone();
100        let parks = crate::Parks::new();
101        // The loop catches a panic in its work itself; this one hears a
102        // panic in how the loop ends.
103        drop(coven_foundation::task::spawn_owned(
104            "the sync loop",
105            sync.database.bugs().clone(),
106            run(sync, codes, receiver, parks.clone(), ended.clone()),
107            |bug| async move {
108                unwound.get_or_init(|| LoopEnd::Bug(bug.clone()));
109                if let Some(owner) = owner.upgrade() {
110                    panicked(&owner, bug).await;
111                }
112            },
113        ));
114        Self {
115            commands,
116            waits,
117            ended,
118            #[cfg(any(test, feature = "test-utils"))]
119            parks,
120            #[cfg(any(test, feature = "test-utils"))]
121            pauses: Arc::new(tokio::sync::watch::channel(0).0),
122        }
123    }
124
125    /// Each time the loop has taken every command, change and wake that
126    /// was ready, and waits for the next.
127    #[cfg(any(test, feature = "test-utils"))]
128    pub fn parked_from_now(&self) -> impl std::future::Future<Output = ()> + Send + 'static {
129        self.parks.parked_from_now()
130    }
131
132    /// Wait until a pause is asked of the loop after this call; the pause
133    /// then waits for the loop to take it, after any pass under way.
134    #[cfg(any(test, feature = "test-utils"))]
135    pub fn pause_asked_from_now(&self) -> impl std::future::Future<Output = ()> + Send + 'static {
136        let mut pauses = self.pauses.subscribe();
137        let now = *pauses.borrow_and_update();
138        async move {
139            coven_foundation::clock::within(pauses.wait_for(|pauses| *pauses > now))
140                .await
141                .expect("a pause is asked")
142                .expect("the loop's pauses are kept");
143        }
144    }
145
146    /// Why a command found the loop gone: a bug it ended on, which the
147    /// sink heard already, as `WorkerFailed`; another failure of its own
148    /// work; its store closing; or its connection released (E15).
149    fn gone(&self) -> SyncError {
150        match self.ended.get() {
151            Some(end) => end.error(),
152            // The loop records why it ends before it lets its commands go,
153            // except on a panic: the panic drops them as it unwinds, before
154            // its bug is recorded, so a command in between hears this.
155            None => DbError::StoreClosed.into(),
156        }
157    }
158    /// Start and request an immediate pass. From now on the person's retry
159    /// requests wait for this loop's passes.
160    pub async fn start(&self) -> Result<(), SyncError> {
161        self.waits.sync_started().await?;
162        if self.commands.send(Command::Start).is_err() {
163            self.waits.sync_stopped().await?;
164            return Err(self.gone());
165        }
166        Ok(())
167    }
168    /// Retain one pass request, including while a pass is running.
169    pub fn sync_now(&self) -> Result<(), SyncError> {
170        self.commands.send(Command::Now).map_err(|_| self.gone())
171    }
172    /// Queue stopping after the active pass and this connection's transfers.
173    /// Stopping writes nothing the sync state reads; a release failure is
174    /// recorded as the last pass's failure.
175    /// Retry requests no pass has taken are answered `NotSyncing` now. The
176    /// loop stops even when writing that fails, and the call returns why.
177    pub async fn stop(&self) -> Result<(), SyncError> {
178        let stopped = self.waits.sync_stopped().await;
179        self.commands.send(Command::Stop).map_err(|_| self.gone())?;
180        Ok(stopped?)
181    }
182    /// Wait for the active pass, retaining its started state, pending work and
183    /// retry schedule while the owner prepares a connection or custody change.
184    pub async fn pause(&self) -> Result<(), SyncError> {
185        let (send, receive) = oneshot::channel();
186        self.commands.send(Command::Pause(send)).map_err(|_| self.gone())?;
187        #[cfg(any(test, feature = "test-utils"))]
188        self.pauses.send_modify(|pauses| *pauses += 1);
189        receive.await.map_err(|_| self.gone())
190    }
191    /// Continue the retained loop after a failed connection or custody change.
192    pub fn resume(&self) -> Result<(), SyncError> {
193        self.commands.send(Command::Resume).map_err(|_| self.gone())
194    }
195    /// Finish active work and this connection's transfers, then end this
196    /// loop. The store's owner closes its transfers and database.
197    pub async fn close(&self) -> Result<(), SyncError> {
198        let (send, receive) = oneshot::channel();
199        self.commands.send(Command::Close(send)).map_err(|_| self.gone())?;
200        receive.await.map_err(|_| self.gone())?
201    }
202}
203/// Coven's own operation whose last attempt failed, if any, with the cause
204/// its record keeps: it retries under its own backoff, but the pass that
205/// left it did not end synced (E5).
206async fn failing(sync: &SequenceSync) -> Result<Option<crate::PendingRecord>, SyncError> {
207    let records = sync.database.pending::<crate::PendingRecord>().await?;
208    Ok(records.into_iter().find(|record| {
209        let coven_owned = matches!(
210            record.subject,
211            crate::PendingSubject::Operation { kind, .. } if kind.coven_owned()
212        );
213        coven_owned
214            && matches!(
215                record.reason.retry_cause(),
216                crate::PendingReason::Storage { .. }
217                    | crate::PendingReason::Refused(_)
218                    | crate::PendingReason::Failed(_)
219            )
220    }))
221}
222
223/// Complete a pass and member-key custody (§11.5).
224async fn pass(
225    sync: &SequenceSync,
226    codes: &crate::RestoreCodes,
227) -> Result<Option<crate::PendingRecord>, SyncError> {
228    sync.pass().await?;
229    codes.resume_member_keys(sync).await?;
230    failing(sync).await
231}
232
233/// How a failed pass ended: a blocked slot, with the cause its record
234/// keeps, an attempt that received no provider response, or the failure
235/// itself (E5).
236fn failed(sync: &SequenceSync, error: SyncError) -> PassFailure {
237    let blocked = match sync.state.status() {
238        SequenceStatus::Blocked(record) => Some(*record),
239        SequenceStatus::Opening | SequenceStatus::Synced { .. } | SequenceStatus::Stopped(_) => {
240            None
241        }
242    };
243    let Some(record) = blocked else {
244        return crate::connection_failure(error);
245    };
246    let crate::PendingSubject::Slot(slot) = record.subject else {
247        return crate::connection_failure(error);
248    };
249    PassFailure::Blocked { slot, reason: record.reason, error: record.cause }
250}
251
252/// `error`, once a bug it carries has gone to the store's bug sink: nobody
253/// waits for the loop, so its failures reach no caller (E15).
254fn reported(sync: &SequenceSync, error: impl Into<SyncError>) -> SyncError {
255    coven_foundation::task::reported(sync.database.bugs().as_ref(), error.into())
256}
257
258/// `result` of a database call the loop makes on its own, once a bug its
259/// failure carries has gone to the store's bug sink: no caller hears of it
260/// (E15). Any other failure of the loop's own writes has failed the
261/// store's handle, which every call and live query now report.
262fn own_write<T>(sync: &SequenceSync, result: Result<T, DbError>) -> Result<T, DbError> {
263    result.map_err(|error| coven_foundation::task::reported(sync.database.bugs().as_ref(), error))
264}
265
266/// Write how sync last failed outside a pass's end, in the step that fails.
267/// A failed write ends the loop: its bug has gone to the sink, or it has
268/// failed the store's handle.
269async fn show(sync: &SequenceSync, failure: LastPass) -> Result<(), DbError> {
270    own_write(sync, failure.record(&sync.database).await)
271}
272
273/// Write how the pass ended, `now` by the store's clock, and that it moves
274/// no more. A failed write ends the loop, as `show`'s does.
275async fn ended(sync: &SequenceSync, last: LastPass) -> Result<(), DbError> {
276    own_write(sync, last.record_end(&sync.database, sync.clock.now()).await)
277}
278
279/// Run the pass, writing what it moved as it reads a slot it hadn't or claims
280/// one, each time that changes: until then the sync state shows what it did
281/// before, so a pass that finds nothing shows nothing (§19.1). A failed write
282/// has failed the store's handle; the pass still finishes, and the first
283/// failure comes back beside its result.
284async fn moving_pass(
285    sync: &SequenceSync,
286    codes: &crate::RestoreCodes,
287) -> (Result<Option<crate::PendingRecord>, SyncError>, Result<(), DbError>) {
288    let mut moves = sync.state.pass_begins();
289    let passing = pass(sync, codes);
290    tokio::pin!(passing);
291    let mut written = Ok(());
292    loop {
293        tokio::select! {
294            biased;
295            passed = &mut passing => return (passed, written),
296            moved = moves.moved() => {
297                let recorded = sync.database.record_moved(moved).await;
298                written = written.and(recorded);
299            }
300        }
301    }
302}
303
304/// The loop's own work failed with `error`: its bug goes to the sink, the
305/// last pass shows it, and the loop ends on it (E15).
306async fn give_up(sync: &SequenceSync, error: impl Into<SyncError>) -> LoopEnd {
307    let (end, shown) = LoopEnd::on(reported(sync, error));
308    #[expect(
309        clippy::let_underscore_must_use,
310        reason = "the loop ends either way; a bug has gone to the sink, and any other failed write has failed the handle, which reports it"
311    )]
312    let _ = show(sync, LastPass::failed(shown)).await;
313    end
314}
315
316/// Sync stopped as the loop ends or gives up: the waiting retries hear so.
317/// A failed write has answered them with its failure, and its bug has gone
318/// to the sink or it has failed the handle, so the loop goes on to end
319/// either way.
320async fn stopped(sync: &SequenceSync) {
321    if let Err(error) = own_write(sync, sync.waits.sync_stopped().await) {
322        tracing::debug!("stopping sync failed: {error}");
323    }
324}
325
326/// Wait until `interval` after `from`, by the store's monotonic clock.
327fn idle_wait(
328    sync: &SequenceSync,
329    from: Duration,
330    interval: Duration,
331) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> {
332    sync.clock.deadline((from + interval).saturating_sub(sync.clock.elapsed()))
333}
334
335/// The loop panicked with `bug`, which has gone to the sink: the last pass
336/// is recorded failed with it, sync stops, and the retries waiting, those
337/// the panicking pass took among them, are answered with it as
338/// `SyncError::WorkerFailed` (E15).
339async fn panicked(sync: &SequenceSync, bug: coven_foundation::task::Bug) {
340    #[expect(
341        clippy::let_underscore_must_use,
342        reason = "the loop has ended either way; a bug has gone to the sink, and any other failed write has failed the handle, which reports it"
343    )]
344    let _ = show(sync, LastPass::failed(SyncError::WorkerFailed(bug.clone()))).await;
345    if let Err(error) = own_write(sync, sync.waits.sync_panicked(bug).await) {
346        tracing::debug!("stopping sync after the loop's panic failed: {error}");
347    }
348}
349
350/// The upload worker's panic halted sync with `bug`, which the sink heard:
351/// the loop ends as on its own panic, so the last pass shows the bug, even
352/// over a pass that ended since, and later commands hear it as
353/// `SyncError::WorkerFailed` (E15).
354async fn halted(sync: &SequenceSync, bug: coven_foundation::task::Bug) -> LoopEnd {
355    panicked(sync, bug.clone()).await;
356    LoopEnd::Bug(bug)
357}
358
359/// Run the loop, catching a panic in its work while it still holds its
360/// commands and a closer's reply: it records why it ended before it lets
361/// either go, so each hears that, a panic's bug as `WorkerFailed` (E15).
362async fn run(
363    sync: Arc<SequenceSync>,
364    codes: crate::RestoreCodes,
365    mut commands: mpsc::UnboundedReceiver<Command>,
366    parks: crate::Parks,
367    why_ended: Arc<std::sync::OnceLock<LoopEnd>>,
368) {
369    let _running = parks.running();
370    let bugs = sync.database.bugs().clone();
371    let owner = sync.clone();
372    let mut closing = None;
373    let served = coven_foundation::task::caught(
374        "the sync loop",
375        serve(sync, codes, &mut commands, &parks, &mut closing),
376    )
377    .await;
378    let (end, closed) = match served {
379        Ok(served) => served,
380        Err(bug) => {
381            coven_foundation::task::report(bugs.as_ref(), &bug);
382            panicked(&owner, bug.clone()).await;
383            (LoopEnd::Bug(bug.clone()), Err(SyncError::WorkerFailed(bug)))
384        }
385    };
386    // The connection's owners are released before the closer learns so.
387    drop(owner);
388    // Commands from now on hear why the loop ended; those it never took
389    // go as `commands` does, after this.
390    why_ended.get_or_init(|| end);
391    if let Some(reply) = closing {
392        // A closer that stopped waiting leaves a bug to the sink and any
393        // other failure to the log.
394        if let Err(Err(error)) = reply.send(closed) {
395            if !coven_foundation::task::report_carried(bugs.as_ref(), &error) {
396                tracing::warn!("closing sync failed after its caller stopped waiting: {error}");
397            }
398        }
399    }
400}
401
402/// The loop's work: why it ended, and how a closer's release of its
403/// transfers went. A closer's reply goes in `closing` as the loop takes it.
404async fn serve(
405    sync: Arc<SequenceSync>,
406    codes: crate::RestoreCodes,
407    commands: &mut mpsc::UnboundedReceiver<Command>,
408    parks: &crate::Parks,
409    closing: &mut Option<oneshot::Sender<Result<(), SyncError>>>,
410) -> (LoopEnd, Result<(), SyncError>) {
411    let mut changes = sync.database.sync_changes();
412    let mut rearmed = sync.waits.rearmed();
413    let mut retry = match crate::ConnectionRetry::open(&sync.database).await {
414        Ok(retry) => retry,
415        Err(error) => {
416            crate::ConnectionRetry::Never(crate::connection_failure(reported(&sync, error)))
417        }
418    };
419    let mut closed = Ok(());
420    let end = 'run: {
421        let held = retry.held(&sync.waits);
422        if let Err(error) = sync.waits.holds(held).await {
423            break 'run give_up(&sync, error).await;
424        }
425        if let Some(failure) = held {
426            if let Err(error) = show(&sync, failure.clone().into()).await {
427                break 'run LoopEnd::on(error.into()).0;
428            }
429        }
430        let mut started = false;
431        let mut paused = false;
432        let mut pending = false;
433        // An idle pass comes the idle interval after the last pass ended, or
434        // after the loop began; an app holding a shorter one moves it (E5).
435        let mut interval = sync.state.idle_interval();
436        let mut idle_from = sync.clock.elapsed();
437        let mut idle = idle_wait(&sync, idle_from, interval.idle_wait());
438        loop {
439            // The network report ends the connection's and operations'
440            // waits for the network, and the person's retry all their waits; each
441            // asks for a pass. Reports during a pass are handled after it, so
442            // they end the waits it failed into (§6.7).
443            if !paused {
444                match sync.waits.end_reported_waits(crate::NetworkOwner::Connection).await {
445                    Ok(reported) => pending |= reported && started,
446                    Err(error) => break give_up(&sync, error).await,
447                }
448            }
449            // The cooldown's end wakes the loop to write it. Taken before the
450            // write below, so a cooldown that write still finds live, but that
451            // ends before the loop waits, wakes it at once instead of leaving
452            // the row saying it holds.
453            let cooldown = sync.waits.cooldown_wake();
454            // A pass held by a cooldown, or by a failure only the person can
455            // clear, answers the retries acted on at once (IO `held_answers`),
456            // and what a retry would do is written again, with a cooldown that
457            // ended, paused too.
458            if started {
459                if let Err(error) = sync.waits.answer_held(retry.held(&sync.waits)).await {
460                    break give_up(&sync, error).await;
461                }
462            }
463            // A requested pass waits only for the connection's backoff; an idle
464            // loop also wakes for operations' waits and their evidence.
465            let mut wakes = if pending { Vec::new() } else { sync.waits.evidence_wakes() };
466            wakes.extend(sync.waits.next_wake(|subject| {
467                *subject == RetrySubject::Sync
468                    || (!pending && matches!(subject, RetrySubject::Operation(_)))
469            }));
470            let work_wake = async {
471                if wakes.is_empty() {
472                    std::future::pending::<()>().await;
473                } else {
474                    futures_util::future::select_all(wakes).await;
475                }
476            };
477            let previous = match if paused {
478                Ok(None)
479            } else {
480                sync.database.retry_wait(RetrySubject::Sync).await
481            } {
482                Ok(wait) => wait,
483                Err(error) => break give_up(&sync, error).await,
484            };
485            let due = !paused
486                && retry.held(&sync.waits).is_none()
487                && sync.waits.due(RetrySubject::Sync, previous.as_ref());
488            tokio::select! {
489                biased;
490                never = parks.polling() => match never {},
491                bug = sync.waits.halt() => break halted(&sync, bug).await,
492                command = commands.recv() => match command {
493                    Some(Command::Start) => {
494                        if !started {
495                            let ready = sync.files.set_storage(Some(sync.storage.clone()), std::future::ready(Ok(())));
496                            match ready.await {
497                                Ok(()) => {
498                                    started = true;
499                                    pending = true;
500                                    sync.waits.connection_changed();
501                                }
502                                Err(error) => {
503                                    stopped(&sync).await;
504                                    if let Err(error) = show(&sync, LastPass::failed(reported(&sync, error))).await { break LoopEnd::on(error.into()).0; }
505                                }
506                            }
507                        }
508                    }
509                    Some(Command::Now) => pending |= started,
510                    Some(Command::Pause(reply)) => {
511                        paused = true;
512                        #[expect(
513                            clippy::let_underscore_must_use,
514                            reason = "the caller may have stopped waiting for the pause"
515                        )]
516                        let _ = reply.send(());
517                    }
518                    Some(Command::Resume) => paused = false,
519                    Some(Command::Stop) => {
520                        started = false;
521                        pending = false;
522                        // No caller waits for stopping; a failure is recorded
523                        // as the last pass's.
524                        if let Err(error) = sync.files.set_storage(None, std::future::ready(Ok(()))).await {
525                            if let Err(error) = show(&sync, LastPass::failed(reported(&sync, error))).await { break LoopEnd::on(error.into()).0; }
526                        }
527                    }
528                    // Transfers outlive this connection's loop; the store's
529                    // owner closes them. This connection's transfers finish first.
530                    Some(Command::Close(reply)) => {
531                        // Held outside the work, so a panic below answers it.
532                        *closing = Some(reply);
533                        if started && !paused {
534                            closed = sync.files.set_storage(None, std::future::ready(Ok(()))).await;
535                        }
536                        break LoopEnd::Released;
537                    }
538                    None => {
539                        // Nobody is left to receive the outcome, so a failure
540                        // to release the transfers is logged.
541                        if started && !paused {
542                            let released = sync.files.set_storage(None, std::future::ready(Ok(())));
543                            if let Err(error) = released.await {
544                                if !coven_foundation::task::report_carried(sync.database.bugs().as_ref(), &error) {
545                                    tracing::warn!("releasing transfers without a handle failed: {error}");
546                                }
547                            }
548                        }
549                        break LoopEnd::Released;
550                    }
551                },
552                changed = changes.next(), if started && !paused => match changed {
553                    Ok(()) => pending = true,
554                    Err(DbError::StoreClosed) => break LoopEnd::StoreClosed,
555                    Err(error) => {
556                        started = false;
557                        stopped(&sync).await;
558                        if let Err(error) = show(&sync, LastPass::failed(reported(&sync, error))).await { break LoopEnd::on(error.into()).0; }
559                    }
560                },
561                () = interval.changed() => idle = idle_wait(&sync, idle_from, interval.idle_wait()),
562                _ = &mut idle, if started && !paused && !pending => pending = true,
563                _ = sync.state.pass_requested(), if started && !paused && !pending => pending = true,
564                // A wait armed outside this loop changes the schedule to wake on.
565                // The waits, which send it, go only with the store.
566                changed = rearmed.changed(), if started && !paused => if changed.is_err() { break LoopEnd::StoreClosed },
567                _ = work_wake, if started && !paused && !(pending && due) => pending = true,
568                // The cooldown's end, which the loop writes as it plans again,
569                // lets a requested pass run, and changes what the person's
570                // retry would do even with no pass requested or while paused.
571                _ = async {
572                    if let Some(wake) = cooldown { wake.await } else { std::future::pending::<()>().await }
573                }, if started => {}
574                _ = std::future::ready(()), if started && !paused && pending && due => {
575                    pending = false;
576                    if let Err(error) = changes.take_changed() {
577                        break give_up(&sync, error).await;
578                    }
579                    // The pass answers the retries acted on before it began
580                    // (IO `pass_answers_acted_on`). It begins only as the
581                    // retry effect's row lets it: a cooldown that began since
582                    // the plan holds it, and the loop plans again with the pass
583                    // still requested.
584                    let requests = match sync.waits.begin_pass().await {
585                        Ok(Some(requests)) => requests,
586                        Ok(None) => {
587                            pending = true;
588                            continue;
589                        }
590                        Err(error) => break give_up(&sync, error).await,
591                    };
592                    let connection_version = sync.waits.connection_version();
593                    let (passed, moving) = moving_pass(&sync, &codes).await;
594                    if let Err(error) = moving {
595                        let (end, shown) = LoopEnd::on(reported(&sync, error));
596                        requests.answer(&PassFailure::Failed { error: shown }.into());
597                        break end;
598                    }
599                    // The idle interval counts from the pass's end, before the
600                    // steps that record how it ended.
601                    idle_from = sync.clock.elapsed();
602                    let outcome = match passed {
603                        Ok(failing) => match retry.succeeded(&sync.database, &sync.waits).await {
604                            Ok(()) => match failing {
605                                None => {
606                                    let finished_at = sync.clock.now();
607                                    if let Err(error) = LastPass::Synced.record_end(&sync.database, finished_at).await {
608                                        let (end, shown) = LoopEnd::on(reported(&sync, error));
609                                        requests.answer(&PassFailure::Failed { error: shown }.into());
610                                        break end;
611                                    }
612                                    RetryOutcome::Synced { finished_at }
613                                }
614                                Some(record) => {
615                                    // Work that never clears stops sync until a
616                                    // person resets it (E5).
617                                    started = record.retry() != crate::Retry::Never;
618                                    pending = false;
619                                    PassFailure::Failed { error: SyncFailure::Operation(Box::new(record)) }.into()
620                                }
621                            },
622                            Err(error) => {
623                                started = false;
624                                PassFailure::Failed { error: reported(&sync, error).into() }.into()
625                            }
626                        },
627                        Err(error) => {
628                            let error = reported(&sync, error);
629                            let reason = sync.error_reason(&error);
630                            match retry.failed(&sync.database, &sync.waits, reason, connection_version, error, |error| failed(&sync, error)).await {
631                                Ok(failure) => {
632                                    pending = true;
633                                    failure.into()
634                                }
635                                Err(failure) => {
636                                    started = false;
637                                    failure.into()
638                                }
639                            }
640                        }
641                    };
642                    // What a retry would do changes with the pass's failure, or
643                    // with sync stopping, before the row records it or the
644                    // person hears it.
645                    if let Err(error) = sync.waits.holds(retry.held(&sync.waits)).await {
646                        let (end, shown) = LoopEnd::on(reported(&sync, error));
647                        requests.answer(&PassFailure::Failed { error: shown }.into());
648                        break end;
649                    }
650                    if !started {
651                        stopped(&sync).await;
652                    }
653                    if let RetryOutcome::Failed(failure) = &outcome {
654                        // `ended` has sent a bug it failed on to the sink.
655                        if let Err(error) = ended(&sync, failure.clone().into()).await {
656                            let (end, shown) = LoopEnd::on(error.into());
657                            requests.answer(&PassFailure::Failed { error: shown }.into());
658                            break end;
659                        }
660                    }
661                    requests.answer(&outcome);
662                    idle = idle_wait(&sync, idle_from, interval.idle_wait());
663                }
664                never = parks.parked() => match never {},
665            }
666        }
667    };
668    // Closing hands waiting retries to the connection that replaces this
669    // one, or its owner answers them as it releases; any other end leaves no
670    // pass to answer them.
671    if closing.is_none() {
672        stopped(&sync).await;
673    }
674    (end, closed)
675}
676
677#[cfg(test)]
678#[path = "sync_loop_tests.rs"]
679mod tests;