Skip to main content

coven_sync/
retry_answer.rs

1//! The answer to the person's retry (§6.7; IO `RetryAnswer`).
2//!
3//! What decides it is `coven_retry_effect`'s row: whether a started
4//! connection will plan, whether a failure only new credentials, an update
5//! or a reset can clear holds every pass, and the provider cooldown holding
6//! them. Each step below changes that row and acts on the requests in one
7//! step on the store's writer (`StoreDatabase::change_retry_state`), so the
8//! answer a request gets is the effect the row shows when it is made (IO
9//! `ask_answers_by_effect`), and a request, stopping and a pass taking its
10//! requests each happen at once (IO `never_stranded`). Each step writes the
11//! cooldown as it stands then, and the row keeps no cooldown while sync is
12//! stopped, since no owner would write its end. The cooldown's end is a
13//! write too: the connection's owner wakes when it ends, and its next step
14//! writes it.
15//!
16//! A retry request is a report, so it takes the next report number. A
17//! request the row answers at once, stopped, held or cooling down, is
18//! answered so; any other waits for a pass. The connection's owner, the
19//! connection's first attempt and then its sync loop, acts on reports when
20//! it plans between passes. A pass begins only while the row lets it, and
21//! takes every waiting request the owner has acted on, answering them when
22//! it ends; a request made after the owner last acted waits for a later
23//! pass. When a failure or cooldown begins to hold the pass, the owner's
24//! next plan answers the requests it acted on at once. Stopping sync
25//! answers every waiting request `NotSyncing`. A pass that ends without
26//! answering the requests it took, because it panicked or was dropped,
27//! leaves them waiting again; an owner that panicked stops sync and answers
28//! every waiting request with its bug, as `SyncError::WorkerFailed` (E15).
29//!
30//! A failed write has failed the store's handle, which every call and live
31//! query then reports, and no later step can commit; the requests waiting
32//! then are answered with that failure. A write that panicked fails alone:
33//! its bug goes to the step's caller, or to the sink when the connection's
34//! owner wrote on its own, and that owner stops (E15).
35
36use crate::{NetworkOwner, PassFailure, RetryWaits, SyncFailure};
37use coven_database::{DbError, RetryState, RetrySync};
38use coven_foundation::clock::ClockRef;
39use coven_foundation::locks::lock_unpoisoned;
40use std::{
41    future::Future,
42    pin::Pin,
43    sync::{Arc, Mutex, Weak},
44    task::{Context, Poll},
45    time::SystemTime,
46};
47use tokio::sync::oneshot;
48
49/// How a step commits: a change of the retry state alone, or a start of
50/// sync, which `coven_sync_status` counts in the same transaction.
51#[derive(Clone, Copy)]
52enum Step {
53    Change,
54    Start,
55}
56
57/// What the first sync pass after the person's retry came to (E5).
58#[derive(Clone, Debug)]
59pub enum RetryOutcome {
60    /// The pass reached its read end.
61    Synced {
62        /// The store's clock when the pass finished.
63        finished_at: SystemTime,
64    },
65    /// The pass failed, or a failure only new credentials, an update or a
66    /// reset can clear holds every pass; the failure says which.
67    Failed(PassFailure),
68    /// A provider cooldown holds every pass until then. The pass runs once
69    /// it ends, with no further request.
70    CoolingDown {
71        /// When the cooldown ends, by the store's clock.
72        until: SystemTime,
73    },
74    /// Sync is stopped or the handle closed, so no pass follows the request.
75    NotSyncing,
76}
77
78impl From<PassFailure> for RetryOutcome {
79    fn from(failure: PassFailure) -> Self {
80        Self::Failed(failure)
81    }
82}
83
84/// What the person's retry would do if made now (E5), as
85/// `coven_retry_effect` holds it: run a pass, or be answered at once
86/// without one. An app offers a retry only while it is `Pass`; the status
87/// says why it is not.
88#[derive(Clone, Copy, Debug, PartialEq, Eq)]
89pub enum RetryEffect {
90    /// A pass runs for the retry, and its end answers it.
91    Pass,
92    /// A provider cooldown holds every pass until then, by the store's
93    /// clock: the retry is answered `CoolingDown` at once, and the pass runs
94    /// once the cooldown ends.
95    CoolingDown {
96        /// When the cooldown ends, by the store's clock.
97        until: SystemTime,
98    },
99    /// A failure only new credentials, an update or a reset can clear holds
100    /// every pass: the retry is answered `Failed` with it at once. The
101    /// status shows the failure.
102    Failed,
103    /// Sync is stopped or the handle closed: the retry is answered
104    /// `NotSyncing` at once.
105    NotSyncing,
106}
107
108impl From<RetryState> for RetryEffect {
109    /// A held failure answers before a cooldown: the cooldown's end would
110    /// not let the pass run. `coven_retry_effect.effect` is computed alike.
111    fn from(state: RetryState) -> Self {
112        match (state.sync, state.held) {
113            (RetrySync::Stopped, _) => Self::NotSyncing,
114            (RetrySync::Started { .. }, true) => Self::Failed,
115            (RetrySync::Started { cooling_until: Some(until) }, false) => {
116                Self::CoolingDown { until }
117            }
118            (RetrySync::Started { cooling_until: None }, false) => Self::Pass,
119        }
120    }
121}
122
123impl RetryEffect {
124    /// Read `coven_retry_effect`, as an app's live query does:
125    /// `handle.subscribe(RetryEffect::read)`.
126    pub fn read(sql: coven_database::SqlReadContext<'_>) -> Result<Self, DbError> {
127        let (effect, until) =
128            sql.query_row("SELECT effect,cooling_until FROM coven_retry_effect", [], |row| {
129                Ok((row.get::<_, String>(0)?, row.get::<_, Option<i64>>(1)?))
130            })?;
131        Self::from_columns(&effect, until)
132    }
133
134    /// The effect that `coven_retry_effect`'s `effect` and `cooling_until`
135    /// columns name, for an app that reads them in a query of its own. A
136    /// pair the view never computes is an invariant error.
137    pub fn from_columns(effect: &str, cooling_until: Option<i64>) -> Result<Self, DbError> {
138        Ok(match (effect, cooling_until) {
139            ("pass", None) => Self::Pass,
140            ("cooling_down", Some(until)) => {
141                Self::CoolingDown { until: coven_database::stored_time(until)? }
142            }
143            ("failed", _) => Self::Failed,
144            ("not_syncing", None) => Self::NotSyncing,
145            _ => return Err(DbError::Invariant { what: "the retry effect is one coven computes" }),
146        })
147    }
148
149    /// Follow `coven_retry_effect` with a live query, as an app does.
150    #[cfg(any(test, feature = "test-utils"))]
151    pub fn follow(
152        database: &coven_database::StoreDatabase,
153    ) -> coven_database::test_utils::Followed<Self, DbError> {
154        coven_database::test_utils::Followed::new(database.subscribe(Self::read))
155    }
156}
157
158/// Resolves when the first sync pass after a retry request ends (E5).
159/// Dropping it leaves the retry in place.
160#[derive(Debug)]
161pub struct RetryAnswer(oneshot::Receiver<RetryOutcome>);
162
163impl Future for RetryAnswer {
164    type Output = RetryOutcome;
165
166    fn poll(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll<RetryOutcome> {
167        // Every waiting request is answered before its owner lets it go; a
168        // request is dropped unanswered only with the store's retry schedule
169        // itself, once the handle is gone.
170        Pin::new(&mut self.0).poll(context).map(|answer| answer.unwrap_or(RetryOutcome::NotSyncing))
171    }
172}
173
174/// Retry requests by report number, each with where its answer goes.
175type Waiting = Vec<(u64, oneshot::Sender<RetryOutcome>)>;
176
177/// Answer every request in `requests` with `outcome`.
178fn answer(requests: Waiting, outcome: &RetryOutcome) {
179    for (_, request) in requests {
180        #[expect(
181            clippy::let_underscore_must_use,
182            reason = "a person who stopped waiting needs no answer"
183        )]
184        let _ = request.send(outcome.clone());
185    }
186}
187
188/// Requests a pass took when it began, answered when it ends. A pass that
189/// panicked, or was dropped, before answering them leaves them waiting
190/// again, for its owner's ending to answer.
191#[must_use = "the pass answers its requests when it ends"]
192pub struct PassRequests {
193    taken: Waiting,
194    /// The schedule they wait in again, unless it is gone.
195    waits: Weak<RetryWaits>,
196}
197
198impl PassRequests {
199    /// Answer every request with how the pass ended.
200    pub fn answer(mut self, outcome: &RetryOutcome) {
201        answer(std::mem::take(&mut self.taken), outcome);
202    }
203}
204
205impl Drop for PassRequests {
206    fn drop(&mut self) {
207        if self.taken.is_empty() {
208            return;
209        }
210        let Some(waits) = self.waits.upgrade() else {
211            return;
212        };
213        lock_unpoisoned(&waits.requests).waiting.append(&mut self.taken);
214    }
215}
216
217/// Waiting requests by report number, and the failure the row says holds
218/// every pass: the error objects a `Failed` answer carries, which the row
219/// can't hold. Both change only in a step on the writer, once the row they
220/// go with commits.
221pub(crate) struct RetryRequests {
222    held: Option<PassFailure>,
223    waiting: Waiting,
224}
225
226impl RetryRequests {
227    /// No request waiting, and no failure held.
228    pub(crate) fn new() -> Self {
229        Self { held: None, waiting: Vec::new() }
230    }
231
232    /// The answer a request gets at once in `state`, none when a pass
233    /// answers it: the effect's.
234    fn at_once(&self, state: RetryState) -> Option<RetryOutcome> {
235        match RetryEffect::from(state) {
236            RetryEffect::Pass => None,
237            RetryEffect::CoolingDown { until } => Some(RetryOutcome::CoolingDown { until }),
238            RetryEffect::Failed => Some(RetryOutcome::Failed(
239                self.held.clone().expect("the row holds a failure only with its failure"),
240            )),
241            RetryEffect::NotSyncing => Some(RetryOutcome::NotSyncing),
242        }
243    }
244
245    /// The waiting requests the connection's owner has acted on, which
246    /// wait in `waits` again if a pass drops them unanswered.
247    fn acted_on(&mut self, waits: &Arc<RetryWaits>, handled: u64) -> PassRequests {
248        let (taken, later) = std::mem::take(&mut self.waiting)
249            .into_iter()
250            .partition::<Vec<_>, _>(|(number, _)| *number <= handled);
251        self.waiting = later;
252        PassRequests { taken, waits: Arc::downgrade(waits) }
253    }
254}
255
256/// When the live provider cooldown ends, by the store's clock as it was
257/// armed, so writing one cooldown again names the same instant. `None` also
258/// when the clock cannot represent that time; the pass then answers its
259/// requests once the cooldown ends.
260fn cooldown_until(schedule: &Mutex<super::WaitSchedule>, clock: &ClockRef) -> Option<SystemTime> {
261    let elapsed = clock.elapsed();
262    lock_unpoisoned(schedule)
263        .armed
264        .get(&coven_database::RetrySubject::Provider)
265        .filter(|armed| !armed.over(elapsed))?
266        .ends
267}
268
269/// The answer to the requests waiting when a step fails, carrying the typed
270/// error the step's caller gets, never its text (F.5), or `NotSyncing` once
271/// the store closed. A step fails only as its handle does, whose failure
272/// every later call reports from one shared cause, so the answer carries that
273/// same failure; as its runtime shuts down, saying how far it got; or as it
274/// panics, with the bug.
275fn failed(error: &DbError) -> RetryOutcome {
276    let failure = match error {
277        DbError::StoreClosed => return RetryOutcome::NotSyncing,
278        DbError::StatePersistence(cause) => DbError::StatePersistence(cause.clone()),
279        DbError::RollbackFailed => DbError::RollbackFailed,
280        DbError::CheckpointFailed(cause) => DbError::CheckpointFailed(cause.clone()),
281        DbError::StatisticsFailed(cause) => DbError::StatisticsFailed(cause.clone()),
282        DbError::WallTimer(cause) => DbError::WallTimer(cause.clone()),
283        DbError::StoreFailed(bug) => DbError::StoreFailed(bug.clone()),
284        DbError::Shutdown(shutdown) => DbError::Shutdown(*shutdown),
285        DbError::Bug(bug) => DbError::Bug(bug.clone()),
286        _ => DbError::Invariant { what: "a retry step fails only as its handle or runtime does" },
287    };
288    PassFailure::Failed { error: SyncFailure::Other(Arc::new(failure)) }.into()
289}
290
291impl RetryWaits {
292    /// The row with the cooldown live as the step that writes it commits,
293    /// while sync is started; while it is stopped, none.
294    pub(super) fn live_cooldown(&self) -> impl FnOnce(RetryState) -> RetryState + Send + 'static {
295        let (schedule, clock) = (self.state.clone(), self.clock.clone());
296        move |state| match state.sync {
297            RetrySync::Stopped => state,
298            RetrySync::Started { .. } => RetryState {
299                sync: RetrySync::Started { cooling_until: cooldown_until(&schedule, &clock) },
300                ..state
301            },
302        }
303    }
304
305    /// Change the row with `change` and the live cooldown, then run `then`
306    /// with the requests and the committed state, in one step on the
307    /// writer. A failed write answers every waiting request with it.
308    async fn step<T: Send + 'static>(
309        self: &Arc<Self>,
310        change: impl FnOnce(RetryState) -> RetryState + Send + 'static,
311        then: impl FnOnce(&Arc<RetryWaits>, &mut RetryRequests, RetryState) -> T + Send + 'static,
312    ) -> Result<T, DbError> {
313        self.step_as(Step::Change, change, then).await
314    }
315
316    /// `step`, as `kind` commits it.
317    async fn step_as<T: Send + 'static>(
318        self: &Arc<Self>,
319        kind: Step,
320        change: impl FnOnce(RetryState) -> RetryState + Send + 'static,
321        then: impl FnOnce(&Arc<RetryWaits>, &mut RetryRequests, RetryState) -> T + Send + 'static,
322    ) -> Result<T, DbError> {
323        let Some(database) = &self.database else {
324            return Err(DbError::Invariant { what: "only a handle that writes the store syncs" });
325        };
326        let cooling = self.live_cooldown();
327        let waits = self.clone();
328        let change = move |state| cooling(change(state));
329        let then = move |state: RetryState| {
330            waits.syncing.send_if_modified(|syncing| {
331                let started = matches!(state.sync, RetrySync::Started { .. });
332                std::mem::replace(syncing, started) != started
333            });
334            then(&waits, &mut lock_unpoisoned(&waits.requests), state)
335        };
336        let result = match kind {
337            Step::Change => database.change_retry_state(change, then).await,
338            Step::Start => database.start_retry_state(change, then).await,
339        };
340        if let Err(error) = &result {
341            let waiting = std::mem::take(&mut lock_unpoisoned(&self.requests).waiting);
342            answer(waiting, &failed(error));
343        }
344        result
345    }
346
347    /// The person asked to retry everything now (§6.7): each owner ends
348    /// every ordinary wait it retries when it next plans, as for a network
349    /// report, answered or not, and the provider cooldown holds. Stopped,
350    /// held or cooling down, as the row says, answers at once; otherwise the
351    /// first pass that begins after the connection's owner acted on the
352    /// request answers it. A closed store answers `NotSyncing`: no pass
353    /// follows.
354    pub async fn retry_requested(self: &Arc<Self>) -> Result<RetryAnswer, DbError> {
355        let (answer, answered) = oneshot::channel();
356        let asked = self
357            .step(
358                |state| state,
359                move |waits, requests, state| {
360                    let number = waits.report(crate::retry_waits::Ending::Every);
361                    match requests.at_once(state) {
362                        None => requests.waiting.push((number, answer)),
363                        Some(outcome) => {
364                            #[expect(
365                                clippy::let_underscore_must_use,
366                                reason = "the receiver is still held here"
367                            )]
368                            let _ = answer.send(outcome);
369                        }
370                    }
371                },
372            )
373            .await;
374        match asked {
375            Ok(()) => Ok(RetryAnswer(answered)),
376            Err(DbError::StoreClosed) => {
377                let (answer, answered) = oneshot::channel();
378                #[expect(
379                    clippy::let_underscore_must_use,
380                    reason = "the receiver is still held here"
381                )]
382                let _ = answer.send(RetryOutcome::NotSyncing);
383                Ok(RetryAnswer(answered))
384            }
385            Err(error) => Err(error),
386        }
387    }
388
389    /// The upload worker's own work failed with `error`, which is no bug
390    /// and which no caller hears: write it as how the last pass ended, so
391    /// the sync state shows it (§19.1). A failed write has failed the
392    /// handle.
393    pub async fn show_worker_failure(&self, error: crate::SyncError) -> Result<(), DbError> {
394        let Some(database) = &self.database else {
395            return Err(DbError::Invariant { what: "only a handle that writes the store syncs" });
396        };
397        crate::LastPass::failed(error).record(database).await
398    }
399
400    /// A started connection will plan: requests wait for its passes, and
401    /// the failure its owner last found still holds them. Once a worker's
402    /// panic has halted sync (`sync_halted`), it fails with that bug as
403    /// `SyncError::WorkerFailed` until the store reopens (E15).
404    pub async fn sync_started(self: &Arc<Self>) -> Result<(), crate::SyncError> {
405        if let Some(bug) = self.halted() {
406            return Err(crate::SyncError::WorkerFailed(bug));
407        }
408        Ok(self
409            .step_as(
410                Step::Start,
411                |state| RetryState { sync: RetrySync::Started { cooling_until: None }, ..state },
412                |_, _, _| (),
413            )
414            .await?)
415    }
416
417    /// Sync stopped, or no connection will plan: no pass follows a waiting
418    /// request, so each is answered `NotSyncing`. A pass already running
419    /// answers the requests it took.
420    pub async fn sync_stopped(self: &Arc<Self>) -> Result<(), DbError> {
421        let stopped = self
422            .step(
423                |state| RetryState { sync: RetrySync::Stopped, ..state },
424                |_, requests, _| std::mem::take(&mut requests.waiting),
425            )
426            .await?;
427        answer(stopped, &RetryOutcome::NotSyncing);
428        Ok(())
429    }
430
431    /// The upload worker, which serves the whole store, panicked with
432    /// `bug`, which the sink heard: sync halts for the rest of this store's
433    /// session, as the connection owner's own panic halts it (E15). The last
434    /// pass shows the bug with nothing moving, sync stops, each waiting
435    /// request hears it as `SyncError::WorkerFailed`, the connection's owner
436    /// ends on it, and every start fails with it until the store reopens.
437    pub async fn sync_halted(
438        self: &Arc<Self>,
439        bug: coven_foundation::task::Bug,
440    ) -> Result<(), DbError> {
441        self.halted.send_replace(Some(bug.clone()));
442        if let Some(database) = &self.database {
443            crate::LastPass::failed(crate::SyncError::WorkerFailed(bug.clone()))
444                .record_halt(database)
445                .await?;
446        }
447        self.sync_panicked(bug).await
448    }
449
450    /// The connection's owner panicked with `bug`, which the sink heard, so
451    /// no pass follows a waiting request: sync stops, and each request,
452    /// those the panicking pass took among them, is answered `Failed` with
453    /// `SyncError::WorkerFailed` carrying the bug (E15).
454    pub async fn sync_panicked(
455        self: &Arc<Self>,
456        bug: coven_foundation::task::Bug,
457    ) -> Result<(), DbError> {
458        let waiting = self
459            .step(
460                |state| RetryState { sync: RetrySync::Stopped, ..state },
461                |_, requests, _| std::mem::take(&mut requests.waiting),
462            )
463            .await?;
464        let error = crate::SyncError::WorkerFailed(bug).into();
465        answer(waiting, &PassFailure::Failed { error }.into());
466        Ok(())
467    }
468
469    /// A pass begins, if the row lets one: sync is started, and neither a
470    /// failure nor a cooldown holds it. It takes every waiting request the
471    /// connection's owner has acted on (IO `pass_answers_acted_on`). `None`
472    /// when the row holds the pass, and the owner plans again.
473    pub async fn begin_pass(self: &Arc<Self>) -> Result<Option<PassRequests>, DbError> {
474        self.step(
475            |state| state,
476            |waits, requests, state| {
477                (RetryEffect::from(state) == RetryEffect::Pass)
478                    .then(|| requests.acted_on(waits, waits.handled(NetworkOwner::Connection)))
479            },
480        )
481        .await
482    }
483
484    /// The connection's owner found what holds every pass: a failure only
485    /// new credentials, an update or a reset can clear, or none. It says so
486    /// as it opens and after each pass, before writing the status a failed
487    /// pass leaves, so the effect never trails that status; each time it
488    /// plans, `answer_held` says so too.
489    pub async fn holds(self: &Arc<Self>, failure: Option<&PassFailure>) -> Result<(), DbError> {
490        let failure = failure.cloned();
491        let held = failure.is_some();
492        self.step(
493            move |state| RetryState { held, ..state },
494            move |_, requests, _| requests.held = failure,
495        )
496        .await
497    }
498
499    /// The owner plans: it records what holds every pass, as `holds` does,
500    /// and answers the requests it has acted on now if no pass will run for
501    /// them (IO `held_answers`). A failure only new credentials, an update
502    /// or a reset can clear answers `Failed`; otherwise a live provider
503    /// cooldown answers `CoolingDown`, and anything else that holds the
504    /// pass leaves them for it. The answer is the effect's.
505    pub async fn answer_held(
506        self: &Arc<Self>,
507        failure: Option<&PassFailure>,
508    ) -> Result<(), DbError> {
509        let failure = failure.cloned();
510        let held = failure.is_some();
511        let answered = self
512            .step(
513                move |state| RetryState { held, ..state },
514                move |waits, requests, state| {
515                    requests.held = failure;
516                    // With no answer at once, a pass runs and answers them.
517                    let outcome = requests.at_once(state)?;
518                    let handled = waits.handled(NetworkOwner::Connection);
519                    Some((requests.acted_on(waits, handled), outcome, RetryEffect::from(state)))
520                },
521            )
522            .await?;
523        let Some((acted_on, outcome, effect)) = answered else {
524            return Ok(());
525        };
526        // No pass runs for these requests, so nothing else says what came of
527        // the person's retry.
528        if !acted_on.taken.is_empty() {
529            tracing::info!(
530                requests = acted_on.taken.len(),
531                ?effect,
532                "no pass runs for the person's retry; answering it at once"
533            );
534        }
535        acted_on.answer(&outcome);
536        Ok(())
537    }
538
539    /// The connection's first attempt connected: its requests wait for the
540    /// loop's first pass, or are answered `NotSyncing` if sync stopped
541    /// meanwhile.
542    pub async fn return_requests(self: &Arc<Self>, mut taken: PassRequests) -> Result<(), DbError> {
543        let stopped = self
544            .step(
545                |state| state,
546                move |_, requests, state| match state.sync {
547                    RetrySync::Started { .. } => {
548                        requests.waiting.append(&mut taken.taken);
549                        None
550                    }
551                    RetrySync::Stopped => Some(taken),
552                },
553            )
554            .await?;
555        if let Some(stopped) = stopped {
556            stopped.answer(&RetryOutcome::NotSyncing);
557        }
558        Ok(())
559    }
560}
561
562#[cfg(test)]
563#[path = "retry_answer_tests.rs"]
564mod tests;