Skip to main content

coven_sync/
retry_waits.rs

1//! The shared retry schedule (§6.7, §21.2).
2//!
3//! Durable rows hold each wait's wall-clock deadline `T` and full wait `W`;
4//! this owner keeps the monotonic timer a running session enforces: a wait
5//! is over when the clock's elapsed time reaches its deadline or its timer
6//! fires, whichever the clock reports first.
7//! Opening arms `max(0, min(T - now, W))`, so a clock moved back cannot
8//! stretch a wait and one moved forward can only shorten it. The provider
9//! cooldown applies to every consumer of the store's connection: no
10//! subject is due while it lasts, whatever its own wait, and an app retry
11//! overrides only the subject's ordinary wait.
12
13use crate::{PendingReason, SyncError};
14use coven_database::{DbError, RetrySubject, RetryWait, StoreDatabase};
15use coven_foundation::clock::ClockRef;
16use coven_foundation::locks::lock_unpoisoned;
17use coven_storage::{retry_delay, StorageError, StorageFailure};
18use futures_util::FutureExt;
19use std::{
20    collections::{btree_map::Entry, BTreeMap},
21    future::Future,
22    pin::Pin,
23    sync::{Arc, Mutex},
24    time::{Duration, SystemTime},
25};
26
27type Timer = futures_util::future::Shared<Pin<Box<dyn Future<Output = ()> + Send>>>;
28
29/// A wake for the work an owner retries next.
30pub(crate) type Wake = Pin<Box<dyn Future<Output = ()> + Send>>;
31
32/// The wait a reopened store arms for a persisted `T` and `W`.
33pub(crate) fn restart_wait(not_before: SystemTime, now: SystemTime, wait: Duration) -> Duration {
34    not_before.duration_since(now).unwrap_or(Duration::ZERO).min(wait)
35}
36
37/// One installation's retry schedule. Construct when the store opens and
38/// inject the same owner into operations, file transfers, the sync loop and
39/// every provider connection.
40pub struct RetryWaits {
41    clock: ClockRef,
42    /// Absent for a read-only handle, whose cooldowns last for its session.
43    database: Option<StoreDatabase>,
44    /// Shared with the writer's steps that write `coven_retry_effect`,
45    /// which read the live cooldown as they commit (`retry_answer.rs`).
46    state: Arc<Mutex<WaitSchedule>>,
47    /// What each waiting operation's wait names, and whether it changed.
48    evidence: Mutex<operation_evidence::OperationEvidence>,
49    /// Bumped whenever a failed attempt arms a new wait, so an owner waiting
50    /// on the earlier schedule plans again.
51    rearmed: tokio::sync::watch::Sender<u64>,
52    /// The network reports, and how far each owner acted on them.
53    network: network_report::Reports,
54    /// The person's retry requests waiting for a pass (`retry_answer.rs`).
55    requests: Mutex<retry_answer::RetryRequests>,
56    /// Whether `coven_retry_effect` says sync is started, as its last step
57    /// committed it: uploads run only then (§17.4).
58    syncing: tokio::sync::watch::Sender<bool>,
59    /// The reason last logged for each subject's failure.
60    failures: crate::failure_log::FailureLog,
61    /// The bug of a store-wide worker's panic that halted sync for the rest
62    /// of this store's session (`sync_halted`), once one has.
63    halted: tokio::sync::watch::Sender<Option<coven_foundation::task::Bug>>,
64}
65
66/// Hears each wait a failed attempt arms, so an owner waiting on the
67/// earlier schedule plans again (`RetryWaits::rearmed`).
68pub struct RearmWatch(tokio::sync::watch::Receiver<u64>);
69
70impl RearmWatch {
71    /// Resolves once another wait is armed; an error once the schedule is
72    /// gone.
73    pub async fn changed(&mut self) -> Result<(), tokio::sync::watch::error::RecvError> {
74        self.0.changed().await
75    }
76}
77
78/// Hears sync start and stop, so the upload worker runs only while it is
79/// started (`RetryWaits::syncing`, §17.4).
80pub(crate) struct SyncingWatch(tokio::sync::watch::Receiver<bool>);
81
82impl SyncingWatch {
83    /// Resolves at the next start or stop; an error once the schedule is
84    /// gone.
85    pub(crate) async fn changed(&mut self) -> Result<(), tokio::sync::watch::error::RecvError> {
86        self.0.changed().await
87    }
88}
89
90#[derive(Default)]
91struct WaitSchedule {
92    armed: BTreeMap<RetrySubject, Armed>,
93}
94
95struct Armed {
96    /// The persisted `T` this deadline enforces.
97    not_before: SystemTime,
98    /// Monotonic deadline and its timer; `None` waits for the app.
99    timer: Option<(Duration, Timer)>,
100    /// When the wait ends by the store's clock, fixed as it is armed so
101    /// every report of one wait names one instant; the timer, not this,
102    /// ends it. `None` without a timer, or when the clock cannot represent
103    /// the instant.
104    ends: Option<SystemTime>,
105    /// Its owner has evaluated it since its wait ended, so the ended wait
106    /// needs no further wake.
107    seen: bool,
108}
109
110impl Armed {
111    fn over(&self, elapsed: Duration) -> bool {
112        self.timer
113            .as_ref()
114            .is_some_and(|(due, timer)| *due <= elapsed || timer.clone().now_or_never().is_some())
115    }
116}
117
118impl RetryWaits {
119    /// Arm every persisted wait with the restart rule (§6.7).
120    pub async fn open(database: StoreDatabase, clock: ClockRef) -> Result<Self, DbError> {
121        let waits = database.retry_waits().await?;
122        let now = clock.now();
123        let owner = Self {
124            clock,
125            database: Some(database),
126            state: Arc::default(),
127            evidence: Mutex::new(operation_evidence::OperationEvidence::new()),
128            rearmed: tokio::sync::watch::channel(0).0,
129            network: network_report::Reports::new(),
130            requests: Mutex::new(retry_answer::RetryRequests::new()),
131            failures: crate::failure_log::FailureLog::new(),
132            syncing: tokio::sync::watch::channel(false).0,
133            halted: tokio::sync::watch::channel(None).0,
134        };
135        for (subject, wait) in waits {
136            owner.arm_for(subject, wait.not_before, restart_wait(wait.not_before, now, wait.wait));
137        }
138        owner.watch_persisted().await?;
139        Ok(owner)
140    }
141
142    /// A read-only handle cannot persist: its cooldowns hold only its own requests.
143    pub fn in_memory(clock: ClockRef) -> Self {
144        Self {
145            clock,
146            database: None,
147            state: Arc::default(),
148            evidence: Mutex::new(operation_evidence::OperationEvidence::new()),
149            rearmed: tokio::sync::watch::channel(0).0,
150            network: network_report::Reports::new(),
151            requests: Mutex::new(retry_answer::RetryRequests::new()),
152            failures: crate::failure_log::FailureLog::new(),
153            syncing: tokio::sync::watch::channel(false).0,
154            halted: tokio::sync::watch::channel(None).0,
155        }
156    }
157
158    /// Log that `subject` failed with `reason` because of `error`, when its
159    /// reason appears or changes (`FailureLog::failed`). Returns whether it
160    /// was logged.
161    pub(crate) fn log_failure(
162        &self,
163        subject: impl Into<crate::failure_log::Logged>,
164        reason: &PendingReason,
165        error: &(dyn std::error::Error + 'static),
166    ) -> bool {
167        self.failures.failed(subject.into(), reason, error)
168    }
169
170    /// How many failures were logged so far.
171    #[cfg(test)]
172    pub(crate) fn failures_logged(&self) -> usize {
173        self.failures.count()
174    }
175
176    /// The subject's work advanced past a failure `ended` accepts: a later
177    /// failure is new and logs again.
178    pub(crate) fn failure_cleared(
179        &self,
180        subject: impl Into<crate::failure_log::Logged>,
181        ended: impl Fn(&PendingReason) -> bool,
182    ) {
183        self.failures.cleared(&subject.into(), ended);
184    }
185
186    /// Forget the logged failures of pin requests not in `live`, which have
187    /// ended.
188    pub(crate) fn retain_kept_failures(&self, live: &std::collections::BTreeSet<i64>) {
189        self.failures.retain_kept(live);
190    }
191
192    fn lock(&self) -> std::sync::MutexGuard<'_, WaitSchedule> {
193        lock_unpoisoned(&self.state)
194    }
195
196    fn cooling(state: &WaitSchedule, elapsed: Duration) -> bool {
197        state.armed.get(&RetrySubject::Provider).is_some_and(|armed| !armed.over(elapsed))
198    }
199
200    /// The next wait after an unsuccessful attempt: the backoff continues
201    /// from `previous`, and `T` is persisted with `W` before waiting.
202    pub(crate) fn next(&self, previous: Option<&RetryWait>) -> Result<RetryWait, SyncError> {
203        let failures = previous.map_or(0, |wait| wait.failures).saturating_add(1);
204        let wait = retry_delay(failures - 1);
205        let not_before = self.clock.now().checked_add(wait).ok_or_else(|| {
206            StorageFailure::InvalidConfiguration
207                .with_source("clock cannot represent a retry deadline")
208        })?;
209        Ok(RetryWait { failures, not_before, wait })
210    }
211
212    /// Enforce a persisted wait monotonically from now, by the restart rule:
213    /// time that passed since its `T` was computed counts toward it, so it
214    /// ends no later than the `T` it reports.
215    pub(crate) fn arm(&self, subject: RetrySubject, wait: &RetryWait) {
216        let left = restart_wait(wait.not_before, self.clock.now(), wait.wait);
217        self.arm_for(subject, wait.not_before, left);
218        self.rearmed.send_modify(|count| *count = count.wrapping_add(1));
219    }
220
221    fn arm_for(&self, subject: RetrySubject, not_before: SystemTime, wait: Duration) {
222        let armed = self.armed(not_before, wait);
223        self.lock().armed.insert(subject, armed);
224    }
225
226    /// A wait enforcing `not_before`, over once `wait` has passed from now.
227    fn armed(&self, not_before: SystemTime, wait: Duration) -> Armed {
228        let timer = (self.clock.elapsed() + wait, self.clock.wait_out(wait).shared());
229        let ends = self.clock.now().checked_add(wait);
230        Armed { not_before, timer: Some(timer), ends, seen: false }
231    }
232
233    /// Hear sync start and stop, as `coven_retry_effect` commits it.
234    pub(crate) fn syncing(&self) -> SyncingWatch {
235        SyncingWatch(self.syncing.subscribe())
236    }
237
238    /// Whether sync is started now, as `coven_retry_effect` last committed
239    /// it: starting and stopping write the row. A store opens stopped, and
240    /// a read-only handle never starts.
241    pub(crate) fn started(&self) -> bool {
242        *self.syncing.borrow()
243    }
244
245    /// The bug that halted sync for the rest of this store's session, if
246    /// one has (`sync_halted`).
247    pub fn halted(&self) -> Option<coven_foundation::task::Bug> {
248        self.halted.borrow().clone()
249    }
250
251    /// Resolves with the bug that halts sync, once one has: the connection's
252    /// owner then ends as on its own panic.
253    pub fn halt(
254        &self,
255    ) -> impl std::future::Future<Output = coven_foundation::task::Bug> + Send + 'static {
256        let mut halted = self.halted.subscribe();
257        async move {
258            let bug = halted.wait_for(Option::is_some).await.ok().and_then(|bug| bug.clone());
259            match bug {
260                Some(bug) => bug,
261                // The sender goes only with the store, and the owner with it.
262                None => std::future::pending().await,
263            }
264        }
265    }
266
267    /// Changes whenever a failed attempt arms a new wait.
268    pub fn rearmed(&self) -> RearmWatch {
269        RearmWatch(self.rearmed.subscribe())
270    }
271
272    /// A wait that could not be persisted holds until the app retries; it is
273    /// never permission to retry at once (§6.7).
274    pub(crate) fn hold(&self, subject: RetrySubject) {
275        self.lock().armed.insert(
276            subject,
277            Armed { not_before: SystemTime::UNIX_EPOCH, timer: None, ends: None, seen: false },
278        );
279    }
280
281    /// The subject's work succeeded, changed or waits for the app instead.
282    pub(crate) fn forget(&self, subject: RetrySubject) {
283        self.lock().armed.remove(&subject);
284    }
285
286    /// The operation advanced or finished: its wait, the evidence its wait
287    /// named and its logged failure all end.
288    pub(crate) fn operation_advanced(&self, id: coven_database::OperationId) {
289        self.clear_evidence(id);
290        self.forget(RetrySubject::Operation(id));
291        self.failures.cleared_operation(id);
292    }
293
294    /// The app retried the operation: its wait ends, and a later failure is
295    /// new.
296    pub(crate) fn retried(&self, id: coven_database::OperationId) {
297        self.forget(RetrySubject::Operation(id));
298        self.failures.cleared_operation(id);
299    }
300
301    /// Keep only the waits `keep` accepts: an owner lets go of those whose
302    /// work has ended.
303    pub(crate) fn retain(&self, keep: impl Fn(&RetrySubject) -> bool) {
304        self.lock().armed.retain(|subject, _| keep(subject));
305    }
306
307    /// Forget operation waits whose operations no longer exist.
308    pub(crate) fn retain_operations(
309        &self,
310        live: &std::collections::BTreeSet<coven_database::OperationId>,
311    ) {
312        self.retain(|subject| match subject {
313            RetrySubject::Operation(id) => live.contains(id),
314            _ => true,
315        });
316        self.retain_evidence(live);
317        self.failures.retain_operations(live);
318    }
319
320    /// Whether automatic work on the subject may start now, given its
321    /// persisted wait. No subject is due while the provider cooldown lasts.
322    /// An absent row means the work advanced or the app retried it, unless
323    /// this session holds the subject because persisting its wait failed.
324    /// An operation's wait can also end by evidence (`operation_evidence.rs`).
325    pub fn due(&self, subject: RetrySubject, persisted: Option<&RetryWait>) -> bool {
326        let elapsed = self.clock.elapsed();
327        let Some(persisted) = persisted else {
328            let mut state = self.lock();
329            if state.armed.get(&subject).is_some_and(|armed| armed.timer.is_none()) {
330                return false;
331            }
332            state.armed.remove(&subject);
333            return !Self::cooling(&state, elapsed);
334        };
335        // The persisted wait is enforced in this session, and read, under
336        // one lock: a row armed before this session, or replaced since, is
337        // armed by the restart rule. Between two locks another owner could
338        // forget it.
339        let restarted = || {
340            let wait = restart_wait(persisted.not_before, self.clock.now(), persisted.wait);
341            self.armed(persisted.not_before, wait)
342        };
343        let mut state = self.lock();
344        let armed = match state.armed.entry(subject) {
345            Entry::Occupied(entry) => {
346                let armed = entry.into_mut();
347                if armed.not_before != persisted.not_before && armed.timer.is_some() {
348                    *armed = restarted();
349                }
350                armed
351            }
352            Entry::Vacant(entry) => entry.insert(restarted()),
353        };
354        let over = armed.over(elapsed);
355        armed.seen |= over;
356        !Self::cooling(&state, elapsed) && over
357    }
358
359    /// The owner evaluated the subject's work without running it, e.g.
360    /// because its condition waits for the app: an ended wait needs no wake.
361    pub(crate) fn seen(&self, subject: RetrySubject) {
362        let elapsed = self.clock.elapsed();
363        if let Some(armed) = self.lock().armed.get_mut(&subject) {
364            armed.seen |= armed.over(elapsed);
365        }
366    }
367
368    /// The live provider cooldown's remaining wait.
369    pub(crate) fn cooldown(&self) -> Option<Duration> {
370        let elapsed = self.clock.elapsed();
371        let state = self.lock();
372        let armed = state.armed.get(&RetrySubject::Provider)?;
373        if armed.over(elapsed) {
374            return None;
375        }
376        let (due, _) = armed.timer.as_ref()?;
377        Some(due.saturating_sub(elapsed).max(Duration::from_nanos(1)))
378    }
379
380    /// How long until the subject's own wait is over, cooldown aside; `None`
381    /// when it waits on no timer.
382    #[cfg(test)]
383    pub(crate) fn remaining(&self, subject: RetrySubject) -> Option<Duration> {
384        let elapsed = self.clock.elapsed();
385        let state = self.lock();
386        let armed = state.armed.get(&subject)?;
387        let (due, _) = armed.timer.as_ref()?;
388        Some(if armed.over(elapsed) { Duration::ZERO } else { due.saturating_sub(elapsed) })
389    }
390
391    /// End one subject's wait now, as the person's retry ends every ordinary
392    /// wait (§6.7): it keeps its failure count, so the next failure continues
393    /// the backoff, and is saved ended, so no reopen arms it again. A wait
394    /// this session holds because saving it failed is released. The provider
395    /// cooldown holds.
396    pub(crate) async fn end_wait(&self, subject: RetrySubject) -> Result<(), SyncError> {
397        let Some(database) = &self.database else {
398            self.forget(subject);
399            return Ok(());
400        };
401        let select = move |waits: &[(RetrySubject, RetryWait)],
402                           _: &[crate::PendingRecord],
403                           _: &[coven_database::FileUpload],
404                           _: &[coven_database::PinRequest]| {
405            waits
406                .iter()
407                .filter(|(held, wait)| *held == subject && !wait.wait.is_zero())
408                .map(|(held, _)| *held)
409                .collect()
410        };
411        match database
412            .end_retry_waits::<crate::PendingRecord>(self.clock.now(), select)
413            .await?
414            .first()
415        {
416            Some((subject, wait)) => self.arm(*subject, wait),
417            None => self.forget(subject),
418        }
419        Ok(())
420    }
421
422    /// How long until the earliest armed subject `matching` is due by its
423    /// deadline, cooldown included; `None` when nothing it covers waits on a timer.
424    #[cfg(test)]
425    pub(crate) fn next_due(&self, matching: impl Fn(&RetrySubject) -> bool) -> Option<Duration> {
426        let elapsed = self.clock.elapsed();
427        let state = self.lock();
428        let cooldown = state
429            .armed
430            .get(&RetrySubject::Provider)
431            .and_then(|armed| armed.timer.as_ref())
432            .map_or(Duration::ZERO, |(due, _)| *due);
433        state
434            .armed
435            .iter()
436            .filter(|(subject, _)| **subject != RetrySubject::Provider && matching(subject))
437            .filter_map(|(_, armed)| armed.timer.as_ref())
438            .map(|(due, _)| (*due).max(cooldown).saturating_sub(elapsed))
439            .min()
440    }
441
442    /// Resolves when the earliest armed subject `matching` is due: its timer
443    /// and any live provider cooldown have both fired. A wait that ended
444    /// after its owner last evaluated it wakes at once; one the owner has
445    /// evaluated since wakes nothing until new evidence or a new wait.
446    /// `None` when nothing it covers waits on a timer.
447    pub fn next_wake(&self, matching: impl Fn(&RetrySubject) -> bool) -> Option<Wake> {
448        let elapsed = self.clock.elapsed();
449        let state = self.lock();
450        let earliest = state
451            .armed
452            .iter()
453            .filter(|(subject, armed)| {
454                **subject != RetrySubject::Provider
455                    && matching(subject)
456                    && !(armed.seen && armed.over(elapsed))
457            })
458            .filter_map(|(_, armed)| armed.timer.clone())
459            .min_by_key(|(due, _)| *due)?;
460        let cooldown = state
461            .armed
462            .get(&RetrySubject::Provider)
463            .filter(|armed| !armed.over(elapsed))
464            .and_then(|armed| armed.timer.clone());
465        Some(Box::pin(async move {
466            earliest.1.await;
467            if let Some((_, cooldown)) = cooldown {
468                cooldown.await;
469            }
470        }))
471    }
472
473    /// Resolves when the live provider cooldown ends.
474    pub fn cooldown_wake(&self) -> Option<Wake> {
475        let elapsed = self.clock.elapsed();
476        let state = self.lock();
477        let armed =
478            state.armed.get(&RetrySubject::Provider).filter(|armed| !armed.over(elapsed))?;
479        let (_, timer) = armed.timer.clone()?;
480        Some(Box::pin(timer))
481    }
482
483    /// Persist the subject's next wait after an unsuccessful attempt, then
484    /// enforce it. A persistence failure holds the subject for the app.
485    pub(crate) async fn fail(
486        &self,
487        subject: RetrySubject,
488        previous: Option<&RetryWait>,
489    ) -> Result<(), SyncError> {
490        let wait = self.next(previous)?;
491        let Some(database) = &self.database else {
492            self.arm(subject, &wait);
493            return Ok(());
494        };
495        match database.record_retry(subject, wait).await {
496            Ok(()) => {
497                self.arm(subject, &wait);
498                Ok(())
499            }
500            Err(error) => {
501                self.hold(subject);
502                Err(error.into())
503            }
504        }
505    }
506}
507
508#[async_trait::async_trait]
509impl coven_storage::Cooldown for RetryWaits {
510    fn remaining(&self) -> Option<Duration> {
511        self.cooldown()
512    }
513
514    /// Arm first, so no consumer in this session retries, then persist it
515    /// with what it makes a retry do, in one transaction
516    /// (`retry_answer.rs`). A cooldown never shortens one already live.
517    async fn record(&self, wait: Duration) -> Result<(), StorageError> {
518        if self.cooldown().is_some_and(|remaining| remaining >= wait) {
519            return Ok(());
520        }
521        let not_before = self.clock.now().checked_add(wait).ok_or_else(|| {
522            StorageFailure::InvalidConfiguration.with_source("clock cannot represent a cooldown")
523        })?;
524        let cooldown = RetryWait { failures: 0, not_before, wait };
525        self.arm(RetrySubject::Provider, &cooldown);
526        match &self.database {
527            Some(database) => database
528                .record_cooldown(cooldown, self.live_cooldown())
529                .await
530                .map_err(|error| match error {
531                    coven_database::DbError::Bug(bug) => bug.into(),
532                    error => StorageFailure::File.with_source(error),
533                }),
534            None => Ok(()),
535        }
536    }
537}
538
539#[path = "operation_evidence.rs"]
540mod operation_evidence;
541
542#[path = "network_report.rs"]
543mod network_report;
544pub(crate) use network_report::Ending;
545pub use network_report::NetworkOwner;
546
547#[path = "retry_answer.rs"]
548mod retry_answer;
549pub use retry_answer::{PassRequests, RetryAnswer, RetryEffect, RetryOutcome};
550
551#[cfg(test)]
552#[path = "retry_waits_tests.rs"]
553pub(crate) mod tests;