coven_sync/retry_answer.rs
1//! The answer to the person's retry (§6.7; IO `RetryAnswer`).
2//!
3//! What decides it is `coven_retry_effect`'s row: whether a started
4//! connection will plan, whether a failure only new credentials, an update
5//! or a reset can clear holds every pass, and the provider cooldown holding
6//! them. Each step below changes that row and acts on the requests in one
7//! step on the store's writer (`StoreDatabase::change_retry_state`), so the
8//! answer a request gets is the effect the row shows when it is made (IO
9//! `ask_answers_by_effect`), and a request, stopping and a pass taking its
10//! requests each happen at once (IO `never_stranded`). Each step writes the
11//! cooldown as it stands then, and the row keeps no cooldown while sync is
12//! stopped, since no owner would write its end. The cooldown's end is a
13//! write too: the connection's owner wakes when it ends, and its next step
14//! writes it.
15//!
16//! A retry request is a report, so it takes the next report number. A
17//! request the row answers at once, stopped, held or cooling down, is
18//! answered so; any other waits for a pass. The connection's owner, the
19//! connection's first attempt and then its sync loop, acts on reports when
20//! it plans between passes. A pass begins only while the row lets it, and
21//! takes every waiting request the owner has acted on, answering them when
22//! it ends; a request made after the owner last acted waits for a later
23//! pass. When a failure or cooldown begins to hold the pass, the owner's
24//! next plan answers the requests it acted on at once. Stopping sync
25//! answers every waiting request `NotSyncing`. A pass that ends without
26//! answering the requests it took, because it panicked or was dropped,
27//! leaves them waiting again; an owner that panicked stops sync and answers
28//! every waiting request with its bug, as `SyncError::WorkerFailed` (E15).
29//!
30//! A failed write has failed the store's handle, which every call and live
31//! query then reports, and no later step can commit; the requests waiting
32//! then are answered with that failure. A write that panicked fails alone:
33//! its bug goes to the step's caller, or to the sink when the connection's
34//! owner wrote on its own, and that owner stops (E15).
35
36use crate::{NetworkOwner, PassFailure, RetryWaits, SyncFailure};
37use coven_database::{DbError, RetryState, RetrySync};
38use coven_foundation::clock::ClockRef;
39use coven_foundation::locks::lock_unpoisoned;
40use std::{
41 future::Future,
42 pin::Pin,
43 sync::{Arc, Mutex, Weak},
44 task::{Context, Poll},
45 time::SystemTime,
46};
47use tokio::sync::oneshot;
48
49/// How a step commits: a change of the retry state alone, or a start of
50/// sync, which `coven_sync_status` counts in the same transaction.
51#[derive(Clone, Copy)]
52enum Step {
53 Change,
54 Start,
55}
56
57/// What the first sync pass after the person's retry came to (E5).
58#[derive(Clone, Debug)]
59pub enum RetryOutcome {
60 /// The pass reached its read end.
61 Synced {
62 /// The store's clock when the pass finished.
63 finished_at: SystemTime,
64 },
65 /// The pass failed, or a failure only new credentials, an update or a
66 /// reset can clear holds every pass; the failure says which.
67 Failed(PassFailure),
68 /// A provider cooldown holds every pass until then. The pass runs once
69 /// it ends, with no further request.
70 CoolingDown {
71 /// When the cooldown ends, by the store's clock.
72 until: SystemTime,
73 },
74 /// Sync is stopped or the handle closed, so no pass follows the request.
75 NotSyncing,
76}
77
78impl From<PassFailure> for RetryOutcome {
79 fn from(failure: PassFailure) -> Self {
80 Self::Failed(failure)
81 }
82}
83
84/// What the person's retry would do if made now (E5), as
85/// `coven_retry_effect` holds it: run a pass, or be answered at once
86/// without one. An app offers a retry only while it is `Pass`; the status
87/// says why it is not.
88#[derive(Clone, Copy, Debug, PartialEq, Eq)]
89pub enum RetryEffect {
90 /// A pass runs for the retry, and its end answers it.
91 Pass,
92 /// A provider cooldown holds every pass until then, by the store's
93 /// clock: the retry is answered `CoolingDown` at once, and the pass runs
94 /// once the cooldown ends.
95 CoolingDown {
96 /// When the cooldown ends, by the store's clock.
97 until: SystemTime,
98 },
99 /// A failure only new credentials, an update or a reset can clear holds
100 /// every pass: the retry is answered `Failed` with it at once. The
101 /// status shows the failure.
102 Failed,
103 /// Sync is stopped or the handle closed: the retry is answered
104 /// `NotSyncing` at once.
105 NotSyncing,
106}
107
108impl From<RetryState> for RetryEffect {
109 /// A held failure answers before a cooldown: the cooldown's end would
110 /// not let the pass run. `coven_retry_effect.effect` is computed alike.
111 fn from(state: RetryState) -> Self {
112 match (state.sync, state.held) {
113 (RetrySync::Stopped, _) => Self::NotSyncing,
114 (RetrySync::Started { .. }, true) => Self::Failed,
115 (RetrySync::Started { cooling_until: Some(until) }, false) => {
116 Self::CoolingDown { until }
117 }
118 (RetrySync::Started { cooling_until: None }, false) => Self::Pass,
119 }
120 }
121}
122
123impl RetryEffect {
124 /// Read `coven_retry_effect`, as an app's live query does:
125 /// `handle.subscribe(RetryEffect::read)`.
126 pub fn read(sql: coven_database::SqlReadContext<'_>) -> Result<Self, DbError> {
127 let (effect, until) =
128 sql.query_row("SELECT effect,cooling_until FROM coven_retry_effect", [], |row| {
129 Ok((row.get::<_, String>(0)?, row.get::<_, Option<i64>>(1)?))
130 })?;
131 Self::from_columns(&effect, until)
132 }
133
134 /// The effect that `coven_retry_effect`'s `effect` and `cooling_until`
135 /// columns name, for an app that reads them in a query of its own. A
136 /// pair the view never computes is an invariant error.
137 pub fn from_columns(effect: &str, cooling_until: Option<i64>) -> Result<Self, DbError> {
138 Ok(match (effect, cooling_until) {
139 ("pass", None) => Self::Pass,
140 ("cooling_down", Some(until)) => {
141 Self::CoolingDown { until: coven_database::stored_time(until)? }
142 }
143 ("failed", _) => Self::Failed,
144 ("not_syncing", None) => Self::NotSyncing,
145 _ => return Err(DbError::Invariant { what: "the retry effect is one coven computes" }),
146 })
147 }
148
149 /// Follow `coven_retry_effect` with a live query, as an app does.
150 #[cfg(any(test, feature = "test-utils"))]
151 pub fn follow(
152 database: &coven_database::StoreDatabase,
153 ) -> coven_database::test_utils::Followed<Self, DbError> {
154 coven_database::test_utils::Followed::new(database.subscribe(Self::read))
155 }
156}
157
158/// Resolves when the first sync pass after a retry request ends (E5).
159/// Dropping it leaves the retry in place.
160#[derive(Debug)]
161pub struct RetryAnswer(oneshot::Receiver<RetryOutcome>);
162
163impl Future for RetryAnswer {
164 type Output = RetryOutcome;
165
166 fn poll(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll<RetryOutcome> {
167 // Every waiting request is answered before its owner lets it go; a
168 // request is dropped unanswered only with the store's retry schedule
169 // itself, once the handle is gone.
170 Pin::new(&mut self.0).poll(context).map(|answer| answer.unwrap_or(RetryOutcome::NotSyncing))
171 }
172}
173
174/// Retry requests by report number, each with where its answer goes.
175type Waiting = Vec<(u64, oneshot::Sender<RetryOutcome>)>;
176
177/// Answer every request in `requests` with `outcome`.
178fn answer(requests: Waiting, outcome: &RetryOutcome) {
179 for (_, request) in requests {
180 #[expect(
181 clippy::let_underscore_must_use,
182 reason = "a person who stopped waiting needs no answer"
183 )]
184 let _ = request.send(outcome.clone());
185 }
186}
187
188/// Requests a pass took when it began, answered when it ends. A pass that
189/// panicked, or was dropped, before answering them leaves them waiting
190/// again, for its owner's ending to answer.
191#[must_use = "the pass answers its requests when it ends"]
192pub struct PassRequests {
193 taken: Waiting,
194 /// The schedule they wait in again, unless it is gone.
195 waits: Weak<RetryWaits>,
196}
197
198impl PassRequests {
199 /// Answer every request with how the pass ended.
200 pub fn answer(mut self, outcome: &RetryOutcome) {
201 answer(std::mem::take(&mut self.taken), outcome);
202 }
203}
204
205impl Drop for PassRequests {
206 fn drop(&mut self) {
207 if self.taken.is_empty() {
208 return;
209 }
210 let Some(waits) = self.waits.upgrade() else {
211 return;
212 };
213 lock_unpoisoned(&waits.requests).waiting.append(&mut self.taken);
214 }
215}
216
217/// Waiting requests by report number, and the failure the row says holds
218/// every pass: the error objects a `Failed` answer carries, which the row
219/// can't hold. Both change only in a step on the writer, once the row they
220/// go with commits.
221pub(crate) struct RetryRequests {
222 held: Option<PassFailure>,
223 waiting: Waiting,
224}
225
226impl RetryRequests {
227 /// No request waiting, and no failure held.
228 pub(crate) fn new() -> Self {
229 Self { held: None, waiting: Vec::new() }
230 }
231
232 /// The answer a request gets at once in `state`, none when a pass
233 /// answers it: the effect's.
234 fn at_once(&self, state: RetryState) -> Option<RetryOutcome> {
235 match RetryEffect::from(state) {
236 RetryEffect::Pass => None,
237 RetryEffect::CoolingDown { until } => Some(RetryOutcome::CoolingDown { until }),
238 RetryEffect::Failed => Some(RetryOutcome::Failed(
239 self.held.clone().expect("the row holds a failure only with its failure"),
240 )),
241 RetryEffect::NotSyncing => Some(RetryOutcome::NotSyncing),
242 }
243 }
244
245 /// The waiting requests the connection's owner has acted on, which
246 /// wait in `waits` again if a pass drops them unanswered.
247 fn acted_on(&mut self, waits: &Arc<RetryWaits>, handled: u64) -> PassRequests {
248 let (taken, later) = std::mem::take(&mut self.waiting)
249 .into_iter()
250 .partition::<Vec<_>, _>(|(number, _)| *number <= handled);
251 self.waiting = later;
252 PassRequests { taken, waits: Arc::downgrade(waits) }
253 }
254}
255
256/// When the live provider cooldown ends, by the store's clock as it was
257/// armed, so writing one cooldown again names the same instant. `None` also
258/// when the clock cannot represent that time; the pass then answers its
259/// requests once the cooldown ends.
260fn cooldown_until(schedule: &Mutex<super::WaitSchedule>, clock: &ClockRef) -> Option<SystemTime> {
261 let elapsed = clock.elapsed();
262 lock_unpoisoned(schedule)
263 .armed
264 .get(&coven_database::RetrySubject::Provider)
265 .filter(|armed| !armed.over(elapsed))?
266 .ends
267}
268
269/// The answer to the requests waiting when a step fails, carrying the typed
270/// error the step's caller gets, never its text (F.5), or `NotSyncing` once
271/// the store closed. A step fails only as its handle does, whose failure
272/// every later call reports from one shared cause, so the answer carries that
273/// same failure; as its runtime shuts down, saying how far it got; or as it
274/// panics, with the bug.
275fn failed(error: &DbError) -> RetryOutcome {
276 let failure = match error {
277 DbError::StoreClosed => return RetryOutcome::NotSyncing,
278 DbError::StatePersistence(cause) => DbError::StatePersistence(cause.clone()),
279 DbError::RollbackFailed => DbError::RollbackFailed,
280 DbError::CheckpointFailed(cause) => DbError::CheckpointFailed(cause.clone()),
281 DbError::StatisticsFailed(cause) => DbError::StatisticsFailed(cause.clone()),
282 DbError::WallTimer(cause) => DbError::WallTimer(cause.clone()),
283 DbError::StoreFailed(bug) => DbError::StoreFailed(bug.clone()),
284 DbError::Shutdown(shutdown) => DbError::Shutdown(*shutdown),
285 DbError::Bug(bug) => DbError::Bug(bug.clone()),
286 _ => DbError::Invariant { what: "a retry step fails only as its handle or runtime does" },
287 };
288 PassFailure::Failed { error: SyncFailure::Other(Arc::new(failure)) }.into()
289}
290
291impl RetryWaits {
292 /// The row with the cooldown live as the step that writes it commits,
293 /// while sync is started; while it is stopped, none.
294 pub(super) fn live_cooldown(&self) -> impl FnOnce(RetryState) -> RetryState + Send + 'static {
295 let (schedule, clock) = (self.state.clone(), self.clock.clone());
296 move |state| match state.sync {
297 RetrySync::Stopped => state,
298 RetrySync::Started { .. } => RetryState {
299 sync: RetrySync::Started { cooling_until: cooldown_until(&schedule, &clock) },
300 ..state
301 },
302 }
303 }
304
305 /// Change the row with `change` and the live cooldown, then run `then`
306 /// with the requests and the committed state, in one step on the
307 /// writer. A failed write answers every waiting request with it.
308 async fn step<T: Send + 'static>(
309 self: &Arc<Self>,
310 change: impl FnOnce(RetryState) -> RetryState + Send + 'static,
311 then: impl FnOnce(&Arc<RetryWaits>, &mut RetryRequests, RetryState) -> T + Send + 'static,
312 ) -> Result<T, DbError> {
313 self.step_as(Step::Change, change, then).await
314 }
315
316 /// `step`, as `kind` commits it.
317 async fn step_as<T: Send + 'static>(
318 self: &Arc<Self>,
319 kind: Step,
320 change: impl FnOnce(RetryState) -> RetryState + Send + 'static,
321 then: impl FnOnce(&Arc<RetryWaits>, &mut RetryRequests, RetryState) -> T + Send + 'static,
322 ) -> Result<T, DbError> {
323 let Some(database) = &self.database else {
324 return Err(DbError::Invariant { what: "only a handle that writes the store syncs" });
325 };
326 let cooling = self.live_cooldown();
327 let waits = self.clone();
328 let change = move |state| cooling(change(state));
329 let then = move |state: RetryState| {
330 waits.syncing.send_if_modified(|syncing| {
331 let started = matches!(state.sync, RetrySync::Started { .. });
332 std::mem::replace(syncing, started) != started
333 });
334 then(&waits, &mut lock_unpoisoned(&waits.requests), state)
335 };
336 let result = match kind {
337 Step::Change => database.change_retry_state(change, then).await,
338 Step::Start => database.start_retry_state(change, then).await,
339 };
340 if let Err(error) = &result {
341 let waiting = std::mem::take(&mut lock_unpoisoned(&self.requests).waiting);
342 answer(waiting, &failed(error));
343 }
344 result
345 }
346
347 /// The person asked to retry everything now (§6.7): each owner ends
348 /// every ordinary wait it retries when it next plans, as for a network
349 /// report, answered or not, and the provider cooldown holds. Stopped,
350 /// held or cooling down, as the row says, answers at once; otherwise the
351 /// first pass that begins after the connection's owner acted on the
352 /// request answers it. A closed store answers `NotSyncing`: no pass
353 /// follows.
354 pub async fn retry_requested(self: &Arc<Self>) -> Result<RetryAnswer, DbError> {
355 let (answer, answered) = oneshot::channel();
356 let asked = self
357 .step(
358 |state| state,
359 move |waits, requests, state| {
360 let number = waits.report(crate::retry_waits::Ending::Every);
361 match requests.at_once(state) {
362 None => requests.waiting.push((number, answer)),
363 Some(outcome) => {
364 #[expect(
365 clippy::let_underscore_must_use,
366 reason = "the receiver is still held here"
367 )]
368 let _ = answer.send(outcome);
369 }
370 }
371 },
372 )
373 .await;
374 match asked {
375 Ok(()) => Ok(RetryAnswer(answered)),
376 Err(DbError::StoreClosed) => {
377 let (answer, answered) = oneshot::channel();
378 #[expect(
379 clippy::let_underscore_must_use,
380 reason = "the receiver is still held here"
381 )]
382 let _ = answer.send(RetryOutcome::NotSyncing);
383 Ok(RetryAnswer(answered))
384 }
385 Err(error) => Err(error),
386 }
387 }
388
389 /// The upload worker's own work failed with `error`, which is no bug
390 /// and which no caller hears: write it as how the last pass ended, so
391 /// the sync state shows it (§19.1). A failed write has failed the
392 /// handle.
393 pub async fn show_worker_failure(&self, error: crate::SyncError) -> Result<(), DbError> {
394 let Some(database) = &self.database else {
395 return Err(DbError::Invariant { what: "only a handle that writes the store syncs" });
396 };
397 crate::LastPass::failed(error).record(database).await
398 }
399
400 /// A started connection will plan: requests wait for its passes, and
401 /// the failure its owner last found still holds them. Once a worker's
402 /// panic has halted sync (`sync_halted`), it fails with that bug as
403 /// `SyncError::WorkerFailed` until the store reopens (E15).
404 pub async fn sync_started(self: &Arc<Self>) -> Result<(), crate::SyncError> {
405 if let Some(bug) = self.halted() {
406 return Err(crate::SyncError::WorkerFailed(bug));
407 }
408 Ok(self
409 .step_as(
410 Step::Start,
411 |state| RetryState { sync: RetrySync::Started { cooling_until: None }, ..state },
412 |_, _, _| (),
413 )
414 .await?)
415 }
416
417 /// Sync stopped, or no connection will plan: no pass follows a waiting
418 /// request, so each is answered `NotSyncing`. A pass already running
419 /// answers the requests it took.
420 pub async fn sync_stopped(self: &Arc<Self>) -> Result<(), DbError> {
421 let stopped = self
422 .step(
423 |state| RetryState { sync: RetrySync::Stopped, ..state },
424 |_, requests, _| std::mem::take(&mut requests.waiting),
425 )
426 .await?;
427 answer(stopped, &RetryOutcome::NotSyncing);
428 Ok(())
429 }
430
431 /// The upload worker, which serves the whole store, panicked with
432 /// `bug`, which the sink heard: sync halts for the rest of this store's
433 /// session, as the connection owner's own panic halts it (E15). The last
434 /// pass shows the bug with nothing moving, sync stops, each waiting
435 /// request hears it as `SyncError::WorkerFailed`, the connection's owner
436 /// ends on it, and every start fails with it until the store reopens.
437 pub async fn sync_halted(
438 self: &Arc<Self>,
439 bug: coven_foundation::task::Bug,
440 ) -> Result<(), DbError> {
441 self.halted.send_replace(Some(bug.clone()));
442 if let Some(database) = &self.database {
443 crate::LastPass::failed(crate::SyncError::WorkerFailed(bug.clone()))
444 .record_halt(database)
445 .await?;
446 }
447 self.sync_panicked(bug).await
448 }
449
450 /// The connection's owner panicked with `bug`, which the sink heard, so
451 /// no pass follows a waiting request: sync stops, and each request,
452 /// those the panicking pass took among them, is answered `Failed` with
453 /// `SyncError::WorkerFailed` carrying the bug (E15).
454 pub async fn sync_panicked(
455 self: &Arc<Self>,
456 bug: coven_foundation::task::Bug,
457 ) -> Result<(), DbError> {
458 let waiting = self
459 .step(
460 |state| RetryState { sync: RetrySync::Stopped, ..state },
461 |_, requests, _| std::mem::take(&mut requests.waiting),
462 )
463 .await?;
464 let error = crate::SyncError::WorkerFailed(bug).into();
465 answer(waiting, &PassFailure::Failed { error }.into());
466 Ok(())
467 }
468
469 /// A pass begins, if the row lets one: sync is started, and neither a
470 /// failure nor a cooldown holds it. It takes every waiting request the
471 /// connection's owner has acted on (IO `pass_answers_acted_on`). `None`
472 /// when the row holds the pass, and the owner plans again.
473 pub async fn begin_pass(self: &Arc<Self>) -> Result<Option<PassRequests>, DbError> {
474 self.step(
475 |state| state,
476 |waits, requests, state| {
477 (RetryEffect::from(state) == RetryEffect::Pass)
478 .then(|| requests.acted_on(waits, waits.handled(NetworkOwner::Connection)))
479 },
480 )
481 .await
482 }
483
484 /// The connection's owner found what holds every pass: a failure only
485 /// new credentials, an update or a reset can clear, or none. It says so
486 /// as it opens and after each pass, before writing the status a failed
487 /// pass leaves, so the effect never trails that status; each time it
488 /// plans, `answer_held` says so too.
489 pub async fn holds(self: &Arc<Self>, failure: Option<&PassFailure>) -> Result<(), DbError> {
490 let failure = failure.cloned();
491 let held = failure.is_some();
492 self.step(
493 move |state| RetryState { held, ..state },
494 move |_, requests, _| requests.held = failure,
495 )
496 .await
497 }
498
499 /// The owner plans: it records what holds every pass, as `holds` does,
500 /// and answers the requests it has acted on now if no pass will run for
501 /// them (IO `held_answers`). A failure only new credentials, an update
502 /// or a reset can clear answers `Failed`; otherwise a live provider
503 /// cooldown answers `CoolingDown`, and anything else that holds the
504 /// pass leaves them for it. The answer is the effect's.
505 pub async fn answer_held(
506 self: &Arc<Self>,
507 failure: Option<&PassFailure>,
508 ) -> Result<(), DbError> {
509 let failure = failure.cloned();
510 let held = failure.is_some();
511 let answered = self
512 .step(
513 move |state| RetryState { held, ..state },
514 move |waits, requests, state| {
515 requests.held = failure;
516 // With no answer at once, a pass runs and answers them.
517 let outcome = requests.at_once(state)?;
518 let handled = waits.handled(NetworkOwner::Connection);
519 Some((requests.acted_on(waits, handled), outcome, RetryEffect::from(state)))
520 },
521 )
522 .await?;
523 let Some((acted_on, outcome, effect)) = answered else {
524 return Ok(());
525 };
526 // No pass runs for these requests, so nothing else says what came of
527 // the person's retry.
528 if !acted_on.taken.is_empty() {
529 tracing::info!(
530 requests = acted_on.taken.len(),
531 ?effect,
532 "no pass runs for the person's retry; answering it at once"
533 );
534 }
535 acted_on.answer(&outcome);
536 Ok(())
537 }
538
539 /// The connection's first attempt connected: its requests wait for the
540 /// loop's first pass, or are answered `NotSyncing` if sync stopped
541 /// meanwhile.
542 pub async fn return_requests(self: &Arc<Self>, mut taken: PassRequests) -> Result<(), DbError> {
543 let stopped = self
544 .step(
545 |state| state,
546 move |_, requests, state| match state.sync {
547 RetrySync::Started { .. } => {
548 requests.waiting.append(&mut taken.taken);
549 None
550 }
551 RetrySync::Stopped => Some(taken),
552 },
553 )
554 .await?;
555 if let Some(stopped) = stopped {
556 stopped.answer(&RetryOutcome::NotSyncing);
557 }
558 Ok(())
559 }
560}
561
562#[cfg(test)]
563#[path = "retry_answer_tests.rs"]
564mod tests;