1use crate::{PendingReason, SyncError};
14use coven_database::{DbError, RetrySubject, RetryWait, StoreDatabase};
15use coven_foundation::clock::ClockRef;
16use coven_foundation::locks::lock_unpoisoned;
17use coven_storage::{retry_delay, StorageError, StorageFailure};
18use futures_util::FutureExt;
19use std::{
20 collections::{btree_map::Entry, BTreeMap},
21 future::Future,
22 pin::Pin,
23 sync::{Arc, Mutex},
24 time::{Duration, SystemTime},
25};
26
27type Timer = futures_util::future::Shared<Pin<Box<dyn Future<Output = ()> + Send>>>;
28
29pub(crate) type Wake = Pin<Box<dyn Future<Output = ()> + Send>>;
31
32pub(crate) fn restart_wait(not_before: SystemTime, now: SystemTime, wait: Duration) -> Duration {
34 not_before.duration_since(now).unwrap_or(Duration::ZERO).min(wait)
35}
36
37pub struct RetryWaits {
41 clock: ClockRef,
42 database: Option<StoreDatabase>,
44 state: Arc<Mutex<WaitSchedule>>,
47 evidence: Mutex<operation_evidence::OperationEvidence>,
49 rearmed: tokio::sync::watch::Sender<u64>,
52 network: network_report::Reports,
54 requests: Mutex<retry_answer::RetryRequests>,
56 syncing: tokio::sync::watch::Sender<bool>,
59 failures: crate::failure_log::FailureLog,
61 halted: tokio::sync::watch::Sender<Option<coven_foundation::task::Bug>>,
64}
65
66pub struct RearmWatch(tokio::sync::watch::Receiver<u64>);
69
70impl RearmWatch {
71 pub async fn changed(&mut self) -> Result<(), tokio::sync::watch::error::RecvError> {
74 self.0.changed().await
75 }
76}
77
78pub(crate) struct SyncingWatch(tokio::sync::watch::Receiver<bool>);
81
82impl SyncingWatch {
83 pub(crate) async fn changed(&mut self) -> Result<(), tokio::sync::watch::error::RecvError> {
86 self.0.changed().await
87 }
88}
89
90#[derive(Default)]
91struct WaitSchedule {
92 armed: BTreeMap<RetrySubject, Armed>,
93}
94
95struct Armed {
96 not_before: SystemTime,
98 timer: Option<(Duration, Timer)>,
100 ends: Option<SystemTime>,
105 seen: bool,
108}
109
110impl Armed {
111 fn over(&self, elapsed: Duration) -> bool {
112 self.timer
113 .as_ref()
114 .is_some_and(|(due, timer)| *due <= elapsed || timer.clone().now_or_never().is_some())
115 }
116}
117
118impl RetryWaits {
119 pub async fn open(database: StoreDatabase, clock: ClockRef) -> Result<Self, DbError> {
121 let waits = database.retry_waits().await?;
122 let now = clock.now();
123 let owner = Self {
124 clock,
125 database: Some(database),
126 state: Arc::default(),
127 evidence: Mutex::new(operation_evidence::OperationEvidence::new()),
128 rearmed: tokio::sync::watch::channel(0).0,
129 network: network_report::Reports::new(),
130 requests: Mutex::new(retry_answer::RetryRequests::new()),
131 failures: crate::failure_log::FailureLog::new(),
132 syncing: tokio::sync::watch::channel(false).0,
133 halted: tokio::sync::watch::channel(None).0,
134 };
135 for (subject, wait) in waits {
136 owner.arm_for(subject, wait.not_before, restart_wait(wait.not_before, now, wait.wait));
137 }
138 owner.watch_persisted().await?;
139 Ok(owner)
140 }
141
142 pub fn in_memory(clock: ClockRef) -> Self {
144 Self {
145 clock,
146 database: None,
147 state: Arc::default(),
148 evidence: Mutex::new(operation_evidence::OperationEvidence::new()),
149 rearmed: tokio::sync::watch::channel(0).0,
150 network: network_report::Reports::new(),
151 requests: Mutex::new(retry_answer::RetryRequests::new()),
152 failures: crate::failure_log::FailureLog::new(),
153 syncing: tokio::sync::watch::channel(false).0,
154 halted: tokio::sync::watch::channel(None).0,
155 }
156 }
157
158 pub(crate) fn log_failure(
162 &self,
163 subject: impl Into<crate::failure_log::Logged>,
164 reason: &PendingReason,
165 error: &(dyn std::error::Error + 'static),
166 ) -> bool {
167 self.failures.failed(subject.into(), reason, error)
168 }
169
170 #[cfg(test)]
172 pub(crate) fn failures_logged(&self) -> usize {
173 self.failures.count()
174 }
175
176 pub(crate) fn failure_cleared(
179 &self,
180 subject: impl Into<crate::failure_log::Logged>,
181 ended: impl Fn(&PendingReason) -> bool,
182 ) {
183 self.failures.cleared(&subject.into(), ended);
184 }
185
186 pub(crate) fn retain_kept_failures(&self, live: &std::collections::BTreeSet<i64>) {
189 self.failures.retain_kept(live);
190 }
191
192 fn lock(&self) -> std::sync::MutexGuard<'_, WaitSchedule> {
193 lock_unpoisoned(&self.state)
194 }
195
196 fn cooling(state: &WaitSchedule, elapsed: Duration) -> bool {
197 state.armed.get(&RetrySubject::Provider).is_some_and(|armed| !armed.over(elapsed))
198 }
199
200 pub(crate) fn next(&self, previous: Option<&RetryWait>) -> Result<RetryWait, SyncError> {
203 let failures = previous.map_or(0, |wait| wait.failures).saturating_add(1);
204 let wait = retry_delay(failures - 1);
205 let not_before = self.clock.now().checked_add(wait).ok_or_else(|| {
206 StorageFailure::InvalidConfiguration
207 .with_source("clock cannot represent a retry deadline")
208 })?;
209 Ok(RetryWait { failures, not_before, wait })
210 }
211
212 pub(crate) fn arm(&self, subject: RetrySubject, wait: &RetryWait) {
216 let left = restart_wait(wait.not_before, self.clock.now(), wait.wait);
217 self.arm_for(subject, wait.not_before, left);
218 self.rearmed.send_modify(|count| *count = count.wrapping_add(1));
219 }
220
221 fn arm_for(&self, subject: RetrySubject, not_before: SystemTime, wait: Duration) {
222 let armed = self.armed(not_before, wait);
223 self.lock().armed.insert(subject, armed);
224 }
225
226 fn armed(&self, not_before: SystemTime, wait: Duration) -> Armed {
228 let timer = (self.clock.elapsed() + wait, self.clock.wait_out(wait).shared());
229 let ends = self.clock.now().checked_add(wait);
230 Armed { not_before, timer: Some(timer), ends, seen: false }
231 }
232
233 pub(crate) fn syncing(&self) -> SyncingWatch {
235 SyncingWatch(self.syncing.subscribe())
236 }
237
238 pub(crate) fn started(&self) -> bool {
242 *self.syncing.borrow()
243 }
244
245 pub fn halted(&self) -> Option<coven_foundation::task::Bug> {
248 self.halted.borrow().clone()
249 }
250
251 pub fn halt(
254 &self,
255 ) -> impl std::future::Future<Output = coven_foundation::task::Bug> + Send + 'static {
256 let mut halted = self.halted.subscribe();
257 async move {
258 let bug = halted.wait_for(Option::is_some).await.ok().and_then(|bug| bug.clone());
259 match bug {
260 Some(bug) => bug,
261 None => std::future::pending().await,
263 }
264 }
265 }
266
267 pub fn rearmed(&self) -> RearmWatch {
269 RearmWatch(self.rearmed.subscribe())
270 }
271
272 pub(crate) fn hold(&self, subject: RetrySubject) {
275 self.lock().armed.insert(
276 subject,
277 Armed { not_before: SystemTime::UNIX_EPOCH, timer: None, ends: None, seen: false },
278 );
279 }
280
281 pub(crate) fn forget(&self, subject: RetrySubject) {
283 self.lock().armed.remove(&subject);
284 }
285
286 pub(crate) fn operation_advanced(&self, id: coven_database::OperationId) {
289 self.clear_evidence(id);
290 self.forget(RetrySubject::Operation(id));
291 self.failures.cleared_operation(id);
292 }
293
294 pub(crate) fn retried(&self, id: coven_database::OperationId) {
297 self.forget(RetrySubject::Operation(id));
298 self.failures.cleared_operation(id);
299 }
300
301 pub(crate) fn retain(&self, keep: impl Fn(&RetrySubject) -> bool) {
304 self.lock().armed.retain(|subject, _| keep(subject));
305 }
306
307 pub(crate) fn retain_operations(
309 &self,
310 live: &std::collections::BTreeSet<coven_database::OperationId>,
311 ) {
312 self.retain(|subject| match subject {
313 RetrySubject::Operation(id) => live.contains(id),
314 _ => true,
315 });
316 self.retain_evidence(live);
317 self.failures.retain_operations(live);
318 }
319
320 pub fn due(&self, subject: RetrySubject, persisted: Option<&RetryWait>) -> bool {
326 let elapsed = self.clock.elapsed();
327 let Some(persisted) = persisted else {
328 let mut state = self.lock();
329 if state.armed.get(&subject).is_some_and(|armed| armed.timer.is_none()) {
330 return false;
331 }
332 state.armed.remove(&subject);
333 return !Self::cooling(&state, elapsed);
334 };
335 let restarted = || {
340 let wait = restart_wait(persisted.not_before, self.clock.now(), persisted.wait);
341 self.armed(persisted.not_before, wait)
342 };
343 let mut state = self.lock();
344 let armed = match state.armed.entry(subject) {
345 Entry::Occupied(entry) => {
346 let armed = entry.into_mut();
347 if armed.not_before != persisted.not_before && armed.timer.is_some() {
348 *armed = restarted();
349 }
350 armed
351 }
352 Entry::Vacant(entry) => entry.insert(restarted()),
353 };
354 let over = armed.over(elapsed);
355 armed.seen |= over;
356 !Self::cooling(&state, elapsed) && over
357 }
358
359 pub(crate) fn seen(&self, subject: RetrySubject) {
362 let elapsed = self.clock.elapsed();
363 if let Some(armed) = self.lock().armed.get_mut(&subject) {
364 armed.seen |= armed.over(elapsed);
365 }
366 }
367
368 pub(crate) fn cooldown(&self) -> Option<Duration> {
370 let elapsed = self.clock.elapsed();
371 let state = self.lock();
372 let armed = state.armed.get(&RetrySubject::Provider)?;
373 if armed.over(elapsed) {
374 return None;
375 }
376 let (due, _) = armed.timer.as_ref()?;
377 Some(due.saturating_sub(elapsed).max(Duration::from_nanos(1)))
378 }
379
380 #[cfg(test)]
383 pub(crate) fn remaining(&self, subject: RetrySubject) -> Option<Duration> {
384 let elapsed = self.clock.elapsed();
385 let state = self.lock();
386 let armed = state.armed.get(&subject)?;
387 let (due, _) = armed.timer.as_ref()?;
388 Some(if armed.over(elapsed) { Duration::ZERO } else { due.saturating_sub(elapsed) })
389 }
390
391 pub(crate) async fn end_wait(&self, subject: RetrySubject) -> Result<(), SyncError> {
397 let Some(database) = &self.database else {
398 self.forget(subject);
399 return Ok(());
400 };
401 let select = move |waits: &[(RetrySubject, RetryWait)],
402 _: &[crate::PendingRecord],
403 _: &[coven_database::FileUpload],
404 _: &[coven_database::PinRequest]| {
405 waits
406 .iter()
407 .filter(|(held, wait)| *held == subject && !wait.wait.is_zero())
408 .map(|(held, _)| *held)
409 .collect()
410 };
411 match database
412 .end_retry_waits::<crate::PendingRecord>(self.clock.now(), select)
413 .await?
414 .first()
415 {
416 Some((subject, wait)) => self.arm(*subject, wait),
417 None => self.forget(subject),
418 }
419 Ok(())
420 }
421
422 #[cfg(test)]
425 pub(crate) fn next_due(&self, matching: impl Fn(&RetrySubject) -> bool) -> Option<Duration> {
426 let elapsed = self.clock.elapsed();
427 let state = self.lock();
428 let cooldown = state
429 .armed
430 .get(&RetrySubject::Provider)
431 .and_then(|armed| armed.timer.as_ref())
432 .map_or(Duration::ZERO, |(due, _)| *due);
433 state
434 .armed
435 .iter()
436 .filter(|(subject, _)| **subject != RetrySubject::Provider && matching(subject))
437 .filter_map(|(_, armed)| armed.timer.as_ref())
438 .map(|(due, _)| (*due).max(cooldown).saturating_sub(elapsed))
439 .min()
440 }
441
442 pub fn next_wake(&self, matching: impl Fn(&RetrySubject) -> bool) -> Option<Wake> {
448 let elapsed = self.clock.elapsed();
449 let state = self.lock();
450 let earliest = state
451 .armed
452 .iter()
453 .filter(|(subject, armed)| {
454 **subject != RetrySubject::Provider
455 && matching(subject)
456 && !(armed.seen && armed.over(elapsed))
457 })
458 .filter_map(|(_, armed)| armed.timer.clone())
459 .min_by_key(|(due, _)| *due)?;
460 let cooldown = state
461 .armed
462 .get(&RetrySubject::Provider)
463 .filter(|armed| !armed.over(elapsed))
464 .and_then(|armed| armed.timer.clone());
465 Some(Box::pin(async move {
466 earliest.1.await;
467 if let Some((_, cooldown)) = cooldown {
468 cooldown.await;
469 }
470 }))
471 }
472
473 pub fn cooldown_wake(&self) -> Option<Wake> {
475 let elapsed = self.clock.elapsed();
476 let state = self.lock();
477 let armed =
478 state.armed.get(&RetrySubject::Provider).filter(|armed| !armed.over(elapsed))?;
479 let (_, timer) = armed.timer.clone()?;
480 Some(Box::pin(timer))
481 }
482
483 pub(crate) async fn fail(
486 &self,
487 subject: RetrySubject,
488 previous: Option<&RetryWait>,
489 ) -> Result<(), SyncError> {
490 let wait = self.next(previous)?;
491 let Some(database) = &self.database else {
492 self.arm(subject, &wait);
493 return Ok(());
494 };
495 match database.record_retry(subject, wait).await {
496 Ok(()) => {
497 self.arm(subject, &wait);
498 Ok(())
499 }
500 Err(error) => {
501 self.hold(subject);
502 Err(error.into())
503 }
504 }
505 }
506}
507
508#[async_trait::async_trait]
509impl coven_storage::Cooldown for RetryWaits {
510 fn remaining(&self) -> Option<Duration> {
511 self.cooldown()
512 }
513
514 async fn record(&self, wait: Duration) -> Result<(), StorageError> {
518 if self.cooldown().is_some_and(|remaining| remaining >= wait) {
519 return Ok(());
520 }
521 let not_before = self.clock.now().checked_add(wait).ok_or_else(|| {
522 StorageFailure::InvalidConfiguration.with_source("clock cannot represent a cooldown")
523 })?;
524 let cooldown = RetryWait { failures: 0, not_before, wait };
525 self.arm(RetrySubject::Provider, &cooldown);
526 match &self.database {
527 Some(database) => database
528 .record_cooldown(cooldown, self.live_cooldown())
529 .await
530 .map_err(|error| match error {
531 coven_database::DbError::Bug(bug) => bug.into(),
532 error => StorageFailure::File.with_source(error),
533 }),
534 None => Ok(()),
535 }
536 }
537}
538
539#[path = "operation_evidence.rs"]
540mod operation_evidence;
541
542#[path = "network_report.rs"]
543mod network_report;
544pub(crate) use network_report::Ending;
545pub use network_report::NetworkOwner;
546
547#[path = "retry_answer.rs"]
548mod retry_answer;
549pub use retry_answer::{PassRequests, RetryAnswer, RetryEffect, RetryOutcome};
550
551#[cfg(test)]
552#[path = "retry_waits_tests.rs"]
553pub(crate) mod tests;