1use crate::{
3 LastPass, PassFailure, RetryOutcome, RetryWaits, SequenceStatus, SequenceSync, SyncError,
4 SyncFailure,
5};
6use coven_database::{DbError, RetrySubject};
7use std::{sync::Arc, time::Duration};
8use tokio::sync::{mpsc, oneshot};
9
10#[derive(Clone)]
11pub struct SyncLoop {
14 commands: mpsc::UnboundedSender<Command>,
15 waits: Arc<RetryWaits>,
17 ended: Arc<std::sync::OnceLock<LoopEnd>>,
20 #[cfg(any(test, feature = "test-utils"))]
22 parks: crate::Parks,
23 #[cfg(any(test, feature = "test-utils"))]
25 pauses: Arc<tokio::sync::watch::Sender<usize>>,
26}
27#[derive(Clone)]
29enum LoopEnd {
30 Bug(coven_foundation::task::Bug),
32 Failed { failure: crate::LocalFailure, source: Arc<dyn std::error::Error + Send + Sync> },
35 StoreClosed,
37 Released,
40}
41
42impl LoopEnd {
43 fn on(error: SyncError) -> (Self, SyncFailure) {
46 use coven_foundation::task::CarriesBug;
47 if let Some(bug) = error.bug().cloned() {
48 return (Self::Bug(bug), error.into());
49 }
50 let failure = error.local_failure();
51 let shown = SyncFailure::from(error);
52 let source: Arc<dyn std::error::Error + Send + Sync> = match &shown {
53 SyncFailure::Other(source) => source.clone(),
54 SyncFailure::Storage(source) => source.clone(),
55 shown => Arc::new(shown.clone()),
56 };
57 (Self::Failed { failure, source }, shown)
58 }
59
60 fn error(&self) -> SyncError {
62 match self {
63 Self::Bug(bug) => SyncError::WorkerFailed(bug.clone()),
64 Self::Failed { failure, source } => {
65 SyncError::Failed { failure: *failure, source: source.clone() }
66 }
67 Self::StoreClosed => DbError::StoreClosed.into(),
68 Self::Released => SyncError::NoStorage,
69 }
70 }
71}
72
73enum Command {
74 Start,
75 Now,
76 Stop,
77 Pause(oneshot::Sender<()>),
78 Resume,
79 Close(oneshot::Sender<Result<(), SyncError>>),
80}
81#[expect(
82 clippy::map_err_ignore,
83 reason = "a command fails only when the loop has ended, and `gone` says why"
84)]
85impl SyncLoop {
86 pub fn new(sync: Arc<SequenceSync>, codes: crate::RestoreCodes) -> Self {
94 let (commands, receiver) = mpsc::unbounded_channel();
95 let waits = sync.waits.clone();
96 let owner = Arc::downgrade(&sync);
98 let ended = Arc::new(std::sync::OnceLock::new());
99 let unwound = ended.clone();
100 let parks = crate::Parks::new();
101 drop(coven_foundation::task::spawn_owned(
104 "the sync loop",
105 sync.database.bugs().clone(),
106 run(sync, codes, receiver, parks.clone(), ended.clone()),
107 |bug| async move {
108 unwound.get_or_init(|| LoopEnd::Bug(bug.clone()));
109 if let Some(owner) = owner.upgrade() {
110 panicked(&owner, bug).await;
111 }
112 },
113 ));
114 Self {
115 commands,
116 waits,
117 ended,
118 #[cfg(any(test, feature = "test-utils"))]
119 parks,
120 #[cfg(any(test, feature = "test-utils"))]
121 pauses: Arc::new(tokio::sync::watch::channel(0).0),
122 }
123 }
124
125 #[cfg(any(test, feature = "test-utils"))]
128 pub fn parked_from_now(&self) -> impl std::future::Future<Output = ()> + Send + 'static {
129 self.parks.parked_from_now()
130 }
131
132 #[cfg(any(test, feature = "test-utils"))]
135 pub fn pause_asked_from_now(&self) -> impl std::future::Future<Output = ()> + Send + 'static {
136 let mut pauses = self.pauses.subscribe();
137 let now = *pauses.borrow_and_update();
138 async move {
139 coven_foundation::clock::within(pauses.wait_for(|pauses| *pauses > now))
140 .await
141 .expect("a pause is asked")
142 .expect("the loop's pauses are kept");
143 }
144 }
145
146 fn gone(&self) -> SyncError {
150 match self.ended.get() {
151 Some(end) => end.error(),
152 None => DbError::StoreClosed.into(),
156 }
157 }
158 pub async fn start(&self) -> Result<(), SyncError> {
161 self.waits.sync_started().await?;
162 if self.commands.send(Command::Start).is_err() {
163 self.waits.sync_stopped().await?;
164 return Err(self.gone());
165 }
166 Ok(())
167 }
168 pub fn sync_now(&self) -> Result<(), SyncError> {
170 self.commands.send(Command::Now).map_err(|_| self.gone())
171 }
172 pub async fn stop(&self) -> Result<(), SyncError> {
178 let stopped = self.waits.sync_stopped().await;
179 self.commands.send(Command::Stop).map_err(|_| self.gone())?;
180 Ok(stopped?)
181 }
182 pub async fn pause(&self) -> Result<(), SyncError> {
185 let (send, receive) = oneshot::channel();
186 self.commands.send(Command::Pause(send)).map_err(|_| self.gone())?;
187 #[cfg(any(test, feature = "test-utils"))]
188 self.pauses.send_modify(|pauses| *pauses += 1);
189 receive.await.map_err(|_| self.gone())
190 }
191 pub fn resume(&self) -> Result<(), SyncError> {
193 self.commands.send(Command::Resume).map_err(|_| self.gone())
194 }
195 pub async fn close(&self) -> Result<(), SyncError> {
198 let (send, receive) = oneshot::channel();
199 self.commands.send(Command::Close(send)).map_err(|_| self.gone())?;
200 receive.await.map_err(|_| self.gone())?
201 }
202}
203async fn failing(sync: &SequenceSync) -> Result<Option<crate::PendingRecord>, SyncError> {
207 let records = sync.database.pending::<crate::PendingRecord>().await?;
208 Ok(records.into_iter().find(|record| {
209 let coven_owned = matches!(
210 record.subject,
211 crate::PendingSubject::Operation { kind, .. } if kind.coven_owned()
212 );
213 coven_owned
214 && matches!(
215 record.reason.retry_cause(),
216 crate::PendingReason::Storage { .. }
217 | crate::PendingReason::Refused(_)
218 | crate::PendingReason::Failed(_)
219 )
220 }))
221}
222
223async fn pass(
225 sync: &SequenceSync,
226 codes: &crate::RestoreCodes,
227) -> Result<Option<crate::PendingRecord>, SyncError> {
228 sync.pass().await?;
229 codes.resume_member_keys(sync).await?;
230 failing(sync).await
231}
232
233fn failed(sync: &SequenceSync, error: SyncError) -> PassFailure {
237 let blocked = match sync.state.status() {
238 SequenceStatus::Blocked(record) => Some(*record),
239 SequenceStatus::Opening | SequenceStatus::Synced { .. } | SequenceStatus::Stopped(_) => {
240 None
241 }
242 };
243 let Some(record) = blocked else {
244 return crate::connection_failure(error);
245 };
246 let crate::PendingSubject::Slot(slot) = record.subject else {
247 return crate::connection_failure(error);
248 };
249 PassFailure::Blocked { slot, reason: record.reason, error: record.cause }
250}
251
252fn reported(sync: &SequenceSync, error: impl Into<SyncError>) -> SyncError {
255 coven_foundation::task::reported(sync.database.bugs().as_ref(), error.into())
256}
257
258fn own_write<T>(sync: &SequenceSync, result: Result<T, DbError>) -> Result<T, DbError> {
263 result.map_err(|error| coven_foundation::task::reported(sync.database.bugs().as_ref(), error))
264}
265
266async fn show(sync: &SequenceSync, failure: LastPass) -> Result<(), DbError> {
270 own_write(sync, failure.record(&sync.database).await)
271}
272
273async fn ended(sync: &SequenceSync, last: LastPass) -> Result<(), DbError> {
276 own_write(sync, last.record_end(&sync.database, sync.clock.now()).await)
277}
278
279async fn moving_pass(
285 sync: &SequenceSync,
286 codes: &crate::RestoreCodes,
287) -> (Result<Option<crate::PendingRecord>, SyncError>, Result<(), DbError>) {
288 let mut moves = sync.state.pass_begins();
289 let passing = pass(sync, codes);
290 tokio::pin!(passing);
291 let mut written = Ok(());
292 loop {
293 tokio::select! {
294 biased;
295 passed = &mut passing => return (passed, written),
296 moved = moves.moved() => {
297 let recorded = sync.database.record_moved(moved).await;
298 written = written.and(recorded);
299 }
300 }
301 }
302}
303
304async fn give_up(sync: &SequenceSync, error: impl Into<SyncError>) -> LoopEnd {
307 let (end, shown) = LoopEnd::on(reported(sync, error));
308 #[expect(
309 clippy::let_underscore_must_use,
310 reason = "the loop ends either way; a bug has gone to the sink, and any other failed write has failed the handle, which reports it"
311 )]
312 let _ = show(sync, LastPass::failed(shown)).await;
313 end
314}
315
316async fn stopped(sync: &SequenceSync) {
321 if let Err(error) = own_write(sync, sync.waits.sync_stopped().await) {
322 tracing::debug!("stopping sync failed: {error}");
323 }
324}
325
326fn idle_wait(
328 sync: &SequenceSync,
329 from: Duration,
330 interval: Duration,
331) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> {
332 sync.clock.deadline((from + interval).saturating_sub(sync.clock.elapsed()))
333}
334
335async fn panicked(sync: &SequenceSync, bug: coven_foundation::task::Bug) {
340 #[expect(
341 clippy::let_underscore_must_use,
342 reason = "the loop has ended either way; a bug has gone to the sink, and any other failed write has failed the handle, which reports it"
343 )]
344 let _ = show(sync, LastPass::failed(SyncError::WorkerFailed(bug.clone()))).await;
345 if let Err(error) = own_write(sync, sync.waits.sync_panicked(bug).await) {
346 tracing::debug!("stopping sync after the loop's panic failed: {error}");
347 }
348}
349
350async fn halted(sync: &SequenceSync, bug: coven_foundation::task::Bug) -> LoopEnd {
355 panicked(sync, bug.clone()).await;
356 LoopEnd::Bug(bug)
357}
358
359async fn run(
363 sync: Arc<SequenceSync>,
364 codes: crate::RestoreCodes,
365 mut commands: mpsc::UnboundedReceiver<Command>,
366 parks: crate::Parks,
367 why_ended: Arc<std::sync::OnceLock<LoopEnd>>,
368) {
369 let _running = parks.running();
370 let bugs = sync.database.bugs().clone();
371 let owner = sync.clone();
372 let mut closing = None;
373 let served = coven_foundation::task::caught(
374 "the sync loop",
375 serve(sync, codes, &mut commands, &parks, &mut closing),
376 )
377 .await;
378 let (end, closed) = match served {
379 Ok(served) => served,
380 Err(bug) => {
381 coven_foundation::task::report(bugs.as_ref(), &bug);
382 panicked(&owner, bug.clone()).await;
383 (LoopEnd::Bug(bug.clone()), Err(SyncError::WorkerFailed(bug)))
384 }
385 };
386 drop(owner);
388 why_ended.get_or_init(|| end);
391 if let Some(reply) = closing {
392 if let Err(Err(error)) = reply.send(closed) {
395 if !coven_foundation::task::report_carried(bugs.as_ref(), &error) {
396 tracing::warn!("closing sync failed after its caller stopped waiting: {error}");
397 }
398 }
399 }
400}
401
402async fn serve(
405 sync: Arc<SequenceSync>,
406 codes: crate::RestoreCodes,
407 commands: &mut mpsc::UnboundedReceiver<Command>,
408 parks: &crate::Parks,
409 closing: &mut Option<oneshot::Sender<Result<(), SyncError>>>,
410) -> (LoopEnd, Result<(), SyncError>) {
411 let mut changes = sync.database.sync_changes();
412 let mut rearmed = sync.waits.rearmed();
413 let mut retry = match crate::ConnectionRetry::open(&sync.database).await {
414 Ok(retry) => retry,
415 Err(error) => {
416 crate::ConnectionRetry::Never(crate::connection_failure(reported(&sync, error)))
417 }
418 };
419 let mut closed = Ok(());
420 let end = 'run: {
421 let held = retry.held(&sync.waits);
422 if let Err(error) = sync.waits.holds(held).await {
423 break 'run give_up(&sync, error).await;
424 }
425 if let Some(failure) = held {
426 if let Err(error) = show(&sync, failure.clone().into()).await {
427 break 'run LoopEnd::on(error.into()).0;
428 }
429 }
430 let mut started = false;
431 let mut paused = false;
432 let mut pending = false;
433 let mut interval = sync.state.idle_interval();
436 let mut idle_from = sync.clock.elapsed();
437 let mut idle = idle_wait(&sync, idle_from, interval.idle_wait());
438 loop {
439 if !paused {
444 match sync.waits.end_reported_waits(crate::NetworkOwner::Connection).await {
445 Ok(reported) => pending |= reported && started,
446 Err(error) => break give_up(&sync, error).await,
447 }
448 }
449 let cooldown = sync.waits.cooldown_wake();
454 if started {
459 if let Err(error) = sync.waits.answer_held(retry.held(&sync.waits)).await {
460 break give_up(&sync, error).await;
461 }
462 }
463 let mut wakes = if pending { Vec::new() } else { sync.waits.evidence_wakes() };
466 wakes.extend(sync.waits.next_wake(|subject| {
467 *subject == RetrySubject::Sync
468 || (!pending && matches!(subject, RetrySubject::Operation(_)))
469 }));
470 let work_wake = async {
471 if wakes.is_empty() {
472 std::future::pending::<()>().await;
473 } else {
474 futures_util::future::select_all(wakes).await;
475 }
476 };
477 let previous = match if paused {
478 Ok(None)
479 } else {
480 sync.database.retry_wait(RetrySubject::Sync).await
481 } {
482 Ok(wait) => wait,
483 Err(error) => break give_up(&sync, error).await,
484 };
485 let due = !paused
486 && retry.held(&sync.waits).is_none()
487 && sync.waits.due(RetrySubject::Sync, previous.as_ref());
488 tokio::select! {
489 biased;
490 never = parks.polling() => match never {},
491 bug = sync.waits.halt() => break halted(&sync, bug).await,
492 command = commands.recv() => match command {
493 Some(Command::Start) => {
494 if !started {
495 let ready = sync.files.set_storage(Some(sync.storage.clone()), std::future::ready(Ok(())));
496 match ready.await {
497 Ok(()) => {
498 started = true;
499 pending = true;
500 sync.waits.connection_changed();
501 }
502 Err(error) => {
503 stopped(&sync).await;
504 if let Err(error) = show(&sync, LastPass::failed(reported(&sync, error))).await { break LoopEnd::on(error.into()).0; }
505 }
506 }
507 }
508 }
509 Some(Command::Now) => pending |= started,
510 Some(Command::Pause(reply)) => {
511 paused = true;
512 #[expect(
513 clippy::let_underscore_must_use,
514 reason = "the caller may have stopped waiting for the pause"
515 )]
516 let _ = reply.send(());
517 }
518 Some(Command::Resume) => paused = false,
519 Some(Command::Stop) => {
520 started = false;
521 pending = false;
522 if let Err(error) = sync.files.set_storage(None, std::future::ready(Ok(()))).await {
525 if let Err(error) = show(&sync, LastPass::failed(reported(&sync, error))).await { break LoopEnd::on(error.into()).0; }
526 }
527 }
528 Some(Command::Close(reply)) => {
531 *closing = Some(reply);
533 if started && !paused {
534 closed = sync.files.set_storage(None, std::future::ready(Ok(()))).await;
535 }
536 break LoopEnd::Released;
537 }
538 None => {
539 if started && !paused {
542 let released = sync.files.set_storage(None, std::future::ready(Ok(())));
543 if let Err(error) = released.await {
544 if !coven_foundation::task::report_carried(sync.database.bugs().as_ref(), &error) {
545 tracing::warn!("releasing transfers without a handle failed: {error}");
546 }
547 }
548 }
549 break LoopEnd::Released;
550 }
551 },
552 changed = changes.next(), if started && !paused => match changed {
553 Ok(()) => pending = true,
554 Err(DbError::StoreClosed) => break LoopEnd::StoreClosed,
555 Err(error) => {
556 started = false;
557 stopped(&sync).await;
558 if let Err(error) = show(&sync, LastPass::failed(reported(&sync, error))).await { break LoopEnd::on(error.into()).0; }
559 }
560 },
561 () = interval.changed() => idle = idle_wait(&sync, idle_from, interval.idle_wait()),
562 _ = &mut idle, if started && !paused && !pending => pending = true,
563 _ = sync.state.pass_requested(), if started && !paused && !pending => pending = true,
564 changed = rearmed.changed(), if started && !paused => if changed.is_err() { break LoopEnd::StoreClosed },
567 _ = work_wake, if started && !paused && !(pending && due) => pending = true,
568 _ = async {
572 if let Some(wake) = cooldown { wake.await } else { std::future::pending::<()>().await }
573 }, if started => {}
574 _ = std::future::ready(()), if started && !paused && pending && due => {
575 pending = false;
576 if let Err(error) = changes.take_changed() {
577 break give_up(&sync, error).await;
578 }
579 let requests = match sync.waits.begin_pass().await {
585 Ok(Some(requests)) => requests,
586 Ok(None) => {
587 pending = true;
588 continue;
589 }
590 Err(error) => break give_up(&sync, error).await,
591 };
592 let connection_version = sync.waits.connection_version();
593 let (passed, moving) = moving_pass(&sync, &codes).await;
594 if let Err(error) = moving {
595 let (end, shown) = LoopEnd::on(reported(&sync, error));
596 requests.answer(&PassFailure::Failed { error: shown }.into());
597 break end;
598 }
599 idle_from = sync.clock.elapsed();
602 let outcome = match passed {
603 Ok(failing) => match retry.succeeded(&sync.database, &sync.waits).await {
604 Ok(()) => match failing {
605 None => {
606 let finished_at = sync.clock.now();
607 if let Err(error) = LastPass::Synced.record_end(&sync.database, finished_at).await {
608 let (end, shown) = LoopEnd::on(reported(&sync, error));
609 requests.answer(&PassFailure::Failed { error: shown }.into());
610 break end;
611 }
612 RetryOutcome::Synced { finished_at }
613 }
614 Some(record) => {
615 started = record.retry() != crate::Retry::Never;
618 pending = false;
619 PassFailure::Failed { error: SyncFailure::Operation(Box::new(record)) }.into()
620 }
621 },
622 Err(error) => {
623 started = false;
624 PassFailure::Failed { error: reported(&sync, error).into() }.into()
625 }
626 },
627 Err(error) => {
628 let error = reported(&sync, error);
629 let reason = sync.error_reason(&error);
630 match retry.failed(&sync.database, &sync.waits, reason, connection_version, error, |error| failed(&sync, error)).await {
631 Ok(failure) => {
632 pending = true;
633 failure.into()
634 }
635 Err(failure) => {
636 started = false;
637 failure.into()
638 }
639 }
640 }
641 };
642 if let Err(error) = sync.waits.holds(retry.held(&sync.waits)).await {
646 let (end, shown) = LoopEnd::on(reported(&sync, error));
647 requests.answer(&PassFailure::Failed { error: shown }.into());
648 break end;
649 }
650 if !started {
651 stopped(&sync).await;
652 }
653 if let RetryOutcome::Failed(failure) = &outcome {
654 if let Err(error) = ended(&sync, failure.clone().into()).await {
656 let (end, shown) = LoopEnd::on(error.into());
657 requests.answer(&PassFailure::Failed { error: shown }.into());
658 break end;
659 }
660 }
661 requests.answer(&outcome);
662 idle = idle_wait(&sync, idle_from, interval.idle_wait());
663 }
664 never = parks.parked() => match never {},
665 }
666 }
667 };
668 if closing.is_none() {
672 stopped(&sync).await;
673 }
674 (end, closed)
675}
676
677#[cfg(test)]
678#[path = "sync_loop_tests.rs"]
679mod tests;