coven_sync/network_report.rs
1//! The reports that end waits early (§6.7; IO `WaitingNetwork`).
2//!
3//! A network report says the device reached a usable network: the store's
4//! watch of the network saw one appear, or the device move to another
5//! (`network_follow.rs`). A wait whose last
6//! attempt received no provider response waits for the network, so the
7//! report ends it: the connection's backoff, an operation's and a queued
8//! upload's. A wait after a provider's answer holds. A download has no wait:
9//! only the app's next read or pin tries it again.
10//! A retry report says the person asked to retry everything now: it ends
11//! every ordinary wait, answered or not. Neither ends the provider cooldown,
12//! and an ended wait keeps its failure count.
13//!
14//! Each owner acts on reports for the waits it retries, when it next plans:
15//! the connection and its sync loop after the pass in flight, and the upload
16//! worker each time it starts uploads. A report during an attempt also ends
17//! the wait that attempt fails into (IO `signal_during_attempt_wakes`): the
18//! passes act on it after the attempt, and the upload worker ends that wait
19//! as the attempt ends. Reports an owner has not acted on yet coalesce into one, which ends
20//! every ordinary wait if any of them was a retry report.
21
22use crate::{PendingRecord, PendingSubject, RetryWaits, SyncError};
23use coven_database::{FileUpload, PinRequest, RetrySubject, RetryWait};
24use std::{
25 collections::{BTreeMap, BTreeSet},
26 sync::atomic::{AtomicU64, Ordering},
27 time::SystemTime,
28};
29
30/// An owner of waits that acts on reports.
31#[derive(Clone, Copy, Debug, PartialEq, Eq)]
32pub enum NetworkOwner {
33 /// The connection and its sync loop: the connection's backoff and
34 /// coven's own operations.
35 Connection,
36 /// The upload worker: queued uploads, and the pins coven keeps, which it
37 /// makes after each run.
38 Uploads,
39}
40
41impl NetworkOwner {
42 fn retries(self, subject: &RetrySubject) -> bool {
43 match self {
44 Self::Connection => matches!(subject, RetrySubject::Sync | RetrySubject::Operation(_)),
45 Self::Uploads => matches!(subject, RetrySubject::Upload(_) | RetrySubject::Pin(_)),
46 }
47 }
48
49 fn index(self) -> usize {
50 match self {
51 Self::Connection => 0,
52 Self::Uploads => 1,
53 }
54 }
55}
56
57/// Which waits a report ends.
58#[derive(Clone, Copy, Debug, PartialEq, Eq)]
59pub(crate) enum Ending {
60 /// Waits whose last attempt received no provider response.
61 Network,
62 /// Every ordinary wait: the person asked to retry everything now.
63 Every,
64}
65
66/// Reports so far, the latest retry report among them, and the count each
67/// owner has acted on; an owner acts on a report only once the waits it
68/// ends have ended.
69/// Hears each network or retry report, for an owner that
70/// waits on its own timers (`RetryWaits::network_reports`).
71pub struct NetworkReportWatch(tokio::sync::watch::Receiver<u64>);
72
73impl NetworkReportWatch {
74 /// Resolves once another report arrives; an error once the schedule is
75 /// gone.
76 pub async fn changed(&mut self) -> Result<(), tokio::sync::watch::error::RecvError> {
77 self.0.changed().await
78 }
79}
80
81pub(crate) struct Reports {
82 reported: tokio::sync::watch::Sender<u64>,
83 /// The number of the latest retry report; zero before any.
84 retried: AtomicU64,
85 handled: [AtomicU64; 2],
86}
87
88impl Reports {
89 /// No reports yet, and none acted on.
90 pub(crate) fn new() -> Self {
91 Self {
92 reported: tokio::sync::watch::channel(0).0,
93 retried: AtomicU64::new(0),
94 handled: [AtomicU64::new(0), AtomicU64::new(0)],
95 }
96 }
97
98 /// Reports so far, and which waits those after `seen` end; `None` when
99 /// none arrived since.
100 fn since(&self, seen: u64) -> (u64, Option<Ending>) {
101 let reported = self.reported.borrow();
102 let ending = if *reported <= seen {
103 None
104 } else if self.retried.load(Ordering::SeqCst) > seen {
105 Some(Ending::Every)
106 } else {
107 Some(Ending::Network)
108 };
109 (*reported, ending)
110 }
111}
112
113/// The pending record a wait belongs to: the one naming its failure.
114fn record_for<'a>(
115 subject: RetrySubject,
116 pending: &'a [PendingRecord],
117 uploads: &[FileUpload],
118 pins: &[PinRequest],
119) -> Option<&'a PendingRecord> {
120 let wanted = match subject {
121 RetrySubject::Sync => PendingSubject::Connection,
122 RetrySubject::Operation(id) => {
123 return pending.iter().find(|record| {
124 matches!(record.subject, PendingSubject::Operation { id: owned, .. } if owned == id)
125 })
126 }
127 RetrySubject::Upload(id) => {
128 let location = &uploads.iter().find(|upload| upload.id == id)?.location;
129 PendingSubject::Upload { device: location.uploader(), object: location.object() }
130 }
131 RetrySubject::Pin(id) => {
132 PendingSubject::Keep { file: pins.iter().find(|request| request.id == id)?.file }
133 }
134 RetrySubject::Join | RetrySubject::Provider => return None,
135 };
136 pending.iter().find(|record| record.subject == wanted)
137}
138
139impl RetryWaits {
140 /// The device reached a usable network: one appeared, or it moved to
141 /// another (§6.7). Each owner acts on the report when it next plans;
142 /// reports before then coalesce.
143 pub fn report_network(&self) {
144 self.report(Ending::Network);
145 }
146
147 /// Report and return its number. The person's retry
148 /// (`retry_requested`) reports this way too.
149 pub(crate) fn report(&self, ending: Ending) -> u64 {
150 let mut number = 0;
151 self.network.reported.send_modify(|count| {
152 *count += 1;
153 number = *count;
154 if ending == Ending::Every {
155 self.network.retried.store(*count, Ordering::SeqCst);
156 }
157 });
158 // The owners plan again on a changed schedule.
159 self.rearmed.send_modify(|count| *count = count.wrapping_add(1));
160 number
161 }
162
163 /// How many reports `owner` has acted on.
164 pub(crate) fn handled(&self, owner: NetworkOwner) -> u64 {
165 self.network.handled[owner.index()].load(Ordering::SeqCst)
166 }
167
168 /// Changes with every report, for an owner that waits on its own timers.
169 pub fn network_reports(&self) -> NetworkReportWatch {
170 NetworkReportWatch(self.network.reported.subscribe())
171 }
172
173 /// End the waits the reports since `owner` last acted end (§6.7): its
174 /// waits for the network, or every ordinary wait it retries after a
175 /// retry report. Each ended wait keeps its failure count, so the next
176 /// failure continues the backoff, and is saved ended, so no reopen arms
177 /// it again (IO `ended_reopens_due`). The provider cooldown holds, and a
178 /// network report leaves a wait after a provider's answer and one held
179 /// because saving it failed. Returns whether a report arrived, so the
180 /// owner also runs the work it asks for. If reading or saving fails,
181 /// every wait holds, the owner reports the error, and the reports stay
182 /// unacted on, so the owner's next plan acts on them again.
183 pub async fn end_reported_waits(&self, owner: NetworkOwner) -> Result<bool, SyncError> {
184 let handled = &self.network.handled[owner.index()];
185 let (reported, ending) = self.network.since(handled.load(Ordering::SeqCst));
186 let Some(ending) = ending else {
187 return Ok(false);
188 };
189 self.end_waits(owner, move |_| Some(ending)).await?;
190 // Reports since `reported` was read stay for the next plan.
191 handled.fetch_max(reported, Ordering::SeqCst);
192 Ok(true)
193 }
194
195 /// How many reports have arrived so far. An attempt notes it as it
196 /// starts, so it can tell whether one arrived while it ran.
197 pub(crate) fn network_reported(&self) -> u64 {
198 *self.network.reported.borrow()
199 }
200
201 /// Which waits the reports since `seen` end; `None` when none arrived.
202 pub(crate) fn reports_since(&self, seen: u64) -> Option<Ending> {
203 self.network.since(seen).1
204 }
205
206 /// End each subject's wait as the reports that arrived while its attempt
207 /// ran say, whatever reports their owner already acted on: the report
208 /// ends the wait the attempt failed into (IO
209 /// `signal_during_attempt_wakes`). Fails as `end_reported_waits` does.
210 pub(crate) async fn end_waits_after_report(
211 &self,
212 owner: NetworkOwner,
213 subjects: &BTreeMap<RetrySubject, Ending>,
214 ) -> Result<(), SyncError> {
215 let subjects = subjects.clone();
216 self.end_waits(owner, move |subject| subjects.get(subject).copied()).await
217 }
218
219 /// End `owner`'s waits as `ending` says for each subject.
220 async fn end_waits(
221 &self,
222 owner: NetworkOwner,
223 ending: impl Fn(&RetrySubject) -> Option<Ending> + Send + 'static + Clone,
224 ) -> Result<(), SyncError> {
225 // A wait this session holds for the app stays held unless the person
226 // retries; one already over needs no end unless a new failure
227 // replaced it.
228 let (held, over) = {
229 let elapsed = self.clock.elapsed();
230 let state = self.lock();
231 let mut held = BTreeSet::new();
232 let mut over = BTreeMap::new();
233 for (subject, armed) in &state.armed {
234 if !owner.retries(subject) {
235 continue;
236 }
237 if armed.timer.is_none() {
238 held.insert(*subject);
239 } else if armed.over(elapsed) {
240 over.insert(*subject, armed.not_before);
241 }
242 }
243 (held, over)
244 };
245 // The person's retry releases a held wait: it was holding for them.
246 let released: Vec<RetrySubject> =
247 held.iter().filter(|subject| ending(subject) == Some(Ending::Every)).copied().collect();
248 let Some(database) = &self.database else {
249 // A handle that cannot persist keeps its waits in memory: the
250 // person's retry forgets them.
251 let every: Vec<RetrySubject> = self
252 .lock()
253 .armed
254 .keys()
255 .filter(|subject| owner.retries(subject) && ending(subject) == Some(Ending::Every))
256 .copied()
257 .collect();
258 for subject in every {
259 self.forget(subject);
260 }
261 return Ok(());
262 };
263 let chosen = ending.clone();
264 let select = move |waits: &[(RetrySubject, RetryWait)],
265 pending: &[PendingRecord],
266 uploads: &[FileUpload],
267 pins: &[PinRequest]| {
268 waits
269 .iter()
270 .filter(|(subject, wait)| {
271 let Some(ending) = chosen(subject) else {
272 return false;
273 };
274 owner.retries(subject)
275 && over.get(subject) != Some(&wait.not_before)
276 && !wait.wait.is_zero()
277 && match ending {
278 Ending::Every => true,
279 Ending::Network => {
280 !held.contains(subject)
281 && record_for(*subject, pending, uploads, pins)
282 .is_some_and(|record| record.reason.offline())
283 }
284 }
285 })
286 .map(|(subject, _)| *subject)
287 .collect()
288 };
289 let now: SystemTime = self.clock.now();
290 let mut ended = BTreeSet::new();
291 for (subject, wait) in database.end_retry_waits::<PendingRecord>(now, select).await? {
292 self.arm(subject, &wait);
293 ended.insert(subject);
294 }
295 // A held wait with no saved row to end is simply forgotten.
296 for subject in released {
297 if !ended.contains(&subject) {
298 self.forget(subject);
299 }
300 }
301 Ok(())
302 }
303}