Skip to main content

coven_sync/
network_report.rs

1//! The reports that end waits early (§6.7; IO `WaitingNetwork`).
2//!
3//! A network report says the device reached a usable network: the store's
4//! watch of the network saw one appear, or the device move to another
5//! (`network_follow.rs`). A wait whose last
6//! attempt received no provider response waits for the network, so the
7//! report ends it: the connection's backoff, an operation's and a queued
8//! upload's. A wait after a provider's answer holds. A download has no wait:
9//! only the app's next read or pin tries it again.
10//! A retry report says the person asked to retry everything now: it ends
11//! every ordinary wait, answered or not. Neither ends the provider cooldown,
12//! and an ended wait keeps its failure count.
13//!
14//! Each owner acts on reports for the waits it retries, when it next plans:
15//! the connection and its sync loop after the pass in flight, and the upload
16//! worker each time it starts uploads. A report during an attempt also ends
17//! the wait that attempt fails into (IO `signal_during_attempt_wakes`): the
18//! passes act on it after the attempt, and the upload worker ends that wait
19//! as the attempt ends. Reports an owner has not acted on yet coalesce into one, which ends
20//! every ordinary wait if any of them was a retry report.
21
22use crate::{PendingRecord, PendingSubject, RetryWaits, SyncError};
23use coven_database::{FileUpload, PinRequest, RetrySubject, RetryWait};
24use std::{
25    collections::{BTreeMap, BTreeSet},
26    sync::atomic::{AtomicU64, Ordering},
27    time::SystemTime,
28};
29
30/// An owner of waits that acts on reports.
31#[derive(Clone, Copy, Debug, PartialEq, Eq)]
32pub enum NetworkOwner {
33    /// The connection and its sync loop: the connection's backoff and
34    /// coven's own operations.
35    Connection,
36    /// The upload worker: queued uploads, and the pins coven keeps, which it
37    /// makes after each run.
38    Uploads,
39}
40
41impl NetworkOwner {
42    fn retries(self, subject: &RetrySubject) -> bool {
43        match self {
44            Self::Connection => matches!(subject, RetrySubject::Sync | RetrySubject::Operation(_)),
45            Self::Uploads => matches!(subject, RetrySubject::Upload(_) | RetrySubject::Pin(_)),
46        }
47    }
48
49    fn index(self) -> usize {
50        match self {
51            Self::Connection => 0,
52            Self::Uploads => 1,
53        }
54    }
55}
56
57/// Which waits a report ends.
58#[derive(Clone, Copy, Debug, PartialEq, Eq)]
59pub(crate) enum Ending {
60    /// Waits whose last attempt received no provider response.
61    Network,
62    /// Every ordinary wait: the person asked to retry everything now.
63    Every,
64}
65
66/// Reports so far, the latest retry report among them, and the count each
67/// owner has acted on; an owner acts on a report only once the waits it
68/// ends have ended.
69/// Hears each network or retry report, for an owner that
70/// waits on its own timers (`RetryWaits::network_reports`).
71pub struct NetworkReportWatch(tokio::sync::watch::Receiver<u64>);
72
73impl NetworkReportWatch {
74    /// Resolves once another report arrives; an error once the schedule is
75    /// gone.
76    pub async fn changed(&mut self) -> Result<(), tokio::sync::watch::error::RecvError> {
77        self.0.changed().await
78    }
79}
80
81pub(crate) struct Reports {
82    reported: tokio::sync::watch::Sender<u64>,
83    /// The number of the latest retry report; zero before any.
84    retried: AtomicU64,
85    handled: [AtomicU64; 2],
86}
87
88impl Reports {
89    /// No reports yet, and none acted on.
90    pub(crate) fn new() -> Self {
91        Self {
92            reported: tokio::sync::watch::channel(0).0,
93            retried: AtomicU64::new(0),
94            handled: [AtomicU64::new(0), AtomicU64::new(0)],
95        }
96    }
97
98    /// Reports so far, and which waits those after `seen` end; `None` when
99    /// none arrived since.
100    fn since(&self, seen: u64) -> (u64, Option<Ending>) {
101        let reported = self.reported.borrow();
102        let ending = if *reported <= seen {
103            None
104        } else if self.retried.load(Ordering::SeqCst) > seen {
105            Some(Ending::Every)
106        } else {
107            Some(Ending::Network)
108        };
109        (*reported, ending)
110    }
111}
112
113/// The pending record a wait belongs to: the one naming its failure.
114fn record_for<'a>(
115    subject: RetrySubject,
116    pending: &'a [PendingRecord],
117    uploads: &[FileUpload],
118    pins: &[PinRequest],
119) -> Option<&'a PendingRecord> {
120    let wanted = match subject {
121        RetrySubject::Sync => PendingSubject::Connection,
122        RetrySubject::Operation(id) => {
123            return pending.iter().find(|record| {
124                matches!(record.subject, PendingSubject::Operation { id: owned, .. } if owned == id)
125            })
126        }
127        RetrySubject::Upload(id) => {
128            let location = &uploads.iter().find(|upload| upload.id == id)?.location;
129            PendingSubject::Upload { device: location.uploader(), object: location.object() }
130        }
131        RetrySubject::Pin(id) => {
132            PendingSubject::Keep { file: pins.iter().find(|request| request.id == id)?.file }
133        }
134        RetrySubject::Join | RetrySubject::Provider => return None,
135    };
136    pending.iter().find(|record| record.subject == wanted)
137}
138
139impl RetryWaits {
140    /// The device reached a usable network: one appeared, or it moved to
141    /// another (§6.7). Each owner acts on the report when it next plans;
142    /// reports before then coalesce.
143    pub fn report_network(&self) {
144        self.report(Ending::Network);
145    }
146
147    /// Report and return its number. The person's retry
148    /// (`retry_requested`) reports this way too.
149    pub(crate) fn report(&self, ending: Ending) -> u64 {
150        let mut number = 0;
151        self.network.reported.send_modify(|count| {
152            *count += 1;
153            number = *count;
154            if ending == Ending::Every {
155                self.network.retried.store(*count, Ordering::SeqCst);
156            }
157        });
158        // The owners plan again on a changed schedule.
159        self.rearmed.send_modify(|count| *count = count.wrapping_add(1));
160        number
161    }
162
163    /// How many reports `owner` has acted on.
164    pub(crate) fn handled(&self, owner: NetworkOwner) -> u64 {
165        self.network.handled[owner.index()].load(Ordering::SeqCst)
166    }
167
168    /// Changes with every report, for an owner that waits on its own timers.
169    pub fn network_reports(&self) -> NetworkReportWatch {
170        NetworkReportWatch(self.network.reported.subscribe())
171    }
172
173    /// End the waits the reports since `owner` last acted end (§6.7): its
174    /// waits for the network, or every ordinary wait it retries after a
175    /// retry report. Each ended wait keeps its failure count, so the next
176    /// failure continues the backoff, and is saved ended, so no reopen arms
177    /// it again (IO `ended_reopens_due`). The provider cooldown holds, and a
178    /// network report leaves a wait after a provider's answer and one held
179    /// because saving it failed. Returns whether a report arrived, so the
180    /// owner also runs the work it asks for. If reading or saving fails,
181    /// every wait holds, the owner reports the error, and the reports stay
182    /// unacted on, so the owner's next plan acts on them again.
183    pub async fn end_reported_waits(&self, owner: NetworkOwner) -> Result<bool, SyncError> {
184        let handled = &self.network.handled[owner.index()];
185        let (reported, ending) = self.network.since(handled.load(Ordering::SeqCst));
186        let Some(ending) = ending else {
187            return Ok(false);
188        };
189        self.end_waits(owner, move |_| Some(ending)).await?;
190        // Reports since `reported` was read stay for the next plan.
191        handled.fetch_max(reported, Ordering::SeqCst);
192        Ok(true)
193    }
194
195    /// How many reports have arrived so far. An attempt notes it as it
196    /// starts, so it can tell whether one arrived while it ran.
197    pub(crate) fn network_reported(&self) -> u64 {
198        *self.network.reported.borrow()
199    }
200
201    /// Which waits the reports since `seen` end; `None` when none arrived.
202    pub(crate) fn reports_since(&self, seen: u64) -> Option<Ending> {
203        self.network.since(seen).1
204    }
205
206    /// End each subject's wait as the reports that arrived while its attempt
207    /// ran say, whatever reports their owner already acted on: the report
208    /// ends the wait the attempt failed into (IO
209    /// `signal_during_attempt_wakes`). Fails as `end_reported_waits` does.
210    pub(crate) async fn end_waits_after_report(
211        &self,
212        owner: NetworkOwner,
213        subjects: &BTreeMap<RetrySubject, Ending>,
214    ) -> Result<(), SyncError> {
215        let subjects = subjects.clone();
216        self.end_waits(owner, move |subject| subjects.get(subject).copied()).await
217    }
218
219    /// End `owner`'s waits as `ending` says for each subject.
220    async fn end_waits(
221        &self,
222        owner: NetworkOwner,
223        ending: impl Fn(&RetrySubject) -> Option<Ending> + Send + 'static + Clone,
224    ) -> Result<(), SyncError> {
225        // A wait this session holds for the app stays held unless the person
226        // retries; one already over needs no end unless a new failure
227        // replaced it.
228        let (held, over) = {
229            let elapsed = self.clock.elapsed();
230            let state = self.lock();
231            let mut held = BTreeSet::new();
232            let mut over = BTreeMap::new();
233            for (subject, armed) in &state.armed {
234                if !owner.retries(subject) {
235                    continue;
236                }
237                if armed.timer.is_none() {
238                    held.insert(*subject);
239                } else if armed.over(elapsed) {
240                    over.insert(*subject, armed.not_before);
241                }
242            }
243            (held, over)
244        };
245        // The person's retry releases a held wait: it was holding for them.
246        let released: Vec<RetrySubject> =
247            held.iter().filter(|subject| ending(subject) == Some(Ending::Every)).copied().collect();
248        let Some(database) = &self.database else {
249            // A handle that cannot persist keeps its waits in memory: the
250            // person's retry forgets them.
251            let every: Vec<RetrySubject> = self
252                .lock()
253                .armed
254                .keys()
255                .filter(|subject| owner.retries(subject) && ending(subject) == Some(Ending::Every))
256                .copied()
257                .collect();
258            for subject in every {
259                self.forget(subject);
260            }
261            return Ok(());
262        };
263        let chosen = ending.clone();
264        let select = move |waits: &[(RetrySubject, RetryWait)],
265                           pending: &[PendingRecord],
266                           uploads: &[FileUpload],
267                           pins: &[PinRequest]| {
268            waits
269                .iter()
270                .filter(|(subject, wait)| {
271                    let Some(ending) = chosen(subject) else {
272                        return false;
273                    };
274                    owner.retries(subject)
275                        && over.get(subject) != Some(&wait.not_before)
276                        && !wait.wait.is_zero()
277                        && match ending {
278                            Ending::Every => true,
279                            Ending::Network => {
280                                !held.contains(subject)
281                                    && record_for(*subject, pending, uploads, pins)
282                                        .is_some_and(|record| record.reason.offline())
283                            }
284                        }
285                })
286                .map(|(subject, _)| *subject)
287                .collect()
288        };
289        let now: SystemTime = self.clock.now();
290        let mut ended = BTreeSet::new();
291        for (subject, wait) in database.end_retry_waits::<PendingRecord>(now, select).await? {
292            self.arm(subject, &wait);
293            ended.insert(subject);
294        }
295        // A held wait with no saved row to end is simply forgotten.
296        for subject in released {
297            if !ended.contains(&subject) {
298                self.forget(subject);
299            }
300        }
301        Ok(())
302    }
303}