Skip to main content

coven_sync/
sequence_join.rs

1//! The joining device's wait for its approving device's admission (§13.2).
2
3use super::{
4    sequence_bootstrap::{Bootstrap, KeyScan},
5    SequenceSync,
6};
7use crate::{HeldIntervals, JoiningIdentity, SyncError};
8use coven_foundation::progress::ProgressReport;
9use coven_storage::{CloudProvider, StorageFailure, StorageInvitation};
10use std::time::Duration;
11use tokio::sync::watch;
12
13/// The joining member's outcome after checking storage (§13.2).
14#[derive(Clone, Copy, Debug, PartialEq, Eq)]
15pub enum JoinOutcome {
16    /// The request exists and no copy for it has opened.
17    Waiting,
18    /// This request's copies opened, in the admission that carries them:
19    /// the join publishes the store, whose first load applies that admission
20    /// (§13.2).
21    Admitted,
22    /// The request is gone and no copy for it has opened.
23    Declined,
24}
25
26impl SequenceSync {
27    /// Complete provider acceptance before submitting a join request.
28    pub async fn accept_join(&self, invitation: &StorageInvitation) -> Result<(), SyncError> {
29        Ok(self.storage.join(self.store(), invitation).await?)
30    }
31
32    /// Take back `accept_join`'s provider acceptance, for a join cancelled
33    /// before its request was sent; one already taken back is done (§13.2).
34    pub async fn leave_join(&self, invitation: &StorageInvitation) -> Result<(), SyncError> {
35        Ok(self.storage.leave(invitation).await?)
36    }
37
38    /// Whether the invite still lets the request be sent first now: no
39    /// first request goes out once the invite has ended by this device's
40    /// clock (§13.2).
41    pub fn join_open(&self, request: &JoiningIdentity) -> bool {
42        request.open_at(self.clock.now())
43    }
44
45    /// Publish the recorded immutable request after provider acceptance.
46    /// `false` when the invite ended before it was sent: its marker holds
47    /// the request's path, and the join ends as declined (§13.2).
48    /// `false` too when the join's access to the location is gone
49    /// (`access_lost`).
50    pub async fn submit_join(&self, request: &JoiningIdentity) -> Result<bool, SyncError> {
51        self.unless_access_lost(request.publish(self.storage.as_ref()).await)
52    }
53
54    /// Send the request of a join that recorded its send in an earlier call
55    /// and may have stopped before the create, while the invite is open.
56    /// `false` when the join's access to the location is gone
57    /// (`access_lost`).
58    pub async fn resend_join(&self, request: &JoiningIdentity) -> Result<bool, SyncError> {
59        if self.join_open(request) {
60            return self
61                .unless_access_lost(request.resend(self.storage.as_ref()).await.map(|()| true));
62        }
63        Ok(true)
64    }
65
66    /// Whether `error` says the access a join was shown to hold, to the
67    /// invite's own location, is gone: the provider refused the account or
68    /// found the location missing, or an S3 key the invite carries was
69    /// deleted or disabled. A sign-in the provider rejects is no such answer:
70    /// the person signs in again (§13.2).
71    pub fn access_lost(&self, error: &SyncError) -> bool {
72        let SyncError::Storage(error) = error else { return false };
73        let failure = error.failure();
74        failure.access_ended()
75            || (self.storage.config().provider() == CloudProvider::S3
76                && failure == StorageFailure::Authentication)
77    }
78
79    /// `result`, with a lost access as the join's end: `false`.
80    fn unless_access_lost(&self, result: Result<bool, SyncError>) -> Result<bool, SyncError> {
81        match result {
82            Err(error) if self.access_lost(&error) => Ok(false),
83            result => result,
84        }
85    }
86
87    /// Inspect the slots after the invite for copies routed to the request's
88    /// pair; once they open a store key, in the admission that carries them,
89    /// record that and keep the keys in custody: the join has found its
90    /// admission and publishes the store. Copies alone do not admit: the
91    /// published store's first load applies the admission, and stops the
92    /// store if its history refused it (§13.2). A check reports nothing to
93    /// `subscribe_progress` while it scans for the copies: the scan is part
94    /// of the wait. Once they open, it reports the scan as loading's first
95    /// step (E10).
96    ///
97    /// A check that could not reach the shared location before the keys open
98    /// has lost the join's access, and ends it as an absent request does
99    /// (`access_lost`, `JoinScan.outcome`); one of a join whose keys already
100    /// opened is the published store's lost access, which is no ending.
101    pub async fn join_outcome(
102        &self,
103        request: &mut JoiningIdentity,
104    ) -> Result<JoinOutcome, SyncError> {
105        let opened = request.opened();
106        match self.check_join(request).await {
107            Err(error) if !opened && self.access_lost(&error) => Ok(JoinOutcome::Declined),
108            result => result,
109        }
110    }
111
112    async fn check_join(&self, request: &mut JoiningIdentity) -> Result<JoinOutcome, SyncError> {
113        let mut session = self.session.lock().await;
114        let member = self.member.read()?.ok_or(StorageFailure::MemberKeysMissing)?;
115        if member.keys().public_keys() != request.identity().keys().public_keys() {
116            return Err(SyncError::InvitationChanged);
117        }
118        self.scan_after(&mut session, request.read_through())?;
119        let held = self.keys.read()?;
120        // The keys first open in a check whose custody held none: a new
121        // installation's, or a resumed join's, which may have applied slots
122        // before it stopped. Each call's custody starts empty (§13.2).
123        let opening = held.is_none();
124        // Such a check scans for this member's copies, and once they open
125        // the store, loads it, or the rest of it, reporting each step as
126        // `unlock` does (§13.4). The scan counts aside until then.
127        let mut progress = opening.then(|| ProgressReport::of(&self.progress, self.clock.clone()));
128        let (counted, scanned) = watch::channel(None);
129        let mut counting = ProgressReport::new(counted, self.clock.clone());
130        let mut scan = KeyScan::new(&mut counting);
131        let ring = match held {
132            Some(ring) => ring,
133            None => match self.bootstrap_keys(&mut session, &member, Some(&mut scan)).await? {
134                Bootstrap::Keys(ring) => ring,
135                // The joining pair was bound and then replaced (§12.3).
136                Bootstrap::Replaced => {
137                    return Err(SyncError::MemberKeysReplaced { member: member.member() })
138                }
139                Bootstrap::Missing(_) | Bootstrap::Empty => {
140                    if request.pending(self.storage.as_ref()).await? {
141                        return Ok(JoinOutcome::Waiting);
142                    }
143                    // Approval may have raced the ending miss. Scan again
144                    // after observing request deletion; bootstrap follows any
145                    // floor that advances during this final scan as well.
146                    match self.bootstrap_keys(&mut session, &member, Some(&mut scan)).await? {
147                        Bootstrap::Keys(ring) => ring,
148                        Bootstrap::Replaced => {
149                            return Err(SyncError::MemberKeysReplaced { member: member.member() })
150                        }
151                        Bootstrap::Missing(_) | Bootstrap::Empty => {
152                            return Ok(JoinOutcome::Declined)
153                        }
154                    }
155                }
156            },
157        };
158        // The admission carrying its copies is found. The store the join
159        // publishes keeps the keys, and loads with them; they are kept
160        // before the join records the admission found, so a join whose
161        // custody holds them has found it in this call (§13.2).
162        self.keys.persist(&ring)?;
163        // The join keeps that before it reports loading, so it never reports
164        // waiting for approval again, even resumed (E10).
165        if !request.opened() {
166            request.record_opened()?;
167        }
168        // The scan that found the copies is loading's first step, reported
169        // whole now that it has opened them.
170        if let (Some(progress), Some(scanned)) = (progress.as_mut(), scanned.borrow().clone()) {
171            progress.begin(scanned.step, scanned.total);
172            progress.done(scanned.done);
173        }
174        Ok(JoinOutcome::Admitted)
175    }
176}
177
178/// The wait between an approval's checks, which a join and a pairing keep:
179/// its persisted delay, armed again after reopening, and any provider
180/// cooldown (§6.7).
181#[derive(Clone)]
182pub struct ApprovalWaits {
183    database: coven_database::StoreDatabase,
184    waits: std::sync::Arc<crate::RetryWaits>,
185    clock: coven_foundation::clock::ClockRef,
186}
187
188impl ApprovalWaits {
189    /// The waits of an installation with this database, waits and clock.
190    pub fn new(
191        database: coven_database::StoreDatabase,
192        waits: std::sync::Arc<crate::RetryWaits>,
193        clock: coven_foundation::clock::ClockRef,
194    ) -> Self {
195        Self { database, waits, clock }
196    }
197
198    /// A poll that found the request still waiting persists its next delay
199    /// before waiting, continuing the backoff across reopening.
200    pub async fn waiting(&self) -> Result<(), SyncError> {
201        let subject = coven_database::RetrySubject::Join;
202        let previous = self.database.retry_wait(subject).await?;
203        self.waits.fail(subject, previous.as_ref()).await
204    }
205
206    /// The outcome is known: approval polling ends.
207    pub async fn settled(&self) -> Result<(), SyncError> {
208        let subject = coven_database::RetrySubject::Join;
209        if self.database.retry_wait(subject).await?.is_some() {
210            self.database.clear_retry(subject).await?;
211        }
212        self.waits.forget(subject);
213        Ok(())
214    }
215}
216
217/// An approval's wait for its next poll: its persisted delay, armed again
218/// after reopening, any provider cooldown (§6.7), and the intervals the app
219/// holds (§13.2).
220pub struct JoinPoll {
221    waits: ApprovalWaits,
222    held: HeldIntervals,
223    parks: crate::Parks,
224}
225
226impl JoinPoll {
227    /// The wait on `waits`, honoring the intervals `held` reads and telling
228    /// `parks` where it parks.
229    pub fn new(waits: ApprovalWaits, held: HeldIntervals, parks: crate::Parks) -> Self {
230        Self { waits, held, parks }
231    }
232
233    /// Wait until the next approval poll is due. While a guard is held, the
234    /// wait also ends once the shortest interval held has passed since the
235    /// last check ended, keeping its failure count; a provider cooldown still
236    /// holds (§13.2).
237    pub async fn due(&mut self) -> Result<(), SyncError> {
238        let sync = &self.waits;
239        let subject = coven_database::RetrySubject::Join;
240        loop {
241            let persisted = sync.database.retry_wait(subject).await?;
242            if sync.waits.due(subject, persisted.as_ref()) {
243                return Ok(());
244            }
245            let wake = sync
246                .waits
247                .next_wake(|candidate| *candidate == subject)
248                .or_else(|| sync.waits.cooldown_wake())
249                .ok_or_else(|| {
250                    StorageFailure::File.with_source("the approval poll's delay was not persisted")
251                })?;
252            let asked = Self::held_check(sync, persisted.as_ref(), self.held.shortest());
253            tokio::select! {
254                biased;
255                never = self.parks.polling() => match never {},
256                () = wake => (),
257                asked = asked => {
258                    asked?;
259                    sync.waits.end_wait(subject).await?
260                }
261                () = self.held.changed() => (),
262                never = self.parks.parked() => match never {},
263            }
264        }
265    }
266
267    /// Resolves when a guard's interval has passed since the last check ended,
268    /// which the persisted wait's start, `T - W`, marks; never when no guard
269    /// is held or the wait has ended already.
270    async fn held_check(
271        sync: &ApprovalWaits,
272        persisted: Option<&coven_database::RetryWait>,
273        shortest: Option<Duration>,
274    ) -> Result<(), SyncError> {
275        let (Some(wait), Some(interval)) =
276            (persisted.filter(|wait| !wait.wait.is_zero()), shortest)
277        else {
278            return std::future::pending().await;
279        };
280        let checked = wait.not_before.checked_sub(wait.wait);
281        let at = checked.and_then(|checked| checked.checked_add(interval)).ok_or_else(|| {
282            StorageFailure::InvalidConfiguration
283                .with_source("the clock cannot represent the next approval check")
284        })?;
285        sync.clock
286            .sleep_until(at)
287            .await
288            .map_err(|error| StorageFailure::InvalidConfiguration.with_source(error).into())
289    }
290}
291
292#[cfg(test)]
293#[path = "sequence_join_tests.rs"]
294mod tests;