coven_sync/sequence_join.rs
1//! The joining device's wait for its approving device's admission (§13.2).
2
3use super::{
4 sequence_bootstrap::{Bootstrap, KeyScan},
5 SequenceSync,
6};
7use crate::{HeldIntervals, JoiningIdentity, SyncError};
8use coven_foundation::progress::ProgressReport;
9use coven_storage::{CloudProvider, StorageFailure, StorageInvitation};
10use std::time::Duration;
11use tokio::sync::watch;
12
13/// The joining member's outcome after checking storage (§13.2).
14#[derive(Clone, Copy, Debug, PartialEq, Eq)]
15pub enum JoinOutcome {
16 /// The request exists and no copy for it has opened.
17 Waiting,
18 /// This request's copies opened, in the admission that carries them:
19 /// the join publishes the store, whose first load applies that admission
20 /// (§13.2).
21 Admitted,
22 /// The request is gone and no copy for it has opened.
23 Declined,
24}
25
26impl SequenceSync {
27 /// Complete provider acceptance before submitting a join request.
28 pub async fn accept_join(&self, invitation: &StorageInvitation) -> Result<(), SyncError> {
29 Ok(self.storage.join(self.store(), invitation).await?)
30 }
31
32 /// Take back `accept_join`'s provider acceptance, for a join cancelled
33 /// before its request was sent; one already taken back is done (§13.2).
34 pub async fn leave_join(&self, invitation: &StorageInvitation) -> Result<(), SyncError> {
35 Ok(self.storage.leave(invitation).await?)
36 }
37
38 /// Whether the invite still lets the request be sent first now: no
39 /// first request goes out once the invite has ended by this device's
40 /// clock (§13.2).
41 pub fn join_open(&self, request: &JoiningIdentity) -> bool {
42 request.open_at(self.clock.now())
43 }
44
45 /// Publish the recorded immutable request after provider acceptance.
46 /// `false` when the invite ended before it was sent: its marker holds
47 /// the request's path, and the join ends as declined (§13.2).
48 /// `false` too when the join's access to the location is gone
49 /// (`access_lost`).
50 pub async fn submit_join(&self, request: &JoiningIdentity) -> Result<bool, SyncError> {
51 self.unless_access_lost(request.publish(self.storage.as_ref()).await)
52 }
53
54 /// Send the request of a join that recorded its send in an earlier call
55 /// and may have stopped before the create, while the invite is open.
56 /// `false` when the join's access to the location is gone
57 /// (`access_lost`).
58 pub async fn resend_join(&self, request: &JoiningIdentity) -> Result<bool, SyncError> {
59 if self.join_open(request) {
60 return self
61 .unless_access_lost(request.resend(self.storage.as_ref()).await.map(|()| true));
62 }
63 Ok(true)
64 }
65
66 /// Whether `error` says the access a join was shown to hold, to the
67 /// invite's own location, is gone: the provider refused the account or
68 /// found the location missing, or an S3 key the invite carries was
69 /// deleted or disabled. A sign-in the provider rejects is no such answer:
70 /// the person signs in again (§13.2).
71 pub fn access_lost(&self, error: &SyncError) -> bool {
72 let SyncError::Storage(error) = error else { return false };
73 let failure = error.failure();
74 failure.access_ended()
75 || (self.storage.config().provider() == CloudProvider::S3
76 && failure == StorageFailure::Authentication)
77 }
78
79 /// `result`, with a lost access as the join's end: `false`.
80 fn unless_access_lost(&self, result: Result<bool, SyncError>) -> Result<bool, SyncError> {
81 match result {
82 Err(error) if self.access_lost(&error) => Ok(false),
83 result => result,
84 }
85 }
86
87 /// Inspect the slots after the invite for copies routed to the request's
88 /// pair; once they open a store key, in the admission that carries them,
89 /// record that and keep the keys in custody: the join has found its
90 /// admission and publishes the store. Copies alone do not admit: the
91 /// published store's first load applies the admission, and stops the
92 /// store if its history refused it (§13.2). A check reports nothing to
93 /// `subscribe_progress` while it scans for the copies: the scan is part
94 /// of the wait. Once they open, it reports the scan as loading's first
95 /// step (E10).
96 ///
97 /// A check that could not reach the shared location before the keys open
98 /// has lost the join's access, and ends it as an absent request does
99 /// (`access_lost`, `JoinScan.outcome`); one of a join whose keys already
100 /// opened is the published store's lost access, which is no ending.
101 pub async fn join_outcome(
102 &self,
103 request: &mut JoiningIdentity,
104 ) -> Result<JoinOutcome, SyncError> {
105 let opened = request.opened();
106 match self.check_join(request).await {
107 Err(error) if !opened && self.access_lost(&error) => Ok(JoinOutcome::Declined),
108 result => result,
109 }
110 }
111
112 async fn check_join(&self, request: &mut JoiningIdentity) -> Result<JoinOutcome, SyncError> {
113 let mut session = self.session.lock().await;
114 let member = self.member.read()?.ok_or(StorageFailure::MemberKeysMissing)?;
115 if member.keys().public_keys() != request.identity().keys().public_keys() {
116 return Err(SyncError::InvitationChanged);
117 }
118 self.scan_after(&mut session, request.read_through())?;
119 let held = self.keys.read()?;
120 // The keys first open in a check whose custody held none: a new
121 // installation's, or a resumed join's, which may have applied slots
122 // before it stopped. Each call's custody starts empty (§13.2).
123 let opening = held.is_none();
124 // Such a check scans for this member's copies, and once they open
125 // the store, loads it, or the rest of it, reporting each step as
126 // `unlock` does (§13.4). The scan counts aside until then.
127 let mut progress = opening.then(|| ProgressReport::of(&self.progress, self.clock.clone()));
128 let (counted, scanned) = watch::channel(None);
129 let mut counting = ProgressReport::new(counted, self.clock.clone());
130 let mut scan = KeyScan::new(&mut counting);
131 let ring = match held {
132 Some(ring) => ring,
133 None => match self.bootstrap_keys(&mut session, &member, Some(&mut scan)).await? {
134 Bootstrap::Keys(ring) => ring,
135 // The joining pair was bound and then replaced (§12.3).
136 Bootstrap::Replaced => {
137 return Err(SyncError::MemberKeysReplaced { member: member.member() })
138 }
139 Bootstrap::Missing(_) | Bootstrap::Empty => {
140 if request.pending(self.storage.as_ref()).await? {
141 return Ok(JoinOutcome::Waiting);
142 }
143 // Approval may have raced the ending miss. Scan again
144 // after observing request deletion; bootstrap follows any
145 // floor that advances during this final scan as well.
146 match self.bootstrap_keys(&mut session, &member, Some(&mut scan)).await? {
147 Bootstrap::Keys(ring) => ring,
148 Bootstrap::Replaced => {
149 return Err(SyncError::MemberKeysReplaced { member: member.member() })
150 }
151 Bootstrap::Missing(_) | Bootstrap::Empty => {
152 return Ok(JoinOutcome::Declined)
153 }
154 }
155 }
156 },
157 };
158 // The admission carrying its copies is found. The store the join
159 // publishes keeps the keys, and loads with them; they are kept
160 // before the join records the admission found, so a join whose
161 // custody holds them has found it in this call (§13.2).
162 self.keys.persist(&ring)?;
163 // The join keeps that before it reports loading, so it never reports
164 // waiting for approval again, even resumed (E10).
165 if !request.opened() {
166 request.record_opened()?;
167 }
168 // The scan that found the copies is loading's first step, reported
169 // whole now that it has opened them.
170 if let (Some(progress), Some(scanned)) = (progress.as_mut(), scanned.borrow().clone()) {
171 progress.begin(scanned.step, scanned.total);
172 progress.done(scanned.done);
173 }
174 Ok(JoinOutcome::Admitted)
175 }
176}
177
178/// The wait between an approval's checks, which a join and a pairing keep:
179/// its persisted delay, armed again after reopening, and any provider
180/// cooldown (§6.7).
181#[derive(Clone)]
182pub struct ApprovalWaits {
183 database: coven_database::StoreDatabase,
184 waits: std::sync::Arc<crate::RetryWaits>,
185 clock: coven_foundation::clock::ClockRef,
186}
187
188impl ApprovalWaits {
189 /// The waits of an installation with this database, waits and clock.
190 pub fn new(
191 database: coven_database::StoreDatabase,
192 waits: std::sync::Arc<crate::RetryWaits>,
193 clock: coven_foundation::clock::ClockRef,
194 ) -> Self {
195 Self { database, waits, clock }
196 }
197
198 /// A poll that found the request still waiting persists its next delay
199 /// before waiting, continuing the backoff across reopening.
200 pub async fn waiting(&self) -> Result<(), SyncError> {
201 let subject = coven_database::RetrySubject::Join;
202 let previous = self.database.retry_wait(subject).await?;
203 self.waits.fail(subject, previous.as_ref()).await
204 }
205
206 /// The outcome is known: approval polling ends.
207 pub async fn settled(&self) -> Result<(), SyncError> {
208 let subject = coven_database::RetrySubject::Join;
209 if self.database.retry_wait(subject).await?.is_some() {
210 self.database.clear_retry(subject).await?;
211 }
212 self.waits.forget(subject);
213 Ok(())
214 }
215}
216
217/// An approval's wait for its next poll: its persisted delay, armed again
218/// after reopening, any provider cooldown (§6.7), and the intervals the app
219/// holds (§13.2).
220pub struct JoinPoll {
221 waits: ApprovalWaits,
222 held: HeldIntervals,
223 parks: crate::Parks,
224}
225
226impl JoinPoll {
227 /// The wait on `waits`, honoring the intervals `held` reads and telling
228 /// `parks` where it parks.
229 pub fn new(waits: ApprovalWaits, held: HeldIntervals, parks: crate::Parks) -> Self {
230 Self { waits, held, parks }
231 }
232
233 /// Wait until the next approval poll is due. While a guard is held, the
234 /// wait also ends once the shortest interval held has passed since the
235 /// last check ended, keeping its failure count; a provider cooldown still
236 /// holds (§13.2).
237 pub async fn due(&mut self) -> Result<(), SyncError> {
238 let sync = &self.waits;
239 let subject = coven_database::RetrySubject::Join;
240 loop {
241 let persisted = sync.database.retry_wait(subject).await?;
242 if sync.waits.due(subject, persisted.as_ref()) {
243 return Ok(());
244 }
245 let wake = sync
246 .waits
247 .next_wake(|candidate| *candidate == subject)
248 .or_else(|| sync.waits.cooldown_wake())
249 .ok_or_else(|| {
250 StorageFailure::File.with_source("the approval poll's delay was not persisted")
251 })?;
252 let asked = Self::held_check(sync, persisted.as_ref(), self.held.shortest());
253 tokio::select! {
254 biased;
255 never = self.parks.polling() => match never {},
256 () = wake => (),
257 asked = asked => {
258 asked?;
259 sync.waits.end_wait(subject).await?
260 }
261 () = self.held.changed() => (),
262 never = self.parks.parked() => match never {},
263 }
264 }
265 }
266
267 /// Resolves when a guard's interval has passed since the last check ended,
268 /// which the persisted wait's start, `T - W`, marks; never when no guard
269 /// is held or the wait has ended already.
270 async fn held_check(
271 sync: &ApprovalWaits,
272 persisted: Option<&coven_database::RetryWait>,
273 shortest: Option<Duration>,
274 ) -> Result<(), SyncError> {
275 let (Some(wait), Some(interval)) =
276 (persisted.filter(|wait| !wait.wait.is_zero()), shortest)
277 else {
278 return std::future::pending().await;
279 };
280 let checked = wait.not_before.checked_sub(wait.wait);
281 let at = checked.and_then(|checked| checked.checked_add(interval)).ok_or_else(|| {
282 StorageFailure::InvalidConfiguration
283 .with_source("the clock cannot represent the next approval check")
284 })?;
285 sync.clock
286 .sleep_until(at)
287 .await
288 .map_err(|error| StorageFailure::InvalidConfiguration.with_source(error).into())
289 }
290}
291
292#[cfg(test)]
293#[path = "sequence_join_tests.rs"]
294mod tests;