Skip to main content

coven_sync/
operation_evidence.rs

1//! What each waiting operation's wait names, and the evidence that it changed
2//! (§6.7, §21.2; IO `WaitingEvidence`). A wait is due by evidence only
3//! when what it names changes: its rows commit outside the attempt that ended
4//! in it, the key it lacks arrives, or the storage connection it lacks changes.
5//! Nothing else wakes it, so a wait whose prerequisite never changes keeps
6//! §21.2's unchanged-wait rate (`unchanged_rate`).
7
8use crate::{PendingReason, PendingRecord, Prerequisite, Retry, RetryWaits, SyncError};
9use coven_database::operation_kind;
10use coven_database::{
11    AttemptCommits, CommitScope, DatabaseChanges, DbError, OperationId, PrerequisiteRows,
12    RetrySubject,
13};
14use coven_foundation::id_source::KeyId;
15use coven_foundation::locks::lock_unpoisoned;
16use coven_merge::Audience;
17use coven_storage::ObjectPath;
18use std::collections::{BTreeMap, BTreeSet};
19
20pub(crate) struct OperationEvidence {
21    /// Counts storage connection changes.
22    connection: u64,
23    waits: BTreeMap<OperationId, Wait>,
24}
25
26impl OperationEvidence {
27    /// No wait yet, and no connection change.
28    pub(crate) fn new() -> Self {
29        Self { connection: 0, waits: BTreeMap::new() }
30    }
31}
32
33enum Wait {
34    /// An attempt runs in `scope`. Other work's commits since it began are
35    /// kept until it names what it waits on.
36    Attempt { scope: CommitScope, commits: AttemptCommits },
37    /// The wait an attempt ended in.
38    Waiting {
39        rows: DatabaseChanges<DbError>,
40        /// The key it lacks; whether this device held it when the wait
41        /// began, unknown for a wait armed on opening.
42        key: Option<(Audience, KeyId, Option<bool>)>,
43        /// The connection count when a wait for storage began.
44        connection: Option<u64>,
45        woken: bool,
46    },
47}
48
49/// What a wait for `reason` names (IO `Armed.prereq`).
50#[derive(Default)]
51struct Needs {
52    rows: Vec<PrerequisiteRows>,
53    key: Option<(Audience, KeyId)>,
54    connection: bool,
55}
56
57fn object_rows(path: &ObjectPath) -> Vec<PrerequisiteRows> {
58    if path.slot_number().is_some() {
59        vec![PrerequisiteRows::Slot]
60    } else {
61        Vec::new()
62    }
63}
64
65/// What a wait for `reason` watches; a reload wait also names the pending
66/// reload operation, which advances or finishes (IO `PrerequisiteRows.needs`).
67fn needs(reason: &PendingReason, reload: Option<OperationId>) -> Needs {
68    let rows = |rows| Needs { rows, ..Needs::default() };
69    let mut needs = match reason.retry_cause() {
70        PendingReason::Missing { path } => rows(object_rows(path)),
71        PendingReason::Waits(prerequisite) => match prerequisite {
72            Prerequisite::SequenceCatchUp | Prerequisite::Floor => {
73                rows(vec![PrerequisiteRows::Slot])
74            }
75            Prerequisite::Reload(audience) => {
76                rows(vec![PrerequisiteRows::Reload(audience.clone())])
77            }
78        },
79        PendingReason::KeyUnavailable { audience, key, .. } => {
80            Needs { key: Some((audience.clone(), *key)), ..Needs::default() }
81        }
82        PendingReason::ProviderPending { account, .. }
83        | PendingReason::AccountInUse { account } => {
84            rows(vec![PrerequisiteRows::ProviderAccess(account.clone())])
85        }
86        PendingReason::Storage { .. } | PendingReason::NoStorage => {
87            Needs { connection: true, ..Needs::default() }
88        }
89        // Every other condition waits for time, the app or an update.
90        _ => Needs::default(),
91    };
92    needs.rows.extend(reload.map(PrerequisiteRows::Operation));
93    needs
94}
95
96impl RetryWaits {
97    /// Opening watches every persisted operation wait from now: commits after
98    /// the store opened, by any work, are evidence for it. One satisfied
99    /// before is unseen, and the wait is due within its persisted full wait
100    /// (IO `reopened_due`, `reopen_rate`).
101    pub(crate) async fn watch_persisted(&self) -> Result<(), DbError> {
102        let Some(database) = &self.database else {
103            return Ok(());
104        };
105        for (subject, _) in database.retry_waits().await? {
106            let RetrySubject::Operation(id) = subject else {
107                continue;
108            };
109            let Some(pending) = database.operation_pending::<PendingRecord>(id).await? else {
110                continue;
111            };
112            if pending.retry() == Retry::Automatic {
113                let reload = match pending.reason.retry_cause() {
114                    PendingReason::Waits(Prerequisite::Reload(_)) => database
115                        .first_operation(operation_kind::SEQUENCE)
116                        .await?
117                        .map(|record| record.id),
118                    _ => None,
119                };
120                self.arm_evidence(id, &pending.reason, None, reload);
121            }
122        }
123        Ok(())
124    }
125
126    /// Begin an attempt at the operation: commits the attempt makes belong to
127    /// the returned scope and are no evidence against the wait it ends in.
128    pub(crate) fn begin_attempt(&self, id: OperationId) -> CommitScope {
129        let scope = CommitScope::new();
130        if let Some(database) = &self.database {
131            let commits = database.attempt_commits(scope);
132            lock_unpoisoned(&self.evidence).waits.insert(id, Wait::Attempt { scope, commits });
133        }
134        scope
135    }
136
137    /// The scope of the operation's running attempt, or a fresh one.
138    pub(crate) fn attempt_scope(&self, id: OperationId) -> CommitScope {
139        match lock_unpoisoned(&self.evidence).waits.get(&id) {
140            Some(Wait::Attempt { scope, .. }) => *scope,
141            _ => CommitScope::new(),
142        }
143    }
144
145    /// The attempt ended waiting on `reason`: watch what that names. Other
146    /// work's commits to it while the attempt ran wake the wait at once (IO
147    /// `racing_commit_wakes`). `held` says whether this device holds a key
148    /// the reason lacks; `reload` is the reload operation a reload wait names.
149    pub(crate) fn arm_evidence(
150        &self,
151        id: OperationId,
152        reason: &PendingReason,
153        held: Option<bool>,
154        reload: Option<OperationId>,
155    ) {
156        let Some(database) = &self.database else {
157            return;
158        };
159        let needs = needs(reason, reload);
160        // The wait is built with the lock let go, so a panic in building it
161        // leaves the operation with no evidence wait, a whole state: its
162        // retry waits for its timer.
163        let (attempt, connection) = {
164            let mut evidence = lock_unpoisoned(&self.evidence);
165            (evidence.waits.remove(&id), evidence.connection)
166        };
167        let commits = match attempt {
168            Some(Wait::Attempt { commits, .. }) => commits,
169            _ => database.attempt_commits(CommitScope::new()),
170        };
171        let wait = Wait::Waiting {
172            rows: database.watch_attempt(commits, &needs.rows),
173            key: needs.key.map(|(audience, key)| (audience, key, held)),
174            connection: needs.connection.then_some(connection),
175            woken: false,
176        };
177        lock_unpoisoned(&self.evidence).waits.insert(id, wait);
178    }
179
180    /// What a wait for `reason` watches: its prerequisites' rows, the key it
181    /// lacks, and whether it waits for the storage connection; `reload` is the
182    /// reload operation a reload wait names (IO `PrerequisiteRows.needs`).
183    pub(crate) fn watches(
184        reason: &PendingReason,
185        reload: Option<OperationId>,
186    ) -> (Vec<PrerequisiteRows>, Option<(Audience, KeyId)>, bool) {
187        let needs = needs(reason, reload);
188        (needs.rows, needs.key, needs.connection)
189    }
190
191    /// The key the operation's wait lacks, if it lacks one.
192    pub(crate) fn lacks_key(reason: &PendingReason) -> Option<(Audience, KeyId)> {
193        Self::watches(reason, None).1
194    }
195
196    pub(crate) fn clear_evidence(&self, id: OperationId) {
197        lock_unpoisoned(&self.evidence).waits.remove(&id);
198    }
199
200    pub(crate) fn retain_evidence(&self, live: &BTreeSet<OperationId>) {
201        lock_unpoisoned(&self.evidence).waits.retain(|id, _| live.contains(id));
202    }
203
204    /// Whether what the operation's wait names changed since it began: its
205    /// rows, the key it lacks (`holds` reads this device's keys), or the
206    /// connection. It stays woken until the next attempt begins.
207    pub(crate) fn evidence_woken(
208        &self,
209        id: OperationId,
210        holds: impl Fn(&Audience, KeyId) -> Result<bool, SyncError>,
211    ) -> Result<bool, SyncError> {
212        let mut evidence = lock_unpoisoned(&self.evidence);
213        let connection = evidence.connection;
214        let Some(Wait::Waiting { rows, key, connection: began, woken }) =
215            evidence.waits.get_mut(&id)
216        else {
217            return Ok(false);
218        };
219        *woken |= rows.take_changed()?;
220        if let Some((audience, key, held)) = key {
221            if *held != Some(true) {
222                *woken |= holds(audience, *key)?;
223            }
224        }
225        *woken |= began.is_some_and(|began| began != connection);
226        Ok(*woken)
227    }
228
229    /// The storage connection changed: waits for storage are woken.
230    pub fn connection_changed(&self) {
231        lock_unpoisoned(&self.evidence).connection += 1;
232        self.rearmed.send_modify(|count| *count = count.wrapping_add(1));
233    }
234
235    /// The credential/connection version a connection failure waits to change.
236    pub fn connection_version(&self) -> u64 {
237        lock_unpoisoned(&self.evidence).connection
238    }
239
240    /// Wakes for waits not yet woken, when their rows commit.
241    pub(crate) fn evidence_wakes(&self) -> Vec<crate::retry_waits::Wake> {
242        lock_unpoisoned(&self.evidence)
243            .waits
244            .values()
245            .filter_map(|wait| match wait {
246                Wait::Waiting { rows, woken: false, .. } => {
247                    Some(Box::pin(rows.changed()) as crate::retry_waits::Wake)
248                }
249                _ => None,
250            })
251            .collect()
252    }
253}