Skip to main content

coven_sync/
connection_retry.rs

1//! Connection attempts and passes obey the same failure classes and durable wait.
2
3use crate::{
4    PassFailure, PendingRecord, PendingSubject, Retry, RetryWaits, SyncError, SyncFailure,
5};
6use coven_database::{RetrySubject, StoreDatabase};
7
8/// What permits this connection to attempt another pass. Time is held by
9/// `RetryWaits`. A failure that waits for something other than time holds
10/// every pass, and keeps what the status showed for it.
11pub enum ConnectionRetry {
12    /// The persisted backoff and provider cooldown decide when to try.
13    Automatic,
14    /// Only new credentials or a new connection can clear this failure.
15    AppAction {
16        /// The connection the failure happened on.
17        version: u64,
18        /// What the status showed for it.
19        failure: PassFailure,
20    },
21    /// Only reopening with an applicable update can retry the input.
22    AfterUpdate(PassFailure),
23    /// This installation must be reset before it can retry.
24    Never(PassFailure),
25}
26
27impl ConnectionRetry {
28    /// Reopening is a new connection, but cannot clear a permanent failure or
29    /// recheck immutable input with the same software.
30    pub async fn open(database: &StoreDatabase) -> Result<Self, SyncError> {
31        let record = database
32            .pending::<PendingRecord>()
33            .await?
34            .into_iter()
35            .find(|record| record.subject == PendingSubject::Connection);
36        let Some(record) = record else {
37            return Ok(Self::Automatic);
38        };
39        let never = record.retry() == Retry::Never;
40        let after_update = !never
41            && record.retry() == Retry::AfterUpdate
42            && !database.pending_can_retry::<PendingRecord>(PendingSubject::Connection).await?;
43        if !never && !after_update {
44            return Ok(Self::Automatic);
45        }
46        let failure = PassFailure::Failed {
47            error: match record.reason {
48                crate::PendingReason::Storage { failure, .. } => {
49                    SyncFailure::Storage(std::sync::Arc::new(failure.into()))
50                }
51                crate::PendingReason::Removed(removal) => SyncFailure::Removed(removal),
52                crate::PendingReason::UpdateRequired(update) => SyncFailure::UpdateRequired(update),
53                _ => SyncError::Pending(Box::new(record)).into(),
54            },
55        };
56        Ok(if never { Self::Never(failure) } else { Self::AfterUpdate(failure) })
57    }
58
59    /// The failure holding every pass, if evidence does not permit an
60    /// attempt; the caller also enforces its time wait.
61    pub fn held(&self, waits: &RetryWaits) -> Option<&PassFailure> {
62        match self {
63            Self::Automatic => None,
64            Self::AppAction { version, failure } => {
65                (*version == waits.connection_version()).then_some(failure)
66            }
67            Self::AfterUpdate(failure) | Self::Never(failure) => Some(failure),
68        }
69    }
70
71    /// Save the failure and next deadline in one transaction before waiting,
72    /// and return what the status shows for it: `shown` turns the error into
73    /// that failure. If saving fails, the returned error is that failure
74    /// together with the save's, and only new credentials or a new connection
75    /// retry it. The connection's owner saves on its own, so a bug the save
76    /// fails on goes to the store's bug sink (E15).
77    pub async fn failed(
78        &mut self,
79        database: &StoreDatabase,
80        waits: &RetryWaits,
81        reason: crate::PendingReason,
82        connection_version: u64,
83        error: SyncError,
84        shown: impl FnOnce(SyncError) -> PassFailure,
85    ) -> Result<PassFailure, PassFailure> {
86        if matches!(error, SyncError::DeviceIdInUse(_)) {
87            let failure = shown(error);
88            *self = Self::Never(failure.clone());
89            return Ok(failure);
90        }
91        // The loop runs the pass on its own, so the record keeps the
92        // error's text as its cause, as an operation's attempt does (E5).
93        let record = PendingRecord {
94            subject: PendingSubject::Connection,
95            reason,
96            cause: Some(error.cause().to_string()),
97        };
98        let retry = match &error {
99            SyncError::Pending(record) => record.retry(),
100            _ => record.retry(),
101        };
102        // These failures already belong to a required input; a second
103        // connection record would duplicate that input's pending condition.
104        let connection = !matches!(
105            error,
106            SyncError::Pending(_)
107                | SyncError::Damaged(_)
108                | SyncError::KeyUnavailable { .. }
109                | SyncError::UpdateRequired(_)
110        );
111        // Whether the save itself failed: its wait is then held for the app.
112        let saved = async {
113            let previous = database
114                .retry_wait(RetrySubject::Sync)
115                .await
116                .map_err(|error| (error.into(), false))?;
117            let wait = match retry {
118                Retry::Automatic => {
119                    Some(waits.next(previous.as_ref()).map_err(|error| (error, false))?)
120                }
121                _ => None,
122            };
123            database
124                .record_sync_retry(PendingSubject::Connection, connection.then_some(record), wait)
125                .await
126                .map_err(|error| (error.into(), true))?;
127            Ok::<_, (SyncError, bool)>(wait)
128        }
129        .await;
130        let wait = match saved {
131            Ok(wait) => wait,
132            Err((cleanup, held)) => {
133                let cleanup = coven_foundation::task::reported(database.bugs().as_ref(), cleanup);
134                let failure =
135                    PassFailure::Failed { error: SyncError::cleanup(error, cleanup).into() };
136                if held {
137                    waits.hold(RetrySubject::Sync);
138                    *self =
139                        Self::AppAction { version: connection_version, failure: failure.clone() };
140                }
141                return Err(failure);
142            }
143        };
144        match wait {
145            Some(wait) => waits.arm(RetrySubject::Sync, &wait),
146            None => waits.forget(RetrySubject::Sync),
147        }
148        let failure = shown(error);
149        *self = match retry {
150            Retry::Automatic => Self::Automatic,
151            Retry::AppAction => {
152                Self::AppAction { version: connection_version, failure: failure.clone() }
153            }
154            Retry::AfterUpdate => Self::AfterUpdate(failure.clone()),
155            Retry::Never => Self::Never(failure.clone()),
156        };
157        Ok(failure)
158    }
159
160    /// A completed pass clears its condition and exponent together.
161    pub async fn succeeded(
162        &mut self,
163        database: &StoreDatabase,
164        waits: &RetryWaits,
165    ) -> Result<(), SyncError> {
166        database.clear_sync_retry(PendingSubject::Connection).await?;
167        waits.forget(RetrySubject::Sync);
168        *self = Self::Automatic;
169        Ok(())
170    }
171}
172
173/// The last attempt is offline only when its network failure received no reply.
174pub fn connection_failure(error: SyncError) -> PassFailure {
175    match SyncFailure::from(error) {
176        SyncFailure::Storage(error)
177            if crate::pending::offline(error.failure(), error.answered()) =>
178        {
179            PassFailure::Offline { error }
180        }
181        error => PassFailure::Failed { error },
182    }
183}
184
185#[cfg(test)]
186#[path = "connection_retry_tests.rs"]
187mod tests;