Skip to main content

coven_database/
sync_facts.rs

1//! What the sync state is computed from (§19.1, api.md E5), read in one
2//! committed snapshot by one list of SQL expressions (`facts!`):
3//! `coven_sync_status`'s row, whether sync runs and the cooldown
4//! `coven_retry_effect` holds, the send queue, the bound uploads (the uploads
5//! a write waits for, each with its retry wait and record), and the pending
6//! records the state reads. The `coven_sync_state` view passes them to
7//! `_coven_sync_state` as its arguments (`sync_functions.rs`), and coven
8//! reads them as a row where sync computes the state in Rust: a closed
9//! store's state, and each upload's phase (`QueueFacts`). Both decode them
10//! with `SyncFacts::decode`. Sync computes the state; this module reads
11//! what sync needs. The `coven_upload_phases` view passes each upload's own
12//! facts to `_coven_upload_phase` in the same way (`UploadPhaseFacts`).
13//!
14//! A trailing upload, a download and a pin coven keeps are not sync's: this
15//! module reads no upload of theirs and no record about them, and the state
16//! never sees them (§17.4, §19.1). Which upload is bound comes from the
17//! queue's own column (`holds`), and the read leaves a record out when its
18//! row names such an upload, a download or a pin, or a move none of whose
19//! uploads is bound. Nor does the state read the pause, the network or a
20//! self-replacement's write pause, which hold trailing uploads only: an
21//! upload's phase reads them beside the facts (`TransferHolds`).
22//!
23//! A live query over these facts runs again whenever one of them changes:
24//! `facts!` reads the files in flight through `coven_sending`, the view
25//! that a transfer's start and end change (`transfers_in_flight.rs`). `facts!` reads each
26//! table through a column and never only counts its rows, so that SQLite
27//! names the view to the authorizer for each read the view makes
28//! (`internal_schema_views.rs`).
29
30use crate::sqlite::StoreSql;
31use crate::DbError;
32use sqlive::rusqlite::types::{FromSql, ValueRef};
33use std::collections::BTreeMap;
34use std::time::{Duration, SystemTime, UNIX_EPOCH};
35
36/// The facts, as SQL expressions over `coven_sync_status AS s` and
37/// `coven_retry_effect AS r`, in the order `SyncFacts::decode` reads them:
38/// the status row's columns; whether sync runs and its cooldown; the send
39/// queue's writes and the upload queue's uploads, counted; whether a bound
40/// upload has bytes to send; the bound uploads the state can show, as a JSON
41/// array; the pending records the state reads, as a JSON array; the sync
42/// scope's backoff; and each operation's backoff, as a JSON array.
43///
44/// `facts!` reads each kind of bound upload the state can show from the
45/// table where that kind has the fewest rows: the uploads with a record from
46/// the pending records, the first one sending from the files in flight, the
47/// first queued one with no record and no retry by walking the bound queue
48/// in id order to the first match, and the first with no record that waits
49/// out a retry from the retry rows.
50/// The outer read then reads only those rows, and only bound ones. Each is
51/// `[id, stored, sending, retry seconds, retry nanos, record position]`, the
52/// retry time as `_coven_unix_seconds` and `_coven_unix_nanos` give it, since
53/// JSON holds no bytes. A record is `[position, record, cause]`, in the order
54/// the records arose, and an operation's backoff `[operation, seconds,
55/// nanos]`.
56macro_rules! facts {
57    () => {
58        concat!(
59            "s.storage,s.provider,s.last_pass,s.moved,s.last_synced,r.syncing,r.cooling_until,",
60            crate::file_queue::unsent_counts!(),
61            ",EXISTS(SELECT 1 FROM _coven_file_uploads AS b
62                 WHERE b.holds IS NOT NULL AND NOT b.stored
63                   AND EXISTS(SELECT 1 FROM _coven_upload_files AS l WHERE l.upload=b.id)),
64             (SELECT json_group_array(json_array(u.id,u.stored,
65                     u.id IN (SELECT c.position FROM coven_sending AS c),
66                     iif(t.upload IS NULL,NULL,_coven_unix_seconds(t.not_before)),
67                     iif(t.upload IS NULL,NULL,_coven_unix_nanos(t.not_before)),e.id)
68                     ORDER BY u.id)
69                 FROM _coven_file_uploads AS u
70                 LEFT JOIN _coven_retries AS t ON t.upload=u.id
71                 LEFT JOIN _coven_pending AS e
72                     ON e.work='upload' AND e.device=u.device AND e.object=u.object
73                 WHERE u.id IN (
74                     SELECT f.id FROM _coven_pending AS p
75                     JOIN _coven_file_uploads AS f ON f.device=p.device AND f.object=p.object
76                     WHERE p.work='upload'
77                     UNION
78                     SELECT MIN(g.position) FROM coven_sending AS g
79                     JOIN _coven_file_uploads AS b ON b.id=g.position
80                     WHERE b.holds IS NOT NULL
81                     UNION
82                     SELECT * FROM (
83                         SELECT f.id FROM _coven_file_uploads AS f
84                         WHERE NOT f.stored AND f.holds IS NOT NULL
85                           AND f.id NOT IN (SELECT position FROM coven_sending)
86                           AND EXISTS(SELECT 1 FROM _coven_upload_files AS l WHERE l.upload=f.id)
87                           AND NOT EXISTS(SELECT 1 FROM _coven_pending AS p
88                                          WHERE p.work='upload' AND p.device=f.device
89                                            AND p.object=f.object)
90                           AND NOT EXISTS(SELECT 1 FROM _coven_retries AS t WHERE t.upload=f.id)
91                         ORDER BY f.id LIMIT 1)
92                     UNION
93                     SELECT MIN(f.id) FROM _coven_retries AS t
94                     CROSS JOIN _coven_file_uploads AS f ON f.id=t.upload
95                     WHERE t.upload IS NOT NULL
96                       AND NOT f.stored AND f.holds IS NOT NULL
97                       AND f.id NOT IN (SELECT position FROM coven_sending)
98                       AND EXISTS(SELECT 1 FROM _coven_upload_files AS l WHERE l.upload=f.id)
99                       AND NOT EXISTS(SELECT 1 FROM _coven_pending AS p
100                                      WHERE p.work='upload' AND p.device=f.device
101                                        AND p.object=f.object))
102                   AND u.holds IS NOT NULL
103                   AND EXISTS(SELECT 1 FROM _coven_upload_files AS l WHERE l.upload=u.id)),
104             (SELECT json_group_array(json_array(p.id,json(CAST(p.value AS TEXT)),p.cause)
105                     ORDER BY p.id)
106                 FROM _coven_pending AS p
107                 WHERE (p.pin IS NOT NULL
108                     OR p.work='download'
109                     OR (p.work='upload' AND NOT EXISTS(
110                         SELECT 1 FROM _coven_file_uploads AS f
111                         WHERE f.device=p.device AND f.object=p.object AND f.holds IS NOT NULL))
112                     OR (p.move IS NOT NULL AND NOT EXISTS(
113                         SELECT 1 FROM _coven_file_uploads AS f
114                         WHERE f.move=p.move AND f.holds IS NOT NULL))) IS NOT TRUE),
115             (SELECT t.not_before FROM _coven_retries AS t WHERE t.scope='sync'),
116             (SELECT json_group_array(json_array(t.operation,_coven_unix_seconds(t.not_before),
117                     _coven_unix_nanos(t.not_before)))
118                 FROM _coven_retries AS t WHERE t.operation>0)"
119        )
120    };
121}
122pub(crate) use facts;
123
124/// How many expressions `facts!` lists: the arguments `_coven_sync_state`
125/// takes.
126pub(crate) const FACTS: i32 = 14;
127
128/// The facts in one row.
129const READ: &str =
130    concat!("SELECT ", facts!(), " FROM coven_sync_status AS s JOIN coven_retry_effect AS r");
131
132/// What the sync state is computed from, in one committed snapshot.
133#[derive(Clone, Debug, PartialEq)]
134pub struct SyncFacts<P> {
135    /// What the sync loop knows: `coven_sync_status`'s row.
136    pub row: crate::SyncStatusRow,
137    /// Whether a started connection will plan, and the cooldown holding
138    /// every request, as `coven_retry_effect` holds them.
139    pub sync: crate::RetrySync,
140    /// What the send queue and the upload queue hold (`UnsentWork`): a
141    /// write waits when it counts one.
142    pub unsent: crate::UnsentWork,
143    /// The bound uploads the state can show, in queue order: the first one
144    /// sending, the first queued one (neither sending nor stored) with no
145    /// record and none waiting out a retry, the first with no record that
146    /// does, and every one with a record. The state shows the first
147    /// candidate of the highest rank, so no other upload changes it: later
148    /// sending ones tie with the first, a later queued one ties with the
149    /// first of its kind (syncing for its turn, waiting for its retry), and a
150    /// stored one names none.
151    pub uploads: Vec<UploadFacts<P>>,
152    /// Whether the upload queue holds a bound upload with bytes to send,
153    /// whether `uploads` lists it or not.
154    pub queued: bool,
155    /// The pending records the state reads, each with its position in
156    /// `coven_pending`, in the order they arose: every one but those about a
157    /// trailing upload, a download, a pin coven keeps or a move of trailing
158    /// uploads alone.
159    pub records: Vec<(i64, P)>,
160    /// When the sync scope's backoff ends, if a failed pass armed one
161    /// (§6.7): the next try of a pass or of a slot's application.
162    pub pass_retry: Option<SystemTime>,
163    /// When each journal operation's own backoff ends: the next try of
164    /// coven's own operation a failed pass names.
165    pub operation_retries: Vec<(crate::OperationId, SystemTime)>,
166}
167
168/// One bound upload in the queue, as the sync state reads it.
169#[derive(Clone, Debug, PartialEq)]
170pub struct UploadFacts<P> {
171    /// This process is sending it now.
172    pub sending: bool,
173    /// Storage holds its bytes; it waits for the rest of its move.
174    pub stored: bool,
175    /// Its last attempt failed, and it retries then (§6.7).
176    pub retry_at: Option<SystemTime>,
177    /// Its own pending record, if it has one.
178    pub record: Option<P>,
179}
180
181/// What holds trailing uploads, and no bound upload or the sync state: an
182/// upload's phase reads them beside the facts (§17.4, §17.9, §12.3).
183#[derive(Clone, Copy, Debug, PartialEq, Eq)]
184pub struct TransferHolds {
185    /// Whether the app paused trailing uploads.
186    pub paused: bool,
187    /// Whether the app moves files only on unmetered networks.
188    pub unmetered_only: bool,
189    /// Whether the device's network is metered, as the store's watch last
190    /// said.
191    pub metered: bool,
192    /// Whether a self-replacement of this member's keys pauses writes
193    /// (§12.3).
194    pub keys_changing: bool,
195}
196
197/// Where the facts are read from: a row of the facts' SELECT, or the
198/// arguments of one call of `_coven_sync_state`.
199pub(crate) trait FactValues {
200    /// Value `index`, counted from zero, as SQLite holds it.
201    fn value(&self, index: usize) -> sqlive::rusqlite::Result<ValueRef<'_>>;
202
203    /// Value `index`, converted as a row's column is.
204    fn get<T: FromSql>(&self, index: usize) -> Result<T, DbError> {
205        let value = self.value(index)?;
206        T::column_result(value).map_err(|error| {
207            DbError::from(sqlive::rusqlite::Error::FromSqlConversionFailure(
208                index,
209                value.data_type(),
210                Box::new(error),
211            ))
212        })
213    }
214}
215
216impl FactValues for sqlive::rusqlite::Row<'_> {
217    fn value(&self, index: usize) -> sqlive::rusqlite::Result<ValueRef<'_>> {
218        self.get_ref(index)
219    }
220}
221
222impl FactValues for sqlive::rusqlite::functions::Context<'_> {
223    fn value(&self, index: usize) -> sqlive::rusqlite::Result<ValueRef<'_>> {
224        Ok(self.get_raw(index))
225    }
226}
227
228/// One listed upload as `facts!` gives it.
229type ListedUpload = (i64, i64, i64, Option<i64>, Option<u32>, Option<i64>);
230
231impl<P: crate::PendingValue + Clone> SyncFacts<P> {
232    /// The facts from `facts!`'s values, as a row or a call's arguments hold
233    /// them. A value that doesn't decode is `DamagedDatabase`, and an upload's
234    /// record that isn't among the records an invariant failure.
235    pub(crate) fn decode(values: &impl FactValues) -> Result<Self, DbError> {
236        let row = crate::internal_schema::sync_status::decode(values, 0)?;
237        let sync = crate::internal_schema::retry_effect::sync(values.get(5)?, values.get(6)?)?;
238        let count = |index| {
239            u64::try_from(values.get::<i64>(index)?).map_err(|_negative| DbError::DamagedDatabase)
240        };
241        let unsent = crate::UnsentWork { writes: count(7)?, files: count(8)? };
242        let records: Vec<(i64, P, Option<String>)> = json(&values.get::<String>(11)?)?;
243        let records: Vec<(i64, P)> = records
244            .into_iter()
245            .map(|(position, record, cause)| (position, record.with_cause(cause)))
246            .collect();
247        let by_position: BTreeMap<i64, &P> =
248            records.iter().map(|(position, record)| (*position, record)).collect();
249        let uploads = json::<Vec<ListedUpload>>(&values.get::<String>(10)?)?
250            .into_iter()
251            .map(|(_id, stored, sending, seconds, nanos, position)| {
252                let record = position
253                    .map(|position| {
254                        by_position.get(&position).map(|record| (*record).clone()).ok_or(
255                            DbError::Invariant {
256                                what: "an upload's record is among the pending records",
257                            },
258                        )
259                    })
260                    .transpose()?;
261                let retry_at = match (seconds, nanos) {
262                    (Some(seconds), Some(nanos)) => Some(unix_time(seconds, nanos)?),
263                    (None, None) => None,
264                    _ => return Err(DbError::DamagedDatabase),
265                };
266                Ok(UploadFacts { sending: sending != 0, stored: stored != 0, retry_at, record })
267            })
268            .collect::<Result<_, DbError>>()?;
269        let operation_retries = json::<Vec<(i64, i64, u32)>>(&values.get::<String>(13)?)?
270            .into_iter()
271            .map(|(operation, seconds, nanos)| {
272                Ok((crate::OperationId(operation), unix_time(seconds, nanos)?))
273            })
274            .collect::<Result<_, DbError>>()?;
275        Ok(Self {
276            row,
277            sync,
278            unsent,
279            uploads,
280            queued: values.get(9)?,
281            records,
282            pass_retry: values
283                .get::<Option<Vec<u8>>>(12)?
284                .as_deref()
285                .map(crate::user_file::decode_time)
286                .transpose()?,
287            operation_retries,
288        })
289    }
290}
291
292/// The value a JSON array of `facts!` holds.
293fn json<T: serde::de::DeserializeOwned>(text: &str) -> Result<T, DbError> {
294    serde_json::from_str(text).map_err(|_damaged| DbError::DamagedDatabase)
295}
296
297/// The time `seconds` and `nanos` name, as `unix_seconds_and_nanos` splits
298/// it: whole seconds since the Unix epoch, rounded down, and the nanoseconds
299/// past that second.
300fn unix_time(seconds: i64, nanos: u32) -> Result<SystemTime, DbError> {
301    let whole = Duration::from_secs(seconds.unsigned_abs());
302    let second =
303        if seconds >= 0 { UNIX_EPOCH.checked_add(whole) } else { UNIX_EPOCH.checked_sub(whole) };
304    second
305        .and_then(|second| second.checked_add(Duration::from_nanos(u64::from(nanos))))
306        .filter(|_| nanos < 1_000_000_000)
307        .ok_or(DbError::DamagedDatabase)
308}
309
310/// Read the facts in the caller's snapshot.
311pub(crate) fn read<P: crate::PendingValue + Clone>(db: &StoreSql) -> Result<SyncFacts<P>, DbError> {
312    db.query_row(READ, [], |row| Ok(SyncFacts::decode(row)))?
313}
314
315/// What the phase of each upload in the queue is computed from (§17.9, api.md
316/// E7): the facts the sync state reads, what holds trailing uploads, the
317/// whole queue, and what is recorded about each upload, in one committed
318/// snapshot.
319#[derive(Clone, Debug)]
320pub struct QueueFacts<P> {
321    /// What the sync state is computed from.
322    pub sync: SyncFacts<P>,
323    /// What holds trailing uploads.
324    pub holds: TransferHolds,
325    /// The queue as the app observes it, every file in it.
326    pub queue: crate::UploadQueueState,
327    /// What is recorded about each upload in the queue, by queue id.
328    pub recorded: Vec<UploadRecords<P>>,
329}
330
331/// What is recorded about one queued upload.
332#[derive(Clone, Debug)]
333pub struct UploadRecords<P> {
334    /// The upload's queue id.
335    pub upload: i64,
336    /// The path of the user's original it uploads, if it uploads one.
337    pub original: Option<std::path::PathBuf>,
338    /// The record about the upload's stored bytes, if it has one.
339    pub own: Option<P>,
340    /// The record about the move the upload belongs to, if it has one: it
341    /// names an original, which may be this upload's or a sibling's.
342    pub moving: Option<P>,
343}
344
345/// How many values `_coven_upload_phase` takes, in the order
346/// `UploadPhaseFacts::decode` reads them: whether sync runs and its
347/// cooldown, as `coven_retry_effect` holds them; how the last pass ended, as
348/// `coven_sync_status` holds it; the pause, the setting to move files only
349/// on unmetered networks, whether the network is metered and whether a
350/// self-replacement pauses writes; whether this process sends the upload,
351/// whether storage holds its bytes and whether it is bound; when its retry
352/// wait ends, as `_coven_retries` stores it; the position and value of its
353/// own pending record and of its move's; and the path of the original it
354/// uploads, as `_coven_file_uploads` stores it.
355pub(crate) const UPLOAD_PHASE_FACTS: i32 = 16;
356
357/// What one upload's phase is computed from (§17.9, api.md E4), as
358/// `coven_upload_phases` passes it to `_coven_upload_phase` for each upload
359/// in this device's queue, in one committed snapshot: what holds every
360/// transfer, what holds trailing uploads, and the upload's own facts.
361#[derive(Clone, Debug)]
362pub struct UploadPhaseFacts<P> {
363    /// Whether a started connection will plan, and the cooldown holding
364    /// every request.
365    pub sync: crate::RetrySync,
366    /// How the last pass ended, as sync encodes it; none until a pass ends
367    /// on this device.
368    pub last_pass: Option<Vec<u8>>,
369    /// What holds trailing uploads.
370    pub holds: TransferHolds,
371    /// This process is sending it now.
372    pub sending: bool,
373    /// Storage holds its bytes; it waits for the rest of its move.
374    pub stored: bool,
375    /// A bound reference's, which the pause, a metered network and a
376    /// self-replacement don't hold (§17.4).
377    pub bound: bool,
378    /// Its last attempt failed, and it retries then (§6.7).
379    pub retry_at: Option<SystemTime>,
380    /// The record about its stored bytes, with its position in
381    /// `coven_pending`, if it has one.
382    pub own: Option<(i64, P)>,
383    /// The record about its move, with its position, if it has one: it
384    /// names an original, which may be this upload's or a sibling's.
385    pub moving: Option<(i64, P)>,
386    /// The path of the user's original it uploads, if it uploads one.
387    pub original: Option<std::path::PathBuf>,
388}
389
390impl<P: crate::PendingValue> UploadPhaseFacts<P> {
391    /// The facts from the values `coven_upload_phases` passes, in the order
392    /// `UPLOAD_PHASE_FACTS` lists them. A value that doesn't decode is
393    /// `DamagedDatabase`, and a record that doesn't decode `StateEncoding`.
394    pub(crate) fn decode(values: &impl FactValues) -> Result<Self, DbError> {
395        let record = |first: usize| -> Result<Option<(i64, P)>, DbError> {
396            match (values.get::<Option<i64>>(first)?, values.get::<Option<Vec<u8>>>(first + 1)?) {
397                (Some(position), Some(value)) => Ok(Some((
398                    position,
399                    serde_json::from_slice(&value).map_err(DbError::StateEncoding)?,
400                ))),
401                (None, None) => Ok(None),
402                _ => Err(DbError::DamagedDatabase),
403            }
404        };
405        Ok(Self {
406            sync: crate::internal_schema::retry_effect::sync(values.get(0)?, values.get(1)?)?,
407            last_pass: values.get(2)?,
408            holds: TransferHolds {
409                paused: values.get(3)?,
410                unmetered_only: values.get(4)?,
411                metered: values.get(5)?,
412                keys_changing: values.get(6)?,
413            },
414            sending: values.get(7)?,
415            stored: values.get(8)?,
416            bound: values.get(9)?,
417            retry_at: values
418                .get::<Option<Vec<u8>>>(10)?
419                .as_deref()
420                .map(crate::user_file::decode_time)
421                .transpose()?,
422            own: record(11)?,
423            moving: record(13)?,
424            original: values
425                .get::<Option<Vec<u8>>>(15)?
426                .map(crate::user_file::decode_path)
427                .transpose()?,
428        })
429    }
430}
431
432/// Read the facts, what holds trailing uploads, and the queue with what is
433/// recorded about each upload, in the caller's snapshot.
434pub(crate) fn read_queue<P: crate::PendingValue + Clone>(
435    db: &StoreSql,
436) -> Result<QueueFacts<P>, DbError> {
437    let records: BTreeMap<i64, P> = crate::pending::positioned::<P>(db)?.into_iter().collect();
438    let recorded = db
439        .query(
440            "SELECT u.id,u.original,o.id,m.id FROM _coven_file_uploads AS u
441             LEFT JOIN _coven_pending AS o
442                 ON o.work='upload' AND o.device=u.device AND o.object=u.object
443             LEFT JOIN _coven_pending AS m ON m.move=u.move
444             ORDER BY u.id",
445            [],
446            |row| {
447                Ok((
448                    row.get::<_, i64>(0)?,
449                    row.get::<_, Option<Vec<u8>>>(1)?,
450                    row.get::<_, Option<i64>>(2)?,
451                    row.get::<_, Option<i64>>(3)?,
452                ))
453            },
454        )?
455        .into_iter()
456        .map(|(upload, original, own, moving)| {
457            let find = |position: Option<i64>| {
458                position
459                    .map(|position| {
460                        records.get(&position).cloned().ok_or(DbError::Invariant {
461                            what: "an upload's record is among the pending records",
462                        })
463                    })
464                    .transpose()
465            };
466            Ok(UploadRecords {
467                upload,
468                original: original.map(crate::user_file::decode_path).transpose()?,
469                own: find(own)?,
470                moving: find(moving)?,
471            })
472        })
473        .collect::<Result<_, DbError>>()?;
474    let (metered, unmetered_only) =
475        db.query_row("SELECT metered,unmetered_only FROM coven_network", [], |row| {
476            Ok((row.get::<_, bool>(0)?, row.get::<_, bool>(1)?))
477        })?;
478    let holds = TransferHolds {
479        paused: crate::file_queue::paused(db)?,
480        unmetered_only,
481        metered,
482        keys_changing: crate::operation::writes_paused(db)?,
483    };
484    Ok(QueueFacts { sync: read(db)?, holds, queue: crate::file_queue::state(db)?, recorded })
485}
486
487#[cfg(test)]
488#[path = "sync_facts_tests.rs"]
489pub(crate) mod tests;