coven_database/sync_facts.rs
1//! What the sync state is computed from (§19.1, api.md E5), read in one
2//! committed snapshot by one list of SQL expressions (`facts!`):
3//! `coven_sync_status`'s row, whether sync runs and the cooldown
4//! `coven_retry_effect` holds, the send queue, the bound uploads (the uploads
5//! a write waits for, each with its retry wait and record), and the pending
6//! records the state reads. The `coven_sync_state` view passes them to
7//! `_coven_sync_state` as its arguments (`sync_functions.rs`), and coven
8//! reads them as a row where sync computes the state in Rust: a closed
9//! store's state, and each upload's phase (`QueueFacts`). Both decode them
10//! with `SyncFacts::decode`. Sync computes the state; this module reads
11//! what sync needs. The `coven_upload_phases` view passes each upload's own
12//! facts to `_coven_upload_phase` in the same way (`UploadPhaseFacts`).
13//!
14//! A trailing upload, a download and a pin coven keeps are not sync's: this
15//! module reads no upload of theirs and no record about them, and the state
16//! never sees them (§17.4, §19.1). Which upload is bound comes from the
17//! queue's own column (`holds`), and the read leaves a record out when its
18//! row names such an upload, a download or a pin, or a move none of whose
19//! uploads is bound. Nor does the state read the pause, the network or a
20//! self-replacement's write pause, which hold trailing uploads only: an
21//! upload's phase reads them beside the facts (`TransferHolds`).
22//!
23//! A live query over these facts runs again whenever one of them changes:
24//! `facts!` reads the files in flight through `coven_sending`, the view
25//! that a transfer's start and end change (`transfers_in_flight.rs`). `facts!` reads each
26//! table through a column and never only counts its rows, so that SQLite
27//! names the view to the authorizer for each read the view makes
28//! (`internal_schema_views.rs`).
29
30use crate::sqlite::StoreSql;
31use crate::DbError;
32use sqlive::rusqlite::types::{FromSql, ValueRef};
33use std::collections::BTreeMap;
34use std::time::{Duration, SystemTime, UNIX_EPOCH};
35
36/// The facts, as SQL expressions over `coven_sync_status AS s` and
37/// `coven_retry_effect AS r`, in the order `SyncFacts::decode` reads them:
38/// the status row's columns; whether sync runs and its cooldown; the send
39/// queue's writes and the upload queue's uploads, counted; whether a bound
40/// upload has bytes to send; the bound uploads the state can show, as a JSON
41/// array; the pending records the state reads, as a JSON array; the sync
42/// scope's backoff; and each operation's backoff, as a JSON array.
43///
44/// `facts!` reads each kind of bound upload the state can show from the
45/// table where that kind has the fewest rows: the uploads with a record from
46/// the pending records, the first one sending from the files in flight, the
47/// first queued one with no record and no retry by walking the bound queue
48/// in id order to the first match, and the first with no record that waits
49/// out a retry from the retry rows.
50/// The outer read then reads only those rows, and only bound ones. Each is
51/// `[id, stored, sending, retry seconds, retry nanos, record position]`, the
52/// retry time as `_coven_unix_seconds` and `_coven_unix_nanos` give it, since
53/// JSON holds no bytes. A record is `[position, record, cause]`, in the order
54/// the records arose, and an operation's backoff `[operation, seconds,
55/// nanos]`.
56macro_rules! facts {
57 () => {
58 concat!(
59 "s.storage,s.provider,s.last_pass,s.moved,s.last_synced,r.syncing,r.cooling_until,",
60 crate::file_queue::unsent_counts!(),
61 ",EXISTS(SELECT 1 FROM _coven_file_uploads AS b
62 WHERE b.holds IS NOT NULL AND NOT b.stored
63 AND EXISTS(SELECT 1 FROM _coven_upload_files AS l WHERE l.upload=b.id)),
64 (SELECT json_group_array(json_array(u.id,u.stored,
65 u.id IN (SELECT c.position FROM coven_sending AS c),
66 iif(t.upload IS NULL,NULL,_coven_unix_seconds(t.not_before)),
67 iif(t.upload IS NULL,NULL,_coven_unix_nanos(t.not_before)),e.id)
68 ORDER BY u.id)
69 FROM _coven_file_uploads AS u
70 LEFT JOIN _coven_retries AS t ON t.upload=u.id
71 LEFT JOIN _coven_pending AS e
72 ON e.work='upload' AND e.device=u.device AND e.object=u.object
73 WHERE u.id IN (
74 SELECT f.id FROM _coven_pending AS p
75 JOIN _coven_file_uploads AS f ON f.device=p.device AND f.object=p.object
76 WHERE p.work='upload'
77 UNION
78 SELECT MIN(g.position) FROM coven_sending AS g
79 JOIN _coven_file_uploads AS b ON b.id=g.position
80 WHERE b.holds IS NOT NULL
81 UNION
82 SELECT * FROM (
83 SELECT f.id FROM _coven_file_uploads AS f
84 WHERE NOT f.stored AND f.holds IS NOT NULL
85 AND f.id NOT IN (SELECT position FROM coven_sending)
86 AND EXISTS(SELECT 1 FROM _coven_upload_files AS l WHERE l.upload=f.id)
87 AND NOT EXISTS(SELECT 1 FROM _coven_pending AS p
88 WHERE p.work='upload' AND p.device=f.device
89 AND p.object=f.object)
90 AND NOT EXISTS(SELECT 1 FROM _coven_retries AS t WHERE t.upload=f.id)
91 ORDER BY f.id LIMIT 1)
92 UNION
93 SELECT MIN(f.id) FROM _coven_retries AS t
94 CROSS JOIN _coven_file_uploads AS f ON f.id=t.upload
95 WHERE t.upload IS NOT NULL
96 AND NOT f.stored AND f.holds IS NOT NULL
97 AND f.id NOT IN (SELECT position FROM coven_sending)
98 AND EXISTS(SELECT 1 FROM _coven_upload_files AS l WHERE l.upload=f.id)
99 AND NOT EXISTS(SELECT 1 FROM _coven_pending AS p
100 WHERE p.work='upload' AND p.device=f.device
101 AND p.object=f.object))
102 AND u.holds IS NOT NULL
103 AND EXISTS(SELECT 1 FROM _coven_upload_files AS l WHERE l.upload=u.id)),
104 (SELECT json_group_array(json_array(p.id,json(CAST(p.value AS TEXT)),p.cause)
105 ORDER BY p.id)
106 FROM _coven_pending AS p
107 WHERE (p.pin IS NOT NULL
108 OR p.work='download'
109 OR (p.work='upload' AND NOT EXISTS(
110 SELECT 1 FROM _coven_file_uploads AS f
111 WHERE f.device=p.device AND f.object=p.object AND f.holds IS NOT NULL))
112 OR (p.move IS NOT NULL AND NOT EXISTS(
113 SELECT 1 FROM _coven_file_uploads AS f
114 WHERE f.move=p.move AND f.holds IS NOT NULL))) IS NOT TRUE),
115 (SELECT t.not_before FROM _coven_retries AS t WHERE t.scope='sync'),
116 (SELECT json_group_array(json_array(t.operation,_coven_unix_seconds(t.not_before),
117 _coven_unix_nanos(t.not_before)))
118 FROM _coven_retries AS t WHERE t.operation>0)"
119 )
120 };
121}
122pub(crate) use facts;
123
124/// How many expressions `facts!` lists: the arguments `_coven_sync_state`
125/// takes.
126pub(crate) const FACTS: i32 = 14;
127
128/// The facts in one row.
129const READ: &str =
130 concat!("SELECT ", facts!(), " FROM coven_sync_status AS s JOIN coven_retry_effect AS r");
131
132/// What the sync state is computed from, in one committed snapshot.
133#[derive(Clone, Debug, PartialEq)]
134pub struct SyncFacts<P> {
135 /// What the sync loop knows: `coven_sync_status`'s row.
136 pub row: crate::SyncStatusRow,
137 /// Whether a started connection will plan, and the cooldown holding
138 /// every request, as `coven_retry_effect` holds them.
139 pub sync: crate::RetrySync,
140 /// What the send queue and the upload queue hold (`UnsentWork`): a
141 /// write waits when it counts one.
142 pub unsent: crate::UnsentWork,
143 /// The bound uploads the state can show, in queue order: the first one
144 /// sending, the first queued one (neither sending nor stored) with no
145 /// record and none waiting out a retry, the first with no record that
146 /// does, and every one with a record. The state shows the first
147 /// candidate of the highest rank, so no other upload changes it: later
148 /// sending ones tie with the first, a later queued one ties with the
149 /// first of its kind (syncing for its turn, waiting for its retry), and a
150 /// stored one names none.
151 pub uploads: Vec<UploadFacts<P>>,
152 /// Whether the upload queue holds a bound upload with bytes to send,
153 /// whether `uploads` lists it or not.
154 pub queued: bool,
155 /// The pending records the state reads, each with its position in
156 /// `coven_pending`, in the order they arose: every one but those about a
157 /// trailing upload, a download, a pin coven keeps or a move of trailing
158 /// uploads alone.
159 pub records: Vec<(i64, P)>,
160 /// When the sync scope's backoff ends, if a failed pass armed one
161 /// (§6.7): the next try of a pass or of a slot's application.
162 pub pass_retry: Option<SystemTime>,
163 /// When each journal operation's own backoff ends: the next try of
164 /// coven's own operation a failed pass names.
165 pub operation_retries: Vec<(crate::OperationId, SystemTime)>,
166}
167
168/// One bound upload in the queue, as the sync state reads it.
169#[derive(Clone, Debug, PartialEq)]
170pub struct UploadFacts<P> {
171 /// This process is sending it now.
172 pub sending: bool,
173 /// Storage holds its bytes; it waits for the rest of its move.
174 pub stored: bool,
175 /// Its last attempt failed, and it retries then (§6.7).
176 pub retry_at: Option<SystemTime>,
177 /// Its own pending record, if it has one.
178 pub record: Option<P>,
179}
180
181/// What holds trailing uploads, and no bound upload or the sync state: an
182/// upload's phase reads them beside the facts (§17.4, §17.9, §12.3).
183#[derive(Clone, Copy, Debug, PartialEq, Eq)]
184pub struct TransferHolds {
185 /// Whether the app paused trailing uploads.
186 pub paused: bool,
187 /// Whether the app moves files only on unmetered networks.
188 pub unmetered_only: bool,
189 /// Whether the device's network is metered, as the store's watch last
190 /// said.
191 pub metered: bool,
192 /// Whether a self-replacement of this member's keys pauses writes
193 /// (§12.3).
194 pub keys_changing: bool,
195}
196
197/// Where the facts are read from: a row of the facts' SELECT, or the
198/// arguments of one call of `_coven_sync_state`.
199pub(crate) trait FactValues {
200 /// Value `index`, counted from zero, as SQLite holds it.
201 fn value(&self, index: usize) -> sqlive::rusqlite::Result<ValueRef<'_>>;
202
203 /// Value `index`, converted as a row's column is.
204 fn get<T: FromSql>(&self, index: usize) -> Result<T, DbError> {
205 let value = self.value(index)?;
206 T::column_result(value).map_err(|error| {
207 DbError::from(sqlive::rusqlite::Error::FromSqlConversionFailure(
208 index,
209 value.data_type(),
210 Box::new(error),
211 ))
212 })
213 }
214}
215
216impl FactValues for sqlive::rusqlite::Row<'_> {
217 fn value(&self, index: usize) -> sqlive::rusqlite::Result<ValueRef<'_>> {
218 self.get_ref(index)
219 }
220}
221
222impl FactValues for sqlive::rusqlite::functions::Context<'_> {
223 fn value(&self, index: usize) -> sqlive::rusqlite::Result<ValueRef<'_>> {
224 Ok(self.get_raw(index))
225 }
226}
227
228/// One listed upload as `facts!` gives it.
229type ListedUpload = (i64, i64, i64, Option<i64>, Option<u32>, Option<i64>);
230
231impl<P: crate::PendingValue + Clone> SyncFacts<P> {
232 /// The facts from `facts!`'s values, as a row or a call's arguments hold
233 /// them. A value that doesn't decode is `DamagedDatabase`, and an upload's
234 /// record that isn't among the records an invariant failure.
235 pub(crate) fn decode(values: &impl FactValues) -> Result<Self, DbError> {
236 let row = crate::internal_schema::sync_status::decode(values, 0)?;
237 let sync = crate::internal_schema::retry_effect::sync(values.get(5)?, values.get(6)?)?;
238 let count = |index| {
239 u64::try_from(values.get::<i64>(index)?).map_err(|_negative| DbError::DamagedDatabase)
240 };
241 let unsent = crate::UnsentWork { writes: count(7)?, files: count(8)? };
242 let records: Vec<(i64, P, Option<String>)> = json(&values.get::<String>(11)?)?;
243 let records: Vec<(i64, P)> = records
244 .into_iter()
245 .map(|(position, record, cause)| (position, record.with_cause(cause)))
246 .collect();
247 let by_position: BTreeMap<i64, &P> =
248 records.iter().map(|(position, record)| (*position, record)).collect();
249 let uploads = json::<Vec<ListedUpload>>(&values.get::<String>(10)?)?
250 .into_iter()
251 .map(|(_id, stored, sending, seconds, nanos, position)| {
252 let record = position
253 .map(|position| {
254 by_position.get(&position).map(|record| (*record).clone()).ok_or(
255 DbError::Invariant {
256 what: "an upload's record is among the pending records",
257 },
258 )
259 })
260 .transpose()?;
261 let retry_at = match (seconds, nanos) {
262 (Some(seconds), Some(nanos)) => Some(unix_time(seconds, nanos)?),
263 (None, None) => None,
264 _ => return Err(DbError::DamagedDatabase),
265 };
266 Ok(UploadFacts { sending: sending != 0, stored: stored != 0, retry_at, record })
267 })
268 .collect::<Result<_, DbError>>()?;
269 let operation_retries = json::<Vec<(i64, i64, u32)>>(&values.get::<String>(13)?)?
270 .into_iter()
271 .map(|(operation, seconds, nanos)| {
272 Ok((crate::OperationId(operation), unix_time(seconds, nanos)?))
273 })
274 .collect::<Result<_, DbError>>()?;
275 Ok(Self {
276 row,
277 sync,
278 unsent,
279 uploads,
280 queued: values.get(9)?,
281 records,
282 pass_retry: values
283 .get::<Option<Vec<u8>>>(12)?
284 .as_deref()
285 .map(crate::user_file::decode_time)
286 .transpose()?,
287 operation_retries,
288 })
289 }
290}
291
292/// The value a JSON array of `facts!` holds.
293fn json<T: serde::de::DeserializeOwned>(text: &str) -> Result<T, DbError> {
294 serde_json::from_str(text).map_err(|_damaged| DbError::DamagedDatabase)
295}
296
297/// The time `seconds` and `nanos` name, as `unix_seconds_and_nanos` splits
298/// it: whole seconds since the Unix epoch, rounded down, and the nanoseconds
299/// past that second.
300fn unix_time(seconds: i64, nanos: u32) -> Result<SystemTime, DbError> {
301 let whole = Duration::from_secs(seconds.unsigned_abs());
302 let second =
303 if seconds >= 0 { UNIX_EPOCH.checked_add(whole) } else { UNIX_EPOCH.checked_sub(whole) };
304 second
305 .and_then(|second| second.checked_add(Duration::from_nanos(u64::from(nanos))))
306 .filter(|_| nanos < 1_000_000_000)
307 .ok_or(DbError::DamagedDatabase)
308}
309
310/// Read the facts in the caller's snapshot.
311pub(crate) fn read<P: crate::PendingValue + Clone>(db: &StoreSql) -> Result<SyncFacts<P>, DbError> {
312 db.query_row(READ, [], |row| Ok(SyncFacts::decode(row)))?
313}
314
315/// What the phase of each upload in the queue is computed from (§17.9, api.md
316/// E7): the facts the sync state reads, what holds trailing uploads, the
317/// whole queue, and what is recorded about each upload, in one committed
318/// snapshot.
319#[derive(Clone, Debug)]
320pub struct QueueFacts<P> {
321 /// What the sync state is computed from.
322 pub sync: SyncFacts<P>,
323 /// What holds trailing uploads.
324 pub holds: TransferHolds,
325 /// The queue as the app observes it, every file in it.
326 pub queue: crate::UploadQueueState,
327 /// What is recorded about each upload in the queue, by queue id.
328 pub recorded: Vec<UploadRecords<P>>,
329}
330
331/// What is recorded about one queued upload.
332#[derive(Clone, Debug)]
333pub struct UploadRecords<P> {
334 /// The upload's queue id.
335 pub upload: i64,
336 /// The path of the user's original it uploads, if it uploads one.
337 pub original: Option<std::path::PathBuf>,
338 /// The record about the upload's stored bytes, if it has one.
339 pub own: Option<P>,
340 /// The record about the move the upload belongs to, if it has one: it
341 /// names an original, which may be this upload's or a sibling's.
342 pub moving: Option<P>,
343}
344
345/// How many values `_coven_upload_phase` takes, in the order
346/// `UploadPhaseFacts::decode` reads them: whether sync runs and its
347/// cooldown, as `coven_retry_effect` holds them; how the last pass ended, as
348/// `coven_sync_status` holds it; the pause, the setting to move files only
349/// on unmetered networks, whether the network is metered and whether a
350/// self-replacement pauses writes; whether this process sends the upload,
351/// whether storage holds its bytes and whether it is bound; when its retry
352/// wait ends, as `_coven_retries` stores it; the position and value of its
353/// own pending record and of its move's; and the path of the original it
354/// uploads, as `_coven_file_uploads` stores it.
355pub(crate) const UPLOAD_PHASE_FACTS: i32 = 16;
356
357/// What one upload's phase is computed from (§17.9, api.md E4), as
358/// `coven_upload_phases` passes it to `_coven_upload_phase` for each upload
359/// in this device's queue, in one committed snapshot: what holds every
360/// transfer, what holds trailing uploads, and the upload's own facts.
361#[derive(Clone, Debug)]
362pub struct UploadPhaseFacts<P> {
363 /// Whether a started connection will plan, and the cooldown holding
364 /// every request.
365 pub sync: crate::RetrySync,
366 /// How the last pass ended, as sync encodes it; none until a pass ends
367 /// on this device.
368 pub last_pass: Option<Vec<u8>>,
369 /// What holds trailing uploads.
370 pub holds: TransferHolds,
371 /// This process is sending it now.
372 pub sending: bool,
373 /// Storage holds its bytes; it waits for the rest of its move.
374 pub stored: bool,
375 /// A bound reference's, which the pause, a metered network and a
376 /// self-replacement don't hold (§17.4).
377 pub bound: bool,
378 /// Its last attempt failed, and it retries then (§6.7).
379 pub retry_at: Option<SystemTime>,
380 /// The record about its stored bytes, with its position in
381 /// `coven_pending`, if it has one.
382 pub own: Option<(i64, P)>,
383 /// The record about its move, with its position, if it has one: it
384 /// names an original, which may be this upload's or a sibling's.
385 pub moving: Option<(i64, P)>,
386 /// The path of the user's original it uploads, if it uploads one.
387 pub original: Option<std::path::PathBuf>,
388}
389
390impl<P: crate::PendingValue> UploadPhaseFacts<P> {
391 /// The facts from the values `coven_upload_phases` passes, in the order
392 /// `UPLOAD_PHASE_FACTS` lists them. A value that doesn't decode is
393 /// `DamagedDatabase`, and a record that doesn't decode `StateEncoding`.
394 pub(crate) fn decode(values: &impl FactValues) -> Result<Self, DbError> {
395 let record = |first: usize| -> Result<Option<(i64, P)>, DbError> {
396 match (values.get::<Option<i64>>(first)?, values.get::<Option<Vec<u8>>>(first + 1)?) {
397 (Some(position), Some(value)) => Ok(Some((
398 position,
399 serde_json::from_slice(&value).map_err(DbError::StateEncoding)?,
400 ))),
401 (None, None) => Ok(None),
402 _ => Err(DbError::DamagedDatabase),
403 }
404 };
405 Ok(Self {
406 sync: crate::internal_schema::retry_effect::sync(values.get(0)?, values.get(1)?)?,
407 last_pass: values.get(2)?,
408 holds: TransferHolds {
409 paused: values.get(3)?,
410 unmetered_only: values.get(4)?,
411 metered: values.get(5)?,
412 keys_changing: values.get(6)?,
413 },
414 sending: values.get(7)?,
415 stored: values.get(8)?,
416 bound: values.get(9)?,
417 retry_at: values
418 .get::<Option<Vec<u8>>>(10)?
419 .as_deref()
420 .map(crate::user_file::decode_time)
421 .transpose()?,
422 own: record(11)?,
423 moving: record(13)?,
424 original: values
425 .get::<Option<Vec<u8>>>(15)?
426 .map(crate::user_file::decode_path)
427 .transpose()?,
428 })
429 }
430}
431
432/// Read the facts, what holds trailing uploads, and the queue with what is
433/// recorded about each upload, in the caller's snapshot.
434pub(crate) fn read_queue<P: crate::PendingValue + Clone>(
435 db: &StoreSql,
436) -> Result<QueueFacts<P>, DbError> {
437 let records: BTreeMap<i64, P> = crate::pending::positioned::<P>(db)?.into_iter().collect();
438 let recorded = db
439 .query(
440 "SELECT u.id,u.original,o.id,m.id FROM _coven_file_uploads AS u
441 LEFT JOIN _coven_pending AS o
442 ON o.work='upload' AND o.device=u.device AND o.object=u.object
443 LEFT JOIN _coven_pending AS m ON m.move=u.move
444 ORDER BY u.id",
445 [],
446 |row| {
447 Ok((
448 row.get::<_, i64>(0)?,
449 row.get::<_, Option<Vec<u8>>>(1)?,
450 row.get::<_, Option<i64>>(2)?,
451 row.get::<_, Option<i64>>(3)?,
452 ))
453 },
454 )?
455 .into_iter()
456 .map(|(upload, original, own, moving)| {
457 let find = |position: Option<i64>| {
458 position
459 .map(|position| {
460 records.get(&position).cloned().ok_or(DbError::Invariant {
461 what: "an upload's record is among the pending records",
462 })
463 })
464 .transpose()
465 };
466 Ok(UploadRecords {
467 upload,
468 original: original.map(crate::user_file::decode_path).transpose()?,
469 own: find(own)?,
470 moving: find(moving)?,
471 })
472 })
473 .collect::<Result<_, DbError>>()?;
474 let (metered, unmetered_only) =
475 db.query_row("SELECT metered,unmetered_only FROM coven_network", [], |row| {
476 Ok((row.get::<_, bool>(0)?, row.get::<_, bool>(1)?))
477 })?;
478 let holds = TransferHolds {
479 paused: crate::file_queue::paused(db)?,
480 unmetered_only,
481 metered,
482 keys_changing: crate::operation::writes_paused(db)?,
483 };
484 Ok(QueueFacts { sync: read(db)?, holds, queue: crate::file_queue::state(db)?, recorded })
485}
486
487#[cfg(test)]
488#[path = "sync_facts_tests.rs"]
489pub(crate) mod tests;