Skip to main content

coven_database/
database_file_retention.rs

1//! File references used to prove that storage deletion cannot strand a row.
2
3use super::StoreDatabase;
4use crate::file_protection::value as retain_value;
5use crate::sqlite::StoreSql;
6use crate::{
7    DbError, DownloadedPartStream, DownloadedWriteStream, FileEvidence, Slot, UnprovenFiles,
8};
9use coven_format::{value::Value, write::RowChange};
10use coven_foundation::id_source::{DeviceId, ObjectId};
11use coven_merge::{ColumnValue, Operation};
12use sqlive::rusqlite::params;
13use std::collections::{BTreeMap, BTreeSet};
14use std::ops::RangeInclusive;
15
16/// Local references read in one database transaction.
17#[cfg(test)]
18pub(crate) struct FileRetention {
19    /// Fixed paths named by rows, losses, published rows, waiting writes or
20    /// this device's upload queue.
21    pub references: BTreeSet<(DeviceId, ObjectId)>,
22}
23
24impl StoreDatabase {
25    /// A snapshot whose absence was confirmed by an exact read or deletion.
26    pub async fn snapshot_absent(
27        &self,
28        id: coven_format::admin::SnapshotId,
29    ) -> Result<bool, DbError> {
30        self.call(move |inner| {
31                        inner.access.with_reader(|db, _| Ok(db.query_row(
32                "SELECT EXISTS(SELECT 1 FROM _coven_absent_snapshots WHERE audience=?1 AND device=?2 AND number=?3)",
33                params![&id.audience, id.device, id.number.to_be_bytes()], |row| row.get(0))?))
34        }).await
35    }
36
37    /// Confirm that a snapshot no longer protects its checked file references.
38    /// The references become candidates in the same transaction as that fact.
39    pub async fn record_absent_snapshot(
40        &self,
41        id: coven_format::admin::SnapshotId,
42        files: BTreeSet<(DeviceId, ObjectId)>,
43    ) -> Result<(), DbError> {
44        self.call(move |inner| {
45            let number = std::num::NonZeroU64::new(id.number).ok_or(DbError::DamagedDatabase)?;
46            let path = coven_format::path::ObjectPath::snapshot(
47                inner.access.directory.id(),
48                id.audience.clone(),
49                id.device,
50                number,
51            );
52            inner.access.with_writer(|writer, _| writer.transaction(|db| {
53                for (device, object) in files { insert_candidate(db, device, object)?; }
54                db.execute("INSERT INTO _coven_absent_snapshots(audience,device,number) VALUES(?1,?2,?3) ON CONFLICT DO NOTHING",
55                    params![&id.audience, id.device, id.number.to_be_bytes()])?;
56                Ok::<_, DbError>(())
57            }))?;
58            inner.inputs.forget_path(path.as_str())
59        }).await
60    }
61    /// Forget announcements no rule reads any more (§16.8): their original
62    /// slots at `slots`, and the absences recorded for `snapshots`, the
63    /// bodies only they named. `listings` journals the listings that find
64    /// those bodies in storage, in the same transaction, so a forgotten
65    /// announcement's body is never left without one.
66    pub async fn forget_announcements(
67        &self,
68        slots: BTreeSet<Slot>,
69        snapshots: BTreeSet<coven_format::admin::SnapshotId>,
70        listings: Vec<crate::NewOperation>,
71    ) -> Result<(), DbError> {
72        self.call(move |inner| {
73            inner.access.with_writer(|writer, _| {
74                writer.transaction(|db| {
75                    crate::sequence_apply::forget_announcements(db, &slots)?;
76                    for id in &snapshots {
77                        db.execute(
78                            "DELETE FROM _coven_absent_snapshots WHERE audience=?1 AND device=?2 AND number=?3",
79                            params![&id.audience, id.device, id.number.to_be_bytes()],
80                        )?;
81                    }
82                    for listing in &listings {
83                        crate::operation::insert(db, listing)?;
84                    }
85                    Ok(())
86                })
87            })
88        })
89        .await
90    }
91
92    /// Every fixed path local data protects, in one read transaction.
93    #[cfg(test)]
94    pub(crate) async fn retained_files(&self) -> Result<FileRetention, DbError> {
95        self.call(move |inner| {
96            inner.access.with_committed(|db, _| {
97                let references = db.query("SELECT device,object FROM _coven_row_file_refs UNION SELECT device,object FROM _coven_published_file_refs UNION SELECT device,object FROM _coven_queued_file_refs UNION SELECT device,object FROM _coven_file_uploads", [], |r| Ok((r.get::<_, DeviceId>(0)?,r.get::<_, ObjectId>(1)?)))?.into_iter().collect();
98                Ok(FileRetention { references })
99            })
100        })
101        .await
102    }
103
104    /// Stream the stored paths a write's records name from checked plaintext. The caller
105    /// must authenticate the complete object before using this result. It
106    /// names the audience and key of each part it skipped: a skipped part, or
107    /// a schema version this version cannot interpret, proves no file absent.
108    pub async fn write_file_references<R: std::io::Read + Send + 'static>(
109        &self,
110        write: DownloadedWriteStream<R>,
111        skipped: BTreeSet<(coven_merge::Audience, coven_foundation::id_source::KeyId)>,
112    ) -> Result<FileEvidence, DbError> {
113        self.call(move |inner| {
114            let mut references = BTreeSet::new();
115            write.header.encode()?;
116            let version = write.header.header.schema_version;
117            if !inner.access.write_schema.versions.contains(&version) {
118                return Ok(FileEvidence::Unproven(UnprovenFiles::Schema(version)));
119            }
120            if write.header.parts.len() != write.parts.len() {
121                return Err(crate::SnapshotError::Inconsistent(
122                    "write stream count differs from its header",
123                )
124                .into());
125            }
126            let opened = write
127                .parts
128                .iter()
129                .filter(|part| matches!(part, DownloadedPartStream::Opened(_)))
130                .count();
131            if (opened == write.parts.len()) != skipped.is_empty() {
132                return Err(crate::SnapshotError::Inconsistent(
133                    "skipped parts disagree with the parts named skipped",
134                )
135                .into());
136            }
137            if !skipped.is_empty() {
138                return Ok(FileEvidence::Unproven(UnprovenFiles::Parts(skipped)));
139            }
140            for (header, part) in write.header.parts.into_iter().zip(write.parts) {
141                let DownloadedPartStream::Opened(input) = part else {
142                    unreachable!("every part opened");
143                };
144                crate::download_stream::read_part(input, header, |frame| {
145                    if let coven_format::dismissal::WriteFrame::Change(row) = frame {
146                        retain_change(&row, &mut references)?;
147                    }
148                    Ok::<_, DbError>(())
149                })?;
150            }
151            Ok(FileEvidence::Complete(references))
152        })
153        .await
154    }
155
156    /// The schema versions whose writes and snapshots this version interprets.
157    pub async fn write_schema_versions(&self) -> Result<RangeInclusive<u32>, DbError> {
158        self.call(move |inner| Ok(inner.access.write_schema.versions.clone())).await
159    }
160
161    /// The candidates of these uploaders, each with whether local data still
162    /// protects it (§17.10). Other uploaders' candidates are not read: their
163    /// files are deleted by another device until ownership changes.
164    pub async fn file_candidates(
165        &self,
166        uploaders: BTreeSet<DeviceId>,
167    ) -> Result<Vec<FileCandidate>, DbError> {
168        self.call(move |inner| {
169            inner.access.with_committed(|db, _| {
170                let mut candidates = Vec::new();
171                for device in uploaders {
172                    candidates.extend(db.query(
173                        "SELECT object,EXISTS(SELECT 1 FROM _coven_file_protection p WHERE p.device=c.device AND p.object=c.object),since FROM _coven_file_candidates c WHERE c.device=?1 ORDER BY c.object",
174                        [device],
175                        |r| {
176                            Ok(FileCandidate {
177                                device,
178                                object: r.get(0)?,
179                                protected: r.get(1)?,
180                                since: crate::store_state_tables::slot(r, 2)?,
181                            })
182                        },
183                    )?);
184                }
185                Ok(candidates)
186            })
187        })
188        .await
189    }
190
191    /// The slot count each of these devices had read through when it made its
192    /// last published write, for those that published one. Every write it
193    /// made before is published too (§6.3, §17.10).
194    pub async fn published_reads(
195        &self,
196        devices: BTreeSet<DeviceId>,
197    ) -> Result<BTreeMap<DeviceId, u64>, DbError> {
198        self.call(move |inner| {
199            inner.access.with_committed(|db, _| {
200                let mut reads = BTreeMap::new();
201                for device in devices {
202                    if let Some(read) = crate::write_metadata::last_landed_read(db, device)? {
203                        reads.insert(device, read);
204                    }
205                }
206                Ok(reads)
207            })
208        })
209        .await
210    }
211
212    /// Settle candidates in one transaction. A protected one goes only while
213    /// local data still protects it, so releasing that protection adds it
214    /// again; a deleted one goes after storage confirmed its deletion.
215    pub async fn settle_file_candidates(
216        &self,
217        protected: Vec<(DeviceId, ObjectId)>,
218        deleted: Vec<(DeviceId, ObjectId)>,
219    ) -> Result<(), DbError> {
220        if protected.is_empty() && deleted.is_empty() {
221            return Ok(());
222        }
223        self.call(move |inner| {
224            inner.access.with_writer(|writer, _| {
225                writer.transaction(|db| {
226                    for (device, object) in protected {
227                        db.execute(
228                            "DELETE FROM _coven_file_candidates WHERE device=?1 AND object=?2 AND EXISTS(SELECT 1 FROM _coven_file_protection WHERE device=?1 AND object=?2)",
229                            params![device, object],
230                        )?;
231                    }
232                    for (device, object) in deleted {
233                        db.execute(
234                            "DELETE FROM _coven_file_candidates WHERE device=?1 AND object=?2",
235                            params![device, object],
236                        )?;
237                    }
238                    Ok(())
239                })
240            })
241        })
242        .await
243    }
244
245    /// Paths named by a retained object this device confirmed deleted: its
246    /// protection ended with the object (§17.10).
247    #[cfg(test)]
248    pub async fn add_file_candidates(
249        &self,
250        files: BTreeSet<(DeviceId, ObjectId)>,
251    ) -> Result<(), DbError> {
252        if files.is_empty() {
253            return Ok(());
254        }
255        self.call(move |inner| {
256            inner.access.with_writer(|writer, _| {
257                writer.transaction(|db| {
258                    for (device, object) in files {
259                        insert_candidate(db, device, object)?;
260                    }
261                    Ok(())
262                })
263            })
264        })
265        .await
266    }
267
268    /// The counted entry stopping `device` for good at which this device
269    /// last listed its file folder, if it did (§17.10).
270    pub async fn file_scan(&self, device: DeviceId) -> Result<Option<Slot>, DbError> {
271        self.call(move |inner| {
272            inner.access.with_reader(|db, _| {
273                Ok(db
274                    .query("SELECT slot FROM _coven_file_scans WHERE device=?1", [device], |r| {
275                        crate::store_state_tables::slot(r, 0)
276                    })?
277                    .pop())
278            })
279        })
280        .await
281    }
282
283    /// Record a complete listing of the file folder of an uploader gone for
284    /// good, taken once the entry at `stop` stopped it, with every path it
285    /// found as a candidate. Returns false, recording nothing, when this
286    /// device hasn't applied a counted cut-off at `stop` naming the device or
287    /// its member: the history the caller judged by is not the one applied
288    /// here. Whether a replacement of the member's keys stopped the device,
289    /// which depends on the slot that added it and the entry's author, the
290    /// caller's history decides (§14.1). Nothing the device writes after the
291    /// stop counts, so any listing taken after it is sound, however often it
292    /// is taken (§17.10).
293    pub async fn complete_file_scan(
294        &self,
295        device: DeviceId,
296        stop: Slot,
297        objects: BTreeSet<ObjectId>,
298    ) -> Result<bool, DbError> {
299        self.call(move |inner| {
300            inner.access.with_writer(|writer, _| {
301                writer.transaction(|db| {
302                    let kept: bool = db.query_row(
303                        "SELECT EXISTS(SELECT 1 FROM _coven_cutoffs c JOIN _coven_devices d ON d.device=?1 WHERE c.slot=?2 AND (c.device=d.device OR c.member=d.member))",
304                        params![device, stop.get().to_be_bytes()],
305                        |r| r.get(0),
306                    )?;
307                    if !kept {
308                        return Ok(false);
309                    }
310                    for object in objects {
311                        insert_candidate(db, device, object)?;
312                    }
313                    db.execute(
314                        "INSERT INTO _coven_file_scans(device,slot) VALUES(?1,?2) ON CONFLICT(device) DO UPDATE SET slot=excluded.slot",
315                        params![
316                            device,
317                            stop.get().to_be_bytes()
318                        ],
319                    )?;
320                    Ok(true)
321                })
322            })
323        })
324        .await
325    }
326}
327
328/// A fixed file path awaiting a storage-deletion decision (§17.10).
329#[derive(Clone, Copy, Debug, PartialEq, Eq)]
330pub struct FileCandidate {
331    /// The uploader named by its path.
332    pub device: DeviceId,
333    /// The stored object's id, the last part of its path.
334    pub object: ObjectId,
335    /// A local row, loss, published row, queued write or upload still names it.
336    pub protected: bool,
337    /// This device's applied slot when the path last lost its local
338    /// protection, or a later one: a write that read through it saw every
339    /// change that left the path unnamed (§17.10).
340    pub since: Slot,
341}
342
343fn insert_candidate(db: &StoreSql, device: DeviceId, object: ObjectId) -> Result<(), DbError> {
344    db.execute(
345        "INSERT INTO _coven_file_candidates(device,object,since) VALUES(?1,?2,(SELECT applied FROM _coven_sequence WHERE singleton=1)) ON CONFLICT DO UPDATE SET since=max(since,excluded.since)",
346        params![device, object],
347    )?;
348    Ok(())
349}
350
351/// Count every candidate from the applied slot `slot`, as though each lost
352/// its protection there.
353pub(crate) fn count_candidates_from(db: &StoreSql, slot: u64) -> Result<(), DbError> {
354    db.execute("UPDATE _coven_file_candidates SET since=?1", [slot.to_be_bytes()])?;
355    Ok(())
356}
357
358/// Stored files named by the records and losses of the audience snapshot
359/// just read into this transaction. A location that won't decode is the
360/// snapshot's own failure.
361pub(crate) fn snapshot_references(
362    db: &StoreSql,
363    schema: &crate::write_schema::WriteSchema,
364    audience: &coven_merge::Audience,
365) -> Result<BTreeSet<(DeviceId, ObjectId)>, DbError> {
366    let mut references = BTreeSet::new();
367    crate::snapshot_install::file_locations(db, schema, audience, |value| {
368        retain_value(value, &mut references)
369    })
370    .and_then(|()| {
371        crate::loss_record::visit(db, audience, |loss| {
372            crate::file_protection::loss(&loss, &mut references);
373            Ok::<_, DbError>(())
374        })
375    })
376    .map_err(|error| match error {
377        DbError::DamagedDatabase => {
378            crate::snapshot_error::invalid("snapshot file location is invalid")
379        }
380        error => error,
381    })?;
382    Ok(references)
383}
384
385/// The stored paths a published change names: the coordinates a file record's
386/// insert or update writes, and those its delete removes. An update names no
387/// coordinates it replaced (D5.5): its author protected them only until the
388/// write was published (§17.10).
389fn retain_change(
390    row: &RowChange,
391    references: &mut BTreeSet<(DeviceId, ObjectId)>,
392) -> Result<(), DbError> {
393    if let Operation::Insert(columns) | Operation::Update(columns) = &row.change.operation {
394        retain_columns(&row.row.table, columns, references)?;
395    }
396    if let Some(column) = crate::file_protection::location(&row.row.table) {
397        if let Some(value) = row.old.get(column) {
398            retain_value(value, references)?;
399        }
400    }
401    Ok(())
402}
403
404fn retain_columns(
405    table: &str,
406    columns: &BTreeMap<String, ColumnValue<Value>>,
407    references: &mut BTreeSet<(DeviceId, ObjectId)>,
408) -> Result<(), DbError> {
409    if let Some(column) = crate::file_protection::location(table) {
410        if let Some(value) = columns.get(column) {
411            retain_value(&value.value, references)?;
412        }
413    }
414    Ok(())
415}
416
417#[cfg(test)]
418#[path = "database_file_retention_tests.rs"]
419pub(crate) mod tests;