1use super::StoreDatabase;
4use crate::file_protection::value as retain_value;
5use crate::sqlite::StoreSql;
6use crate::{
7 DbError, DownloadedPartStream, DownloadedWriteStream, FileEvidence, Slot, UnprovenFiles,
8};
9use coven_format::{value::Value, write::RowChange};
10use coven_foundation::id_source::{DeviceId, ObjectId};
11use coven_merge::{ColumnValue, Operation};
12use sqlive::rusqlite::params;
13use std::collections::{BTreeMap, BTreeSet};
14use std::ops::RangeInclusive;
15
16#[cfg(test)]
18pub(crate) struct FileRetention {
19 pub references: BTreeSet<(DeviceId, ObjectId)>,
22}
23
24impl StoreDatabase {
25 pub async fn snapshot_absent(
27 &self,
28 id: coven_format::admin::SnapshotId,
29 ) -> Result<bool, DbError> {
30 self.call(move |inner| {
31 inner.access.with_reader(|db, _| Ok(db.query_row(
32 "SELECT EXISTS(SELECT 1 FROM _coven_absent_snapshots WHERE audience=?1 AND device=?2 AND number=?3)",
33 params![&id.audience, id.device, id.number.to_be_bytes()], |row| row.get(0))?))
34 }).await
35 }
36
37 pub async fn record_absent_snapshot(
40 &self,
41 id: coven_format::admin::SnapshotId,
42 files: BTreeSet<(DeviceId, ObjectId)>,
43 ) -> Result<(), DbError> {
44 self.call(move |inner| {
45 let number = std::num::NonZeroU64::new(id.number).ok_or(DbError::DamagedDatabase)?;
46 let path = coven_format::path::ObjectPath::snapshot(
47 inner.access.directory.id(),
48 id.audience.clone(),
49 id.device,
50 number,
51 );
52 inner.access.with_writer(|writer, _| writer.transaction(|db| {
53 for (device, object) in files { insert_candidate(db, device, object)?; }
54 db.execute("INSERT INTO _coven_absent_snapshots(audience,device,number) VALUES(?1,?2,?3) ON CONFLICT DO NOTHING",
55 params![&id.audience, id.device, id.number.to_be_bytes()])?;
56 Ok::<_, DbError>(())
57 }))?;
58 inner.inputs.forget_path(path.as_str())
59 }).await
60 }
61 pub async fn forget_announcements(
67 &self,
68 slots: BTreeSet<Slot>,
69 snapshots: BTreeSet<coven_format::admin::SnapshotId>,
70 listings: Vec<crate::NewOperation>,
71 ) -> Result<(), DbError> {
72 self.call(move |inner| {
73 inner.access.with_writer(|writer, _| {
74 writer.transaction(|db| {
75 crate::sequence_apply::forget_announcements(db, &slots)?;
76 for id in &snapshots {
77 db.execute(
78 "DELETE FROM _coven_absent_snapshots WHERE audience=?1 AND device=?2 AND number=?3",
79 params![&id.audience, id.device, id.number.to_be_bytes()],
80 )?;
81 }
82 for listing in &listings {
83 crate::operation::insert(db, listing)?;
84 }
85 Ok(())
86 })
87 })
88 })
89 .await
90 }
91
92 #[cfg(test)]
94 pub(crate) async fn retained_files(&self) -> Result<FileRetention, DbError> {
95 self.call(move |inner| {
96 inner.access.with_committed(|db, _| {
97 let references = db.query("SELECT device,object FROM _coven_row_file_refs UNION SELECT device,object FROM _coven_published_file_refs UNION SELECT device,object FROM _coven_queued_file_refs UNION SELECT device,object FROM _coven_file_uploads", [], |r| Ok((r.get::<_, DeviceId>(0)?,r.get::<_, ObjectId>(1)?)))?.into_iter().collect();
98 Ok(FileRetention { references })
99 })
100 })
101 .await
102 }
103
104 pub async fn write_file_references<R: std::io::Read + Send + 'static>(
109 &self,
110 write: DownloadedWriteStream<R>,
111 skipped: BTreeSet<(coven_merge::Audience, coven_foundation::id_source::KeyId)>,
112 ) -> Result<FileEvidence, DbError> {
113 self.call(move |inner| {
114 let mut references = BTreeSet::new();
115 write.header.encode()?;
116 let version = write.header.header.schema_version;
117 if !inner.access.write_schema.versions.contains(&version) {
118 return Ok(FileEvidence::Unproven(UnprovenFiles::Schema(version)));
119 }
120 if write.header.parts.len() != write.parts.len() {
121 return Err(crate::SnapshotError::Inconsistent(
122 "write stream count differs from its header",
123 )
124 .into());
125 }
126 let opened = write
127 .parts
128 .iter()
129 .filter(|part| matches!(part, DownloadedPartStream::Opened(_)))
130 .count();
131 if (opened == write.parts.len()) != skipped.is_empty() {
132 return Err(crate::SnapshotError::Inconsistent(
133 "skipped parts disagree with the parts named skipped",
134 )
135 .into());
136 }
137 if !skipped.is_empty() {
138 return Ok(FileEvidence::Unproven(UnprovenFiles::Parts(skipped)));
139 }
140 for (header, part) in write.header.parts.into_iter().zip(write.parts) {
141 let DownloadedPartStream::Opened(input) = part else {
142 unreachable!("every part opened");
143 };
144 crate::download_stream::read_part(input, header, |frame| {
145 if let coven_format::dismissal::WriteFrame::Change(row) = frame {
146 retain_change(&row, &mut references)?;
147 }
148 Ok::<_, DbError>(())
149 })?;
150 }
151 Ok(FileEvidence::Complete(references))
152 })
153 .await
154 }
155
156 pub async fn write_schema_versions(&self) -> Result<RangeInclusive<u32>, DbError> {
158 self.call(move |inner| Ok(inner.access.write_schema.versions.clone())).await
159 }
160
161 pub async fn file_candidates(
165 &self,
166 uploaders: BTreeSet<DeviceId>,
167 ) -> Result<Vec<FileCandidate>, DbError> {
168 self.call(move |inner| {
169 inner.access.with_committed(|db, _| {
170 let mut candidates = Vec::new();
171 for device in uploaders {
172 candidates.extend(db.query(
173 "SELECT object,EXISTS(SELECT 1 FROM _coven_file_protection p WHERE p.device=c.device AND p.object=c.object),since FROM _coven_file_candidates c WHERE c.device=?1 ORDER BY c.object",
174 [device],
175 |r| {
176 Ok(FileCandidate {
177 device,
178 object: r.get(0)?,
179 protected: r.get(1)?,
180 since: crate::store_state_tables::slot(r, 2)?,
181 })
182 },
183 )?);
184 }
185 Ok(candidates)
186 })
187 })
188 .await
189 }
190
191 pub async fn published_reads(
195 &self,
196 devices: BTreeSet<DeviceId>,
197 ) -> Result<BTreeMap<DeviceId, u64>, DbError> {
198 self.call(move |inner| {
199 inner.access.with_committed(|db, _| {
200 let mut reads = BTreeMap::new();
201 for device in devices {
202 if let Some(read) = crate::write_metadata::last_landed_read(db, device)? {
203 reads.insert(device, read);
204 }
205 }
206 Ok(reads)
207 })
208 })
209 .await
210 }
211
212 pub async fn settle_file_candidates(
216 &self,
217 protected: Vec<(DeviceId, ObjectId)>,
218 deleted: Vec<(DeviceId, ObjectId)>,
219 ) -> Result<(), DbError> {
220 if protected.is_empty() && deleted.is_empty() {
221 return Ok(());
222 }
223 self.call(move |inner| {
224 inner.access.with_writer(|writer, _| {
225 writer.transaction(|db| {
226 for (device, object) in protected {
227 db.execute(
228 "DELETE FROM _coven_file_candidates WHERE device=?1 AND object=?2 AND EXISTS(SELECT 1 FROM _coven_file_protection WHERE device=?1 AND object=?2)",
229 params![device, object],
230 )?;
231 }
232 for (device, object) in deleted {
233 db.execute(
234 "DELETE FROM _coven_file_candidates WHERE device=?1 AND object=?2",
235 params![device, object],
236 )?;
237 }
238 Ok(())
239 })
240 })
241 })
242 .await
243 }
244
245 #[cfg(test)]
248 pub async fn add_file_candidates(
249 &self,
250 files: BTreeSet<(DeviceId, ObjectId)>,
251 ) -> Result<(), DbError> {
252 if files.is_empty() {
253 return Ok(());
254 }
255 self.call(move |inner| {
256 inner.access.with_writer(|writer, _| {
257 writer.transaction(|db| {
258 for (device, object) in files {
259 insert_candidate(db, device, object)?;
260 }
261 Ok(())
262 })
263 })
264 })
265 .await
266 }
267
268 pub async fn file_scan(&self, device: DeviceId) -> Result<Option<Slot>, DbError> {
271 self.call(move |inner| {
272 inner.access.with_reader(|db, _| {
273 Ok(db
274 .query("SELECT slot FROM _coven_file_scans WHERE device=?1", [device], |r| {
275 crate::store_state_tables::slot(r, 0)
276 })?
277 .pop())
278 })
279 })
280 .await
281 }
282
283 pub async fn complete_file_scan(
294 &self,
295 device: DeviceId,
296 stop: Slot,
297 objects: BTreeSet<ObjectId>,
298 ) -> Result<bool, DbError> {
299 self.call(move |inner| {
300 inner.access.with_writer(|writer, _| {
301 writer.transaction(|db| {
302 let kept: bool = db.query_row(
303 "SELECT EXISTS(SELECT 1 FROM _coven_cutoffs c JOIN _coven_devices d ON d.device=?1 WHERE c.slot=?2 AND (c.device=d.device OR c.member=d.member))",
304 params![device, stop.get().to_be_bytes()],
305 |r| r.get(0),
306 )?;
307 if !kept {
308 return Ok(false);
309 }
310 for object in objects {
311 insert_candidate(db, device, object)?;
312 }
313 db.execute(
314 "INSERT INTO _coven_file_scans(device,slot) VALUES(?1,?2) ON CONFLICT(device) DO UPDATE SET slot=excluded.slot",
315 params![
316 device,
317 stop.get().to_be_bytes()
318 ],
319 )?;
320 Ok(true)
321 })
322 })
323 })
324 .await
325 }
326}
327
328#[derive(Clone, Copy, Debug, PartialEq, Eq)]
330pub struct FileCandidate {
331 pub device: DeviceId,
333 pub object: ObjectId,
335 pub protected: bool,
337 pub since: Slot,
341}
342
343fn insert_candidate(db: &StoreSql, device: DeviceId, object: ObjectId) -> Result<(), DbError> {
344 db.execute(
345 "INSERT INTO _coven_file_candidates(device,object,since) VALUES(?1,?2,(SELECT applied FROM _coven_sequence WHERE singleton=1)) ON CONFLICT DO UPDATE SET since=max(since,excluded.since)",
346 params![device, object],
347 )?;
348 Ok(())
349}
350
351pub(crate) fn count_candidates_from(db: &StoreSql, slot: u64) -> Result<(), DbError> {
354 db.execute("UPDATE _coven_file_candidates SET since=?1", [slot.to_be_bytes()])?;
355 Ok(())
356}
357
358pub(crate) fn snapshot_references(
362 db: &StoreSql,
363 schema: &crate::write_schema::WriteSchema,
364 audience: &coven_merge::Audience,
365) -> Result<BTreeSet<(DeviceId, ObjectId)>, DbError> {
366 let mut references = BTreeSet::new();
367 crate::snapshot_install::file_locations(db, schema, audience, |value| {
368 retain_value(value, &mut references)
369 })
370 .and_then(|()| {
371 crate::loss_record::visit(db, audience, |loss| {
372 crate::file_protection::loss(&loss, &mut references);
373 Ok::<_, DbError>(())
374 })
375 })
376 .map_err(|error| match error {
377 DbError::DamagedDatabase => {
378 crate::snapshot_error::invalid("snapshot file location is invalid")
379 }
380 error => error,
381 })?;
382 Ok(references)
383}
384
385fn retain_change(
390 row: &RowChange,
391 references: &mut BTreeSet<(DeviceId, ObjectId)>,
392) -> Result<(), DbError> {
393 if let Operation::Insert(columns) | Operation::Update(columns) = &row.change.operation {
394 retain_columns(&row.row.table, columns, references)?;
395 }
396 if let Some(column) = crate::file_protection::location(&row.row.table) {
397 if let Some(value) = row.old.get(column) {
398 retain_value(value, references)?;
399 }
400 }
401 Ok(())
402}
403
404fn retain_columns(
405 table: &str,
406 columns: &BTreeMap<String, ColumnValue<Value>>,
407 references: &mut BTreeSet<(DeviceId, ObjectId)>,
408) -> Result<(), DbError> {
409 if let Some(column) = crate::file_protection::location(table) {
410 if let Some(value) = columns.get(column) {
411 retain_value(&value.value, references)?;
412 }
413 }
414 Ok(())
415}
416
417#[cfg(test)]
418#[path = "database_file_retention_tests.rs"]
419pub(crate) mod tests;