Skip to main content

coven_sync/
sequence_files.rs

1//! Uploaded-file deletion is derived from checked references in retained data.
2use super::snapshot_file_scan::{Listing, UploaderScan};
3use crate::{
4    history::History,
5    object_input::ObjectInput,
6    sequence_sync::sequence_operations::SequenceOperation,
7    sequence_sync::{SequenceSync, Session},
8    slot_object::{self, SlotBody},
9    snapshot_authorship::SnapshotAuthorship,
10    snapshot_object::{self, SnapshotObject},
11    OperationKind, OperationRejection, PendingReason, PendingRecord, PendingSubject,
12    RequiredUpdate, SyncError,
13};
14use coven_crypto::{MemberIdentity, StoreKeyring};
15use coven_database::operation_kind;
16use coven_database::{DownloadedPartStream, FileEvidence, NewOperation, Slot, UnprovenFiles};
17use coven_format::{sealed_write::WriteObjectPrefix, sequence::SnapshotAnnouncement};
18use coven_foundation::id_source::{DeviceId, ObjectId};
19use coven_storage::{ObjectPath, StorageFailure};
20use std::{collections::BTreeSet, sync::Arc};
21
22impl SequenceSync {
23    /// The uploaders whose files this device deletes: its own, and each
24    /// device gone for good it is responsible for (§17.10).
25    fn file_uploaders(&self, session: &Session, member: &MemberIdentity) -> BTreeSet<DeviceId> {
26        session
27            .history
28            .state
29            .devices
30            .keys()
31            .copied()
32            .filter(|uploader| session.history.deletes_for(self.device, member.member(), *uploader))
33            .collect()
34    }
35
36    /// The local snapshot capture still in progress, if any: its journal owns
37    /// references no longer present in rows until publication or awaited undo.
38    fn unfinished_capture(
39        records: &[coven_database::OperationRecord],
40    ) -> Result<Option<coven_database::OperationId>, SyncError> {
41        for record in records.iter().filter(|record| record.kind == operation_kind::SEQUENCE) {
42            let work: SequenceOperation = serde_json::from_slice(&record.data)?;
43            if work.snapshot().is_some() {
44                return Ok(Some(record.id));
45            }
46        }
47        Ok(None)
48    }
49
50    /// The journal row owning file deletion, started when absent.
51    async fn files_operation(
52        &self,
53        existing: Option<&coven_database::OperationRecord>,
54    ) -> Result<coven_database::OperationId, SyncError> {
55        Ok(match existing {
56            Some(record) => record.id,
57            None => {
58                self.database
59                    .start_operation(NewOperation {
60                        kind: operation_kind::SEQUENCE_FILES.into(),
61                        data: Vec::new(),
62                        started_by: operation_kind::COVEN.into(),
63                        pauses_writes: false,
64                    })
65                    .await?
66            }
67        })
68    }
69
70    /// A pass's file deletion: list each gone uploader's folder once at the
71    /// entry that stopped it, then delete every unreferenced candidate. A
72    /// failure is held by the operation, which retries under backoff (§17.10).
73    pub(crate) async fn retain_files(
74        &self,
75        session: &Session,
76        ring: &StoreKeyring,
77        member: &MemberIdentity,
78    ) -> Result<(), SyncError> {
79        let uploaders = self.file_uploaders(session, member);
80        let scans = self.uploader_scans(&session.history, &uploaders, Listing::AtStop).await?;
81        let candidates = self.database.file_candidates(uploaders.clone()).await?;
82        let records = self.database.operations().await?;
83        let existing = records.iter().find(|record| record.kind == operation_kind::SEQUENCE_FILES);
84        // A candidate still waiting for another device to read past it gives
85        // a pass nothing to do, however long that device stays away.
86        let seen = self.seen_by_all(session).await?;
87        if scans.is_empty()
88            && existing.is_none()
89            && !candidates.iter().any(|file| file.protected || seen(file.since))
90        {
91            return Ok(());
92        }
93        if Self::unfinished_capture(&records)?.is_some() {
94            return Ok(());
95        }
96        if let Some(record) = existing {
97            if !self.operation_ready(record, ring).await? {
98                return Ok(());
99            }
100        }
101        let id = self.files_operation(existing).await?;
102        let result = coven_database::in_commit_scope(
103            self.waits.begin_attempt(id),
104            self.delete_files(session, ring, member, uploaders, scans, id),
105        )
106        .await;
107        match result {
108            Ok(()) => {
109                self.database.finish_operation(id).await?;
110                self.waits.operation_advanced(id);
111            }
112            Err(error) => {
113                self.record_operation_error(
114                    id,
115                    OperationKind::RetainFiles,
116                    crate::sequence_sync::OperationFailure::Held(error),
117                    ring,
118                )
119                .await?
120            }
121        }
122        Ok(())
123    }
124
125    /// The app's cleanup of the files of devices gone for good (§17.10): for
126    /// every such device this device is responsible for, end the uploads it
127    /// left and delete each snapshot body of its no announcement names
128    /// (§16.6), list its file folder again, then delete every file found
129    /// that nothing refers to and every other device has seen lose its last
130    /// name; later passes delete the rest. It
131    /// returns once that is done, or with the failure that stopped it, which
132    /// the operation also records. A device gone for good later uploads only
133    /// files that can never be referred to, so running this at any time is
134    /// safe.
135    pub async fn clean_up_gone_devices_files(&self) -> Result<(), SyncError> {
136        let mut session = self.session.lock().await;
137        let member = self.member.read()?.ok_or(StorageFailure::MemberKeysMissing)?;
138        let mut ring = self.ring(&mut session, &member).await?;
139        self.read_forward(&mut session, &mut ring, &member).await?;
140        self.settle_open(&mut session, &mut ring, &member).await?;
141        if !self.registered(&session) {
142            return Err(SyncError::Rejected(OperationRejection::NotAllowed));
143        }
144        let uploaders = self.file_uploaders(&session, &member);
145        let scans = self.uploader_scans(&session.history, &uploaders, Listing::Every).await?;
146        let records = self.database.operations().await?;
147        if let Some(capture) = Self::unfinished_capture(&records)? {
148            return Err(SyncError::SnapshotPending(capture));
149        }
150        let existing = records.iter().find(|record| record.kind == operation_kind::SEQUENCE_FILES);
151        let id = self.files_operation(existing).await?;
152        let result = coven_database::in_commit_scope(self.waits.begin_attempt(id), async {
153            // A gone device whose key still works may have started
154            // uploads or stored snapshot bodies since this device last
155            // cleared its storage. Its uploads end before its folders are
156            // listed again, its files' among them (§16.6).
157            for device in uploaders.iter().filter(|device| **device != self.device) {
158                self.clear_gone_storage(&session, *device).await?;
159            }
160            self.delete_files(&session, &ring, &member, uploaders.clone(), scans, id).await
161        })
162        .await;
163        match result {
164            Ok(()) => {
165                self.database.finish_operation(id).await?;
166                self.waits.operation_advanced(id);
167                Ok(())
168            }
169            Err(error) => {
170                self.record_operation_error(
171                    id,
172                    OperationKind::RetainFiles,
173                    crate::sequence_sync::OperationFailure::Returned(&error),
174                    &ring,
175                )
176                .await?;
177                Err(error)
178            }
179        }
180    }
181
182    /// Whether every other device not gone for good has published a write
183    /// that read through `since`, or was added after it (§17.10). A device
184    /// sends its writes in the order it made them, so each one it made
185    /// without reading that far is then published, and the retained slots or
186    /// snapshots show what it names. A gone device's later writes don't
187    /// count (§14.1), and this device's own unpublished writes protect what
188    /// they name locally.
189    async fn seen_by_all(&self, session: &Session) -> Result<impl Fn(Slot) -> bool, SyncError> {
190        let writers = session.history.writers(self.device);
191        let reads = self.database.published_reads(writers.devices().collect()).await?;
192        let through = writers.through(&reads);
193        Ok(move |since: Slot| since.get() <= through)
194    }
195
196    /// List the given folders, then delete each candidate that every other
197    /// device has seen lose its last name, that no retained slot or snapshot
198    /// refers to and that nothing local protects.
199    async fn delete_files(
200        &self,
201        session: &Session,
202        ring: &StoreKeyring,
203        member: &MemberIdentity,
204        uploaders: BTreeSet<DeviceId>,
205        scans: Vec<UploaderScan>,
206        id: coven_database::OperationId,
207    ) -> Result<(), SyncError> {
208        self.scan_uploaders(scans).await?;
209        let candidates = self.database.file_candidates(uploaders).await?;
210        let protected: Vec<_> = candidates
211            .iter()
212            .filter(|file| file.protected)
213            .map(|file| (file.device, file.object))
214            .collect();
215        self.database.settle_file_candidates(protected, Vec::new()).await?;
216        // A device's reads only advance, so what it has read now it has read
217        // when the candidates are read again below.
218        let seen = self.seen_by_all(session).await?;
219        if !candidates.iter().any(|file| !file.protected && seen(file.since)) {
220            return Ok(());
221        }
222        let mut referenced = BTreeSet::new();
223        let first =
224            session.floor.as_ref().map_or(coven_merge::Slot::FIRST, |floor| floor.header.first);
225        for number in coven_merge::SlotRange::new(first, session.through) {
226            let path = self.path(number);
227            let input = match ObjectInput::retained(&self.database, &path).await? {
228                Some(input) => input,
229                None => self.fetch(&path).await?.ok_or(StorageFailure::NotFound)?,
230            };
231            let evidence = self.slot_files(&input, &session.history, ring, member).await?;
232            referenced.extend(self.complete_files(evidence, id, &session.history, member)?);
233            input.release().await?;
234        }
235        for announcement in session.history.announcements.values() {
236            if self.database.snapshot_absent(announcement.snapshot.clone()).await? {
237                continue;
238            }
239            let evidence = self.snapshot_files(announcement, &session.history, ring).await?;
240            referenced.extend(self.complete_files(evidence, id, &session.history, member)?);
241        }
242        // Local protection is checked after the asynchronous reference reads.
243        for file in self
244            .database
245            .file_candidates(candidates.iter().map(|file| file.device).collect())
246            .await?
247        {
248            if file.protected
249                || !seen(file.since)
250                || referenced.contains(&(file.device, file.object))
251            {
252                continue;
253            }
254            self.storage.delete(&ObjectPath::file(self.store(), file.device, file.object)).await?;
255            self.database
256                .settle_file_candidates(Vec::new(), vec![(file.device, file.object)])
257                .await?;
258        }
259        Ok(())
260    }
261
262    fn complete_files(
263        &self,
264        evidence: FileEvidence,
265        id: coven_database::OperationId,
266        history: &History,
267        member: &MemberIdentity,
268    ) -> Result<BTreeSet<(DeviceId, ObjectId)>, SyncError> {
269        match evidence {
270            FileEvidence::Complete(files) => Ok(files),
271            FileEvidence::Unproven(reason) => {
272                let reason = match reason {
273                    UnprovenFiles::Parts(parts) => {
274                        let (audience, key) =
275                            parts.into_iter().next().expect("unread parts name a key");
276                        let holders = Some(history.key_holders(&audience, key, member.member()));
277                        PendingReason::KeyUnavailable { audience, key, holders }
278                    }
279                    UnprovenFiles::Schema(version) => {
280                        PendingReason::UpdateRequired(RequiredUpdate::AppSchema { version })
281                    }
282                };
283                Err(SyncError::Pending(Box::new(PendingRecord::new(
284                    PendingSubject::Operation { id, kind: OperationKind::RetainFiles },
285                    reason,
286                ))))
287            }
288        }
289    }
290
291    async fn reuse_files(
292        &self,
293        recorded: Option<FileEvidence>,
294        history: &History,
295        ring: &StoreKeyring,
296    ) -> Result<Option<FileEvidence>, SyncError> {
297        let Some(evidence) = recorded else {
298            return Ok(None);
299        };
300        if let FileEvidence::Unproven(reason) = &evidence {
301            if reason.changed(
302                |audience, key| history.accepted(ring, audience, key),
303                &self.database.write_schema_versions().await?,
304            ) {
305                return Ok(None);
306            }
307        }
308        Ok(Some(evidence))
309    }
310
311    pub(crate) async fn slot_files(
312        &self,
313        input: &ObjectInput,
314        history: &History,
315        ring: &StoreKeyring,
316        member: &MemberIdentity,
317    ) -> Result<FileEvidence, SyncError> {
318        if let Some(evidence) = self.reuse_files(input.files().await?, history, ring).await? {
319            return Ok(evidence);
320        }
321        input.receive(self.storage.as_ref()).await?;
322        let received = slot_object::open(
323            input.local_bytes().await?,
324            &input.object().path,
325            ring,
326            history,
327            |audience| crate::effects::belongs(&history.state, audience, &member.member()),
328        )?;
329        let mut files = BTreeSet::new();
330        let mut unread = BTreeSet::new();
331        let mut schema = None;
332        if let (_, SlotBody::Writes { writes, .. }) = received.into_parts() {
333            for write in writes {
334                let prefix = WriteObjectPrefix::decode(&write.authorship.evidence.prefix)?;
335                let skipped = write
336                    .header
337                    .parts
338                    .iter()
339                    .zip(&write.parts)
340                    .zip(prefix.part_keys)
341                    .filter_map(|((header, part), key)| {
342                        matches!(part, DownloadedPartStream::Skipped)
343                            .then_some((header.audience.clone(), key))
344                    })
345                    .collect();
346                match self.database.write_file_references(write, skipped).await? {
347                    FileEvidence::Complete(references) => files.extend(references),
348                    FileEvidence::Unproven(UnprovenFiles::Parts(parts)) => unread.extend(parts),
349                    FileEvidence::Unproven(UnprovenFiles::Schema(version)) => {
350                        schema = Some(version)
351                    }
352                }
353            }
354        }
355        let evidence = match schema {
356            Some(version) => FileEvidence::Unproven(UnprovenFiles::Schema(version)),
357            None if !unread.is_empty() => FileEvidence::Unproven(UnprovenFiles::Parts(unread)),
358            None => FileEvidence::Complete(files),
359        };
360        input.keep_files(evidence.clone()).await?;
361        Ok(evidence)
362    }
363
364    pub(crate) async fn snapshot_files(
365        &self,
366        announcement: &SnapshotAnnouncement,
367        history: &History,
368        ring: &StoreKeyring,
369    ) -> Result<FileEvidence, SyncError> {
370        let path = snapshot_object::path(self.store(), &announcement.snapshot)?;
371        let object =
372            match SnapshotObject::read_prefix(&self.database, self.storage.as_ref(), &path).await {
373                Ok(object) => object,
374                Err(SyncError::Storage(error)) if error.failure() == StorageFailure::NotFound => {
375                    self.database
376                        .record_absent_snapshot(announcement.snapshot.clone(), BTreeSet::new())
377                        .await?;
378                    return Ok(FileEvidence::Complete(BTreeSet::new()));
379                }
380                Err(error) => return Err(error),
381            };
382        if let Some(evidence) = self.reuse_files(object.files().await?, history, ring).await? {
383            return Ok(evidence);
384        }
385        let prefix = object.prefix();
386        let evidence = if !history.accepted(ring, &prefix.audience, prefix.key) {
387            FileEvidence::Unproven(UnprovenFiles::Parts(BTreeSet::from([(
388                prefix.audience.clone(),
389                prefix.key,
390            )])))
391        } else if !self.database.write_schema_versions().await?.contains(&match prefix.purpose {
392            coven_format::sequence::Purpose::Raise(version) => version,
393            _ => self.database.schema_version().await?,
394        }) {
395            let coven_format::sequence::Purpose::Raise(version) = prefix.purpose else {
396                unreachable!("current schema is supported");
397            };
398            FileEvidence::Unproven(UnprovenFiles::Schema(version))
399        } else {
400            object.check_announced(self.storage.as_ref(), announcement).await?;
401            let staging = self
402                .database
403                .stage_snapshot(announcement.snapshot.clone(), object.prefix().clone())
404                .await?;
405            FileEvidence::Complete(
406                self.snapshot_references(&object, announcement, history, ring, staging).await?,
407            )
408        };
409        object.keep_files(evidence.clone()).await?;
410        object.release().await?;
411        Ok(evidence)
412    }
413}
414
415impl SequenceSync {
416    /// The files a snapshot's rows and losses name, from its body read once
417    /// into `staging` for the loader's checks. The check is the staging's
418    /// last phase, and a failure before it removes what it staged (§16.2).
419    async fn snapshot_references(
420        &self,
421        object: &SnapshotObject,
422        announcement: &SnapshotAnnouncement,
423        history: &History,
424        ring: &StoreKeyring,
425        mut staging: coven_database::SnapshotStaging,
426    ) -> Result<BTreeSet<(DeviceId, ObjectId)>, SyncError> {
427        let path = object.path();
428        let read = async {
429            let body = object
430                .read_body(self.storage.as_ref(), ring, announcement, Some(&mut staging), None)
431                .await?;
432            staging
433                .finish()
434                .await
435                .map_err(|error| crate::write_object::database_failure(path, error))?;
436            Ok(body)
437        }
438        .await;
439        let body = match read {
440            Ok(body) => body,
441            Err(error) => return Err(self.end_staged(error, staging, announcement, ring).await),
442        };
443        let author = match object.authenticate(
444            history,
445            &announcement.snapshot,
446            &body.digest,
447            &announcement.digest,
448        ) {
449            Ok(author) => author,
450            Err(error) => {
451                return Err(match staging.discard().await {
452                    Ok(()) => error,
453                    Err(cleanup) => SyncError::cleanup(error, cleanup.into()),
454                })
455            }
456        };
457        let authorship = Arc::new(SnapshotAuthorship {
458            store: self.store(),
459            history: history.clone(),
460            ring: ring.clone(),
461        });
462        match self
463            .database
464            .validate_snapshot(announcement.snapshot.clone(), staging, author, authorship)
465            .await
466        {
467            Ok(files) => Ok(files),
468            Err(error) => Err(self
469                .judge_staged_body(error, &[(announcement.clone(), path.clone())], ring, |error| {
470                    crate::write_object::database_failure(path, error)
471                })
472                .await),
473        }
474    }
475}
476
477#[cfg(test)]
478#[path = "sequence_files_tests.rs"]
479mod tests;