Skip to main content

coven_database/
transfers_in_flight.rs

1//! The stored objects this process is transferring now (§4, §17.8, §17.9):
2//! the ones it is sending, and the ones it is downloading to keep whole.
3//! Nothing about them is stored: an attempt ends when its task does, a crash
4//! included, so a reopened store starts with none. App SQL reads them through
5//! coven's SQL functions, as `coven_uploads.sending` and the `coven_sending`
6//! and `coven_downloading` views, which name each file over an object. A
7//! transaction reads which objects are in flight as its snapshot saw them
8//! (`memory_facts.rs`), and a live query that reads one runs again as a
9//! transfer it shows starts or ends. How far each has got is progress no
10//! table holds: a `TransferWatch` hears it, by file, and how each ended (E7,
11//! E8).
12
13use crate::memory_facts::{MemoryFacts, SharedFacts};
14use crate::store_failure::StoreHandle;
15use crate::transfer_watch::{changes_since, Heard, TransferChange, TransferEnd};
16use coven_foundation::id_source::{DeviceId, FileId, ObjectId};
17use coven_foundation::locks::lock_unpoisoned;
18use sqlive::rusqlite::functions::FunctionFlags;
19use sqlive::{Setup, SnapshotValue};
20use std::collections::BTreeMap;
21use std::sync::{Arc, Mutex};
22use tokio::sync::watch;
23
24/// How far one transfer has got: the bytes of its encrypted object
25/// handed to the connection or received, in whatever order, out of the
26/// object's whole size. Known from the transfer's start; never stored.
27#[derive(Clone, Copy, Debug, PartialEq, Eq)]
28pub struct TransferProgress {
29    /// The object's bytes moved so far.
30    pub bytes_transferred: u64,
31    /// The object's whole encrypted size.
32    pub bytes_total: u64,
33}
34
35/// One way files go, and the SQL that reads the files going that way.
36struct Direction {
37    /// What a start or end changes, as live queries read it: a view's
38    /// column, or a view whose rows it changes.
39    read_as: &'static [(&'static str, &'static str)],
40    /// `name(object)`, whether the stored object with this id is in flight,
41    /// for a view that asks of each of its rows.
42    contains: Option<&'static str>,
43    /// `name()`, the objects in flight as a JSON array of hex text, each
44    /// one's uploader's eight id bytes and then its own sixteen: what a view
45    /// joins by index.
46    objects: &'static str,
47    /// The objects in flight among the store's facts, by their uploaders.
48    facts: fn(&MemoryFacts) -> &BTreeMap<ObjectId, DeviceId>,
49    /// The same, to change.
50    facts_mut: fn(&mut MemoryFacts) -> &mut BTreeMap<ObjectId, DeviceId>,
51}
52
53const SENDING: Direction = Direction {
54    read_as: &[("coven_uploads", "sending"), ("coven_sending", "")],
55    contains: Some("_coven_sending"),
56    objects: "_coven_sending_objects",
57    facts: |facts| &facts.sending,
58    facts_mut: |facts| &mut facts.sending,
59};
60
61const DOWNLOADING: Direction = Direction {
62    read_as: &[("coven_downloading", "")],
63    contains: None,
64    objects: "_coven_downloading_objects",
65    facts: |facts| &facts.downloading,
66    facts_mut: |facts| &mut facts.downloading,
67};
68
69/// Coven's functions that read the files in flight, which app SQL reaches
70/// only through coven's views.
71pub(crate) const FUNCTIONS: [&str; 3] =
72    ["_coven_sending", "_coven_sending_objects", "_coven_downloading_objects"];
73
74/// Both ways files go: one owner per open store, whose clones share it.
75#[derive(Clone)]
76pub(crate) struct TransfersInFlight {
77    uploads: InFlight,
78    downloads: InFlight,
79}
80
81impl TransfersInFlight {
82    /// No transfer in flight yet. Each start and end changes `facts` and
83    /// tells `observer`.
84    pub(crate) fn new(observer: StoreHandle, facts: SharedFacts) -> Self {
85        Self {
86            uploads: InFlight::new(&SENDING, observer.clone(), facts.clone()),
87            downloads: InFlight::new(&DOWNLOADING, observer, facts),
88        }
89    }
90
91    /// The files this process is sending, as the upload worker marks them.
92    pub(crate) fn uploads(&self) -> InFlight {
93        self.uploads.clone()
94    }
95
96    /// The files this process is downloading to keep whole, as each pin's
97    /// download marks them.
98    pub(crate) fn downloads(&self) -> InFlight {
99        self.downloads.clone()
100    }
101}
102
103/// Register both ways' functions on a connection to the store's database,
104/// reading the objects in flight as each transaction's snapshot saw them.
105/// None is deterministic; all are safe in coven's views.
106pub(crate) fn register(
107    setup: &Setup<'_, crate::app_policy::CovenPolicy>,
108    at_snapshot: &SnapshotValue<Arc<MemoryFacts>>,
109) -> sqlive::rusqlite::Result<()> {
110    register_direction(&SENDING, setup, at_snapshot)?;
111    register_direction(&DOWNLOADING, setup, at_snapshot)
112}
113
114fn register_direction(
115    direction: &'static Direction,
116    setup: &Setup<'_, crate::app_policy::CovenPolicy>,
117    at_snapshot: &SnapshotValue<Arc<MemoryFacts>>,
118) -> sqlive::rusqlite::Result<()> {
119    let flags = FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_INNOCUOUS;
120    if let Some(name) = direction.contains {
121        let facts = at_snapshot.clone();
122        setup.scalar_function(name, 1, flags, move |call| {
123            // Anything but an object id fails the statement.
124            let object = call.context().get::<ObjectId>(0)?;
125            Ok((direction.facts)(&facts.get()).contains_key(&object))
126        })?;
127    }
128    let facts = at_snapshot.clone();
129    setup.scalar_function(direction.objects, 0, flags, move |_| {
130        let objects: Vec<String> = (direction.facts)(&facts.get())
131            .iter()
132            .map(|(object, device)| format!("{:016x}{}", device.0, object.0.simple()))
133            .collect();
134        serde_json::to_string(&objects)
135            .map_err(|error| sqlive::rusqlite::Error::UserFunctionError(Box::new(error)))
136    })
137}
138
139/// The stored objects this process is transferring one way. The worker
140/// transferring them adds each attempt as it starts and removes it as it
141/// ends, saying how, cancelled included. Clones share it.
142#[derive(Clone)]
143pub struct InFlight {
144    shared: Arc<InFlightShared>,
145}
146
147struct InFlightShared {
148    direction: &'static Direction,
149    state: Mutex<InFlightState>,
150    /// Told of each start and end, as of a commit that changed what the
151    /// live queries reading the direction's views read.
152    observer: StoreHandle,
153    /// The store's facts, which hold each object in flight while one of its
154    /// attempts is, changed in the same step as `state`.
155    facts: SharedFacts,
156    /// Tells each watch that a file started, moved or ended, after the state
157    /// changed; a watch reads the change from the state.
158    changed: watch::Sender<()>,
159}
160
161#[derive(Default)]
162struct InFlightState {
163    /// Each object in flight, with its attempts. Its uploader is among the
164    /// store's facts.
165    objects: BTreeMap<ObjectId, Entry>,
166    /// The next attempt's number, so an ended attempt is never taken for a
167    /// later one of the same object.
168    next: u64,
169    /// Each open watch, by number, with what its reader last heard of each
170    /// file: how far it had got, or how it ended since. `None` before the
171    /// watch's first read. An attempt that ends marks its files ended here
172    /// in the same step that takes them out of flight, so no reader can
173    /// hear a file gone before it hears how it ended.
174    watches: BTreeMap<u64, Option<BTreeMap<FileId, Heard>>>,
175    /// The next watch's number.
176    next_watch: u64,
177}
178
179struct Entry {
180    /// Its attempts in flight, oldest first, each with the files it moves
181    /// the object's bytes for and how far it has got. One pin can keep the
182    /// same object twice at once: it stays in flight until both end, and
183    /// shows the newer one's progress.
184    attempts: Vec<(u64, Vec<FileId>, TransferProgress)>,
185}
186
187impl InFlightState {
188    /// An attempt for `files` ended as `end` says: each of them no attempt
189    /// still moves is ended for every watch that heard it.
190    fn ended(&mut self, files: &[FileId], end: TransferEnd) {
191        let now = self.progress();
192        for file in files.iter().filter(|file| !now.contains_key(*file)) {
193            for heard in self.watches.values_mut().flatten() {
194                if let Some(heard) = heard.get_mut(file) {
195                    *heard = Heard::Ended(end);
196                }
197            }
198        }
199    }
200
201    /// Each file in flight with the progress its object's newest attempt
202    /// shows.
203    fn progress(&self) -> BTreeMap<FileId, TransferProgress> {
204        let mut progress = BTreeMap::new();
205        for entry in self.objects.values() {
206            for (_, files, moved) in &entry.attempts {
207                for file in files {
208                    progress.insert(*file, *moved);
209                }
210            }
211        }
212        progress
213    }
214}
215
216impl InFlight {
217    fn new(direction: &'static Direction, observer: StoreHandle, facts: SharedFacts) -> Self {
218        Self {
219            shared: Arc::new(InFlightShared {
220                direction,
221                state: Mutex::new(InFlightState::default()),
222                observer,
223                facts,
224                changed: watch::channel(()).0,
225            }),
226        }
227    }
228
229    fn state(&self) -> std::sync::MutexGuard<'_, InFlightState> {
230        lock_unpoisoned(&self.shared.state)
231    }
232
233    /// The stored objects in flight now.
234    pub fn current(&self) -> std::collections::BTreeSet<ObjectId> {
235        self.state().objects.keys().copied().collect()
236    }
237
238    /// Mark the stored object, which `device` uploaded, as in flight for
239    /// `files` until the returned guard drops, as far as `progress` says.
240    pub fn start(
241        &self,
242        device: DeviceId,
243        object: ObjectId,
244        files: Vec<FileId>,
245        progress: TransferProgress,
246    ) -> Transferring {
247        let (attempt, started) = {
248            let mut state = self.state();
249            let attempt = state.next;
250            state.next += 1;
251            let entry =
252                state.objects.entry(object).or_insert_with(|| Entry { attempts: Vec::new() });
253            entry.attempts.push((attempt, files, progress));
254            let started = entry.attempts.len() == 1;
255            if started {
256                self.shared.facts.change(|facts| {
257                    (self.shared.direction.facts_mut)(facts).insert(object, device)
258                });
259            }
260            (attempt, started)
261        };
262        self.shared.changed.send_replace(());
263        if started {
264            self.changed();
265        }
266        Transferring { owner: self.clone(), object, attempt, end: TransferEnd::Dropped }
267    }
268
269    /// Hear how far each file in flight has got: all of them first, then
270    /// each change, and how each file ended, until the store closes.
271    pub fn subscribe(&self) -> crate::TransferWatch {
272        let number = {
273            let mut state = self.state();
274            let number = state.next_watch;
275            state.next_watch += 1;
276            state.watches.insert(number, None);
277            number
278        };
279        crate::TransferWatch::new(
280            self.clone(),
281            number,
282            self.shared.changed.subscribe(),
283            self.shared.observer.subscribe(),
284        )
285    }
286
287    /// What watch `number` has yet to hear: every file in flight on its
288    /// first read, and after that each change since its last.
289    pub(crate) fn read(&self, number: u64) -> Vec<TransferChange> {
290        let mut state = self.state();
291        let now = state.progress();
292        let heard = state.watches.get_mut(&number).expect("a watch is registered until it drops");
293        match heard {
294            Some(heard) => changes_since(heard, now),
295            None => {
296                *heard = Some(
297                    now.iter().map(|(file, progress)| (*file, Heard::Moved(*progress))).collect(),
298                );
299                vec![TransferChange::Reset(now)]
300            }
301        }
302    }
303
304    /// Watch `number` dropped: nothing hears for it any more.
305    pub(crate) fn forget(&self, number: u64) {
306        self.state().watches.remove(&number);
307    }
308
309    /// The attempt has got `bytes` of the way; an attempt that ended moves
310    /// nothing.
311    fn reached(&self, object: ObjectId, attempt: u64, bytes: u64) {
312        let mut state = self.state();
313        let Some(progress) = state.objects.get_mut(&object).and_then(|entry| {
314            entry
315                .attempts
316                .iter_mut()
317                .find(|(number, _, _)| *number == attempt)
318                .map(|(_, _, progress)| progress)
319        }) else {
320            return;
321        };
322        progress.bytes_transferred = bytes;
323        drop(state);
324        self.shared.changed.send_replace(());
325    }
326
327    /// Tell live queries after the files changed, so a query that ran
328    /// before the change runs again, and one that runs after it reads them.
329    fn changed(&self) {
330        self.shared.observer.changed_without_write(self.shared.direction.read_as);
331    }
332}
333
334/// One attempt to transfer a stored object. Its worker ends it as its
335/// transfer ended; dropping it otherwise, as a cancelled attempt does, ends
336/// it as dropped.
337pub struct Transferring {
338    owner: InFlight,
339    object: ObjectId,
340    attempt: u64,
341    /// How the attempt ends when this drops.
342    end: TransferEnd,
343}
344
345impl Transferring {
346    /// What tells this attempt's progress, for whatever moves its bytes.
347    pub fn meter(&self) -> TransferMeter {
348        TransferMeter { owner: self.owner.clone(), object: self.object, attempt: self.attempt }
349    }
350
351    /// End the attempt as `end` says.
352    pub fn end(mut self, end: TransferEnd) {
353        self.end = end;
354    }
355}
356
357/// Tells one attempt how far it has got. Once the attempt ends, it tells
358/// nothing: a request still finishing can't move a later attempt.
359#[derive(Clone)]
360pub struct TransferMeter {
361    owner: InFlight,
362    object: ObjectId,
363    attempt: u64,
364}
365
366impl TransferMeter {
367    /// The attempt has moved `bytes` of the file's object.
368    pub fn reached(&self, bytes: u64) {
369        self.owner.reached(self.object, self.attempt, bytes);
370    }
371}
372
373impl Drop for Transferring {
374    fn drop(&mut self) {
375        let ended = {
376            let mut state = self.owner.state();
377            let attempt = state.objects.get_mut(&self.object).and_then(|entry| {
378                let at =
379                    entry.attempts.iter().position(|(attempt, _, _)| *attempt == self.attempt)?;
380                let (_, files, _) = entry.attempts.remove(at);
381                Some((files, entry.attempts.is_empty()))
382            });
383            attempt.map(|(files, ended)| {
384                if ended {
385                    state.objects.remove(&self.object);
386                    let shared = &self.owner.shared;
387                    shared
388                        .facts
389                        .change(|facts| (shared.direction.facts_mut)(facts).remove(&self.object));
390                }
391                state.ended(&files, self.end);
392                ended
393            })
394        };
395        match ended {
396            Some(ended) => {
397                self.owner.shared.changed.send_replace(());
398                if ended {
399                    self.owner.changed();
400                }
401            }
402            // An attempt in flight keeps its place in its object's entry;
403            // without it the progress watches read is wrong. This drop may
404            // run during an unwind, so it fails the store's handle instead of
405            // panicking, and the bug goes to the call it runs in or the bug
406            // sink (E15).
407            None => self.owner.shared.observer.fail_with_bug(coven_foundation::task::Bug::new(
408                "end a transfer in flight",
409                "an attempt in flight lost its place in its object's entry",
410            )),
411        }
412    }
413}
414
415#[cfg(test)]
416#[path = "transfers_in_flight_tests.rs"]
417mod tests;