Skip to main content

coven_database/
upload.rs

1//! Original authored plaintext, with format and keys fixed by its first upload attempt.
2
3use crate::internal_schema::WRITE_DEVICE_SQL;
4use crate::sqlite::StoreSql;
5use crate::write_encoding::{counter, encoded};
6use crate::DbError;
7use coven_format::chunks::CHUNK_SIZE;
8use coven_format::sealed_write::WriteObjectPrefix;
9use coven_format::write_stream::{PartHeader, WriteHeaderFrame};
10use coven_foundation::id_source::DeviceId;
11use coven_merge::WriteId;
12use sqlive::rusqlite::params;
13use std::rc::Rc;
14
15/// A waiting write's plaintext, fixed at commit, read under one committed
16/// reader transaction.
17pub struct UploadPlaintext<'a> {
18    /// Identity, causality and the audience stream boundaries.
19    pub header: WriteHeaderFrame,
20    /// The exact plaintext header frame, sealed as one chunk.
21    pub header_frame: Vec<u8>,
22    /// Each audience's record stream, in header order, cut into 64 KiB chunks.
23    pub parts: UploadParts<'a>,
24}
25
26/// A captured write ready to seal. Sealing and transport belong to sync;
27/// these are its input streams and the keys it seals with.
28pub struct WaitingUpload<'a> {
29    /// The plaintext fixed at commit.
30    pub plaintext: UploadPlaintext<'a>,
31    /// Format and keys fixed by the first attempt.
32    pub keys: WriteObjectPrefix,
33}
34
35/// A captured queue prefix whose writes all have their first-attempt keys
36/// fixed, in queue order. Only [`crate::StoreDatabase::prepare_uploads`]
37/// makes one, and [`crate::StoreDatabase::read_uploads`] reads its
38/// writes' plaintext.
39pub struct PreparedUploads {
40    writes: Vec<PreparedUpload>,
41}
42
43/// One captured write: its queue row and the keys it seals with.
44struct PreparedUpload {
45    rowid: i64,
46    write: WriteId,
47    keys: WriteObjectPrefix,
48}
49
50impl PreparedUploads {
51    /// How many writes the prefix captured.
52    pub fn len(&self) -> usize {
53        self.writes.len()
54    }
55
56    /// Whether the prefix captured no write.
57    pub fn is_empty(&self) -> bool {
58        self.writes.is_empty()
59    }
60}
61
62/// A queue read failed in the database or in its consumer.
63#[derive(Debug, thiserror::Error)]
64#[cfg(any(test, feature = "test-utils"))]
65pub enum UploadReadError<E> {
66    /// Reading the committed queue state failed.
67    #[error(transparent)]
68    Database(#[from] DbError),
69    /// The consumer failed while reading or using the stream.
70    #[error("upload consumer: {0}")]
71    Consumer(E),
72}
73
74#[cfg(any(test, feature = "test-utils"))]
75impl<E: coven_foundation::task::CarriesBug> coven_foundation::task::CarriesBug
76    for UploadReadError<E>
77{
78    fn bug(&self) -> Option<&coven_foundation::task::Bug> {
79        match self {
80            Self::Database(error) => error.bug(),
81            Self::Consumer(error) => error.bug(),
82        }
83    }
84}
85
86/// An incremental SQLite BLOB reader. It never collects the waiting write.
87#[derive(Clone)]
88pub struct UploadBytes<'a> {
89    blob: Rc<sqlive::rusqlite::blob::Blob<'a>>,
90    offset: usize,
91    end: usize,
92}
93
94impl<'a> UploadBytes<'a> {
95    pub(crate) fn new(blob: sqlive::rusqlite::blob::Blob<'a>) -> Self {
96        let end = blob.len();
97        Self { blob: Rc::new(blob), offset: 0, end }
98    }
99
100    fn read(&self, offset: usize, length: usize) -> Result<Vec<u8>, DbError> {
101        self.check_length(offset, length)?;
102        let mut bytes = vec![0; length];
103        self.blob.read_at_exact(&mut bytes, offset)?;
104        Ok(bytes)
105    }
106
107    fn check_length(&self, offset: usize, length: usize) -> Result<(), DbError> {
108        if offset > self.end || length > self.end - offset {
109            return Err(DbError::Invariant {
110                what: "upload reads stay inside the queued plaintext",
111            });
112        }
113        Ok(())
114    }
115}
116
117impl Iterator for UploadBytes<'_> {
118    type Item = Result<Vec<u8>, DbError>;
119
120    fn next(&mut self) -> Option<Self::Item> {
121        if self.offset == self.end {
122            return None;
123        }
124        let length = (self.end - self.offset).min(CHUNK_SIZE);
125        let result = self.read(self.offset, length);
126        self.offset = if result.is_ok() { self.offset + length } else { self.end };
127        Some(result)
128    }
129}
130
131/// Audience stream descriptors paired with their bounded plaintext readers.
132#[derive(Clone)]
133pub struct UploadParts<'a> {
134    bytes: UploadBytes<'a>,
135    headers: std::vec::IntoIter<PartHeader>,
136}
137
138impl<'a> Iterator for UploadParts<'a> {
139    type Item = (PartHeader, UploadBytes<'a>);
140
141    fn next(&mut self) -> Option<Self::Item> {
142        let header = self.headers.next()?;
143        let offset = self.bytes.offset;
144        self.bytes.offset += header.plaintext_length as usize;
145        Some((
146            header,
147            UploadBytes { blob: self.bytes.blob.clone(), offset, end: self.bytes.offset },
148        ))
149    }
150}
151
152impl UploadParts<'_> {
153    /// Each part's authorship root (D11), read from the queued plaintext
154    /// without consuming these streams. `hasher` gives the keyed leaf hasher
155    /// of a part, by its index in header order, under its first-attempt key.
156    pub fn roots<E: From<DbError> + From<coven_format::Error>>(
157        &self,
158        mut hasher: impl FnMut(usize, &PartHeader) -> Result<coven_crypto::AuthorshipHasher, E>,
159    ) -> Result<Vec<[u8; 32]>, E> {
160        let mut roots = Vec::new();
161        for (index, (header, mut bytes)) in self.clone().enumerate() {
162            let hasher = hasher(index, &header)?;
163            let mut builder = coven_format::authorship::RootBuilder::new(&hasher);
164            crate::download_stream::visit_part::<E>(
165                header,
166                |chunk| {
167                    let plain = bytes.next().ok_or(DbError::DamagedDatabase)??;
168                    if plain.len() != chunk.len() {
169                        return Err(DbError::DamagedDatabase.into());
170                    }
171                    chunk.copy_from_slice(&plain);
172                    Ok(())
173                },
174                |frame| {
175                    let record = match &frame {
176                        coven_format::dismissal::WriteFrame::Change(row) => {
177                            coven_format::write::PartRecord::Change(row)
178                        }
179                        coven_format::dismissal::WriteFrame::Execution(execution) => {
180                            coven_format::write::PartRecord::Execution(execution)
181                        }
182                        coven_format::dismissal::WriteFrame::Dismissal(_) => return Ok(()),
183                    };
184                    for leaf in coven_format::authorship::record_leaves(record) {
185                        builder.push(&leaf)?;
186                    }
187                    Ok(())
188                },
189            )?;
190            roots.push(builder.finish());
191        }
192        Ok(roots)
193    }
194}
195
196/// A write waiting in `_coven_uploads`: its row, its id, and the keys its
197/// first attempt fixed, when that is known.
198struct QueueEntry {
199    rowid: i64,
200    write: WriteId,
201    keys: Option<Vec<u8>>,
202}
203
204impl QueueEntry {
205    /// An entry selected as `rowid,device,number` from `_coven_uploads`,
206    /// then its upload keys' `sealing_keys`, or NULL for none known.
207    fn read(row: &sqlive::rusqlite::Row<'_>) -> sqlive::rusqlite::Result<Self> {
208        Ok(Self {
209            rowid: row.get(0)?,
210            write: WriteId { device: row.get::<_, DeviceId>(1)?, number: counter(row.get(2)?)? },
211            keys: row.get(3)?,
212        })
213    }
214}
215
216#[cfg(any(test, feature = "test-utils"))]
217fn oldest(database: &StoreSql) -> Result<Option<QueueEntry>, DbError> {
218    Ok(database
219        .query(
220            "SELECT u.rowid,u.device,u.number,k.sealing_keys FROM _coven_uploads u LEFT JOIN _coven_upload_keys k ON k.device=u.device AND k.number=u.number ORDER BY u.rowid LIMIT 1",
221            [],
222            QueueEntry::read,
223        )?
224        .pop())
225}
226
227/// The oldest waiting write and the keys its first attempt fixed, if any.
228#[cfg(test)]
229pub(crate) fn oldest_keys(
230    database: &StoreSql,
231) -> Result<Option<(WriteId, Option<WriteObjectPrefix>)>, DbError> {
232    let Some(entry) = oldest(database)? else {
233        return Ok(None);
234    };
235    let keys =
236        entry.keys.as_deref().map(WriteObjectPrefix::decode).transpose().map_err(DbError::from)?;
237    Ok(Some((entry.write, keys)))
238}
239
240fn plaintext<'a>(
241    database: &'a StoreSql,
242    rowid: i64,
243    write: WriteId,
244) -> Result<(WriteHeaderFrame, Vec<u8>, UploadBytes<'a>), DbError> {
245    let mut bytes =
246        UploadBytes::new(database.open_blob("_coven_uploads", "record", rowid, true)?);
247    let prefix = bytes.read(0, coven_format::FRAME_PREFIX_LEN)?;
248    let length = encoded(coven_format::frame_length(&prefix))?;
249    let header_frame = bytes.read(0, length)?;
250    let header = encoded(WriteHeaderFrame::decode(&header_frame))?;
251    let total =
252        header.parts.iter().try_fold(length as u64, |n, p| n.checked_add(p.plaintext_length));
253    if total != Some(bytes.end as u64) || header.header.position != write {
254        return Err(DbError::DamagedDatabase);
255    }
256    bytes.offset = length;
257    Ok((header, header_frame, bytes))
258}
259
260#[cfg(any(test, feature = "test-utils"))]
261pub(crate) fn read<R, E>(
262    database: &StoreSql,
263    consume: impl FnOnce(UploadPlaintext<'_>) -> Result<R, E>,
264) -> Result<Option<R>, UploadReadError<E>> {
265    let Some(entry) = oldest(database)? else {
266        return Ok(None);
267    };
268    consume(read_plaintext(database, entry.rowid, entry.write)?)
269        .map(Some)
270        .map_err(UploadReadError::Consumer)
271}
272
273fn read_plaintext<'a>(
274    database: &'a StoreSql,
275    rowid: i64,
276    write: WriteId,
277) -> Result<UploadPlaintext<'a>, DbError> {
278    let (header, header_frame, bytes) = plaintext(database, rowid, write)?;
279    let parts = UploadParts { bytes, headers: header.parts.clone().into_iter() };
280    Ok(UploadPlaintext { header, header_frame, parts })
281}
282
283fn batch(database: &StoreSql, device: DeviceId, through: u64) -> Result<Vec<QueueEntry>, DbError> {
284    Ok(database.query(
285        "SELECT u.rowid,u.device,u.number,k.sealing_keys FROM _coven_uploads u
286        LEFT JOIN _coven_upload_keys k ON k.device=u.device AND k.number=u.number
287        WHERE u.device=?1 AND u.number<=?2 ORDER BY u.number",
288        params![device, through.to_be_bytes()],
289        QueueEntry::read,
290    )?)
291}
292
293/// Fix the keys of every write in `device`'s queue through `through` that
294/// has none yet, keeping the keys earlier attempts fixed.
295pub(crate) fn prepare_batch<E: From<DbError> + From<coven_format::Error>>(
296    database: &mut StoreSql,
297    device: DeviceId,
298    through: u64,
299    mut select: impl FnMut(&WriteHeaderFrame) -> Result<WriteObjectPrefix, E>,
300) -> Result<PreparedUploads, E> {
301    let mut writes = Vec::new();
302    let mut selected = Vec::new();
303    for entry in batch(database, device, through)? {
304        let keys = match entry.keys {
305            Some(keys) => WriteObjectPrefix::decode(&keys).map_err(DbError::from)?,
306            None => {
307                let (header, _, _) = plaintext(database, entry.rowid, entry.write)?;
308                let keys = select(&header)?;
309                selected.push((entry.write, keys.encode()?));
310                keys
311            }
312        };
313        writes.push(PreparedUpload { rowid: entry.rowid, write: entry.write, keys });
314    }
315    if !selected.is_empty() {
316        database.stream_transaction(|database| {
317            for (write, keys) in selected {
318                keep_keys(database, write, &keys)?;
319            }
320            Ok::<_, DbError>(())
321        })?;
322    }
323    Ok(PreparedUploads { writes })
324}
325
326/// Fix the keys a queued write is sealed with, at its first attempt.
327fn keep_keys(database: &StoreSql, write: WriteId, keys: &[u8]) -> Result<(), DbError> {
328    database.execute(
329        "INSERT INTO _coven_upload_keys(device,number,sealing_keys) VALUES(?1,?2,?3)",
330        params![write.device, write.number.to_be_bytes(), keys],
331    )?;
332    Ok(())
333}
334
335/// Read each prepared write's plaintext, with the keys it seals with.
336pub(crate) fn read_batch<E: From<DbError>>(
337    database: &StoreSql,
338    prepared: PreparedUploads,
339    mut consume: impl FnMut(WaitingUpload<'_>) -> Result<(), E>,
340) -> Result<(), E> {
341    for PreparedUpload { rowid, write, keys } in prepared.writes {
342        consume(WaitingUpload { plaintext: read_plaintext(database, rowid, write)?, keys })?;
343    }
344    Ok(())
345}
346
347/// Queue this device's write `write`, its plaintext `record` fixed at
348/// commit, to wait for its upload, with the rows its dismissals name,
349/// `dismissed`.
350pub(crate) fn queue<'a>(
351    database: &StoreSql,
352    write: WriteId,
353    record: &[u8],
354    dismissed: impl IntoIterator<Item = &'a coven_merge::RowId>,
355) -> Result<(), DbError> {
356    let number = write.number.to_be_bytes();
357    database.execute(
358        "INSERT INTO _coven_uploads(device,number,record) VALUES(?1,?2,?3)",
359        params![write.device, number.as_slice(), record],
360    )?;
361    for row in dismissed {
362        database.execute(
363            "INSERT INTO _coven_queued_dismissals(device,number,table_name,key,audience) VALUES(?1,?2,?3,?4,?5) ON CONFLICT DO NOTHING",
364            params![write.device, number.as_slice(), row.table, row.key, &row.audience],
365        )?;
366    }
367    Ok(())
368}
369
370/// The rows the dismissals of this device's queued write `write` name.
371pub(crate) fn dismissed(
372    database: &StoreSql,
373    write: WriteId,
374) -> Result<Vec<coven_merge::RowId>, DbError> {
375    Ok(database.query(
376        "SELECT table_name,key,audience FROM _coven_queued_dismissals WHERE device=?1 AND number=?2",
377        params![write.device, write.number.to_be_bytes().as_slice()],
378        crate::row_queries::read_identity,
379    )?)
380}
381
382/// The writes `device`'s landed claim `claim` carried leave the queue.
383pub(crate) fn forget_claimed(
384    database: &StoreSql,
385    device: DeviceId,
386    claim: u64,
387) -> Result<(), DbError> {
388    database.execute(
389        "DELETE FROM _coven_uploads WHERE device=?1 AND number IN
390         (SELECT number FROM _coven_upload_claims WHERE device=?1 AND claim=?2)",
391        params![device, claim.to_be_bytes().as_slice()],
392    )?;
393    Ok(())
394}
395
396/// The header of this device's write `write` while it waits in the queue.
397pub(crate) fn queued(
398    database: &StoreSql,
399    write: WriteId,
400) -> Result<Option<WriteHeaderFrame>, DbError> {
401    let Some(rowid) = database
402        .query(
403            "SELECT rowid FROM _coven_uploads WHERE device=?1 AND number=?2",
404            params![write.device, write.number.to_be_bytes()],
405            |r| r.get::<_, i64>(0),
406        )?
407        .pop()
408    else {
409        return Ok(None);
410    };
411    Ok(Some(plaintext(database, rowid, write)?.0))
412}
413
414/// Read every waiting plaintext, including attempted entries.
415/// The caller holds a reader transaction for a consistent queue view.
416pub(crate) fn each_plaintext(
417    database: &StoreSql,
418    mut consume: impl FnMut(WriteHeaderFrame, UploadParts<'_>) -> Result<(), DbError>,
419) -> Result<(), DbError> {
420    let entries = database.query(
421        "SELECT rowid,device,number,NULL FROM _coven_uploads ORDER BY rowid",
422        [],
423        QueueEntry::read,
424    )?;
425    for entry in entries {
426        let UploadPlaintext { header, parts, .. } =
427            read_plaintext(database, entry.rowid, entry.write)?;
428        consume(header, parts)?;
429    }
430    Ok(())
431}
432
433#[cfg(test)]
434pub(crate) fn prepare(
435    database: &mut StoreSql,
436    call: &crate::store_call::StoreCall<'_>,
437    log: &crate::StoreState,
438    select: impl FnOnce(
439        &crate::StoreState,
440        u32,
441        &WriteHeaderFrame,
442    ) -> Result<WriteObjectPrefix, DbError>,
443) -> Result<Option<WriteId>, DbError> {
444    database.stream_transaction(|database| {
445        let Some(entry) = oldest(database)? else {
446            return Ok(None);
447        };
448        if entry.keys.is_some() {
449            return Ok(Some(entry.write));
450        }
451        let (header, _, _) = plaintext(database, entry.rowid, entry.write)?;
452        let keys =
453            select(log, crate::transaction_constants::schema_version(database, call)?, &header)?;
454        if keys.part_keys.len() != header.parts.len() {
455            return Err(DbError::Invariant { what: "sealing selects one key per write part" });
456        }
457        keep_keys(database, entry.write, &keys.encode()?)?;
458        Ok(Some(entry.write))
459    })
460}
461
462/// The newest of this device's waiting writes it may send now: none past a
463/// raise's captured prefix (§18.5), and none from the first write a bound
464/// upload holds on (§17.4). `None` with no write waiting. The caller holds
465/// one committed read, so a write is never sent past a hold it was queued
466/// with.
467pub(crate) fn sendable_through(db: &StoreSql, device: DeviceId) -> Result<Option<u64>, DbError> {
468    let newest =
469        db.query_row("SELECT max(number) FROM _coven_uploads WHERE device=?1", [device], |r| {
470            r.get::<_, Option<Vec<u8>>>(0)
471        })?;
472    let Some(newest) = newest.map(counter).transpose()? else {
473        return Ok(None);
474    };
475    let captured = crate::raise_capture::captured_through(db)?;
476    let held = crate::file_queue::first_held(db)?.map(|first| first - 1);
477    Ok(Some([captured, held].into_iter().flatten().fold(newest, u64::min)))
478}
479
480/// The first own queued write at or before `through` not yet confirmed
481/// stored. A snapshot covering own writes through `through` is sent only
482/// once this is none: capture may precede confirmation, but neither settles
483/// a queue slot (§16.1).
484pub(crate) fn unconfirmed(db: &StoreSql, through: WriteId) -> Result<Option<WriteId>, DbError> {
485    Ok(db
486        .query(
487            &format!("SELECT u.number FROM _coven_uploads u JOIN _coven_writes w ON {WRITE_DEVICE_SQL}=u.device AND w.number=u.number WHERE u.device=?1 AND u.number<=?2 AND w.slot IS NULL ORDER BY u.number LIMIT 1"),
488            params![
489                through.device,
490                through.number.to_be_bytes().as_slice()
491            ],
492            |r| {
493                Ok(WriteId {
494                    device: through.device,
495                    number: counter(r.get(0)?)?,
496                })
497            },
498        )?
499        .pop())
500}
501
502#[cfg(test)]
503#[path = "upload_tests.rs"]
504pub(crate) mod tests;