1use crate::internal_schema::WRITE_DEVICE_SQL;
4use crate::sqlite::StoreSql;
5use crate::write_encoding::{counter, encoded};
6use crate::DbError;
7use coven_format::chunks::CHUNK_SIZE;
8use coven_format::sealed_write::WriteObjectPrefix;
9use coven_format::write_stream::{PartHeader, WriteHeaderFrame};
10use coven_foundation::id_source::DeviceId;
11use coven_merge::WriteId;
12use sqlive::rusqlite::params;
13use std::rc::Rc;
14
15pub struct UploadPlaintext<'a> {
18 pub header: WriteHeaderFrame,
20 pub header_frame: Vec<u8>,
22 pub parts: UploadParts<'a>,
24}
25
26pub struct WaitingUpload<'a> {
29 pub plaintext: UploadPlaintext<'a>,
31 pub keys: WriteObjectPrefix,
33}
34
35pub struct PreparedUploads {
40 writes: Vec<PreparedUpload>,
41}
42
43struct PreparedUpload {
45 rowid: i64,
46 write: WriteId,
47 keys: WriteObjectPrefix,
48}
49
50impl PreparedUploads {
51 pub fn len(&self) -> usize {
53 self.writes.len()
54 }
55
56 pub fn is_empty(&self) -> bool {
58 self.writes.is_empty()
59 }
60}
61
62#[derive(Debug, thiserror::Error)]
64#[cfg(any(test, feature = "test-utils"))]
65pub enum UploadReadError<E> {
66 #[error(transparent)]
68 Database(#[from] DbError),
69 #[error("upload consumer: {0}")]
71 Consumer(E),
72}
73
74#[cfg(any(test, feature = "test-utils"))]
75impl<E: coven_foundation::task::CarriesBug> coven_foundation::task::CarriesBug
76 for UploadReadError<E>
77{
78 fn bug(&self) -> Option<&coven_foundation::task::Bug> {
79 match self {
80 Self::Database(error) => error.bug(),
81 Self::Consumer(error) => error.bug(),
82 }
83 }
84}
85
86#[derive(Clone)]
88pub struct UploadBytes<'a> {
89 blob: Rc<sqlive::rusqlite::blob::Blob<'a>>,
90 offset: usize,
91 end: usize,
92}
93
94impl<'a> UploadBytes<'a> {
95 pub(crate) fn new(blob: sqlive::rusqlite::blob::Blob<'a>) -> Self {
96 let end = blob.len();
97 Self { blob: Rc::new(blob), offset: 0, end }
98 }
99
100 fn read(&self, offset: usize, length: usize) -> Result<Vec<u8>, DbError> {
101 self.check_length(offset, length)?;
102 let mut bytes = vec![0; length];
103 self.blob.read_at_exact(&mut bytes, offset)?;
104 Ok(bytes)
105 }
106
107 fn check_length(&self, offset: usize, length: usize) -> Result<(), DbError> {
108 if offset > self.end || length > self.end - offset {
109 return Err(DbError::Invariant {
110 what: "upload reads stay inside the queued plaintext",
111 });
112 }
113 Ok(())
114 }
115}
116
117impl Iterator for UploadBytes<'_> {
118 type Item = Result<Vec<u8>, DbError>;
119
120 fn next(&mut self) -> Option<Self::Item> {
121 if self.offset == self.end {
122 return None;
123 }
124 let length = (self.end - self.offset).min(CHUNK_SIZE);
125 let result = self.read(self.offset, length);
126 self.offset = if result.is_ok() { self.offset + length } else { self.end };
127 Some(result)
128 }
129}
130
131#[derive(Clone)]
133pub struct UploadParts<'a> {
134 bytes: UploadBytes<'a>,
135 headers: std::vec::IntoIter<PartHeader>,
136}
137
138impl<'a> Iterator for UploadParts<'a> {
139 type Item = (PartHeader, UploadBytes<'a>);
140
141 fn next(&mut self) -> Option<Self::Item> {
142 let header = self.headers.next()?;
143 let offset = self.bytes.offset;
144 self.bytes.offset += header.plaintext_length as usize;
145 Some((
146 header,
147 UploadBytes { blob: self.bytes.blob.clone(), offset, end: self.bytes.offset },
148 ))
149 }
150}
151
152impl UploadParts<'_> {
153 pub fn roots<E: From<DbError> + From<coven_format::Error>>(
157 &self,
158 mut hasher: impl FnMut(usize, &PartHeader) -> Result<coven_crypto::AuthorshipHasher, E>,
159 ) -> Result<Vec<[u8; 32]>, E> {
160 let mut roots = Vec::new();
161 for (index, (header, mut bytes)) in self.clone().enumerate() {
162 let hasher = hasher(index, &header)?;
163 let mut builder = coven_format::authorship::RootBuilder::new(&hasher);
164 crate::download_stream::visit_part::<E>(
165 header,
166 |chunk| {
167 let plain = bytes.next().ok_or(DbError::DamagedDatabase)??;
168 if plain.len() != chunk.len() {
169 return Err(DbError::DamagedDatabase.into());
170 }
171 chunk.copy_from_slice(&plain);
172 Ok(())
173 },
174 |frame| {
175 let record = match &frame {
176 coven_format::dismissal::WriteFrame::Change(row) => {
177 coven_format::write::PartRecord::Change(row)
178 }
179 coven_format::dismissal::WriteFrame::Execution(execution) => {
180 coven_format::write::PartRecord::Execution(execution)
181 }
182 coven_format::dismissal::WriteFrame::Dismissal(_) => return Ok(()),
183 };
184 for leaf in coven_format::authorship::record_leaves(record) {
185 builder.push(&leaf)?;
186 }
187 Ok(())
188 },
189 )?;
190 roots.push(builder.finish());
191 }
192 Ok(roots)
193 }
194}
195
196struct QueueEntry {
199 rowid: i64,
200 write: WriteId,
201 keys: Option<Vec<u8>>,
202}
203
204impl QueueEntry {
205 fn read(row: &sqlive::rusqlite::Row<'_>) -> sqlive::rusqlite::Result<Self> {
208 Ok(Self {
209 rowid: row.get(0)?,
210 write: WriteId { device: row.get::<_, DeviceId>(1)?, number: counter(row.get(2)?)? },
211 keys: row.get(3)?,
212 })
213 }
214}
215
216#[cfg(any(test, feature = "test-utils"))]
217fn oldest(database: &StoreSql) -> Result<Option<QueueEntry>, DbError> {
218 Ok(database
219 .query(
220 "SELECT u.rowid,u.device,u.number,k.sealing_keys FROM _coven_uploads u LEFT JOIN _coven_upload_keys k ON k.device=u.device AND k.number=u.number ORDER BY u.rowid LIMIT 1",
221 [],
222 QueueEntry::read,
223 )?
224 .pop())
225}
226
227#[cfg(test)]
229pub(crate) fn oldest_keys(
230 database: &StoreSql,
231) -> Result<Option<(WriteId, Option<WriteObjectPrefix>)>, DbError> {
232 let Some(entry) = oldest(database)? else {
233 return Ok(None);
234 };
235 let keys =
236 entry.keys.as_deref().map(WriteObjectPrefix::decode).transpose().map_err(DbError::from)?;
237 Ok(Some((entry.write, keys)))
238}
239
240fn plaintext<'a>(
241 database: &'a StoreSql,
242 rowid: i64,
243 write: WriteId,
244) -> Result<(WriteHeaderFrame, Vec<u8>, UploadBytes<'a>), DbError> {
245 let mut bytes =
246 UploadBytes::new(database.open_blob("_coven_uploads", "record", rowid, true)?);
247 let prefix = bytes.read(0, coven_format::FRAME_PREFIX_LEN)?;
248 let length = encoded(coven_format::frame_length(&prefix))?;
249 let header_frame = bytes.read(0, length)?;
250 let header = encoded(WriteHeaderFrame::decode(&header_frame))?;
251 let total =
252 header.parts.iter().try_fold(length as u64, |n, p| n.checked_add(p.plaintext_length));
253 if total != Some(bytes.end as u64) || header.header.position != write {
254 return Err(DbError::DamagedDatabase);
255 }
256 bytes.offset = length;
257 Ok((header, header_frame, bytes))
258}
259
260#[cfg(any(test, feature = "test-utils"))]
261pub(crate) fn read<R, E>(
262 database: &StoreSql,
263 consume: impl FnOnce(UploadPlaintext<'_>) -> Result<R, E>,
264) -> Result<Option<R>, UploadReadError<E>> {
265 let Some(entry) = oldest(database)? else {
266 return Ok(None);
267 };
268 consume(read_plaintext(database, entry.rowid, entry.write)?)
269 .map(Some)
270 .map_err(UploadReadError::Consumer)
271}
272
273fn read_plaintext<'a>(
274 database: &'a StoreSql,
275 rowid: i64,
276 write: WriteId,
277) -> Result<UploadPlaintext<'a>, DbError> {
278 let (header, header_frame, bytes) = plaintext(database, rowid, write)?;
279 let parts = UploadParts { bytes, headers: header.parts.clone().into_iter() };
280 Ok(UploadPlaintext { header, header_frame, parts })
281}
282
283fn batch(database: &StoreSql, device: DeviceId, through: u64) -> Result<Vec<QueueEntry>, DbError> {
284 Ok(database.query(
285 "SELECT u.rowid,u.device,u.number,k.sealing_keys FROM _coven_uploads u
286 LEFT JOIN _coven_upload_keys k ON k.device=u.device AND k.number=u.number
287 WHERE u.device=?1 AND u.number<=?2 ORDER BY u.number",
288 params![device, through.to_be_bytes()],
289 QueueEntry::read,
290 )?)
291}
292
293pub(crate) fn prepare_batch<E: From<DbError> + From<coven_format::Error>>(
296 database: &mut StoreSql,
297 device: DeviceId,
298 through: u64,
299 mut select: impl FnMut(&WriteHeaderFrame) -> Result<WriteObjectPrefix, E>,
300) -> Result<PreparedUploads, E> {
301 let mut writes = Vec::new();
302 let mut selected = Vec::new();
303 for entry in batch(database, device, through)? {
304 let keys = match entry.keys {
305 Some(keys) => WriteObjectPrefix::decode(&keys).map_err(DbError::from)?,
306 None => {
307 let (header, _, _) = plaintext(database, entry.rowid, entry.write)?;
308 let keys = select(&header)?;
309 selected.push((entry.write, keys.encode()?));
310 keys
311 }
312 };
313 writes.push(PreparedUpload { rowid: entry.rowid, write: entry.write, keys });
314 }
315 if !selected.is_empty() {
316 database.stream_transaction(|database| {
317 for (write, keys) in selected {
318 keep_keys(database, write, &keys)?;
319 }
320 Ok::<_, DbError>(())
321 })?;
322 }
323 Ok(PreparedUploads { writes })
324}
325
326fn keep_keys(database: &StoreSql, write: WriteId, keys: &[u8]) -> Result<(), DbError> {
328 database.execute(
329 "INSERT INTO _coven_upload_keys(device,number,sealing_keys) VALUES(?1,?2,?3)",
330 params![write.device, write.number.to_be_bytes(), keys],
331 )?;
332 Ok(())
333}
334
335pub(crate) fn read_batch<E: From<DbError>>(
337 database: &StoreSql,
338 prepared: PreparedUploads,
339 mut consume: impl FnMut(WaitingUpload<'_>) -> Result<(), E>,
340) -> Result<(), E> {
341 for PreparedUpload { rowid, write, keys } in prepared.writes {
342 consume(WaitingUpload { plaintext: read_plaintext(database, rowid, write)?, keys })?;
343 }
344 Ok(())
345}
346
347pub(crate) fn queue<'a>(
351 database: &StoreSql,
352 write: WriteId,
353 record: &[u8],
354 dismissed: impl IntoIterator<Item = &'a coven_merge::RowId>,
355) -> Result<(), DbError> {
356 let number = write.number.to_be_bytes();
357 database.execute(
358 "INSERT INTO _coven_uploads(device,number,record) VALUES(?1,?2,?3)",
359 params![write.device, number.as_slice(), record],
360 )?;
361 for row in dismissed {
362 database.execute(
363 "INSERT INTO _coven_queued_dismissals(device,number,table_name,key,audience) VALUES(?1,?2,?3,?4,?5) ON CONFLICT DO NOTHING",
364 params![write.device, number.as_slice(), row.table, row.key, &row.audience],
365 )?;
366 }
367 Ok(())
368}
369
370pub(crate) fn dismissed(
372 database: &StoreSql,
373 write: WriteId,
374) -> Result<Vec<coven_merge::RowId>, DbError> {
375 Ok(database.query(
376 "SELECT table_name,key,audience FROM _coven_queued_dismissals WHERE device=?1 AND number=?2",
377 params![write.device, write.number.to_be_bytes().as_slice()],
378 crate::row_queries::read_identity,
379 )?)
380}
381
382pub(crate) fn forget_claimed(
384 database: &StoreSql,
385 device: DeviceId,
386 claim: u64,
387) -> Result<(), DbError> {
388 database.execute(
389 "DELETE FROM _coven_uploads WHERE device=?1 AND number IN
390 (SELECT number FROM _coven_upload_claims WHERE device=?1 AND claim=?2)",
391 params![device, claim.to_be_bytes().as_slice()],
392 )?;
393 Ok(())
394}
395
396pub(crate) fn queued(
398 database: &StoreSql,
399 write: WriteId,
400) -> Result<Option<WriteHeaderFrame>, DbError> {
401 let Some(rowid) = database
402 .query(
403 "SELECT rowid FROM _coven_uploads WHERE device=?1 AND number=?2",
404 params![write.device, write.number.to_be_bytes()],
405 |r| r.get::<_, i64>(0),
406 )?
407 .pop()
408 else {
409 return Ok(None);
410 };
411 Ok(Some(plaintext(database, rowid, write)?.0))
412}
413
414pub(crate) fn each_plaintext(
417 database: &StoreSql,
418 mut consume: impl FnMut(WriteHeaderFrame, UploadParts<'_>) -> Result<(), DbError>,
419) -> Result<(), DbError> {
420 let entries = database.query(
421 "SELECT rowid,device,number,NULL FROM _coven_uploads ORDER BY rowid",
422 [],
423 QueueEntry::read,
424 )?;
425 for entry in entries {
426 let UploadPlaintext { header, parts, .. } =
427 read_plaintext(database, entry.rowid, entry.write)?;
428 consume(header, parts)?;
429 }
430 Ok(())
431}
432
433#[cfg(test)]
434pub(crate) fn prepare(
435 database: &mut StoreSql,
436 call: &crate::store_call::StoreCall<'_>,
437 log: &crate::StoreState,
438 select: impl FnOnce(
439 &crate::StoreState,
440 u32,
441 &WriteHeaderFrame,
442 ) -> Result<WriteObjectPrefix, DbError>,
443) -> Result<Option<WriteId>, DbError> {
444 database.stream_transaction(|database| {
445 let Some(entry) = oldest(database)? else {
446 return Ok(None);
447 };
448 if entry.keys.is_some() {
449 return Ok(Some(entry.write));
450 }
451 let (header, _, _) = plaintext(database, entry.rowid, entry.write)?;
452 let keys =
453 select(log, crate::transaction_constants::schema_version(database, call)?, &header)?;
454 if keys.part_keys.len() != header.parts.len() {
455 return Err(DbError::Invariant { what: "sealing selects one key per write part" });
456 }
457 keep_keys(database, entry.write, &keys.encode()?)?;
458 Ok(Some(entry.write))
459 })
460}
461
462pub(crate) fn sendable_through(db: &StoreSql, device: DeviceId) -> Result<Option<u64>, DbError> {
468 let newest =
469 db.query_row("SELECT max(number) FROM _coven_uploads WHERE device=?1", [device], |r| {
470 r.get::<_, Option<Vec<u8>>>(0)
471 })?;
472 let Some(newest) = newest.map(counter).transpose()? else {
473 return Ok(None);
474 };
475 let captured = crate::raise_capture::captured_through(db)?;
476 let held = crate::file_queue::first_held(db)?.map(|first| first - 1);
477 Ok(Some([captured, held].into_iter().flatten().fold(newest, u64::min)))
478}
479
480pub(crate) fn unconfirmed(db: &StoreSql, through: WriteId) -> Result<Option<WriteId>, DbError> {
485 Ok(db
486 .query(
487 &format!("SELECT u.number FROM _coven_uploads u JOIN _coven_writes w ON {WRITE_DEVICE_SQL}=u.device AND w.number=u.number WHERE u.device=?1 AND u.number<=?2 AND w.slot IS NULL ORDER BY u.number LIMIT 1"),
488 params![
489 through.device,
490 through.number.to_be_bytes().as_slice()
491 ],
492 |r| {
493 Ok(WriteId {
494 device: through.device,
495 number: counter(r.get(0)?)?,
496 })
497 },
498 )?
499 .pop())
500}
501
502#[cfg(test)]
503#[path = "upload_tests.rs"]
504pub(crate) mod tests;