1use crate::file_location::FileLocation;
4use crate::insert_rows::{insert_rows, value};
5use crate::internal_schema::FILE_NAMED_SQL;
6use crate::sqlite::StoreSql;
7use crate::DbError;
8use coven_crypto::{ContentHash, SecretBytes};
9use coven_format::file::FileHeader;
10use coven_foundation::id_source::{DeviceId, FileId, ObjectId};
11use std::time::SystemTime;
12
13#[derive(Debug)]
18pub struct FileUpload {
19 pub id: i64,
21 pub location: FileLocation,
23 pub size: u64,
25 pub content_hash: ContentHash,
27 pub files: Vec<FileId>,
30 pub queued_at: SystemTime,
32 pub retry: Option<crate::RetryWait>,
34 pub of: crate::MoveId,
36 pub stored: bool,
38 pub holds: Option<u64>,
43 pub session: Option<SecretBytes>,
46 pub progress: Option<UploadProgress>,
49 pub bytes_total: u64,
52}
53
54#[derive(Clone, Copy, Debug, PartialEq, Eq)]
57pub struct UploadProgress {
58 pub bytes_confirmed: u64,
60 pub bytes_total: u64,
62}
63impl Clone for FileUpload {
64 fn clone(&self) -> Self {
65 Self {
66 id: self.id,
67 location: self.location.clone(),
68 size: self.size,
69 content_hash: self.content_hash,
70 files: self.files.clone(),
71 queued_at: self.queued_at,
72 retry: self.retry,
73 of: self.of,
74 stored: self.stored,
75 holds: self.holds,
76 session: self
77 .session
78 .as_ref()
79 .map(|session| SecretBytes::new(session.as_bytes().to_vec())),
80 progress: self.progress,
81 bytes_total: self.bytes_total,
82 }
83 }
84}
85impl PartialEq for FileUpload {
86 fn eq(&self, other: &Self) -> bool {
87 self.id == other.id
88 && self.location == other.location
89 && self.size == other.size
90 && self.content_hash == other.content_hash
91 && self.files == other.files
92 && self.queued_at == other.queued_at
93 && self.retry == other.retry
94 && self.of == other.of
95 && self.stored == other.stored
96 && self.holds == other.holds
97 && self.session.as_ref().map(SecretBytes::as_bytes)
98 == other.session.as_ref().map(SecretBytes::as_bytes)
99 && self.progress == other.progress
100 && self.bytes_total == other.bytes_total
101 }
102}
103
104#[derive(Clone, Debug, PartialEq)]
108pub struct UploadQueueState {
109 pub paused: bool,
111 pub uploads: Vec<FileUpload>,
113 pub sending: std::collections::BTreeSet<FileId>,
116}
117
118#[derive(Clone, Copy, Debug, PartialEq, Eq)]
120pub enum UploadCompletion {
121 Stored,
124 Retired,
127}
128
129#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
131pub struct UnsentWork {
132 pub writes: u64,
134 pub files: u64,
136}
137
138impl UnsentWork {
139 pub fn is_empty(&self) -> bool {
141 self.writes == 0 && self.files == 0
142 }
143}
144
145macro_rules! unsent_counts {
150 () => {
151 "(SELECT count(w.number) FROM _coven_uploads AS w),
152 (SELECT count(q.id) FROM _coven_file_uploads AS q)"
153 };
154}
155pub(crate) use unsent_counts;
156
157pub(crate) fn unsent(db: &StoreSql) -> Result<UnsentWork, DbError> {
158 Ok(db.query_row(concat!("SELECT ", unsent_counts!()), [], |r| {
159 Ok(UnsentWork { writes: r.get::<_, i64>(0)? as u64, files: r.get::<_, i64>(1)? as u64 })
160 })?)
161}
162
163pub(crate) fn state(db: &StoreSql) -> Result<UploadQueueState, DbError> {
164 Ok(UploadQueueState {
165 paused: paused(db)?,
166 uploads: read(db)?,
167 sending: db
168 .query("SELECT file FROM coven_sending", [], |r| r.get::<_, FileId>(0))?
169 .into_iter()
170 .collect(),
171 })
172}
173
174pub(crate) fn paused(db: &StoreSql) -> Result<bool, DbError> {
175 Ok(db.query_row("SELECT EXISTS(SELECT 1 FROM _coven_upload_pause)", [], |r| r.get(0))?)
176}
177
178pub(crate) fn unmetered_only(db: &StoreSql) -> Result<bool, DbError> {
180 Ok(db.query_row("SELECT EXISTS(SELECT 1 FROM _coven_unmetered_only)", [], |r| r.get(0))?)
181}
182
183pub(crate) fn set_unmetered_only(db: &StoreSql, unmetered_only: bool) -> Result<(), DbError> {
184 if unmetered_only {
185 db.execute(
186 "INSERT INTO _coven_unmetered_only(unmetered_only) VALUES(1) ON CONFLICT DO NOTHING",
187 [],
188 )?;
189 } else {
190 db.execute("DELETE FROM _coven_unmetered_only", [])?;
191 }
192 Ok(())
193}
194
195pub(crate) fn set_paused(db: &StoreSql, paused: bool) -> Result<(), DbError> {
196 if paused {
197 db.execute("INSERT INTO _coven_upload_pause(paused) VALUES(1) ON CONFLICT DO NOTHING", [])?;
198 } else {
199 db.execute("DELETE FROM _coven_upload_pause", [])?;
200 }
201 Ok(())
202}
203
204pub(crate) fn read(db: &StoreSql) -> Result<Vec<FileUpload>, DbError> {
208 let rows = db.query(
209 "SELECT u.id,u.location,u.size,u.hash,u.queued_at,u.session,u.bytes_confirmed,
210 u.bytes_total,r.failures,r.not_before,r.wait,u.move,u.stored,u.holds
211 FROM _coven_file_uploads AS u LEFT JOIN _coven_retries AS r ON r.upload=u.id
212 WHERE EXISTS(SELECT 1 FROM _coven_upload_files AS l WHERE l.upload=u.id)
213 ORDER BY u.id",
214 [],
215 upload_row,
216 )?;
217 decode_uploads(db, rows)
218}
219
220pub(crate) fn abandoned(db: &StoreSql) -> Result<Vec<FileUpload>, DbError> {
225 let rows = db.query(
226 "SELECT u.id,u.location,u.size,u.hash,u.queued_at,u.session,u.bytes_confirmed,
227 u.bytes_total,r.failures,r.not_before,r.wait,u.move,u.stored,u.holds
228 FROM _coven_file_uploads AS u LEFT JOIN _coven_retries AS r ON r.upload=u.id
229 WHERE u.session IS NOT NULL
230 AND NOT EXISTS(SELECT 1 FROM _coven_upload_files AS l WHERE l.upload=u.id)
231 ORDER BY u.id",
232 [],
233 upload_row,
234 )?;
235 decode_uploads(db, rows)
236}
237
238#[cfg(any(test, feature = "test-utils"))]
240pub(crate) fn read_all(db: &StoreSql) -> Result<Vec<FileUpload>, DbError> {
241 let rows = db.query(
242 "SELECT u.id,u.location,u.size,u.hash,u.queued_at,u.session,u.bytes_confirmed,
243 u.bytes_total,r.failures,r.not_before,r.wait,u.move,u.stored,u.holds
244 FROM _coven_file_uploads AS u LEFT JOIN _coven_retries AS r ON r.upload=u.id ORDER BY u.id",
245 [],
246 upload_row,
247 )?;
248 decode_uploads(db, rows)
249}
250
251type UploadRow = (
252 (i64, String, i64, Vec<u8>, Vec<u8>),
253 (Option<Vec<u8>>, Option<i64>, Option<i64>),
254 (Option<i64>, Option<Vec<u8>>, Option<i64>),
255 (crate::MoveId, bool, Option<Vec<u8>>),
256);
257
258fn upload_row(r: &sqlive::rusqlite::Row<'_>) -> sqlive::rusqlite::Result<UploadRow> {
259 Ok((
260 (r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?),
261 (r.get(5)?, r.get(6)?, r.get(7)?),
262 (r.get(8)?, r.get(9)?, r.get(10)?),
263 (r.get(11)?, r.get(12)?, r.get(13)?),
264 ))
265}
266
267#[expect(
268 clippy::map_err_ignore,
269 reason = "a stored value that won't convert or decode is DamagedDatabase"
270)]
271fn decode_uploads(db: &StoreSql, rows: Vec<UploadRow>) -> Result<Vec<FileUpload>, DbError> {
272 rows.into_iter()
273 .map(|((id, location, size, hash, queued), session, retry, (of, stored, holds))| {
274 let retry = match retry {
275 (None, None, None) => None,
276 (Some(failures), Some(not_before), Some(wait)) => {
277 Some(crate::RetryWait::stored(failures, ¬_before, wait)?)
278 }
279 _ => return Err(DbError::DamagedDatabase),
280 };
281 let (session, progress) = match session {
282 (None, None, None) => (None, None),
283 (Some(session), Some(confirmed), Some(total)) => (
284 Some(SecretBytes::new(session)),
285 Some(UploadProgress {
286 bytes_confirmed: u64::try_from(confirmed)
287 .map_err(|_| DbError::DamagedDatabase)?,
288 bytes_total: u64::try_from(total).map_err(|_| DbError::DamagedDatabase)?,
289 }),
290 ),
291 _ => return Err(DbError::DamagedDatabase),
292 };
293 let size = u64::try_from(size).map_err(|_| DbError::DamagedDatabase)?;
294 let bytes_total =
296 FileHeader::new(size).encrypted_size().map_err(|_| DbError::DamagedDatabase)?;
297 Ok(FileUpload {
298 id,
299 location: FileLocation::decode(&location)?,
300 size,
301 content_hash: crate::file_hashes::stored_hash(hash)?,
302 files: db.query(
303 "SELECT file FROM _coven_upload_files WHERE upload=?1 ORDER BY file",
304 [id],
305 |r| r.get::<_, FileId>(0),
306 )?,
307 queued_at: crate::user_file::decode_time(&queued)?,
308 retry,
309 of,
310 stored,
311 holds: holds.map(crate::write_encoding::counter).transpose()?,
312 session,
313 progress,
314 bytes_total,
315 })
316 })
317 .collect()
318}
319
320pub(crate) fn first_held(db: &StoreSql) -> Result<Option<u64>, DbError> {
324 let holds = db.query_row(
325 "SELECT min(u.holds) FROM _coven_file_uploads AS u WHERE u.holds IS NOT NULL
326 AND EXISTS(SELECT 1 FROM _coven_upload_files AS l WHERE l.upload=u.id)",
327 [],
328 |r| r.get::<_, Option<Vec<u8>>>(0),
329 )?;
330 Ok(holds.map(crate::write_encoding::counter).transpose()?)
331}
332
333pub(crate) fn upload_id(
335 db: &StoreSql,
336 uploader: DeviceId,
337 object: ObjectId,
338) -> Result<Option<i64>, DbError> {
339 Ok(db
340 .query(
341 "SELECT id FROM _coven_file_uploads WHERE device=?1 AND object=?2",
342 (uploader, object),
343 |row| row.get(0),
344 )?
345 .pop())
346}
347
348pub(crate) fn upload_of(db: &StoreSql, file: FileId) -> Result<Option<i64>, DbError> {
350 Ok(db
351 .query("SELECT upload FROM _coven_upload_files WHERE file=?1", [file], |row| row.get(0))?
352 .pop())
353}
354
355pub(crate) fn bound(db: &StoreSql) -> Result<std::collections::BTreeSet<i64>, DbError> {
357 Ok(db
358 .query("SELECT id FROM _coven_file_uploads WHERE holds IS NOT NULL", [], |r| r.get(0))?
359 .into_iter()
360 .collect())
361}
362
363pub(crate) fn held_upload_of(db: &StoreSql, file: FileId) -> Result<bool, DbError> {
366 Ok(db.query_row(
367 "SELECT EXISTS(SELECT 1 FROM _coven_upload_files AS l
368 JOIN _coven_file_uploads AS u ON u.id=l.upload
369 WHERE l.file=?1 AND u.holds IS NOT NULL)",
370 [file],
371 |row| row.get(0),
372 )?)
373}
374
375#[expect(
378 clippy::map_err_ignore,
379 reason = "a stored value that won't convert is damage, a computed one a broken invariant"
380)]
381pub(crate) fn chunk_hash(
382 db: &StoreSql,
383 location: &FileLocation,
384 index: u64,
385) -> Result<ContentHash, DbError> {
386 let index = i64::try_from(index)
387 .map_err(|_| DbError::Invariant { what: "a chunk index fits SQLite" })?;
388 let hash = db
389 .query_row(
390 "SELECT hash FROM _coven_file_chunks WHERE device=?1 AND object=?2 AND chunk=?3",
391 (location.uploader(), location.object(), index),
392 |r| r.get::<_, Vec<u8>>(0),
393 )
394 .map_err(|error| match error {
395 sqlive::Error::Sqlite(sqlive::rusqlite::Error::QueryReturnedNoRows) => {
396 DbError::DamagedDatabase
397 }
398 error => DbError::from(error),
399 })?;
400 crate::file_hashes::stored_hash(hash)
401}
402
403pub(crate) fn retirable(db: &StoreSql) -> Result<Vec<(i64, FileLocation)>, DbError> {
406 db.query(
407 "SELECT u.id,u.location FROM _coven_file_uploads AS u
408 WHERE u.session IS NULL
409 AND NOT EXISTS(SELECT 1 FROM _coven_upload_files AS l WHERE l.upload=u.id)
410 ORDER BY u.id",
411 [],
412 |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?)),
413 )?
414 .into_iter()
415 .map(|(id, location)| Ok((id, FileLocation::decode(&location)?)))
416 .collect()
417}
418
419pub(crate) fn switched(db: &StoreSql, id: i64) -> Result<(), DbError> {
424 for file in
427 db.query("SELECT file FROM _coven_upload_files WHERE upload=?1 AND keep=1", [id], |r| {
428 r.get::<_, FileId>(0)
429 })?
430 {
431 if crate::file_table::Record::read(db, file)?
432 .is_some_and(|record| matches!(record.place, crate::file_location::Place::Cloud(_)))
433 {
434 crate::file_pins::request(db, file)?;
435 }
436 }
437 retire(db, id)
438}
439
440pub(crate) fn retire(db: &StoreSql, id: i64) -> Result<(), DbError> {
446 let copy: Option<String> = db
447 .query_row("SELECT copy FROM _coven_file_uploads WHERE id=?1", [id], |r| r.get(0))
448 .map_err(|error| match error {
449 sqlive::Error::Sqlite(sqlive::rusqlite::Error::QueryReturnedNoRows) => {
450 DbError::Invariant { what: "a retired upload is queued" }
451 }
452 error => DbError::from(error),
453 })?;
454 db.execute("DELETE FROM _coven_upload_files WHERE upload=?1", [id])?;
455 db.execute("DELETE FROM _coven_file_uploads WHERE id=?1", [id])?;
456 if let Some(copy) = copy {
457 release_copy(db, ©)?;
458 }
459 Ok(())
460}
461
462pub(crate) fn release_copy(db: &StoreSql, path: &str) -> Result<(), DbError> {
466 let named: bool = db.query_row(&format!("SELECT {FILE_NAMED_SQL}"), [path], |r| r.get(0))?;
467 if named {
468 return Ok(());
469 }
470 db.execute("DELETE FROM _coven_copy_chunks WHERE path=?1", [path])?;
471 db.execute("DELETE FROM _coven_copy_sizes WHERE path=?1", [path])?;
472 crate::file_removals::record_released(db, path)?;
473 Ok(())
474}
475
476pub(crate) struct Queued<'a> {
481 pub(crate) files: &'a [(FileId, bool)],
482 pub(crate) size: u64,
483 pub(crate) source: &'a crate::file_write::StagedSource,
484 pub(crate) hashes: &'a crate::file_hashes::FileHashes,
485 pub(crate) of: crate::MoveId,
486 pub(crate) holds: Option<u64>,
487}
488
489pub(crate) fn enqueue(
493 db: &StoreSql,
494 queued: Queued<'_>,
495 location: &FileLocation,
496 now: std::time::SystemTime,
497) -> Result<(), DbError> {
498 enqueue_all(db, &[(queued, location)], now)
499}
500
501pub(crate) fn enqueue_all(
505 db: &StoreSql,
506 uploads: &[(Queued<'_>, &FileLocation)],
507 now: std::time::SystemTime,
508) -> Result<(), DbError> {
509 let mut queue = Vec::new();
510 let mut chunks = Vec::new();
511 for (queued, location) in uploads {
512 let Queued { size, source, hashes, of, holds, .. } = queued;
513 let header = FileHeader::new(*size);
514 if hashes.chunks().len() as u64 != header.chunk_count() {
515 return Err(DbError::Invariant { what: "a staged file hashes every plaintext chunk" });
516 }
517 header.encrypted_size().map_err(|_overflow| DbError::Invariant {
519 what: "a queued file's encrypted layout fits in its offsets",
520 })?;
521 let size = crate::sqlite::integer("file size", *size)?;
522 let (original, modified_at, copy) = match source {
523 crate::file_write::StagedSource::Original(original) => (
524 Some(coven_foundation::files::path_bytes(&original.path)),
525 Some(crate::user_file::encode_time(original.modified_at)),
526 None,
527 ),
528 crate::file_write::StagedSource::Copy(name) => (None, None, Some(name.as_str())),
529 };
530 let (device, object) = (location.uploader(), location.object());
531 queue.push(vec![
532 value(&location.text())?,
533 value(&device)?,
534 value(&object)?,
535 value(&size)?,
536 value(&hashes.content.as_bytes().as_slice())?,
537 value(&original)?,
538 value(&modified_at)?,
539 value(©)?,
540 value(of)?,
541 value(&crate::user_file::encode_time(now))?,
542 value(&holds.map(u64::to_be_bytes))?,
543 ]);
544 for (index, hash) in hashes.chunks().iter().enumerate() {
545 chunks.push(vec![
546 value(&device)?,
547 value(&object)?,
548 value(&(index as i64))?,
549 value(&hash.as_bytes().as_slice())?,
550 ]);
551 }
552 }
553 let mut ids = std::collections::BTreeMap::new();
554 for (object, id) in insert_rows(
555 db,
556 11,
557 &queue,
558 |values| {
559 format!("INSERT INTO _coven_file_uploads(location,device,object,size,hash,original,modified_at,copy,move,queued_at,holds) VALUES{values} RETURNING object,id")
560 },
561 |r| Ok((r.get::<_, ObjectId>(0)?, r.get::<_, i64>(1)?)),
562 )? {
563 ids.insert(object, id);
564 }
565 insert_rows(
566 db,
567 4,
568 &chunks,
569 |values| format!("INSERT INTO _coven_file_chunks(device,object,chunk,hash) VALUES{values}"),
570 |_| Ok(()),
571 )?;
572 let mut linked = Vec::new();
573 let mut listed = Vec::new();
574 for (queued, location) in uploads {
575 let upload = ids
576 .get(&location.object())
577 .ok_or(DbError::Invariant { what: "an enqueued upload returns its id" })?;
578 for (file, keep) in queued.files {
579 linked.push(vec![value(file)?, value(upload)?, value(keep)?]);
580 listed.push((queued.of, *file));
581 }
582 }
583 insert_rows(
584 db,
585 3,
586 &linked,
587 |values| format!("INSERT INTO _coven_upload_files(file,upload,keep) VALUES{values}"),
588 |_| Ok(()),
589 )?;
590 crate::file_moves::list_all(db, &listed)?;
591 Ok(())
592}
593
594pub(crate) fn count_copy(db: &StoreSql, path: &str, size: u64) -> Result<(), DbError> {
598 db.execute(
599 "INSERT INTO _coven_copy_sizes(path,size) VALUES(?1,?2) ON CONFLICT(path) DO NOTHING",
600 (path, crate::sqlite::integer("copy size", size)?),
601 )?;
602 Ok(())
603}
604
605pub(crate) fn keep_copy(
609 db: &StoreSql,
610 file: FileId,
611 name: &str,
612 hashes: &crate::file_hashes::FileHashes,
613) -> Result<(), DbError> {
614 keep_copies(db, &[(file, name, hashes)])
615}
616
617pub(crate) fn keep_copies(
620 db: &StoreSql,
621 copies: &[(FileId, &str, &crate::file_hashes::FileHashes)],
622) -> Result<(), DbError> {
623 let mut files = Vec::new();
624 let mut chunks = Vec::new();
625 for (file, name, hashes) in copies {
626 files.push(vec![value(file)?, value(name)?, value(&hashes.content.as_bytes().as_slice())?]);
627 for (index, hash) in hashes.chunks().iter().enumerate() {
628 chunks.push(vec![
629 value(name)?,
630 value(&(index as i64))?,
631 value(&hash.as_bytes().as_slice())?,
632 ]);
633 }
634 }
635 insert_rows(
636 db,
637 3,
638 &files,
639 |values| format!("INSERT INTO _coven_device_files(file,path,hash) VALUES{values}"),
640 |_| Ok(()),
641 )?;
642 insert_rows(
643 db,
644 3,
645 &chunks,
646 |values| format!("INSERT INTO _coven_copy_chunks(path,chunk,hash) VALUES{values}"),
647 |_| Ok(()),
648 )?;
649 Ok(())
650}
651
652pub(crate) fn share(db: &StoreSql, from: FileId, to: FileId) -> Result<(), DbError> {
655 db.execute(
656 "INSERT INTO _coven_upload_files(file,upload,keep)
657 SELECT ?2,upload,keep FROM _coven_upload_files WHERE file=?1",
658 sqlive::rusqlite::params![from, to],
659 )?;
660 db.execute(
661 "INSERT INTO _coven_device_files(file,path,hash)
662 SELECT ?2,path,hash FROM _coven_device_files WHERE file=?1",
663 sqlive::rusqlite::params![from, to],
664 )?;
665 Ok(())
666}
667
668pub(crate) fn forget(db: &StoreSql, file: FileId) -> Result<(), DbError> {
672 db.execute("DELETE FROM _coven_upload_files WHERE file=?1", [file])?;
673 Ok(())
674}
675
676pub(crate) fn forget_copy(db: &StoreSql, file: FileId) -> Result<(), DbError> {
679 for copy in
680 db.query("DELETE FROM _coven_device_files WHERE file=?1 RETURNING path", [file], |r| {
681 r.get::<_, String>(0)
682 })?
683 {
684 release_copy(db, ©)?;
685 }
686 Ok(())
687}
688
689pub(crate) fn forget_session(db: &StoreSql, id: i64) -> Result<(), DbError> {
692 db.execute(
693 "UPDATE _coven_file_uploads SET session=NULL,bytes_confirmed=NULL,bytes_total=NULL
694 WHERE id=?1
695 AND NOT EXISTS(SELECT 1 FROM _coven_upload_files WHERE upload=?1)",
696 [id],
697 )?;
698 Ok(())
699}
700
701pub(crate) fn record_session(
704 db: &StoreSql,
705 id: i64,
706 session: &[u8],
707 confirmed: i64,
708 total: i64,
709) -> Result<usize, DbError> {
710 Ok(db.execute(
711 "UPDATE _coven_file_uploads SET session=?2,bytes_confirmed=?3,bytes_total=?4
712 WHERE id=?1",
713 (id, session, confirmed, total),
714 )?)
715}
716
717pub(crate) fn stored(db: &StoreSql, upload: i64) -> Result<(), DbError> {
719 db.execute("UPDATE _coven_file_uploads SET stored=1 WHERE id=?1", [upload])?;
720 Ok(())
721}
722
723pub(crate) fn unswitch_move(db: &StoreSql, of: crate::MoveId) -> Result<(), DbError> {
726 db.execute(
727 "DELETE FROM _coven_upload_files
728 WHERE upload IN (SELECT id FROM _coven_file_uploads WHERE move=?1)",
729 [of],
730 )?;
731 Ok(())
732}
733
734#[cfg(test)]
735#[path = "file_queue_tests.rs"]
736pub(crate) mod tests;