1use super::*;
4use crate::sqlite::StoreSql;
5use coven_foundation::files::StoreDirAccess;
6use sqlive::{Database, Options};
7use std::num::NonZeroUsize;
8
9const READERS: NonZeroUsize = NonZeroUsize::new(4).expect("four is not zero");
11
12use crate::{migration::MigrationOperation, AutoVacuum, NewOperation, OpenError, OpenStep};
13use coven_foundation::progress::{ProgressChannel, ProgressReport, ProgressWatch};
14
15#[derive(Clone)]
18pub(crate) struct OpenInputs {
19 pub(crate) tables: Vec<SyncedTable>,
20 pub(crate) migrations: Vec<Migration>,
21 pub(crate) namespaces: Vec<crate::Namespace>,
22 pub(crate) sqlite: crate::SqliteSettings,
23 pub(crate) functions: crate::SqlFunctions,
24 pub(crate) sync_functions: Option<crate::SyncFunctions>,
25}
26
27pub struct DatabaseBuilder {
29 directory: StoreDir,
30 tables: Option<Vec<SyncedTable>>,
31 namespaces: Vec<crate::Namespace>,
32 migrations: Option<Vec<Migration>>,
33 migration_operation: Option<Box<MigrationOperation>>,
34 joining_write: Option<Box<crate::joining_write::JoiningHook>>,
35 before_commit: Option<Box<crate::before_commit::BeforeCommit>>,
36 installation: crate::migration_run::Installation,
37 clock: Option<ClockRef>,
38 ids: Option<IdSourceRef>,
39 sqlite: crate::SqliteSettings,
40 functions: crate::SqlFunctions,
41 sync_functions: Option<crate::SyncFunctions>,
42 progress: ProgressChannel<OpenStep>,
43 #[cfg(test)]
44 upkeep_threads: Option<usize>,
45 #[cfg(any(test, feature = "test-utils"))]
46 refuse_close: bool,
47}
48
49impl DatabaseBuilder {
50 pub fn new(directory: StoreDir) -> Self {
52 Self {
53 directory,
54 tables: None,
55 namespaces: Vec::new(),
56 migrations: None,
57 migration_operation: None,
58 joining_write: None,
59 before_commit: None,
60 installation: crate::migration_run::Installation::Own,
61 clock: None,
62 ids: None,
63 sqlite: crate::SqliteSettings::default(),
64 functions: crate::SqlFunctions::default(),
65 sync_functions: None,
66 progress: ProgressChannel::new(),
67 #[cfg(test)]
68 upkeep_threads: None,
69 #[cfg(any(test, feature = "test-utils"))]
70 refuse_close: false,
71 }
72 }
73
74 #[cfg(any(test, feature = "test-utils"))]
76 pub fn refuse_close(mut self) -> Self {
77 self.refuse_close = true;
78 self
79 }
80
81 #[cfg(test)]
83 pub(crate) fn refuse_upkeep_thread_after(mut self, started: usize) -> Self {
84 self.upkeep_threads = Some(started);
85 self
86 }
87
88 pub fn subscribe_progress(&self) -> ProgressWatch<OpenStep> {
91 self.progress.watch()
92 }
93
94 pub fn synced_tables(mut self, tables: Vec<SyncedTable>) -> Self {
96 self.tables = Some(tables);
97 self
98 }
99
100 pub fn namespaces(mut self, namespaces: Vec<crate::Namespace>) -> Self {
102 self.namespaces = namespaces;
103 self
104 }
105
106 pub fn migrations(mut self, migrations: Vec<Migration>) -> Self {
108 self.migrations = Some(migrations);
109 self
110 }
111
112 pub fn loading_store(mut self) -> Self {
117 self.installation = crate::migration_run::Installation::Loading;
118 self
119 }
120
121 pub(crate) fn rebuilding(mut self, header: coven_format::write::WriteHeader) -> Self {
125 self.installation = crate::migration_run::Installation::Rebuilding(header);
126 self
127 }
128
129 pub fn migration_operation<F>(mut self, operation: F) -> Self
134 where
135 F: Fn(u32) -> Result<NewOperation, DbError> + Send + Sync + 'static,
136 {
137 self.migration_operation = Some(Box::new(operation));
138 self
139 }
140
141 pub fn joining_write<F>(mut self, write: F) -> Self
146 where
147 F: Fn(&crate::JoiningWrite<'_>, &crate::Joining) -> Result<(), DbError>
148 + Send
149 + Sync
150 + 'static,
151 {
152 self.joining_write = Some(Box::new(write));
153 self
154 }
155
156 pub fn before_commit<F>(mut self, function: F) -> Self
163 where
164 F: Fn(&crate::CommitSql<'_>) -> Result<(), DbError> + Send + Sync + 'static,
165 {
166 self.before_commit = Some(Box::new(function));
167 self
168 }
169
170 pub fn clock(mut self, clock: ClockRef) -> Self {
172 self.clock = Some(clock);
173 self
174 }
175
176 pub fn id_source(mut self, ids: IdSourceRef) -> Self {
178 self.ids = Some(ids);
179 self
180 }
181
182 pub fn sqlite_settings(mut self, settings: crate::SqliteSettings) -> Self {
186 self.sqlite = settings;
187 self
188 }
189
190 pub fn sql_functions(mut self, functions: crate::SqlFunctions) -> Self {
193 self.functions = functions;
194 self
195 }
196
197 pub fn sync_functions(mut self, functions: crate::SyncFunctions) -> Self {
204 self.sync_functions = Some(functions);
205 self
206 }
207
208 pub(crate) fn sync_functions_as_opened(
211 mut self,
212 functions: Option<crate::SyncFunctions>,
213 ) -> Self {
214 self.sync_functions = functions;
215 self
216 }
217
218 pub async fn open(self) -> Result<StoreDatabase, OpenError> {
223 let lock = self.directory.lock_for_open().await?;
224 self.open_locked(lock).await
225 }
226
227 pub async fn open_locked(self, lock: StoreLock) -> Result<StoreDatabase, OpenError> {
230 let closing = lock.closing_guard();
231 let bugs = self.directory.bugs().clone();
232 let result = finish_blocking(
233 blocking_completing("open the store", bugs, move || self.open_graph(lock)).await,
234 );
235 closing.disarm();
238 result
239 }
240
241 pub async fn open_read_only(self) -> Result<StoreDatabase<ReadOnly>, OpenError> {
245 let bugs = self.directory.bugs().clone();
246 finish_blocking(
247 blocking_completing("open the store read-only", bugs, move || self.open_read_graph())
248 .await,
249 )
250 }
251
252 fn open_graph(self, lock: StoreLock) -> Result<StoreDatabase, OpenError> {
253 let tables =
254 self.tables.ok_or(OpenError::MissingConfiguration { field: "synced_tables" })?;
255 let migrations =
256 self.migrations.ok_or(OpenError::MissingConfiguration { field: "migrations" })?;
257 coven_foundation::task::operation_step("checking the store's lock");
258 coven_foundation::files::verify_lock(&self.directory, &lock)?;
260 let settings = lock.settings()?;
261 let clock = match self.clock {
262 Some(clock) => clock,
263 None => Arc::new(SystemClock),
264 };
265 let ids = match self.ids {
266 Some(ids) => ids,
267 None => Arc::new(UuidIds),
268 };
269 let started = clock.elapsed();
270 let open_inputs = OpenInputs {
271 tables: tables.clone(),
272 migrations: migrations.clone(),
273 namespaces: self.namespaces.clone(),
274 sqlite: self.sqlite.clone(),
275 functions: self.functions.clone(),
276 sync_functions: self.sync_functions.clone(),
277 };
278 let path = self.directory.database_path();
279 let whole = crate::internal_schema::whole_views();
280 let observer = store_observer(self.directory.bugs());
283 #[cfg(test)]
284 observer.refuse_upkeep_thread_after(self.upkeep_threads);
285 let facts = crate::memory_facts::ProcessFacts::new(observer.clone());
286 let views = |setup: &Setup<'_, crate::app_policy::CovenPolicy>| {
287 crate::coven_functions::register_views(
288 setup,
289 settings.device_id,
290 &facts,
291 self.sync_functions.as_ref(),
292 )
293 };
294 let recorded = crate::file_table::synced(&tables)
295 .iter()
296 .map(|table| table.name.to_ascii_lowercase())
297 .collect();
298 let opening = crate::store_call::TestRecords::new();
301 coven_foundation::task::operation_step("opening the store's database");
302 let setup = crate::app_policy::setup(&self.functions, Some(&views));
305 let opened = Database::open_with(
306 &path,
307 observer.clone(),
308 Options {
309 settings: self.sqlite.clone(),
310 readers: READERS,
311 prepare_on_reader: false,
312 whole_views: whole.clone(),
313 policy: &|| CovenPolicy::new(&tables),
314 setup: &setup,
315 capture: Some(recorded),
316 checkpoint_pages: self.sqlite.checkpoint_pages(),
317 statistics: true,
318 },
319 |writer| -> Result<_, OpenError> {
320 let call = &crate::store_call::StoreCall::new(None, &opening);
321 let mut progress = ProgressReport::of(&self.progress, clock.clone());
322 let migrating = clock.elapsed();
323 coven_foundation::task::operation_step(
324 "preparing the schema and running migrations",
325 );
326 let outcomes = crate::schema_preparation::prepare_schema(
327 writer,
328 call,
329 &tables,
330 &migrations,
331 &self.functions.versions(),
332 crate::schema_preparation::Preparation::Writable {
333 author: crate::migration_run::MigrationOrigin(
334 settings.device_id,
335 clock.now(),
336 self.installation.clone(),
337 ),
338 operation: self.migration_operation.as_deref(),
339 audiences: &|live, call| {
340 crate::audience_database::AudienceDatabase::stage(
341 &self.directory,
342 &self.sqlite,
343 &self.functions,
344 self.sync_functions.as_ref(),
345 &observer,
346 settings.device_id,
347 &tables,
348 &whole,
349 live,
350 call,
351 )
352 },
353 before_commit: self.before_commit.as_deref(),
354 },
355 &mut progress,
356 )?;
357 coven_foundation::task::operation_step("reading the store's state after migrating");
358 crate::database::file_database::declare_budgets(writer, &self.namespaces)?;
359 crate::file_removals::finish_at_open(writer, call, &self.directory)?;
360 let db = &*writer;
361 crate::pending::forget_causes(db)?;
362 crate::internal_schema::retry_effect::save(
364 db,
365 crate::internal_schema::retry_effect::RetryState::OPENED,
366 )?;
367 let mut write_schema =
368 crate::write_schema::WriteSchema::read(db, call, tables.clone())?;
369 write_schema.conversions = crate::migration_receive::read(db, &migrations)?;
370 write_schema.namespaces = self.namespaces;
371 crate::write_schema::prepare_evaluators(db, &write_schema)?;
372 let migrated = clock.elapsed();
373 coven_foundation::task::operation_step(
374 "gathering statistics, then starting commit capture and upkeep threads",
375 );
376 Ok((outcomes, write_schema, migrating, migrated))
377 },
378 );
379 drop(setup);
380 let (database, (outcomes, write_schema, migrating, migrated)) = opened?;
381 let opened = clock.elapsed();
382 #[cfg(any(test, feature = "test-utils"))]
383 if self.refuse_close {
384 database.host(|writer| writer.leave_statement_open());
385 }
386 coven_foundation::task::operation_step("opening the input database");
390 let functions = crate::SqlFunctions::default();
391 let input_setup = crate::app_policy::setup(&functions, None);
392 let (input, ()) = Database::open_with(
393 &self.directory.input_database_path(),
394 observer.clone(),
395 Options {
396 settings: coven_only_settings(),
397 readers: NonZeroUsize::MIN,
398 prepare_on_reader: false,
399 whole_views: crate::internal_schema::whole_views(),
400 policy: &|| CovenPolicy::new(&[]),
401 setup: &input_setup,
402 capture: None,
403 checkpoint_pages: Some(sqlive::CHECKPOINT_PAGES),
404 statistics: false,
405 },
406 |input| -> Result<(), DbError> {
407 let version: u32 = input.query_row("PRAGMA application_id", [], |r| r.get(0))?;
408 match version {
409 0 => input.transaction(|db| {
410 Ok::<_, DbError>(db.batch(&crate::internal_schema::initial_input_schema())?)
411 }),
412 1 => Ok(()),
413 _ => Err(DbError::DamagedDatabase),
414 }
415 },
416 )?;
417 let mut sizes = super::library_bytes::DatabaseSizes::new(
420 database.checkpointed_size(),
421 input.checkpointed_size(),
422 );
423 database.host(|writer| super::library_bytes::record_opened(writer, &mut sizes))?;
424 database.host(|writer| super::archive_bytes::record_opened(writer, &self.directory))?;
427 let inputs = input_database::InputDatabase::new(input)?;
428 tracing::info!(
429 migrations = migrations.len(),
430 migrating = ?(migrated - migrating),
431 opening = ?(opened - migrated),
432 total = ?(clock.elapsed() - started),
433 "opened the database"
434 );
435 let store = StoreDatabase {
436 observer: observer.clone(),
437 closing: Arc::new(lock.closing_guard()),
438 writer: WriterParts {
439 facts,
440 file_tasks: Arc::new(tokio::sync::RwLock::new(())),
441 #[cfg(test)]
442 close_waits: Arc::new(Mutex::new(None)),
443 #[cfg(test)]
444 abandoned_stagings: Arc::new(Mutex::new(Vec::new())),
445 },
446 bugs: self.directory.bugs().clone(),
447 inner: Arc::new(RwLock::new(Some(DatabaseInner {
448 inputs,
449 staging: Arc::new(crate::reserved_names::ReservedNames::new(observer.clone())),
450 access: DatabaseAccess {
451 database,
452 observer,
453 write_schema,
454 directory: self.directory,
455 device: settings.device_id,
456 lock,
457 tests: crate::store_call::TestRecords::new(),
458 #[cfg(test)]
459 local_open_gate: Mutex::new(None),
460 },
461 migrations: outcomes,
462 raise_operation: self.migration_operation,
463 joining_write: self.joining_write,
464 before_commit: self.before_commit,
465 open_inputs,
466 clock,
467 ids,
468 copying: Arc::new(Mutex::new(BTreeSet::new())),
469 #[cfg(test)]
470 removal_gate: Mutex::new(None),
471 #[cfg(test)]
472 flush_hook: Mutex::new(None),
473 #[cfg(any(test, feature = "test-utils"))]
474 retry_end_failures: std::sync::atomic::AtomicUsize::new(0),
475 #[cfg(any(test, feature = "test-utils"))]
476 applied_slot_failures: std::sync::atomic::AtomicUsize::new(0),
477 }))),
478 };
479 store.follow_database_sizes(sizes);
480 Ok(store)
481 }
482 fn open_read_graph(self) -> Result<StoreDatabase<ReadOnly>, OpenError> {
483 let tables =
484 self.tables.ok_or(OpenError::MissingConfiguration { field: "synced_tables" })?;
485 let migrations =
486 self.migrations.ok_or(OpenError::MissingConfiguration { field: "migrations" })?;
487 let lock = self.directory.lock_read_only()?;
488 let settings = self.directory.settings()?;
489 let path = self.directory.database_path();
490 let whole = crate::internal_schema::whole_views();
491 let observer = store_observer(self.directory.bugs());
493 let facts = crate::memory_facts::ProcessFacts::new(observer.clone());
496 let views = |setup: &Setup<'_, crate::app_policy::CovenPolicy>| {
497 crate::coven_functions::register_views(
498 setup,
499 settings.device_id,
500 &facts,
501 self.sync_functions.as_ref(),
502 )
503 };
504 let clock = match self.clock {
506 Some(clock) => clock,
507 None => Arc::new(SystemClock),
508 };
509 let setup = crate::app_policy::setup(&self.functions, Some(&views));
510 let (database, schema) = Database::open_with(
513 &path,
514 observer.clone(),
515 Options {
516 settings: self.sqlite.clone(),
517 readers: READERS,
518 prepare_on_reader: true,
519 whole_views: whole,
520 policy: &|| CovenPolicy::read_only(&tables),
521 setup: &setup,
522 capture: None,
523 checkpoint_pages: None,
524 statistics: false,
525 },
526 |first| {
527 crate::store_call::outside_store(|call| {
528 crate::schema_preparation::prepare_schema(
529 first,
530 call,
531 &tables,
532 &migrations,
533 &self.functions.versions(),
534 crate::schema_preparation::Preparation::ReadOnly,
535 &mut ProgressReport::of(&self.progress, clock),
536 )
537 })?;
538 let mut schema = crate::store_call::outside_store(|call| {
539 crate::write_schema::WriteSchema::read(first, call, tables.clone())
540 })?;
541 schema.namespaces = self.namespaces;
542 Ok::<_, OpenError>(schema)
543 },
544 )?;
545 Ok(StoreDatabase {
546 observer: observer.clone(),
547 closing: Arc::new(lock.closing_guard()),
548 writer: (),
549 bugs: self.directory.bugs().clone(),
550 inner: Arc::new(RwLock::new(Some(DatabaseAccess {
551 database,
552 observer,
553 lock,
554 write_schema: schema,
555 directory: self.directory,
556 device: settings.device_id,
557 tests: crate::store_call::TestRecords::new(),
558 #[cfg(test)]
559 local_open_gate: Mutex::new(None),
560 }))),
561 })
562 }
563}
564
565pub async fn discarded_operations(
572 path: std::path::PathBuf,
573 bugs: &coven_foundation::task::BugSinkRef,
574) -> Result<Vec<crate::OperationRecord>, DbError> {
575 finish_blocking(
576 blocking_completing("read the discarded operations", bugs.clone(), move || {
577 let db = coven_only(&path, true, None)?;
578 crate::operation::read(&db)
579 })
580 .await,
581 )
582}
583
584pub async fn read_device_id_in_use(
592 path: std::path::PathBuf,
593 bugs: &coven_foundation::task::BugSinkRef,
594) -> Result<Option<coven_merge::Slot>, DbError> {
595 finish_blocking(
596 blocking_completing("read the device id in use record", bugs.clone(), move || {
597 let db = coven_only(&path, true, None)?;
598 crate::device_reset::read_in_use(&db)
599 })
600 .await,
601 )
602}
603
604pub async fn read_store_name(directory: StoreDir) -> Result<String, OpenError> {
610 let bugs = directory.bugs().clone();
611 finish_blocking(
612 blocking_completing("read the store name", bugs, move || {
613 let _lock = directory.lock_read_only()?;
614 let database = coven_only(&directory.database_path(), true, None)?;
615 Ok(crate::store_state_tables::name(&database)?)
616 })
617 .await,
618 )
619}
620
621pub async fn closed_sync_facts<P: crate::PendingValue + Clone + Send + 'static>(
631 directory: StoreDir,
632 lock: coven_foundation::files::ClosedStoreLock,
633) -> Result<crate::SyncFacts<P>, OpenError> {
634 let bugs = directory.bugs().clone();
635 finish_blocking(
636 blocking_completing("read the closed store's sync status", bugs, move || {
637 let _lock = lock;
638 let settings = directory.settings()?;
639 let observer = store_observer(directory.bugs());
640 let facts = crate::memory_facts::ProcessFacts::new(observer);
641 let views = |setup: &Setup<'_, crate::app_policy::CovenPolicy>| {
642 crate::coven_functions::register_views(setup, settings.device_id, &facts, None)
645 };
646 let mut database = coven_only(&directory.database_path(), true, Some(&views))?;
647 crate::store_call::outside_store(|call| {
648 let database = &mut database;
649 crate::schema_preparation::prepare_internal_schema(database)?;
650 let db = &*database;
651 let _constants = crate::transaction_constants::keep_constants(db, call);
652 db.read_snapshot(|db| Ok::<_, OpenError>(crate::sync_facts::read(db)?))
653 })
654 })
655 .await,
656 )
657}
658
659pub async fn discarded_store_name(
665 path: std::path::PathBuf,
666 bugs: &coven_foundation::task::BugSinkRef,
667) -> Result<String, DbError> {
668 finish_blocking(
669 blocking_completing("read the discarded store name", bugs.clone(), move || {
670 let database = coven_only(&path, true, None)?;
671 crate::store_state_tables::name(&database)
672 })
673 .await,
674 )
675}
676
677fn store_observer(bugs: &coven_foundation::task::BugSinkRef) -> StoreHandle {
680 StoreHandle::new(
681 bugs.clone(),
682 crate::internal_schema::UNRECORDED.iter().map(|table| (*table).to_owned()).collect(),
683 )
684}
685
686fn coven_only_settings() -> crate::SqliteSettings {
690 crate::SqliteSettings { auto_vacuum: Some(AutoVacuum::Off), ..Default::default() }
694}
695
696pub(crate) fn coven_only(
699 path: &std::path::Path,
700 read_only: bool,
701 views: Option<&crate::coven_functions::Views<'_>>,
702) -> Result<StoreSql, DbError> {
703 let functions = crate::SqlFunctions::default();
704 let setup = crate::app_policy::setup(&functions, views);
705 let connection = StoreSql::open(
706 path,
707 read_only,
708 CovenPolicy::new(&[]),
709 crate::internal_schema::whole_views(),
710 &coven_only_settings(),
711 &setup,
712 )?;
713 Ok(connection)
714}
715
716#[cfg(test)]
717#[path = "database_builder_tests.rs"]
718mod tests;