Skip to main content

coven_database/
snapshot_staging.rs

1//! A snapshot's checked records, staged on the input connection while its
2//! body streams, until one load or file check reads them (§16.2, §21.2).
3//! Rows, cells, references, source records and creations are staged as
4//! columns of tables of their own, so installing reads fields rather than
5//! decoding records again; the few other records keep their frames.
6
7use super::InputDatabase;
8use crate::database::{start_call, DatabaseInner, StoreDatabase};
9use crate::snapshot_error::{invalid, SnapshotError};
10use crate::snapshot_layout::{SnapshotLayout, TableLayout};
11use crate::snapshot_metadata::SnapshotMetadata;
12use crate::sqlite::StoreSql;
13use crate::store_call::StoreCall;
14use crate::write_encoding::{encoded, sql_value};
15use crate::write_metadata::MetadataFailure;
16use crate::write_schema::WriteSchema;
17use crate::DbError;
18use coven_format::{
19    admin::SnapshotId,
20    merge_fields,
21    sealed_snapshot::SnapshotObjectPrefix,
22    snapshot::{section, SnapshotHeader, SnapshotRecord},
23    snapshot_rows::{AuthoredRecord, MergeRow},
24    snapshot_stream::SnapshotChunkDecoder,
25    value::Value,
26    write::WriteHeader,
27};
28use coven_foundation::locks::read_unpoisoned;
29use sqlive::rusqlite::{params, params_from_iter};
30use std::collections::BTreeSet;
31use std::sync::Arc;
32
33/// Created with the input connection, in its temporary storage, so staged
34/// records go with the connection: no restart finds them. That storage spills
35/// to files once the connection's cache fills; opening refuses a SQLite built
36/// to keep it in memory. Only the snapshot loader names these tables.
37pub(super) fn create_tables(db: &StoreSql) -> Result<(), DbError> {
38    db.batch("PRAGMA temp_store=FILE;")?;
39    crate::snapshot_metadata::create_table(db)
40}
41
42/// One staging's number and the tables that hold its records on the input
43/// connection.
44struct StagedTables {
45    id: i64,
46    tables: Vec<String>,
47}
48
49/// A staging's tables of its own, beside its rows' tables.
50pub(crate) fn table(staging: i64, name: &str) -> String {
51    format!("temp._coven_staged_{staging}_{name}")
52}
53
54#[path = "staged_damage.rs"]
55mod damage;
56use damage::with_staged_digest;
57#[cfg(any(test, feature = "test-utils"))]
58pub use damage::StagedDamage;
59pub use damage::StagedDigest;
60
61fn create_staged(
62    db: &StoreSql,
63    layout: &SnapshotLayout,
64    staging: i64,
65) -> Result<Vec<String>, DbError> {
66    let mut tables = Vec::new();
67    for table in &layout.tables {
68        db.batch(&table.create_staged(staging))?;
69        tables.push(table.staged(staging));
70    }
71    let name = |name: &str| table(staging, name).trim_start_matches("temp.").to_owned();
72    db.batch(&format!(
73        "CREATE TEMP TABLE {}(table_name TEXT NOT NULL,key BLOB NOT NULL,audience TEXT NOT NULL,generation BLOB NOT NULL,device BLOB NOT NULL,number BLOB NOT NULL);
74         CREATE TEMP TABLE {}(table_name TEXT NOT NULL,column_name TEXT NOT NULL,PRIMARY KEY(table_name,column_name)) WITHOUT ROWID;
75         CREATE TEMP TABLE {}(write_id BLOB NOT NULL,audience TEXT NOT NULL,header BLOB NOT NULL,PRIMARY KEY(write_id,audience)) WITHOUT ROWID;
76         CREATE TEMP TABLE {}(write_id BLOB NOT NULL,audience TEXT NOT NULL,table_name TEXT NOT NULL,key BLOB NOT NULL,record BLOB NOT NULL,admission BLOB,stop BLOB);
77         CREATE TEMP TABLE {}(table_name TEXT NOT NULL,key BLOB NOT NULL,audience TEXT NOT NULL,write_id BLOB NOT NULL,creator BLOB NOT NULL);
78         CREATE TEMP TABLE {}(section INTEGER NOT NULL,evidence INTEGER NOT NULL CHECK(evidence IN (0,1)),record BLOB NOT NULL);",
79        name("generations"),
80        name("columns"),
81        name("parts"),
82        name("authored"),
83        name("creations"),
84        name("records"),
85    ))?;
86    for name in ["generations", "columns", "parts", "authored", "creations", "records"] {
87        tables.push(table(staging, name));
88    }
89    Ok(tables)
90}
91
92/// Drop a staging's tables whole.
93fn remove(db: &StoreSql, staging: &StagedTables) -> Result<(), DbError> {
94    for table in &staging.tables {
95        db.batch(&format!("DROP TABLE {table}"))?;
96    }
97    crate::snapshot_metadata::remove(db, staging.id)
98}
99
100/// Drop the tables of each of `stagings` that still holds its records, in
101/// the transaction of the operation that ends them.
102fn remove_all(db: &StoreSql, stagings: &[SnapshotStaging]) -> Result<(), DbError> {
103    stagings
104        .iter()
105        .filter_map(|staging| staging.tables.as_ref())
106        .try_for_each(|tables| remove(db, tables))
107}
108
109/// The statements staging one snapshot runs for its records, written once
110/// for its tables.
111struct Inserts {
112    layout: SnapshotLayout,
113    /// Each synced table's row insert, in the layout's order.
114    rows: Vec<String>,
115    generations: String,
116    columns: String,
117    parts: String,
118    authored: String,
119    creations: String,
120    records: String,
121}
122
123impl Inserts {
124    fn new(layout: SnapshotLayout, staging: i64) -> Self {
125        let rows = layout
126            .tables
127            .iter()
128            .map(|table| {
129                format!(
130                    "INSERT INTO {}({}) VALUES({})",
131                    table.staged(staging),
132                    table.names().join(","),
133                    vec!["?"; table.width()].join(",")
134                )
135            })
136            .collect();
137        let insert = |name: &str, columns: &str| {
138            let count = columns.split(',').count();
139            format!(
140                "INSERT INTO {}({columns}) VALUES({})",
141                table(staging, name),
142                (1..=count).map(|i| format!("?{i}")).collect::<Vec<_>>().join(",")
143            )
144        };
145        Self {
146            layout,
147            rows,
148            generations: insert("generations", "table_name,key,audience,generation,device,number"),
149            columns: insert("columns", "table_name,column_name"),
150            parts: insert("parts", "write_id,audience,header"),
151            authored: insert("authored", "write_id,audience,table_name,key,record,admission,stop"),
152            creations: insert("creations", "table_name,key,audience,write_id,creator"),
153            records: insert("records", "section,evidence,record"),
154        }
155    }
156}
157
158impl InputDatabase {
159    fn begin_staging(&self, schema: &WriteSchema) -> Result<(StagedTables, Inserts), DbError> {
160        let id = self.stagings.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
161        let layout = SnapshotLayout::new(schema);
162        let tables = self.write(|db, _| create_staged(db, &layout, id))?;
163        Ok((StagedTables { id, tables }, Inserts::new(layout, id)))
164    }
165
166    /// The damage a test asked for to the staging finishing now, if any.
167    #[cfg(any(test, feature = "test-utils"))]
168    fn next_finish_damage(&self) -> Option<StagedDamage> {
169        let mut damage = coven_foundation::locks::lock_unpoisoned(&self.damage_finishing);
170        let (made, count) = (*damage)?;
171        *damage = (count > 1).then_some((made, count - 1));
172        Some(made)
173    }
174
175    /// Remove a staging's records.
176    fn remove_staging(&self, staging: &StagedTables) -> Result<(), DbError> {
177        self.with_writer(|writer, _| writer.transaction(|db| remove(db, staging)))
178    }
179
180    /// Run one load or file check over `stagings`, the last phase of each:
181    /// `consume` reads their records in one input transaction, which removes
182    /// them as it commits. When it fails, the records are removed before the
183    /// failure is reported, so nothing staged outlives its operation (§16.2).
184    pub(crate) fn read_staged<T, S: AsMut<[SnapshotStaging]>>(
185        &self,
186        stagings: &mut S,
187        consume: impl FnOnce(
188            &crate::input_preparation::PreparedInputs<'_>,
189            &mut S,
190        ) -> Result<T, DbError>,
191    ) -> Result<T, DbError> {
192        self.with_writer(|writer, _| {
193            #[cfg(any(test, feature = "test-utils"))]
194            if let Some(made) = coven_foundation::locks::lock_unpoisoned(&self.damage_staged).take()
195            {
196                let first = stagings.as_mut().iter().find_map(|staging| staging.tables.as_ref());
197                let first = first.map(|tables| tables.id).ok_or(DbError::Invariant {
198                    what: "a test damages a staging that holds its records",
199                })?;
200                writer.transaction(|db| damage::damage(db, first, made))?;
201            }
202            let read = writer.transaction(|db| {
203                stagings.as_mut().iter().try_for_each(|staging| damage::unchanged(db, staging))?;
204                let inputs = crate::input_preparation::PreparedInputs::new(db)?;
205                let result = consume(&inputs, stagings)?;
206                inputs.finish()?;
207                remove_all(db, stagings.as_mut())?;
208                Ok(result)
209            });
210            let result = match read {
211                Ok(result) => Ok(result),
212                Err(error) => {
213                    let error = with_staged_digest(writer, stagings.as_mut(), error);
214                    let removed = writer.transaction(|db| {
215                        #[cfg(any(test, feature = "test-utils"))]
216                        if self.fail_removal.swap(false, std::sync::atomic::Ordering::SeqCst) {
217                            return Err(damage::removal_failure());
218                        }
219                        remove_all(db, stagings.as_mut())
220                    });
221                    match removed {
222                        Ok(()) => Err(error),
223                        // The stagings keep their tables, which dropping
224                        // them removes.
225                        Err(removal) => {
226                            return Err(DbError::StagingCleanup {
227                                operation: Box::new(error),
228                                cleanup: Box::new(removal),
229                            })
230                        }
231                    }
232                }
233            };
234            stagings.as_mut().iter_mut().for_each(SnapshotStaging::taken);
235            result
236        })
237    }
238}
239
240/// One snapshot's staging, from its first batch to the load or file check
241/// that reads its records, or the discard that removes them (§16.2). It is
242/// the one holder of what it stages for that whole operation: its steps are
243/// its phases, and the load takes it whole. Dropped while still holding
244/// records, as by a cancelled or panicking caller, it removes them.
245pub struct SnapshotStaging {
246    database: StoreDatabase,
247    expected: SnapshotId,
248    /// Absent once the load or file check that read them removed them.
249    tables: Option<StagedTables>,
250    phase: Phase,
251    /// The batches staged so far, and the time staging them took.
252    batches: usize,
253    time: std::time::Duration,
254}
255
256/// Where a staging is in its operation.
257enum Phase {
258    /// The body streams in: sync hands it each batch of opened plaintext,
259    /// decoded, checked and committed in one input transaction, outside
260    /// the application writer (§21.2). The state, near a kilobyte with its
261    /// decoder, lives on the heap: each batch moves it to the blocking pool
262    /// and back, and every future holding the staging carries what is inline.
263    Streaming(Box<StagingState>),
264    /// The stream checked out to its end marker.
265    Staged {
266        header: SnapshotHeader,
267        /// The digest of the records as staging ended, which every load or
268        /// file check compares with what the disk holds before it reads
269        /// any (§16.2).
270        digest: StagedDigest,
271        /// The first record that failed the loader's checks as it staged,
272        /// and its section: the load or file check refuses the body with it
273        /// where it reaches that section.
274        refusal: Option<(usize, DbError)>,
275    },
276    /// A batch or the end of the stream failed; only discarding is left.
277    Failed,
278}
279
280/// What staging keeps between batches beside its tables: the decoder, the
281/// statements that stage records, the records staged so far, the columns the
282/// snapshot names, the source part whose changes are arriving, and the first
283/// record the loader's checks refuse.
284struct StagingState {
285    decoder: SnapshotChunkDecoder,
286    inserts: Inserts,
287    records: i64,
288    columns: BTreeSet<(String, String)>,
289    part: Option<StagedPart>,
290    refusal: Option<(usize, DbError)>,
291}
292
293/// The part of section 4 whose changes follow it (D7.3). Its row is staged
294/// with its first change, which names its audience.
295struct StagedPart {
296    write: Vec<u8>,
297    header: WriteHeader,
298    staged: bool,
299}
300
301impl Drop for SnapshotStaging {
302    fn drop(&mut self) {
303        let Some(tables) = self.tables.take() else {
304            return;
305        };
306        // Only an abandoned staging gets here: a cancelled or panicking
307        // caller. Its connection's temporary storage goes when the store
308        // closes, which waits for this removal. Nobody waits for the removal,
309        // so a bug in it goes to the store's bug sink.
310        let bugs = self.database.bugs.clone();
311        let remove = move |inner: &DatabaseInner| {
312            let removed =
313                coven_foundation::task::caught_sync("remove an abandoned snapshot staging", || {
314                    inner.call(|inner| inner.inputs.remove_staging(&tables))
315                });
316            match removed {
317                Ok(Ok(())) => {}
318                Ok(Err(error)) => {
319                    if !coven_foundation::task::report_carried(bugs.as_ref(), &error) {
320                        tracing::error!(%error, "removing an abandoned snapshot staging failed");
321                    }
322                }
323                Err(bug) => coven_foundation::task::report(bugs.as_ref(), &bug),
324            }
325            Ok::<_, DbError>(())
326        };
327        if tokio::runtime::Handle::try_current().is_ok() {
328            let removing = start_call(
329                "remove an abandoned snapshot staging",
330                &self.database.bugs,
331                self.database.inner.clone(),
332                remove,
333            );
334            #[cfg(test)]
335            coven_foundation::locks::lock_unpoisoned(&self.database.writer.abandoned_stagings)
336                .push(removing);
337            #[cfg(not(test))]
338            drop(removing);
339        } else if let Some(inner) = read_unpoisoned(&self.database.inner).as_ref() {
340            let _removed = remove(inner);
341        }
342    }
343}
344
345impl StoreDatabase {
346    /// Begin staging the snapshot `id`, whose signed prefix is `prefix`
347    /// (§16.2). Nothing outside the loader reads what it stages.
348    pub async fn stage_snapshot(
349        &self,
350        id: SnapshotId,
351        prefix: SnapshotObjectPrefix,
352    ) -> Result<SnapshotStaging, DbError> {
353        let (tables, inserts) =
354            self.call(|inner| inner.inputs.begin_staging(&inner.access.write_schema)).await?;
355        Ok(SnapshotStaging {
356            database: self.clone(),
357            expected: id,
358            tables: Some(tables),
359            phase: Phase::Streaming(Box::new(StagingState {
360                decoder: SnapshotChunkDecoder::new(prefix),
361                inserts,
362                records: 0,
363                columns: BTreeSet::new(),
364                part: None,
365                refusal: None,
366            })),
367            batches: 0,
368            time: std::time::Duration::ZERO,
369        })
370    }
371
372    /// Where the input connection keeps temporary tables (`PRAGMA temp_store`):
373    /// 1 is in files.
374    #[cfg(any(test, feature = "test-utils"))]
375    pub async fn input_temp_store(&self) -> Result<u8, DbError> {
376        self.call(|inner| {
377            inner.inputs.with_writer(|writer, _| {
378                Ok(writer.query_row("PRAGMA temp_store", [], |row| row.get(0))?)
379            })
380        })
381        .await
382    }
383
384    /// Make `damage` to a staging before the next load or file check reads
385    /// it, as damage to this device's disk would (§19.1).
386    #[cfg(any(test, feature = "test-utils"))]
387    pub async fn damage_next_staged_read(&self, damage: StagedDamage) -> Result<(), DbError> {
388        self.call(move |inner| {
389            *coven_foundation::locks::lock_unpoisoned(&inner.inputs.damage_staged) = Some(damage);
390            Ok::<_, DbError>(())
391        })
392        .await
393    }
394
395    /// Make `damage` to each of the next `count` stagings as it finishes,
396    /// before finishing checks it, as damage to this device's disk would
397    /// (§19.1). Damage to two stagings of one snapshot alike stands for a
398    /// snapshot whose own records fail.
399    #[cfg(any(test, feature = "test-utils"))]
400    pub async fn damage_staged_finishes(
401        &self,
402        damage: StagedDamage,
403        count: u32,
404    ) -> Result<(), DbError> {
405        self.call(move |inner| {
406            *coven_foundation::locks::lock_unpoisoned(&inner.inputs.damage_finishing) =
407                Some((damage, count));
408            Ok::<_, DbError>(())
409        })
410        .await
411    }
412
413    /// The records staged on the input connection now, for every snapshot:
414    /// the rows of every staging's tables, and their applied writes.
415    #[cfg(any(test, feature = "test-utils"))]
416    pub async fn staged_snapshot_records(&self) -> Result<u64, DbError> {
417        self.call(|inner| {
418            inner.inputs.with_writer(|writer, _| {
419                let tables = writer.query(
420                    "SELECT name FROM temp.sqlite_schema WHERE type='table' AND name GLOB '_coven_staged_*'",
421                    [],
422                    |row| row.get::<_, String>(0),
423                )?;
424                let mut count: i64 = writer.query_row(
425                    "SELECT count(*) FROM temp._coven_snapshot_writes",
426                    [],
427                    |row| row.get(0),
428                )?;
429                for table in tables {
430                    count += writer.query_row(
431                        &format!("SELECT count(*) FROM temp.{}", sqlive::sql::identifier(&table)),
432                        [],
433                        |row| row.get::<_, i64>(0),
434                    )?;
435                }
436                Ok::<_, DbError>(count)
437            })
438        })
439        .await
440        .map(|count| count as u64)
441    }
442}
443
444/// The failure that ends `stagings` before their load or file check runs,
445/// reported once each has removed its records, beside any removal that
446/// failed (§16.2).
447pub(crate) async fn refuse(stagings: Vec<SnapshotStaging>, error: DbError) -> DbError {
448    let mut error = error;
449    for staging in stagings {
450        if let Err(cleanup) = staging.discard().await {
451            error =
452                DbError::StagingCleanup { operation: Box::new(error), cleanup: Box::new(cleanup) };
453        }
454    }
455    error
456}
457
458impl SnapshotStaging {
459    fn id(&self) -> Result<i64, DbError> {
460        self.tables.as_ref().map(|tables| tables.id).ok_or(DbError::Invariant {
461            what: "a staged snapshot is read once, before it is discarded",
462        })
463    }
464
465    /// Decode one batch of opened plaintext and commit its records together.
466    /// After a failure, discard this staging.
467    pub async fn stage(&mut self, plaintext: Vec<u8>) -> Result<(), DbError> {
468        let staging = self.id()?;
469        let Phase::Streaming(mut state) = std::mem::replace(&mut self.phase, Phase::Failed) else {
470            return Err(DbError::Invariant {
471                what: "a snapshot stages only until its stream ends or fails",
472            });
473        };
474        let expected = self.expected.clone();
475        let (state, time) = self
476            .database
477            .call(move |inner| {
478                let started = inner.clock.elapsed();
479                inner.inputs.write(|db, call| {
480                    stage(
481                        db,
482                        call,
483                        &inner.access.write_schema,
484                        &expected,
485                        staging,
486                        &mut state,
487                        &plaintext,
488                    )
489                })?;
490                Ok::<_, DbError>((state, inner.clock.elapsed() - started))
491            })
492            .await?;
493        self.phase = Phase::Streaming(state);
494        self.batches += 1;
495        self.time += time;
496        Ok(())
497    }
498
499    /// Refuse a stream without its header, declared records and end marker,
500    /// then check its applied writes. Nothing more stages after this; after a
501    /// failure, discard this staging.
502    pub async fn finish(&mut self) -> Result<(), DbError> {
503        let staging = self.id()?;
504        let Phase::Streaming(state) = std::mem::replace(&mut self.phase, Phase::Failed) else {
505            return Err(DbError::Invariant {
506                what: "a snapshot stages only until its stream ends or fails",
507            });
508        };
509        let expected = self.expected.clone();
510        let records = state.records;
511        let tables = StagedTables {
512            id: staging,
513            tables: self.tables.as_ref().map(|tables| tables.tables.clone()).unwrap_or_default(),
514        };
515        let (header, refusal, digest) = self
516            .database
517            .call(move |inner| {
518                #[cfg(any(test, feature = "test-utils"))]
519                if let Some(made) = inner.inputs.next_finish_damage() {
520                    inner.inputs.write(|db, _| damage::damage(db, staging, made))?;
521                }
522                inner.inputs.write(|db, _| {
523                    state.decoder.finish().map_err(SnapshotError::Format)?;
524                    let header = state
525                        .decoder
526                        .header()
527                        .ok_or_else(|| invalid("snapshot has no header"))?
528                        .clone();
529                    check_header(&inner.access.write_schema, &expected, &header)?;
530                    SnapshotMetadata::staged(db, staging).validate(header.covers)?;
531                    let digest = damage::digest(db, &tables)?;
532                    Ok((header, state.refusal, digest))
533                })
534            })
535            .await?;
536        tracing::info!(
537            audience = ?header.id.audience,
538            records,
539            batches = self.batches,
540            time = ?self.time,
541            "staged snapshot"
542        );
543        self.phase = Phase::Staged { header, digest, refusal };
544        Ok(())
545    }
546
547    /// Remove everything staged so far (§16.2).
548    pub async fn discard(mut self) -> Result<(), DbError> {
549        let Some(tables) = self.tables.take() else {
550            return Ok(());
551        };
552        self.database.call(move |inner| inner.inputs.remove_staging(&tables)).await
553    }
554
555    /// Whether `database` staged it: records on another store's input
556    /// connection are not there to read.
557    pub(crate) fn staged_by(&self, database: &StoreDatabase) -> bool {
558        Arc::ptr_eq(&self.database.inner, &database.inner)
559    }
560
561    /// Authenticate its staged write evidence for the load or file check
562    /// reading it, in that operation's input transaction (D7.5).
563    pub(crate) fn prepare<'a>(
564        &mut self,
565        inputs: &crate::input_preparation::PreparedInputs<'a>,
566        authorship: &dyn crate::Authorship,
567    ) -> Result<crate::snapshot_preparation::PreparedSnapshot<'a>, DbError> {
568        let staging = self.id()?;
569        let Phase::Staged { header, refusal, .. } = &mut self.phase else {
570            return Err(DbError::Invariant {
571                what: "a snapshot is read once its stream checked out to its end",
572            });
573        };
574        inputs.snapshot(staging, header.clone(), refusal.take(), authorship)
575    }
576
577    /// Its records are gone, with the transaction that removed them.
578    fn taken(&mut self) {
579        self.tables = None;
580    }
581}
582
583fn stage(
584    db: &StoreSql,
585    #[cfg_attr(
586        not(any(test, feature = "test-utils")),
587        expect(unused_variables, reason = "only tests count coven's work")
588    )]
589    call: &StoreCall<'_>,
590    schema: &WriteSchema,
591    expected: &SnapshotId,
592    staging: i64,
593    state: &mut StagingState,
594    mut bytes: &[u8],
595) -> Result<(), DbError> {
596    #[cfg(any(test, feature = "test-utils"))]
597    if state.records == 0 {
598        call.record_work(|work| work.snapshot_reads += 1);
599    }
600    let metadata = SnapshotMetadata::staged(db, staging);
601    let StagingState { decoder, inserts, records, columns, part, refusal } = state;
602    loop {
603        let headed = decoder.header().is_some();
604        let record =
605            decoder.next_record(&mut bytes, &metadata).map_err(MetadataFailure::snapshot)?;
606        if !headed {
607            if let Some(header) = decoder.header() {
608                check_header(schema, expected, header)?;
609            }
610        }
611        let Some(record) = record else {
612            return Ok(());
613        };
614        if let SnapshotRecord::Write(write) = &record {
615            metadata.stage(write)?;
616        }
617        *records += 1;
618        // Once a record is refused the body is, but it is still decoded to
619        // its end, so a damaged body is refused for that.
620        if refusal.is_some() {
621            continue;
622        }
623        let staged = RecordStaging { db, schema, inserts };
624        let section = record.section();
625        if let Err(refused) = staged.record(record, decoder.frame(), columns, part)? {
626            *refusal = Some((section, refused));
627        }
628    }
629}
630
631/// Where one staging's records go.
632struct RecordStaging<'a> {
633    db: &'a StoreSql,
634    schema: &'a WriteSchema,
635    inserts: &'a Inserts,
636}
637
638/// A record the loader's checks accept, or the error refusing it.
639type Checked = Result<(), DbError>;
640
641impl RecordStaging<'_> {
642    /// Stage one record. Failing to write it fails the batch; a record the
643    /// loader refuses is the inner error, which refuses the whole body.
644    fn record(
645        &self,
646        record: SnapshotRecord,
647        frame: &[u8],
648        columns: &mut BTreeSet<(String, String)>,
649        part: &mut Option<StagedPart>,
650    ) -> Result<Checked, DbError> {
651        match &record {
652            SnapshotRecord::Write(_) => return Ok(Ok(())),
653            SnapshotRecord::Column(column) => {
654                self.db.execute(&self.inserts.columns, params![column.table, column.column])?;
655                columns.insert((column.table.clone(), column.column.clone()));
656                return Ok(Ok(()));
657            }
658            SnapshotRecord::Merge(row) => return self.merged(row, columns),
659            SnapshotRecord::Authored(authored) => return self.authored(authored, part),
660            SnapshotRecord::Creation(creation) => {
661                if let Err(refused) = self.schema.row_table(&creation.row) {
662                    return Ok(Err(refused));
663                }
664                self.db.execute(
665                    &self.inserts.creations,
666                    params![
667                        creation.row.table,
668                        creation.row.key,
669                        &creation.row.audience,
670                        encoded(merge_fields::encode_write_id(&creation.write))?,
671                        encoded(merge_fields::encode_change_id(&creation.creator))?
672                    ],
673                )?;
674                return Ok(Ok(()));
675            }
676            SnapshotRecord::Loss(_)
677            | SnapshotRecord::Evidence(_)
678            | SnapshotRecord::Admin(_)
679            | SnapshotRecord::KeyCopy(_)
680            | SnapshotRecord::Announcement(_)
681            | SnapshotRecord::Claim(_) => {}
682        }
683        let section = record.section();
684        let evidence = section == section::EVIDENCE;
685        self.db.execute(&self.inserts.records, params![section as i64, evidence, frame])?;
686        Ok(Ok(()))
687    }
688
689    /// A merge row's checks, then its generations, cells and parents.
690    fn merged(
691        &self,
692        merged: &MergeRow,
693        columns: &BTreeSet<(String, String)>,
694    ) -> Result<Checked, DbError> {
695        if let Err(refused) = check_merged(self.schema, merged, columns) {
696            return Ok(Err(refused));
697        }
698        let state = &merged.state;
699        let row = state.row();
700        let at = self.inserts.layout.position(&row.table);
701        let layout = &self.inserts.layout.tables[at];
702        let generation = state.generation();
703        let writer = state.generations()[&generation];
704        let mut values = vec![sqlive::rusqlite::types::Value::Null; layout.width()];
705        values[TableLayout::KEY] = row.key.clone().into();
706        values[TableLayout::AUDIENCE] = (&row.audience).into();
707        values[TableLayout::PRESENT] = state.present().into();
708        values[TableLayout::GENERATION] = generation.to_be_bytes().to_vec().into();
709        values[TableLayout::DEVICE] = writer.device.0.to_be_bytes().to_vec().into();
710        values[TableLayout::NUMBER] = writer.number.to_be_bytes().to_vec().into();
711        for (j, name) in layout.columns.iter().enumerate() {
712            if let Some(cell) = state.cells().get(name) {
713                values[layout.value_at(j)] = sql_value(&cell.value.value);
714                values[layout.setter_at(j)] = cell.write.device.0.to_be_bytes().to_vec().into();
715                values[layout.setter_at(j) + 1] = cell.write.number.to_be_bytes().to_vec().into();
716            }
717        }
718        put_parents(layout, &mut values, |j| {
719            state.cells().get(&layout.columns[j]).map(|cell| &cell.value)
720        });
721        self.db.execute(&self.inserts.rows[at], params_from_iter(values))?;
722        for (earlier, write) in state.generations() {
723            if *earlier == generation {
724                continue;
725            }
726            self.db.execute(
727                &self.inserts.generations,
728                params![
729                    row.table,
730                    row.key,
731                    &row.audience,
732                    earlier.to_be_bytes().as_slice(),
733                    write.device,
734                    write.number.to_be_bytes().as_slice()
735                ],
736            )?;
737        }
738        Ok(Ok(()))
739    }
740
741    /// A source part, or a kept change of the part before it. A change
742    /// holds in full exactly the columns that were references under its
743    /// write's schema, as this device would keep it (D7.3).
744    fn authored(
745        &self,
746        authored: &AuthoredRecord,
747        part: &mut Option<StagedPart>,
748    ) -> Result<Checked, DbError> {
749        let change = match authored {
750            AuthoredRecord::Part(new) => {
751                *part = Some(StagedPart {
752                    write: encoded(merge_fields::encode_write_id(&new.header.position))?,
753                    header: new.header.clone(),
754                    staged: false,
755                });
756                return Ok(Ok(()));
757            }
758            AuthoredRecord::Change(change) => change,
759        };
760        let part = part.as_mut().ok_or(DbError::Invariant {
761            what: "the snapshot decoder gives a kept change after its part",
762        })?;
763        let row = change.record.row();
764        let audience = &row.audience;
765        if !part.staged {
766            self.db.execute(
767                &self.inserts.parts,
768                params![
769                    part.write,
770                    audience,
771                    encoded(merge_fields::encode_write_header(&part.header))?
772                ],
773            )?;
774            part.staged = true;
775        }
776        let form = crate::write_sources::KeptForm::new(self.schema, part.header.schema_version);
777        if !form.conforms(&change.record) {
778            return Ok(Err(invalid("kept change holds its columns unlike its schema")));
779        }
780        self.db.execute(
781            &self.inserts.authored,
782            params![
783                part.write,
784                audience,
785                row.table,
786                row.key,
787                change.record.encode()?,
788                crate::write_sources::stored_verdict(&change.verdict)?,
789                change.stop.map(|stop| stop.get().to_be_bytes().to_vec())
790            ],
791        )?;
792        Ok(Ok(()))
793    }
794}
795
796/// The loader's checks of a merge row on its own.
797fn check_merged(
798    schema: &WriteSchema,
799    merged: &MergeRow,
800    columns: &BTreeSet<(String, String)>,
801) -> Result<(), DbError> {
802    let state = &merged.state;
803    let row = state.row();
804    let table = schema.row_columns(
805        row,
806        state.present(),
807        state.cells().iter().map(|(name, cell)| (name, &cell.value)),
808    )?;
809    for foreign in &table.foreign_keys {
810        let mut setters = foreign
811            .columns
812            .iter()
813            .filter_map(|column| state.cells().get(crate::write_rows::column_name(table, column)));
814        if let Some(first) = setters.next() {
815            if setters.any(|cell| cell.write != first.write) {
816                return Err(invalid("multi-column reference setters disagree"));
817            }
818        }
819    }
820    if state.generations().is_empty() {
821        return Err(invalid("merge row has no generations"));
822    }
823    for name in state.cells().keys() {
824        if !columns.contains(&(row.table.clone(), name.clone())) {
825            return Err(invalid("merge cell has no column record"));
826        }
827    }
828    Ok(())
829}
830
831/// Stage the parents each column's value carries, by reference pair.
832fn put_parents<'a>(
833    layout: &TableLayout,
834    values: &mut [sqlive::rusqlite::types::Value],
835    column: impl Fn(usize) -> Option<&'a coven_merge::ColumnValue<Value>>,
836) {
837    for (q, (j, key)) in layout.pairs.iter().enumerate() {
838        let at = layout.parent_at(q);
839        match column(*j).and_then(|value| value.parents.get(key)) {
840            Some(parent) => {
841                values[at] = parent.row.key.clone().into();
842                values[at + 1] = (&parent.row.audience).into();
843                values[at + 2] = parent.generation.to_be_bytes().to_vec().into();
844            }
845            None => {
846                values[at] = sqlive::rusqlite::types::Value::Null;
847                values[at + 1] = sqlive::rusqlite::types::Value::Null;
848                values[at + 2] = sqlive::rusqlite::types::Value::Null;
849            }
850        }
851    }
852}
853
854fn check_header(
855    schema: &WriteSchema,
856    expected: &SnapshotId,
857    header: &SnapshotHeader,
858) -> Result<(), DbError> {
859    if header.id != *expected {
860        return Err(invalid("snapshot identity differs from the requested object"));
861    }
862    if !schema.versions.contains(&header.schema_version) {
863        return Err(SnapshotError::Schema {
864            snapshot: header.schema_version,
865            database: *schema.versions.end(),
866        }
867        .into());
868    }
869    Ok(())
870}
871
872#[cfg(test)]
873#[path = "snapshot_staging_tests.rs"]
874mod tests;