Skip to main content

coven_foundation/files/
lock.rs

1//! Store locks live beside the directory so Windows can unpublish it.
2
3use std::fs::{File, OpenOptions, TryLockError};
4use std::path::{Path, PathBuf};
5
6use crate::files::file_lock::{FileLock, LockMode};
7use crate::files::{atomic_file, settings, FileError, SettingsError, StoreSettings};
8use crate::id_source::StoreId;
9
10use super::held_open;
11use super::lock_registry as registry;
12use registry::LockKind;
13pub use registry::StoreClosingGuard;
14pub(super) use registry::{bootstrap_running, LockInstance, RegisteredLock};
15
16/// An exclusive writer lock. Dropping it unlocks and closes the OS handle.
17/// Readers use a separate shared lock so they can coexist with the writer.
18#[derive(Debug)]
19pub struct StoreLock {
20    store: LockedStore<RegisteredLock>,
21}
22
23#[derive(Debug)]
24struct LockedStore<L> {
25    _file: L,
26    directory: PathBuf,
27    id: StoreId,
28}
29
30impl StoreLock {
31    /// Mark this instance as closing when the caller's owning scope ends.
32    /// This guard does not retain the OS lock or any database connection.
33    pub fn closing_guard(&self) -> StoreClosingGuard {
34        self.store._file.closing_guard()
35    }
36
37    /// Read the settings while holding the store's writer lock.
38    pub fn settings(&self) -> Result<StoreSettings, SettingsError> {
39        settings::read(&self.store.directory, self.store.id)
40    }
41
42    /// The canonical directory of the store this lock holds.
43    pub(super) fn directory(&self) -> &Path {
44        &self.store.directory
45    }
46
47    pub(crate) fn verify(&self, directory: &Path, id: StoreId) -> Result<(), StoreLockError> {
48        let path = std::fs::canonicalize(directory)
49            .map_err(|source| FileError::at("resolve locked store", directory, source))?;
50        if path != self.store.directory || id != self.store.id {
51            return Err(StoreLockError::WrongDirectory(id));
52        }
53        Ok(())
54    }
55}
56
57/// A shared lock that prevents deletion while a reader holds store files open.
58#[derive(Debug)]
59pub struct StoreReadLock {
60    _file: RegisteredLock,
61}
62
63impl StoreReadLock {
64    /// Mark this reader as closing when its owner begins to close, so a
65    /// deletion in this process waits for its release instead of refusing.
66    /// This guard does not retain the OS lock.
67    pub fn closing_guard(&self) -> StoreClosingGuard {
68        self._file.closing_guard()
69    }
70}
71
72/// The writer lock, held while a closed store is read without opening it:
73/// holding it shows that no handle runs the store's sync. Registered in this
74/// process as closing from the start, so an open waits for its release.
75/// Dropping it unlocks.
76#[derive(Debug)]
77pub struct ClosedStoreLock {
78    _file: RegisteredLock,
79}
80
81/// A held lock on the cache's range writes: shared by a write in progress,
82/// exclusive while opening finds abandoned ones. Dropping it unlocks.
83#[derive(Debug)]
84pub struct CacheWriteLock {
85    _file: FileLock,
86}
87
88/// The cache-write lock file, in the store directory beside the cache folder,
89/// which holds only cache entries.
90const CACHE_WRITES: &str = ".coven-cache-writes.lock";
91
92/// Wait for the cache-write lock, shared or exclusive. Unlike the store
93/// locks it waits: a range write is short, and opening needs them finished.
94pub(super) fn cache_writes(store: &Path, shared: bool) -> Result<CacheWriteLock, FileError> {
95    let path = store.join(CACHE_WRITES);
96    let mode = if shared { LockMode::Shared } else { LockMode::Exclusive };
97    Ok(CacheWriteLock { _file: wait_for_lock(&path, mode, "lock cache writes")? })
98}
99
100/// Wait for the lock on the file at `path`. A test hears when the lock is
101/// held and the call is about to wait for it (`when_reached`).
102fn wait_for_lock(path: &Path, mode: LockMode, what: &'static str) -> Result<FileLock, FileError> {
103    let file = open_lock(path)?;
104    let attempt = file.try_clone().map_err(|source| FileError::at(what, path, source))?;
105    match FileLock::try_lock(attempt, mode) {
106        Ok(file) => return Ok(file),
107        Err(TryLockError::WouldBlock) => {
108            #[cfg(any(test, feature = "test-utils"))]
109            super::reached::reached(path);
110        }
111        Err(TryLockError::Error(source)) => return Err(FileError::at(what, path, source)),
112    }
113    FileLock::lock(file, mode).map_err(|source| FileError::at(what, path, source))
114}
115
116/// The cache-write lock file of the store at `store`, which a test waits on
117/// to know a range write or an open waits for it.
118#[cfg(any(test, feature = "test-utils"))]
119pub fn cache_writes_lock(store: &Path) -> std::path::PathBuf {
120    store.join(CACHE_WRITES)
121}
122
123/// Exclusive access to the writer, readers and bootstrap, authorizing
124/// deletion. Holding the bootstrap lock keeps every restore, join and
125/// replacement of the store out, including a `reset_device` or
126/// `restore_device` that holds only that lock between the steps of an
127/// unfinished swap (§19.3), and lets deletion remove its lock file too.
128#[derive(Debug)]
129pub struct StoreDeletionLock {
130    store: LockedStore<FileLock>,
131    _readers: FileLock,
132    bootstrap: HeldBootstrap,
133}
134
135/// How a deletion holds the store's bootstrap lock.
136#[derive(Debug)]
137enum HeldBootstrap {
138    /// Taken by the deletion itself.
139    Locked(FileLock),
140    /// Handed over by a bootstrap cancelling its own unpublished store.
141    Leased(RegisteredLock),
142}
143
144impl HeldBootstrap {
145    fn release(self) {
146        match self {
147            Self::Locked(file) => drop(file),
148            Self::Leased(lock) => drop(lock),
149        }
150    }
151}
152
153impl StoreDeletionLock {
154    /// Rename the store's directory to its deletion name, durably: the store
155    /// leaves every listing and open, and the directory under that name, or
156    /// the store's lock files without a directory, record that its deletion
157    /// is under way, for the next listing, restore or join of the store to
158    /// finish (§11.6, E1). Done already, it changes nothing.
159    pub fn unpublish(&mut self) -> Result<(), FileError> {
160        let _layout = lock_layout(self.store.directory.parent().expect("store parent"))?;
161        self.unpublish_locked()
162    }
163
164    /// `unpublish`, under the layout lock.
165    fn unpublish_locked(&mut self) -> Result<(), FileError> {
166        let destination = deletion_path(&self.store.directory, self.store.id);
167        if self.store.directory != destination && inspect_directory(&self.store.directory)? {
168            crate::files::atomic_file::rename_new(&self.store.directory, &destination).map_err(
169                |source| {
170                    held_open::directory_failure("unpublish store", &self.store.directory, source)
171                },
172            )?;
173            self.store.directory = destination;
174            #[cfg(unix)]
175            sync_removal(&self.store.directory)?;
176        }
177        Ok(())
178    }
179
180    /// Unpublish the directory, as `unpublish` does, and remove its contents
181    /// before releasing and removing its sibling locks. Retry failures
182    /// through `lock_for_deletion`, including a failure after only the lock
183    /// files remain.
184    pub fn remove_directory(mut self) -> Result<(), FileError> {
185        let root = self.store.directory.parent().expect("store parent");
186        // Every lock acquisition and creation also takes this lock. None can
187        // open a lock file between its release here and the removal of its
188        // name, so no one holds an unlinked lock while another locks a new
189        // file at the same path.
190        let _layout = lock_layout(root)?;
191        let paths = deletion_lock_paths(&self.store.directory, self.store.id);
192        self.unpublish_locked()?;
193        if inspect_directory(&self.store.directory)? {
194            std::fs::remove_dir_all(&self.store.directory).map_err(|source| {
195                held_open::directory_failure(
196                    "remove store directory",
197                    &self.store.directory,
198                    source,
199                )
200            })?;
201        }
202        #[cfg(unix)]
203        sync_removal(&self.store.directory)?;
204        // Release before removing, on every platform. Windows keeps a file
205        // deleted while a handle is open until that handle closes, and
206        // refuses to open its name meanwhile, which would refuse the next
207        // lock of this store. Under the layout lock, releasing first is as
208        // safe as removing a held file.
209        let Self { store, _readers, bootstrap } = self;
210        drop((store, _readers));
211        bootstrap.release();
212        for path in paths {
213            atomic_file::remove(&path)?;
214        }
215        Ok(())
216    }
217}
218
219#[cfg(unix)]
220fn sync_removal(directory: &Path) -> Result<(), FileError> {
221    crate::files::atomic_file::sync_directory(directory.parent().expect("store parent")).map_err(
222        |source| FileError::AfterRemove { path: directory.to_owned(), source: Box::new(source) },
223    )
224}
225
226/// A store in use is distinct from a filesystem failure.
227#[derive(Debug, thiserror::Error)]
228pub enum StoreLockError {
229    /// A live instance or another process prevents the requested access.
230    /// Async opens wait for a closing instance in this process to release it.
231    #[error("store {0} is already open")]
232    AlreadyOpen(StoreId),
233    /// An unfinished first installation must be resumed by the call that
234    /// began it: a restore or join with its code. A restore on a join's
235    /// installation, or a join on a recorded restore's, is refused too (§13.4).
236    #[error("store {0} requires restore or join to finish bootstrap")]
237    BootstrapPending(StoreId),
238    /// A replacement of this device's installation recorded itself, just
239    /// before its fresh device registered, and did not finish. Only the call
240    /// it names finishes it; the other is refused before anything changes.
241    /// Deletion removes it instead (§19.3).
242    #[error("store {store} requires {by} to finish replacing this device's installation")]
243    ReplacementUnfinished {
244        /// The store whose installation is being replaced.
245        store: StoreId,
246        /// The call that began the replacement.
247        by: super::ReplacedBy,
248    },
249    /// This device is deleting the store and leaving it, and its deletion
250    /// recorded itself, just before it journaled the device's removal, and
251    /// did not finish. Opening and replacing the store are refused before
252    /// anything changes; only finishing the deletion, or discarding the
253    /// store, ends it (E1).
254    #[error("store {0} is being deleted from this device")]
255    DeletionUnfinished(StoreId),
256    /// A supplied lock protects a different directory.
257    #[error("lock does not protect store {0}")]
258    WrongDirectory(StoreId),
259    /// Opening or locking the lock file failed for another reason, or did
260    /// not run because its runtime is shutting down (`FileError::Shutdown`).
261    #[error("store lock: {0}")]
262    File(FileError),
263    /// Taking the lock panicked on the blocking pool: a bug, which the
264    /// caller reports (E15).
265    #[error(transparent)]
266    Bug(#[from] crate::task::Bug),
267}
268
269impl From<FileError> for StoreLockError {
270    fn from(error: FileError) -> Self {
271        match error {
272            FileError::Bug(bug) => Self::Bug(bug),
273            error => Self::File(error),
274        }
275    }
276}
277
278impl crate::task::CarriesBug for StoreLockError {
279    fn bug(&self) -> Option<&crate::task::Bug> {
280        match self {
281            Self::Bug(bug) => Some(bug),
282            Self::File(error) => error.bug(),
283            _ => None,
284        }
285    }
286}
287
288fn open_lock(path: &Path) -> Result<File, FileError> {
289    OpenOptions::new()
290        .read(true)
291        .write(true)
292        .create(true)
293        .truncate(false)
294        .open(path)
295        .map_err(|source| FileError::at("open store lock", path, source))
296}
297
298pub(crate) fn try_lock(path: &Path, id: StoreId, shared: bool) -> Result<FileLock, StoreLockError> {
299    let mode = if shared { LockMode::Shared } else { LockMode::Exclusive };
300    match FileLock::try_lock(open_lock(path)?, mode) {
301        Ok(file) => Ok(file),
302        Err(TryLockError::WouldBlock) => Err(StoreLockError::AlreadyOpen(id)),
303        Err(TryLockError::Error(source)) => Err(FileError::at("lock store", path, source).into()),
304    }
305}
306
307// This file belongs to the layout and is never removed. It serializes lock
308// acquisition, creation, and lock-file removal across processes, so no caller
309// can hold an unlinked lock while another locks a new file at the same path.
310pub(crate) fn lock_layout(root: &Path) -> Result<FileLock, FileError> {
311    wait_for_lock(&layout_lock(root), LockMode::Exclusive, "lock store layout")
312}
313
314/// The layout's lock file under `root`, which a test waits on to know a
315/// lock, creation or removal waits for the layout.
316pub(crate) fn layout_lock(root: &Path) -> PathBuf {
317    root.join(".coven-stores.lock")
318}
319
320#[cfg(any(test, feature = "test-utils"))]
321pub(crate) fn acquire(directory: &Path, id: StoreId) -> Result<StoreLock, StoreLockError> {
322    let directory = resolve_directory(directory)?;
323    let file = registry::try_acquire(&directory, id, LockKind::Writer)?;
324    Ok(StoreLock { store: LockedStore { _file: file, directory, id } })
325}
326
327pub(crate) async fn acquire_for_open(
328    directory: &Path,
329    id: StoreId,
330    bugs: &crate::task::BugSinkRef,
331) -> Result<StoreLock, StoreLockError> {
332    let directory = resolve_directory(directory)?;
333    let file = registry::acquire_registered(&directory, id, LockKind::Writer, bugs).await?;
334    Ok(StoreLock { store: LockedStore { _file: file, directory, id } })
335}
336
337/// Take the writer lock of a closed store to read it, without waiting for
338/// another process or a live instance in this one; an instance closing in
339/// this process is waited for first.
340pub(crate) async fn acquire_closed(
341    directory: &Path,
342    id: StoreId,
343    bugs: &crate::task::BugSinkRef,
344) -> Result<ClosedStoreLock, StoreLockError> {
345    let directory = resolve_directory(directory)?;
346    Ok(ClosedStoreLock { _file: registry::acquire_closing_writer(&directory, id, bugs).await? })
347}
348
349/// Wait until no instance of the store at `directory` in this process is
350/// closing. An absent directory has none.
351pub(crate) async fn closed(directory: &Path, id: StoreId) -> Result<(), FileError> {
352    let directory = match std::fs::canonicalize(directory) {
353        Ok(directory) => directory,
354        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
355        Err(source) => return Err(FileError::at("resolve store directory", directory, source)),
356    };
357    registry::closed(&directory, id).await;
358    Ok(())
359}
360
361pub(super) async fn acquire_bootstrap(
362    directory: &Path,
363    id: StoreId,
364    bugs: &crate::task::BugSinkRef,
365) -> Result<RegisteredLock, StoreLockError> {
366    registry::acquire_registered(directory, id, LockKind::Bootstrap, bugs).await
367}
368
369fn resolve_directory(directory: &Path) -> Result<PathBuf, FileError> {
370    std::fs::canonicalize(directory)
371        .map_err(|source| FileError::at("resolve store directory", directory, source))
372}
373
374pub(super) fn exclude_readers(directory: &Path, id: StoreId) -> Result<FileLock, StoreLockError> {
375    let _layout = lock_layout(directory.parent().expect("store parent"))?;
376    try_lock(&lock_paths(directory, id)[1], id, false)
377}
378
379pub(crate) fn acquire_reader(
380    directory: &Path,
381    id: StoreId,
382) -> Result<StoreReadLock, StoreLockError> {
383    let directory = resolve_directory(directory)?;
384    Ok(StoreReadLock { _file: registry::acquire_reader(&directory, id)? })
385}
386
387pub(super) fn lock_existing(
388    directory: &Path,
389    id: StoreId,
390    shared: bool,
391) -> Result<(PathBuf, FileLock), StoreLockError> {
392    let directory = std::fs::canonicalize(directory)
393        .map_err(|source| FileError::at("resolve store directory", directory, source))?;
394    let _layout = lock_layout(directory.parent().expect("store parent"))?;
395    // Deletion may have completed between canonicalization and the layout lock.
396    std::fs::canonicalize(&directory)
397        .map_err(|source| FileError::at("resolve store directory", &directory, source))?;
398    let [writer, readers] = lock_paths(&directory, id);
399    let path = if shared { readers } else { writer };
400    let file = try_lock(&path, id, shared)?;
401    Ok((directory, file))
402}
403
404pub(crate) fn lock_paths(directory: &Path, id: StoreId) -> [PathBuf; 2] {
405    [
406        directory.with_file_name(format!(".{id}.lock")),
407        directory.with_file_name(format!(".{id}.readers.lock")),
408    ]
409}
410
411/// The lock a bootstrap or replacement holds for its whole call, beside the
412/// store's directory.
413fn bootstrap_lock_path(directory: &Path, id: StoreId) -> PathBuf {
414    directory.with_file_name(format!(".coven-bootstrap-{id}.lock"))
415}
416
417/// The writer lock file of the store at `directory`, which a test waits on
418/// (`when_reached`) to know an open finds the store's writer closing and
419/// waits for it.
420#[cfg(any(test, feature = "test-utils"))]
421pub fn writer_lock(directory: &Path, id: StoreId) -> PathBuf {
422    let [writer, _] = lock_paths(directory, id);
423    writer
424}
425
426/// The bootstrap lock file of the store at `directory`, which a test waits
427/// on to know an open finds the store's bootstrap closing and waits for it.
428#[cfg(any(test, feature = "test-utils"))]
429pub fn bootstrap_lock(directory: &Path, id: StoreId) -> PathBuf {
430    bootstrap_lock_path(directory, id)
431}
432
433/// The file `kind` of the store's locks is held on.
434#[cfg(any(test, feature = "test-utils"))]
435pub(super) fn kind_lock(directory: &Path, id: StoreId, kind: LockKind) -> PathBuf {
436    match kind {
437        LockKind::Writer => writer_lock(directory, id),
438        LockKind::Reader => lock_paths(directory, id)[1].clone(),
439        LockKind::Bootstrap => bootstrap_lock(directory, id),
440    }
441}
442
443/// Take the bootstrap lock without waiting. Under the layout lock, so a
444/// deletion never removes the file between this open and this lock.
445pub(super) fn lock_bootstrap(directory: &Path, id: StoreId) -> Result<FileLock, StoreLockError> {
446    let _layout = lock_layout(directory.parent().expect("store parent"))?;
447    try_lock(&bootstrap_lock_path(directory, id), id, false)
448}
449
450/// Every lock file named for a store, in the order deletion removes them.
451/// The bootstrap lock goes first: while the writer or reader lock file
452/// remains, creation refuses to reuse the id (`creation::create`).
453fn deletion_lock_paths(directory: &Path, id: StoreId) -> [PathBuf; 3] {
454    let [writer, readers] = lock_paths(directory, id);
455    [bootstrap_lock_path(directory, id), writer, readers]
456}
457
458pub(crate) fn deletion_path(directory: &Path, id: StoreId) -> PathBuf {
459    directory.with_file_name(format!(".coven-delete-{id}"))
460}
461
462fn inspect_directory(directory: &Path) -> Result<bool, FileError> {
463    match std::fs::symlink_metadata(directory) {
464        Ok(metadata) if metadata.file_type().is_dir() => Ok(true),
465        Ok(_) => Err(FileError::at(
466            "inspect deletion directory",
467            directory,
468            std::io::Error::new(std::io::ErrorKind::InvalidInput, "store is not a directory"),
469        )),
470        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
471            tracing::debug!(path = %directory.display(), "store directory is absent");
472            Ok(false)
473        }
474        Err(source) => Err(FileError::at("inspect store directory", directory, source)),
475    }
476}
477
478/// Lock a store, or what an interrupted deletion left of it, for deletion.
479/// `lease` holds the bootstrap lock of a bootstrap deleting its own store,
480/// which the deletion lock takes from it once it holds the store's other
481/// locks; otherwise deletion takes that lock itself.
482pub(super) fn for_deletion(
483    directory: &Path,
484    id: StoreId,
485    lease: Option<&mut Option<RegisteredLock>>,
486) -> Result<Option<StoreDeletionLock>, StoreLockError> {
487    let root = directory.parent().expect("store parent");
488    let root = match std::fs::canonicalize(root) {
489        Ok(root) => root,
490        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
491            tracing::debug!(path = %root.display(), "store layout is absent");
492            return Ok(None);
493        }
494        Err(source) => return Err(FileError::at("resolve store layout", root, source).into()),
495    };
496    let _layout = lock_layout(&root)?;
497    let published = root.join(id.to_string());
498    let directory =
499        if inspect_directory(&published)? { published } else { deletion_path(&published, id) };
500    let paths = deletion_lock_paths(&directory, id);
501    if !inspect_directory(&directory)? {
502        let mut remaining = false;
503        for path in &paths {
504            remaining |= atomic_file::exists("inspect store lock", path)?;
505        }
506        if !remaining {
507            // Retry the durability barrier even if the preceding attempt
508            // removed the last lock file before its directory sync failed.
509            #[cfg(unix)]
510            sync_removal(&directory)?;
511            tracing::debug!(store = %id, "store and its locks are absent");
512            return Ok(None);
513        }
514    }
515    let [bootstrap, writer, readers] = paths;
516    let others =
517        || Ok::<_, StoreLockError>((try_lock(&writer, id, false)?, try_lock(&readers, id, false)?));
518    let (bootstrap, (writer, readers)) = match lease {
519        // First, so a bootstrap that has not yet made the directory is
520        // refused before any other lock file is created for its id.
521        None => (HeldBootstrap::Locked(try_lock(&bootstrap, id, false)?), others()?),
522        // Last, so a refusal leaves the bootstrap holding its lock.
523        Some(lease) => {
524            let others = others()?;
525            let held = lease.take().expect("a bootstrap hands over the bootstrap lock it holds");
526            (HeldBootstrap::Leased(held), others)
527        }
528    };
529    Ok(Some(StoreDeletionLock {
530        store: LockedStore { _file: writer, directory, id },
531        _readers: readers,
532        bootstrap,
533    }))
534}
535
536/// Entries in a stores directory besides the layout lock, for tests.
537#[cfg(test)]
538pub(crate) fn store_entries(root: &Path) -> usize {
539    std::fs::read_dir(root)
540        .unwrap()
541        .filter(|entry| entry.as_ref().unwrap().file_name() != ".coven-stores.lock")
542        .count()
543}
544
545#[cfg(test)]
546#[path = "lock_tests.rs"]
547mod tests;