Skip to main content

coven_foundation/
task.rs

1//! Work run apart from its caller — on the blocking pool, as its own task, or
2//! as one message of a long-lived loop — whose panic comes back to that
3//! caller as a value instead of an unwind (api.md E15).
4//!
5//! A panic turned into an `Err` at the nearest join lets the caller's own
6//! failure path run: the rollback or cleanup it does on `Err`, which an unwind
7//! would skip. The caller reports the panic as a bug. Work nobody waits for,
8//! a long-lived worker or work its caller stopped waiting for, reports its
9//! panic to a [`BugSink`] instead, the app's own reporter.
10
11use std::future::Future;
12use std::pin::Pin;
13use std::task::{Context, Poll};
14
15#[cfg(any(test, feature = "test-utils"))]
16#[path = "task_in_flight.rs"]
17mod in_flight;
18
19pub use ::panic_as_error::{
20    catching_operation, caught, caught_sync, panic_message, report, Bug, BugKind, BugSink,
21    BugSinkRef, CarriesBug, SourceLocation,
22};
23
24/// Name the step the blocking operation running on this thread has reached.
25/// A test's hang report prints the latest step of each operation still
26/// running, so a wait inside a long operation shows where. Nothing outside
27/// tests.
28pub fn operation_step(name: &'static str) {
29    #[cfg(any(test, feature = "test-utils"))]
30    in_flight::step(name);
31    #[cfg(not(any(test, feature = "test-utils")))]
32    let _ = name;
33}
34
35/// Why work apart from its caller gave no result.
36#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
37pub enum TaskFailed {
38    /// The work panicked: a bug, which the caller reports.
39    #[error(transparent)]
40    Panicked(Bug),
41    /// The runtime it ran on is shutting down and dropped it before it
42    /// finished. Not a bug: the process is ending the work.
43    #[error(transparent)]
44    Shutdown(Shutdown),
45}
46
47/// Work its runtime dropped as it shut down, before the work finished. Not a
48/// bug: the process is ending the work. The caller gets it as a value and
49/// returns it as its own error (E15).
50#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
51#[error("{operation} {outcome}: its runtime is shutting down")]
52pub struct Shutdown {
53    /// What the work was doing, as its caller named it.
54    pub operation: &'static str,
55    /// How far the work got, so whether anything it would change changed.
56    pub outcome: ShutdownOutcome,
57}
58
59/// How far work its runtime dropped got (E15).
60#[derive(Clone, Copy, Debug, PartialEq, Eq)]
61pub enum ShutdownOutcome {
62    /// The work never started, so nothing it would change changed. tokio
63    /// can't stop a closure on the blocking pool once it has started, so a
64    /// closure the pool dropped is always one it never ran.
65    NotRun,
66    /// The work may have stopped at any of its waits, after some of its
67    /// steps: a task the runtime dropped partway. What it would change may
68    /// be changed in part, so the caller reads that state again before
69    /// relying on it.
70    Unknown,
71}
72
73impl std::fmt::Display for ShutdownOutcome {
74    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
75        f.write_str(match self {
76            Self::NotRun => "did not run",
77            Self::Unknown => "may have run in part",
78        })
79    }
80}
81
82impl TaskFailed {
83    /// The failure a joined task's `error` stands for. A task the runtime
84    /// dropped may have run in part: `ShutdownOutcome::Unknown`. A task
85    /// whose work runs under [`caught()`] hands its panic back as a value, so
86    /// its join fails only as its runtime ends it; a panic that reached the
87    /// join uncaught gives a bug that can't say where it began.
88    pub fn of_join(operation: &'static str, error: tokio::task::JoinError) -> Self {
89        if error.is_panic() {
90            Self::Panicked(panic_as_error::bug_of_unlocated(operation, error.into_panic()))
91        } else {
92            Self::Shutdown(Shutdown { operation, outcome: ShutdownOutcome::Unknown })
93        }
94    }
95
96    /// This failure as the error of the call whose work it ended: its
97    /// `Bug` for a panic, its `Shutdown` for the runtime dropping the work.
98    pub fn into_error<E: From<Bug> + From<Shutdown>>(self) -> E {
99        match self {
100            Self::Panicked(bug) => E::from(bug),
101            Self::Shutdown(shutdown) => E::from(shutdown),
102        }
103    }
104
105    /// What the failed work was doing, as its caller named it.
106    pub fn operation(&self) -> &'static str {
107        match self {
108            Self::Panicked(Bug { operation, .. }) | Self::Shutdown(Shutdown { operation, .. }) => {
109                operation
110            }
111        }
112    }
113
114    /// Whether the work panicked, rather than being dropped by its runtime.
115    pub fn is_panic(&self) -> bool {
116        matches!(self, Self::Panicked(_))
117    }
118}
119
120/// Run `work` on the blocking pool. A panic in it, or the runtime dropping it
121/// unstarted (`ShutdownOutcome::NotRun`), comes back as `TaskFailed`, never
122/// as an unwind.
123pub async fn blocking<T: Send + 'static>(
124    operation: &'static str,
125    work: impl FnOnce() -> T + Send + 'static,
126) -> Result<T, TaskFailed> {
127    match tokio::task::spawn_blocking(move || caught_sync(operation, work)).await {
128        Ok(done) => done.map_err(TaskFailed::Panicked),
129        // The closure catches its own panic, and the pool can't stop a
130        // closure once it has started, so the pool dropped it unstarted.
131        Err(_dropped) => {
132            Err(TaskFailed::Shutdown(Shutdown { operation, outcome: ShutdownOutcome::NotRun }))
133        }
134    }
135}
136
137/// Wait for a task whose work runs under [`caught()`], as
138/// `tokio::spawn(caught(operation, work))` starts it. Its panic comes back as
139/// `TaskFailed::Panicked`, naming where it began; the runtime cancelling it,
140/// as `Shutdown`.
141pub async fn joined<T: Send + 'static>(
142    operation: &'static str,
143    task: tokio::task::JoinHandle<Result<T, Bug>>,
144) -> Result<T, TaskFailed> {
145    match task.await {
146        Ok(caught) => caught.map_err(TaskFailed::Panicked),
147        Err(error) => Err(TaskFailed::of_join(operation, error)),
148    }
149}
150
151/// Logs each bug at error: the sink of an app that wants its bugs only
152/// logged.
153pub struct LoggedBugs;
154
155impl BugSink for LoggedBugs {
156    fn report(&self, bug: &Bug) {
157        tracing::error!(
158            operation = bug.operation,
159            location = bug.location.as_ref().map(tracing::field::display),
160            "a bug in work nobody waits for: {bug}"
161        );
162    }
163}
164
165/// Keeps every bug it hears of, and logs each at error, as an app's crash
166/// reporter would: the sink a test gives coven. A test that expects a bug
167/// takes it; one left untaken when the sink drops fails the test, so a bug
168/// no test expected never passes unseen.
169#[cfg(any(test, feature = "in-memory-keychain"))]
170pub struct HeardBugs {
171    heard: std::sync::Mutex<Vec<Bug>>,
172    /// Told of each bug as it is heard.
173    notice: tokio::sync::Notify,
174    /// Runs as the next bug is heard, inside its report.
175    next_report: std::sync::Mutex<Option<ReportHook>>,
176}
177
178/// What a test runs as [`HeardBugs`] hears its next bug.
179#[cfg(any(test, feature = "in-memory-keychain"))]
180type ReportHook = Box<dyn FnOnce(&Bug) + Send>;
181
182#[cfg(any(test, feature = "in-memory-keychain"))]
183impl HeardBugs {
184    /// A sink that has heard nothing yet, to share with coven.
185    pub fn shared() -> std::sync::Arc<Self> {
186        std::sync::Arc::new(Self {
187            heard: std::sync::Mutex::new(Vec::new()),
188            notice: tokio::sync::Notify::new(),
189            next_report: std::sync::Mutex::new(None),
190        })
191    }
192
193    /// Run `hook` as the next bug is heard, inside its report: for a test
194    /// that acts at the moment the reporter hands a bug over.
195    pub fn on_next_report(&self, hook: impl FnOnce(&Bug) + Send + 'static) {
196        *crate::locks::lock_unpoisoned(&self.next_report) = Some(Box::new(hook));
197    }
198
199    /// Every bug heard since the last take, in the order heard.
200    pub fn take(&self) -> Vec<Bug> {
201        std::mem::take(&mut *crate::locks::lock_unpoisoned(&self.heard))
202    }
203
204    /// Wait for the first bug not yet taken, and take it: for a test whose
205    /// bug reaches the sink after the call it made returned.
206    pub async fn next(&self) -> Bug {
207        loop {
208            let notified = self.notice.notified();
209            {
210                let mut heard = crate::locks::lock_unpoisoned(&self.heard);
211                if !heard.is_empty() {
212                    return heard.remove(0);
213                }
214            }
215            notified.await;
216        }
217    }
218}
219
220#[cfg(any(test, feature = "in-memory-keychain"))]
221impl BugSink for HeardBugs {
222    fn report(&self, bug: &Bug) {
223        tracing::error!(
224            operation = bug.operation,
225            location = bug.location.as_ref().map(tracing::field::display),
226            "a bug in work nobody waits for: {bug}"
227        );
228        let hook = crate::locks::lock_unpoisoned(&self.next_report).take();
229        if let Some(hook) = hook {
230            hook(bug);
231        }
232        crate::locks::lock_unpoisoned(&self.heard).push(bug.clone());
233        self.notice.notify_waiters();
234    }
235}
236
237#[cfg(any(test, feature = "in-memory-keychain"))]
238impl Drop for HeardBugs {
239    fn drop(&mut self) {
240        let untaken = self.take();
241        // A test already failing says why itself.
242        if !untaken.is_empty() && !std::thread::panicking() {
243            panic!("coven reported bugs no test took: {untaken:?}");
244        }
245    }
246}
247
248impl CarriesBug for TaskFailed {
249    fn bug(&self) -> Option<&Bug> {
250        match self {
251            Self::Panicked(bug) => Some(bug),
252            Self::Shutdown(_) => None,
253        }
254    }
255}
256
257/// Hand the bug `error` carries, if it carries one, to `bugs`: the error of
258/// work nobody waits for, which no caller hears of. Whether it carried one,
259/// so the work logs an error that carried none.
260pub fn report_carried(bugs: &dyn BugSink, error: &impl CarriesBug) -> bool {
261    let bug = error.bug();
262    if let Some(bug) = bug {
263        report(bugs, bug);
264    }
265    bug.is_some()
266}
267
268/// `error`, once the bug it carries, if any, has gone to `bugs`: the error
269/// of work nobody waits for, which it records where readers see it.
270pub fn reported<E: CarriesBug>(bugs: &dyn BugSink, error: E) -> E {
271    report_carried(bugs, &error);
272    error
273}
274
275/// Run `work`, named `operation`, as its own task that nobody waits for: a
276/// worker its owner stops. A panic in it goes to `bugs`, then `ended` runs,
277/// so the owner ends what the work served in a way its readers see, such as
278/// recording that sync failed. A panic in `ended` goes to `bugs` too, and no
279/// further.
280pub fn spawn_owned<E, F>(
281    operation: &'static str,
282    bugs: BugSinkRef,
283    work: impl Future<Output = ()> + Send + 'static,
284    ended: E,
285) -> OwnedTask
286where
287    E: FnOnce(Bug) -> F + Send + 'static,
288    F: Future<Output = ()> + Send,
289{
290    OwnedTask {
291        operation,
292        task: tokio::spawn(async move {
293            let Err(bug) = caught(operation, work).await else {
294                return;
295            };
296            report(bugs.as_ref(), &bug);
297            if let Err(again) = caught(operation, ended(bug)).await {
298                report(bugs.as_ref(), &again);
299            }
300        }),
301    }
302}
303
304/// A task [`spawn_owned`] started. Dropping it leaves the task running.
305#[derive(Debug)]
306pub struct OwnedTask {
307    operation: &'static str,
308    task: tokio::task::JoinHandle<()>,
309}
310
311impl OwnedTask {
312    /// End the work at its next wait. Its `ended` does not run.
313    pub fn abort(&self) {
314        self.task.abort();
315    }
316
317    /// Whether the work has ended.
318    pub fn is_finished(&self) -> bool {
319        self.task.is_finished()
320    }
321
322    /// Wait until the work has ended: finished, panicked and reported, or
323    /// dropped by its runtime as it shuts down, which ends it too.
324    pub async fn ended(self) {
325        // The task catches every panic in it, so only its runtime ending it
326        // is left.
327        if let Err(error) = self.task.await {
328            tracing::debug!("{}", TaskFailed::of_join(self.operation, error));
329        }
330    }
331}
332
333/// Run `work`, named `operation`, as its own task, so it finishes even when
334/// its caller stops waiting, as closing a store does. Its panic comes back
335/// as `TaskFailed::Panicked` to a caller still waiting. Once none is, the
336/// panic goes to `bugs`, as does a bug its error carries.
337pub fn spawn_completing<T: Send + 'static, E: CarriesBug + Send + 'static>(
338    operation: &'static str,
339    bugs: BugSinkRef,
340    work: impl Future<Output = Result<T, E>> + Send + 'static,
341) -> CompletingTask<Result<T, E>> {
342    spawn_completing_boxed(operation, bugs, Box::pin(work))
343}
344
345/// [`spawn_completing`] with its work as one boxed type, so tokio's spawn and
346/// the task's harness are built once per result type, not once per call site.
347fn spawn_completing_boxed<T: Send + 'static, E: CarriesBug + Send + 'static>(
348    operation: &'static str,
349    bugs: BugSinkRef,
350    work: Pin<Box<dyn Future<Output = Result<T, E>> + Send>>,
351) -> CompletingTask<Result<T, E>> {
352    let (done, completion) = tokio::sync::oneshot::channel();
353    let heard = bugs.clone();
354    #[cfg(any(test, feature = "test-utils"))]
355    let flight = in_flight::Flight::begin(operation, in_flight::State::Task);
356    tokio::spawn(async move {
357        let answer = caught(operation, work).await;
358        #[cfg(any(test, feature = "test-utils"))]
359        drop(flight);
360        if let Err(unheard) = done.send(answer) {
361            report_unheard(heard.as_ref(), &unheard);
362        }
363    });
364    // The runtime drops a task partway as it shuts down.
365    CompletingTask {
366        operation,
367        dropped: ShutdownOutcome::Unknown,
368        bugs,
369        unread: report_unheard::<T, E>,
370        completion,
371    }
372}
373
374/// Run `work`, named `operation`, on the blocking pool, where it runs to its
375/// end even when its caller stops waiting. Its panic comes back as
376/// `TaskFailed::Panicked` to a caller still waiting, and the pool dropping it
377/// unstarted as `Shutdown` with `ShutdownOutcome::NotRun`. Once no caller
378/// waits, the panic goes to `bugs`, as does a bug its error carries.
379pub fn blocking_completing<T: Send + 'static, E: CarriesBug + Send + 'static>(
380    operation: &'static str,
381    bugs: BugSinkRef,
382    work: impl FnOnce() -> Result<T, E> + Send + 'static,
383) -> CompletingTask<Result<T, E>> {
384    blocking_completing_boxed(operation, bugs, Box::new(work))
385}
386
387/// [`blocking_completing`] with its work as one boxed type, so tokio's
388/// `spawn_blocking`, the channel and the harness are built once per result
389/// type, not once per call site.
390fn blocking_completing_boxed<T: Send + 'static, E: CarriesBug + Send + 'static>(
391    operation: &'static str,
392    bugs: BugSinkRef,
393    work: Box<dyn FnOnce() -> Result<T, E> + Send>,
394) -> CompletingTask<Result<T, E>> {
395    let (done, completion) = tokio::sync::oneshot::channel();
396    let heard = bugs.clone();
397    // The pool runs the closure to its end once it starts, so the closure
398    // answers unless the pool dropped it unstarted.
399    #[cfg(any(test, feature = "test-utils"))]
400    let flight = in_flight::Flight::begin(operation, in_flight::State::Queued);
401    tokio::task::spawn_blocking(move || {
402        #[cfg(any(test, feature = "test-utils"))]
403        flight.started();
404        let answer = caught_sync(operation, work);
405        #[cfg(any(test, feature = "test-utils"))]
406        drop(flight);
407        if let Err(unheard) = done.send(answer) {
408            report_unheard(heard.as_ref(), &unheard);
409        }
410    });
411    CompletingTask {
412        operation,
413        dropped: ShutdownOutcome::NotRun,
414        bugs,
415        unread: report_unheard::<T, E>,
416        completion,
417    }
418}
419
420/// Hand `bugs` the panic of work nobody waited for, or the bug its error
421/// carries.
422fn report_unheard<T, E: CarriesBug>(bugs: &dyn BugSink, unheard: &Result<Result<T, E>, Bug>) {
423    match unheard {
424        Err(bug) => report(bugs, bug),
425        Ok(Err(error)) => {
426            report_carried(bugs, error);
427        }
428        Ok(Ok(_)) => {}
429    }
430}
431
432/// The result of work [`spawn_completing`] or [`blocking_completing`] runs.
433/// Dropping it leaves the work running, and the work's panic, or a bug its
434/// error carries, goes to the sink: once the work ends, or as it drops, for
435/// an answer the work gave that no one read.
436pub struct CompletingTask<T> {
437    operation: &'static str,
438    /// How far the work got if its runtime dropped it.
439    dropped: ShutdownOutcome,
440    /// Where an answer no one read reports its bug.
441    bugs: BugSinkRef,
442    /// Hands `bugs` the bug of an answer no one read.
443    unread: fn(&dyn BugSink, &Result<T, Bug>),
444    completion: tokio::sync::oneshot::Receiver<Result<T, Bug>>,
445}
446
447impl<T> std::fmt::Debug for CompletingTask<T> {
448    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
449        f.debug_struct("CompletingTask")
450            .field("operation", &self.operation)
451            .field("dropped", &self.dropped)
452            .finish_non_exhaustive()
453    }
454}
455
456impl<T> Drop for CompletingTask<T> {
457    fn drop(&mut self) {
458        // Closed first, so the work either answered already, and the answer
459        // is read here, or finds no one waiting and reports it itself.
460        self.completion.close();
461        if let Ok(answer) = self.completion.try_recv() {
462            (self.unread)(self.bugs.as_ref(), &answer);
463        }
464    }
465}
466
467impl<T> Future for CompletingTask<T> {
468    type Output = Result<T, TaskFailed>;
469
470    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
471        let (operation, outcome) = (self.operation, self.dropped);
472        Pin::new(&mut self.completion).poll(cx).map(|completed| match completed {
473            Ok(result) => result.map_err(TaskFailed::Panicked),
474            // The runtime dropped the work as it shut down.
475            Err(_dropped) => Err(TaskFailed::Shutdown(Shutdown { operation, outcome })),
476        })
477    }
478}
479
480#[cfg(test)]
481#[path = "task_tests.rs"]
482pub(crate) mod tests;