forge_workspace/healing/
policy.rs

1//! When a dead seat earns a restart, and how long the next one waits.
2//!
3//! Pure: no I/O, no clock of its own. The caller counts a seat's deaths,
4//! tells this policy how many there have been, and asks whether a run has
5//! lasted long enough to count as stable again.
6
7use std::time::Duration;
8
9use crate::config::SessionsConfig;
10
11/// What a seat is, at the moment it died, on the axes that decide whether
12/// the death is heal-eligible.
13#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
14pub struct SeatFacts {
15    /// A project lead, or a worker with a registry row. An ad-hoc session
16    /// has no row, so it is the user's to wake.
17    pub durable: bool,
18    /// The seat was up before this death. A first spawn or connection
19    /// failure is deterministic and already surfaced.
20    pub connected_once: bool,
21    /// A despawn or close owns this seat; healing it would undo the stop.
22    pub stopping: bool,
23    /// Forge itself is going down; the boot re-spawn owns the return.
24    pub shutting_down: bool,
25}
26
27/// The settled heal schedule.
28#[derive(Clone, Debug, PartialEq, Eq)]
29pub struct HealPolicy {
30    pub enabled: bool,
31    pub attempts: usize,
32    pub backoff: Vec<Duration>,
33    pub stable: Duration,
34}
35
36impl HealPolicy {
37    pub(crate) fn from_config(config: &SessionsConfig) -> Self {
38        Self {
39            enabled: config.heal,
40            attempts: config.heal_attempts,
41            backoff: config.heal_backoff.clone(),
42            stable: config.heal_stable,
43        }
44    }
45
46    /// Whether this death may be healed at all.
47    pub fn is_eligible(&self, facts: SeatFacts) -> bool {
48        self.enabled
49            && facts.durable
50            && facts.connected_once
51            && !facts.stopping
52            && !facts.shutting_down
53    }
54
55    /// How long to wait before restarting a seat that has taken `deaths`
56    /// deaths, counting the one just recorded. `None` once the attempt
57    /// budget is spent: the seat stays failed with its last diagnostic.
58    pub fn next_after(&self, deaths: u32) -> Option<Duration> {
59        if deaths == 0 || deaths as usize > self.attempts {
60            return None;
61        }
62        let index = deaths as usize - 1;
63        self.backoff.get(index).copied().or_else(|| self.backoff.last().copied())
64    }
65
66    /// Whether a seat up for `up_for` has run long enough that its next
67    /// death starts the count again.
68    pub fn resets_after(&self, up_for: Duration) -> bool {
69        up_for >= self.stable
70    }
71}
72
73#[cfg(test)]
74mod tests {
75    use super::*;
76
77    fn policy() -> HealPolicy {
78        HealPolicy {
79            enabled: true,
80            attempts: 3,
81            backoff: vec![
82                Duration::from_secs(5),
83                Duration::from_secs(30),
84                Duration::from_secs(120),
85            ],
86            stable: Duration::from_secs(600),
87        }
88    }
89
90    /// The settled schedule: +5 s, +30 s, +2 min, then nothing - the
91    /// fourth death is the escalation rather than a fourth attempt.
92    #[test]
93    fn the_attempt_schedule_runs_out_after_the_third_restart() {
94        let policy = policy();
95        assert_eq!(policy.next_after(1), Some(Duration::from_secs(5)));
96        assert_eq!(policy.next_after(2), Some(Duration::from_secs(30)));
97        assert_eq!(policy.next_after(3), Some(Duration::from_secs(120)));
98        assert_eq!(policy.next_after(4), None, "the fourth death leaves the seat failed");
99        assert_eq!(policy.next_after(0), None, "no death, no attempt");
100    }
101
102    /// The attempt count bounds the schedule rather than the list: a
103    /// shorter budget stops answering once it is spent.
104    #[test]
105    fn the_attempt_count_bounds_the_schedule() {
106        let policy = HealPolicy { attempts: 1, ..policy() };
107        assert_eq!(policy.next_after(1), Some(Duration::from_secs(5)));
108        assert_eq!(policy.next_after(2), None);
109    }
110
111    /// A backoff list shorter than the budget repeats its last interval,
112    /// so the seat keeps spacing its attempts rather than jumping to
113    /// failed.
114    #[test]
115    fn a_short_backoff_repeats_its_last_interval() {
116        let policy = HealPolicy { backoff: vec![Duration::from_secs(7)], ..policy() };
117        assert_eq!(policy.next_after(1), Some(Duration::from_secs(7)));
118        assert_eq!(policy.next_after(2), Some(Duration::from_secs(7)));
119        assert_eq!(policy.next_after(3), Some(Duration::from_secs(7)));
120        assert_eq!(policy.next_after(4), None);
121    }
122
123    /// Eligibility is the conjunction of five facts, each of which alone
124    /// must refuse: heal off, an ad-hoc seat, never connected, a stop
125    /// pending, and forge shutting down.
126    #[test]
127    fn eligibility_refuses_each_ineligible_fact() {
128        let eligible = SeatFacts { durable: true, connected_once: true, ..SeatFacts::default() };
129        assert!(policy().is_eligible(eligible), "a durable seat that was up heals");
130
131        let off = HealPolicy { enabled: false, ..policy() };
132        assert!(!off.is_eligible(eligible), "heal = false turns the whole thing off");
133
134        for (name, facts) in [
135            ("an ad-hoc seat", SeatFacts { durable: false, ..eligible }),
136            ("a seat that never connected", SeatFacts { connected_once: false, ..eligible }),
137            ("a despawn or close", SeatFacts { stopping: true, ..eligible }),
138            ("forge shutting down", SeatFacts { shutting_down: true, ..eligible }),
139        ] {
140            assert!(!policy().is_eligible(facts), "{name} must not heal");
141        }
142    }
143
144    /// A run that has lasted the stable window resets the count; one just
145    /// short of it does not, so a seat that dies every few minutes never
146    /// earns itself a fresh budget.
147    #[test]
148    fn a_stable_run_resets_the_count() {
149        let policy = policy();
150        assert!(policy.resets_after(Duration::from_secs(600)));
151        assert!(policy.resets_after(Duration::from_secs(601)));
152        assert!(!policy.resets_after(Duration::from_secs(599)));
153    }
154
155    /// The policy's schedule is the config table's, so an edit to
156    /// `[sessions]` changes what the supervisor does without a second
157    /// table in between.
158    #[test]
159    fn the_policy_is_built_from_the_sessions_table() {
160        let config = SessionsConfig {
161            heal: false,
162            heal_attempts: 2,
163            heal_backoff: vec![Duration::from_secs(9), Duration::from_secs(60)],
164            heal_stable: Duration::from_secs(120),
165        };
166        let policy = HealPolicy::from_config(&config);
167        assert!(!policy.enabled);
168        assert_eq!(policy.attempts, 2);
169        assert_eq!(policy.next_after(2), Some(Duration::from_secs(60)));
170        assert_eq!(policy.next_after(3), None);
171        assert!(policy.resets_after(Duration::from_secs(120)));
172    }
173}
forge_workspace/healing/policy.rs

forge_workspace/healing/
policy.rs
