Skip to main content

forge_workspace/
crons.rs

1//! The cron cluster on [`Workspace`]: the durable-cron store accessor
2//! and its mutation helpers, the asleep-slot prompt buffers, the fire
3//! router, the scheduler task, and the cross-crate seed helper.
4//!
5//! Everything here stays on `Workspace` as a second `impl` block, so
6//! every caller (the boot path in [`crate::workspace`], the
7//! `mcp::cron` facade, `spawn::deliver_cron_prompt`, a view's boot)
8//! keeps its path. The `crons` and `update_tx` fields these methods own
9//! are `pub(crate)` for the same reason `db` is:
10//! so this sibling module can reach them without a wrapper. A fired cron
11//! with no live session parks in [`crate::parked`]. Schedule math lives in
12//! [`crate::mcp::cron::schedule`]; the MCP tool surface in
13//! [`crate::mcp::cron`]; delivery in [`crate::spawn`].
14
15use std::sync::Arc;
16use std::time::Duration;
17
18use tracing::Instrument;
19
20use crate::protocol::SessionUpdate;
21use crate::target::ProjectKey;
22use crate::workspace::Workspace;
23
24/// How often the cron scheduler wakes to fire due crons. Minute
25/// granularity matches the cron-expression resolution.
26const CRON_TICK_INTERVAL: Duration = Duration::from_secs(60);
27
28/// The gap between warnings while one cooling episode lasts; the ticks in
29/// between repeat at `debug`.
30const COOLING_REWARN_INTERVAL: Duration = Duration::from_secs(30 * 60);
31
32/// India Standard Time's offset from UTC, the clock a reader of the
33/// notice is on.
34pub(crate) const IST_OFFSET_MINUTES: i64 = 5 * 60 + 30;
35
36/// A cron prompt buffered for a sleeping slot: the raw prompt plus whether
37/// this fire is overdue (delivered with a missed marker on drain), and the
38/// entry's identity, which the drained echo names.
39#[derive(Debug)]
40pub(crate) struct PendingCron {
41    pub text: String,
42    pub missed: bool,
43    pub cron_id: String,
44    pub description: Option<String>,
45}
46
47impl Workspace {
48    /// Lock the durable cron list and apply `f`. Every cron-list mutation
49    /// routes through here to take the lock, and the mutation site persists
50    /// what it changed from inside its own closure - a row at a time, never
51    /// the whole in-memory set. A boot read that failed leaves that set
52    /// EMPTY while the db handle stays live, so a whole-set write would
53    /// erase every durable row the read never loaded.
54    pub(crate) fn with_crons_mut<R>(
55        &self,
56        f: impl FnOnce(&mut Vec<forge_primitives::CronEntry>) -> R,
57    ) -> R {
58        let mut crons = self.crons.lock();
59        f(&mut crons)
60    }
61
62    /// Persist one cron row, replacing any prior record for its id. Called
63    /// from inside the mutation's lock.
64    fn persist_cron(&self, cron: &forge_primitives::CronEntry) {
65        let guard = self.db.lock();
66        let Some(db) = guard.as_ref() else { return };
67        if let Err(error) = crate::store::cron::upsert(db, cron) {
68            tracing::error!(
69                target: "forge_workspace::crons",
70                cron_id = %cron.id,
71                %error,
72                "persisting a cron to the store failed; it may be lost on restart",
73            );
74        }
75    }
76
77    /// Delete one cron row by id. Called from inside the mutation's lock.
78    fn delete_stored_cron(&self, id: &forge_primitives::CronId) {
79        let guard = self.db.lock();
80        let Some(db) = guard.as_ref() else { return };
81        if let Err(error) = crate::store::cron::delete(db, id) {
82            tracing::error!(
83                target: "forge_workspace::crons",
84                cron_id = %id,
85                %error,
86                "deleting a cron from the store failed; it may come back on restart",
87            );
88        }
89    }
90
91    /// Append a cron and persist it. Backs `cron__create`.
92    pub(crate) fn push_cron(&self, entry: forge_primitives::CronEntry) {
93        let project_name = entry.project_name.clone();
94        self.with_crons_mut(|crons| {
95            self.persist_cron(&entry);
96            crons.push(entry);
97        });
98        self.announce_cron_schedules_changed(&project_name);
99    }
100
101    /// Remove the cron `id` in `project_name` regardless of who owns it,
102    /// persist, and report whether an entry was removed. Backs the
103    /// fire-router's dead-slot removal; the owner-scoped `cron__delete`
104    /// uses [`Self::remove_cron_owned_by`].
105    pub(crate) fn remove_cron(&self, project_name: &str, id: &forge_primitives::CronId) -> bool {
106        let removed = self.with_crons_mut(|crons| {
107            let before = crons.len();
108            crons.retain(|c| !(c.id == *id && c.project_name == project_name));
109            let removed = crons.len() != before;
110            if removed {
111                self.delete_stored_cron(id);
112            }
113            removed
114        });
115        if removed {
116            self.announce_cron_schedules_changed(project_name);
117        }
118        removed
119    }
120
121    /// Remove the cron `id` in `project_name` only when the slot that
122    /// owns it matches `label` (`None` = a lead cron, `Some(label)` =
123    /// that worker's), persist, and return the entry removed (`None` when
124    /// nothing matched). Backs `cron__delete` so a caller deletes only its
125    /// own crons.
126    pub(crate) fn remove_cron_owned_by(
127        &self,
128        project_name: &str,
129        id: &forge_primitives::CronId,
130        label: Option<&str>,
131    ) -> Option<forge_primitives::CronEntry> {
132        let removed = self.with_crons_mut(|crons| {
133            let pos = crons.iter().position(|c| {
134                c.id == *id && c.project_name == project_name && c.team_role.as_deref() == label
135            });
136            let removed = pos.map(|pos| crons.remove(pos));
137            if removed.is_some() {
138                self.delete_stored_cron(id);
139            }
140            removed
141        });
142        if removed.is_some() {
143            self.announce_cron_schedules_changed(project_name);
144        }
145        removed
146    }
147
148    /// Remove worker `label`'s crons in `project_key` from the in-memory
149    /// set and the store, scoped to `(project name, team_role == Some(label))`.
150    /// Backs `spawn::teardown_worker` so a despawned dynamic worker's crons
151    /// go with its durable row. The key resolves to a name via the same view
152    /// lookup `remove_gotify_subscriptions_for_worker` uses.
153    pub(crate) fn delete_crons_for_worker(&self, project_key: &ProjectKey, label: &str) {
154        let Some(project_name) =
155            self.list_projects().into_iter().find(|v| v.key == *project_key).map(|v| v.name)
156        else {
157            tracing::warn!(
158                target: "forge_workspace::crons",
159                project = %project_key.as_str(),
160                label = %label,
161                "could not resolve a project name at worker teardown; its crons may be stranded",
162            );
163            return;
164        };
165        let removed = self.with_crons_mut(|crons| {
166            let mut removed: Vec<forge_primitives::CronId> = Vec::new();
167            crons.retain(|c| {
168                let taking =
169                    c.project_name == project_name && c.team_role.as_deref() == Some(label);
170                if taking {
171                    removed.push(c.id.clone());
172                }
173                !taking
174            });
175            for id in &removed {
176                self.delete_stored_cron(id);
177            }
178            !removed.is_empty()
179        });
180        if removed {
181            self.announce_cron_schedules_changed(&project_name);
182        }
183    }
184
185    /// Tell every view the project's schedule set moved, as the set the
186    /// write just left, on the project's lead seat. A name no project
187    /// carries has no seat to route on and announces nothing.
188    fn announce_cron_schedules_changed(&self, project_name: &str) {
189        let Some(key) = self.lead_slot_for_project(project_name) else {
190            tracing::debug!(
191                target: "forge_workspace::crons",
192                event_name = "cron_schedules_changed_unroutable",
193                project = %project_name,
194                "a cron write named a project no project carries; nothing was announced",
195            );
196            return;
197        };
198        let crons = self.crons_for_project(project_name);
199        let _ = self.update_tx.send(SessionUpdate::CronSchedulesChanged { key, crons });
200    }
201
202    /// Whether this run's boot could not load the durable crons.
203    /// `cron__list` refuses on it.
204    pub fn durable_crons_unavailable(&self) -> bool {
205        self.crons_load_failed.load(std::sync::atomic::Ordering::SeqCst)
206    }
207
208    /// The crons registered for `project_name`, whatever their owner. The
209    /// two scoping consumers narrow it - `cron__list` to the caller's own
210    /// crons, the Inspector SCHEDULES snapshot to the active tab's - while
211    /// the refused-delete determination reads it whole, because telling
212    /// "not in this project" from "not yours" needs the other owners' rows.
213    pub fn crons_for_project(&self, project_name: &str) -> Vec<forge_primitives::CronEntry> {
214        self.crons.lock().iter().filter(|c| c.project_name == project_name).cloned().collect()
215    }
216
217    /// A snapshot of every cron across all projects. Backs the
218    /// scheduler's per-tick due-check.
219    pub(crate) fn all_crons_snapshot(&self) -> Vec<forge_primitives::CronEntry> {
220        self.crons.lock().clone()
221    }
222
223    /// Remember the schedule a fired prompt came from, so its row can name
224    /// the schedule after a reload the delivery frame does not survive.
225    ///
226    /// Best-effort: a store that will not write costs a reloaded row its
227    /// title, which is exactly what it had before the pairing existed.
228    pub(crate) fn remember_cron_name(
229        &self,
230        slot: &forge_primitives::SessionSlot,
231        uuid: &str,
232        cron_id: &str,
233        description: Option<&str>,
234    ) {
235        let guard = self.db.lock();
236        let Some(db) = guard.as_ref() else {
237            return;
238        };
239        let name = forge_primitives::cron::CronName {
240            cron_id: cron_id.to_owned(),
241            description: description.map(str::to_owned),
242        };
243        if let Err(error) =
244            crate::store::cron_names::put(db, uuid, &name, std::time::SystemTime::now())
245        {
246            tracing::warn!(
247                event_name = "cron_name_store_failed",
248                org = slot.org(),
249                project = slot.project(),
250                label = slot.label(),
251                cron = %cron_id,
252                prompt = %uuid,
253                %error,
254                "the fired prompt's schedule was not recorded; a reload falls back to the prompt"
255            );
256        }
257    }
258
259    /// The schedules the given fired prompts came from, keyed by the prompt's
260    /// own id: the read half of the pairing the delivery frame carries.
261    pub fn cron_names(
262        &self,
263        slot: &forge_primitives::SessionSlot,
264        uuids: &[&str],
265    ) -> std::collections::BTreeMap<String, forge_primitives::cron::CronName> {
266        let guard = self.db.lock();
267        let Some(db) = guard.as_ref() else {
268            return std::collections::BTreeMap::new();
269        };
270        crate::store::cron_names::for_uuids(db, uuids).unwrap_or_else(|error| {
271            tracing::warn!(
272                event_name = "cron_names_read_failed",
273                org = slot.org(),
274                project = slot.project(),
275                label = slot.label(),
276                prompts = uuids.len(),
277                %error,
278                "the fired prompts' schedules could not be read; their rows fall back to the prompt"
279            );
280            std::collections::BTreeMap::new()
281        })
282    }
283
284    /// Advance a fired cron and persist: a recurring cron records
285    /// `last_fire` and moves `next_fire` to the next future slot (removed
286    /// if it somehow has none); a run-once is removed. A direct state
287    /// mutation - the fire's prompt delivery goes through the Command bus
288    /// separately.
289    pub(crate) fn advance_or_remove_cron(
290        &self,
291        id: &forge_primitives::CronId,
292        fired_at: std::time::SystemTime,
293    ) {
294        // The entry's project comes out of the fold, since a run-once the
295        // advance removes is the one thing that can no longer answer it.
296        let mut project_name = None;
297        self.with_crons_mut(|crons| {
298            let Some(pos) = crons.iter().position(|c| &c.id == id) else { return };
299            project_name = Some(crons[pos].project_name.clone());
300            match &crons[pos].kind {
301                forge_primitives::CronKind::Once(_) => {
302                    crons.remove(pos);
303                    self.delete_stored_cron(id);
304                }
305                forge_primitives::CronKind::Recurring(_) => {
306                    if let Some(next) =
307                        crate::mcp::cron::schedule::next_fire_after(&crons[pos].kind, fired_at)
308                    {
309                        crons[pos].last_fire = Some(fired_at);
310                        crons[pos].next_fire = next;
311                        self.persist_cron(&crons[pos]);
312                    } else {
313                        // A recurring expr that parses but never matches
314                        // (e.g. "0 0 30 2 *"). Don't drop it silently; the
315                        // warning below is the only trace.
316                        let removed = crons.remove(pos);
317                        let expr = if let forge_primitives::CronKind::Recurring(e) = &removed.kind {
318                            e.as_str()
319                        } else {
320                            ""
321                        };
322                        tracing::warn!(
323                            target: "forge_workspace::crons",
324                            cron_id = %removed.id,
325                            project = %removed.project_name,
326                            expr = %expr,
327                            "recurring cron has no upcoming occurrence; removed it",
328                        );
329                        self.delete_stored_cron(id);
330                    }
331                }
332            }
333        });
334        if let Some(project_name) = project_name {
335            self.announce_cron_schedules_changed(&project_name);
336        }
337    }
338
339    /// Fire every cron due at `now`: deliver each prompt into its project
340    /// session (spawning it if asleep, unless the entry carries
341    /// `wake: false` for a job that must not wake a sleeping
342    /// owner, in which case a sleeping owner leaves the fire due and
343    /// unadvanced) and advance/remove the entry.
344    /// Delivery routes through the Command bus (a session action); the
345    /// advance is a direct state write - kept separate per the cron
346    /// state-vs-delivery split. `now` is injected so tests are
347    /// deterministic. Also the boot catch-up: calling this once at
348    /// startup fires every cron that came due while forge was down,
349    /// advancing each past its missed slots (catch-up-once).
350    pub fn fire_due_crons(self: &Arc<Self>, now: std::time::SystemTime) {
351        use crate::spawn::CronFireOutcome;
352        // A tick can land inside a shutdown's drain; the session it would
353        // wake would register behind it and outlive forge. The schedule is
354        // untouched, so the next boot catches up.
355        if self.is_shutting_down() {
356            return;
357        }
358        let snapshot = self.all_crons_snapshot();
359        let due = crate::mcp::cron::schedule::due_crons(&snapshot, now);
360        // What THIS pass finds unwakeable, committed at the end so the set
361        // holds only the latest pass's answer.
362        let mut still_unwakeable = std::collections::HashSet::new();
363        // What THIS pass still found cooling, and when each last warned.
364        let mut still_cooling = std::collections::HashMap::new();
365        for id in &due {
366            let Some(cron) = snapshot.iter().find(|c| &c.id == id) else { continue };
367            // Overdue by more than two ticks: forge or the session was down
368            // through the scheduled minute. Two ticks (not one) absorbs the
369            // scheduler's Skip-behaviour jitter so a same-window fire under
370            // load is never mislabelled missed.
371            let missed = now > cron.next_fire + CRON_TICK_INTERVAL * 2;
372            match crate::spawn::deliver_cron_prompt(self, cron, missed) {
373                // Delivered (or spawn kicked off): advance a recurring to
374                // its next slot, remove a fired run-once.
375                CronFireOutcome::Delivered => self.advance_or_remove_cron(id, now),
376                // The slot has no session (project removed from
377                // forge.toml, or a worker label with no durable row):
378                // remove the cron rather than advance a dead entry
379                // forever.
380                CronFireOutcome::TargetGone => {
381                    tracing::warn!(
382                        target: "forge_workspace::crons",
383                        event_name = "cron_owner_gone",
384                        project = %cron.project_name,
385                        cron_id = %id,
386                        label = cron.team_role.as_deref().unwrap_or("lead"),
387                        "cron slot has no session; removing the cron",
388                    );
389                    self.remove_cron(&cron.project_name, id);
390                }
391                // The owner's row is kept but the wave skips it, so the
392                // prompt has nowhere to land right now. A recurring cron
393                // has another slot coming, so this fire is dropped and the
394                // schedule advances; parking it would grow a bucket
395                // nothing drains. A one-shot has no next slot, and
396                // advancing it removes it, so it stays due and fires once
397                // the owner can be woken.
398                CronFireOutcome::TargetCannotBeWoken { directory } => {
399                    let recurring = matches!(cron.kind, forge_primitives::CronKind::Recurring(_));
400                    // A stuck one-shot is re-evaluated every tick, so the
401                    // repeats carry the same event_name at debug rather
402                    // than writing the same warning once a minute for as
403                    // long as the directory is missing.
404                    if self.cron_unwakeable_is_new(id) {
405                        tracing::warn!(
406                            target: "forge_workspace::crons",
407                            event_name = "cron_owner_cannot_be_woken",
408                            project = %cron.project_name,
409                            cron_id = %id,
410                            label = cron.team_role.as_deref().unwrap_or("lead"),
411                            directory = %directory.display(),
412                            recurring,
413                            "cron owner cannot be woken, so this fire did not land; the owner's \
414                             row is kept",
415                        );
416                    } else {
417                        tracing::debug!(
418                            target: "forge_workspace::crons",
419                            event_name = "cron_owner_cannot_be_woken",
420                            project = %cron.project_name,
421                            cron_id = %id,
422                            "cron owner still cannot be woken; the first warning for this \
423                             cron already names it",
424                        );
425                    }
426                    still_unwakeable.insert(id.clone());
427                    if recurring {
428                        self.advance_or_remove_cron(id, now);
429                    }
430                }
431                // The owner is asleep and this cron may not wake it: leave
432                // the fire due, unadvanced, so the next session start (or a
433                // tick while the owner is awake) runs it. Nothing to warn
434                // about - the cron asked for exactly this.
435                CronFireOutcome::SkippedAsleep => {
436                    tracing::debug!(
437                        target: "forge_workspace::crons",
438                        event_name = "cron_skipped_asleep",
439                        project = %cron.project_name,
440                        cron_id = %id,
441                        "the cron's owner is asleep and this cron does not wake it; the fire \
442                         stays due",
443                    );
444                }
445                // Every account serving the project's model is cooling:
446                // the fire stays due, and the reason is said once per
447                // episode rather than on every tick of it.
448                CronFireOutcome::DeferredCooling { cooling } => {
449                    let reset = cooling_reset_text(cooling.reset_in, now);
450                    let warned_before = self.cron_cooling_warned_at(id);
451                    let warn_now = warned_before.is_none_or(|at| {
452                        now.duration_since(at).unwrap_or(Duration::ZERO) >= COOLING_REWARN_INTERVAL
453                    });
454                    if warn_now {
455                        tracing::warn!(
456                            target: "forge_workspace::crons",
457                            event_name = "cron_deferred_cooling",
458                            project = %cron.project_name,
459                            cron_id = %id,
460                            label = cron.team_role.as_deref().unwrap_or("lead"),
461                            org = %cooling.org,
462                            model = %cooling.model,
463                            cooling_accounts = cooling.tried,
464                            reset_in_secs = cooling.reset_in.as_secs(),
465                            resets_at = %reset,
466                            "cron fire deferred: every account serving this project's model is \
467                             cooling, so the wake would be refused; it stays due and retries",
468                        );
469                        let _ = self.update_tx.send(SessionUpdate::ServiceStatus {
470                            severity:
471                                forge_primitives::cloud::service_status::ServiceSeverity::Warning,
472                            message: format!(
473                                "Cron in '{}' deferred: no account serving {} in {} is free \
474                                 ({} account(s) cooling in the org); resets at {reset}",
475                                cron.project_name, cooling.model, cooling.org, cooling.tried,
476                            ),
477                        });
478                    } else {
479                        tracing::debug!(
480                            target: "forge_workspace::crons",
481                            event_name = "cron_deferred_cooling",
482                            project = %cron.project_name,
483                            cron_id = %id,
484                            label = cron.team_role.as_deref().unwrap_or("lead"),
485                            org = %cooling.org,
486                            model = %cooling.model,
487                            cooling_accounts = cooling.tried,
488                            reset_in_secs = cooling.reset_in.as_secs(),
489                            resets_at = %reset,
490                            "cron fire still deferred by the same cooling; the warning for this \
491                             episode already names it",
492                        );
493                    }
494                    // A repeat carries the earlier warning's time forward,
495                    // so the floor counts from that warning, not this tick.
496                    let warned_at = warned_before.filter(|_| !warn_now).unwrap_or(now);
497                    still_cooling.insert(id.clone(), warned_at);
498                }
499                // Command channel closed (shutting down): leave the cron
500                // due so the next boot catch-up re-fires it - don't consume
501                // a fire that never handed off.
502                CronFireOutcome::DispatchFailed => {
503                    tracing::warn!(
504                        target: "forge_workspace::crons",
505                        project = %cron.project_name,
506                        cron_id = %id,
507                        "cron fire deferred; leaving it due to retry",
508                    );
509                    let _ = self.update_tx.send(SessionUpdate::ServiceStatus {
510                        severity: forge_primitives::cloud::service_status::ServiceSeverity::Warning,
511                        message: format!(
512                            "Cron in '{}' could not fire yet (its session is shutting down, or a \
513                             spawn would be refused right now); it stays due and retries",
514                            cron.project_name
515                        ),
516                    });
517                }
518            }
519        }
520        // Commit this pass. Anything not in it - fired, advanced, deleted
521        // or simply not due - keeps no marker, so the next time it is
522        // unwakeable the warning is a WARN again, and the next cooling
523        // episode is a new one.
524        self.cron_unwakeable_commit(still_unwakeable);
525        self.cron_cooling_commit(still_cooling);
526    }
527
528    /// Spawn the cron scheduler: a background task that wakes every
529    /// `CRON_TICK_INTERVAL` (~60s), fires every due cron, and exits when
530    /// the workspace drops. The registry makes a second call a no-op.
531    /// Mirrors [`Workspace::start_usage_poller`]; started once at boot.
532    pub fn start_cron_scheduler(self: &Arc<Self>) {
533        let weak = Arc::downgrade(self);
534        let span = tracing::info_span!("cron_scheduler");
535        self.supervisor.spawn("cron_scheduler", forge_supervise::Policy::backoff(), move || {
536            let weak = weak.clone();
537            let span = span.clone();
538            async move {
539                let mut interval = tokio::time::interval_at(
540                    tokio::time::Instant::now() + CRON_TICK_INTERVAL,
541                    CRON_TICK_INTERVAL,
542                );
543                interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
544                loop {
545                    interval.tick().await;
546                    let Some(workspace) = weak.upgrade() else {
547                        return;
548                    };
549                    workspace.fire_due_crons(std::time::SystemTime::now());
550                }
551            }
552            .instrument(span)
553        });
554    }
555}
556
557/// The reset a deferred cron names: the wall clock in IST plus the wait
558/// ahead, so a reader needs no arithmetic on a duration. A reset no
559/// clock can spell falls back to the wait alone rather than panicking
560/// the scheduler.
561fn cooling_reset_text(reset_in: Duration, now: std::time::SystemTime) -> String {
562    let ist = now
563        .checked_add(reset_in)
564        .and_then(|at| at.duration_since(std::time::SystemTime::UNIX_EPOCH).ok())
565        .and_then(|since_epoch| {
566            chrono::DateTime::from_timestamp(
567                i64::try_from(since_epoch.as_secs()).ok()?,
568                since_epoch.subsec_nanos(),
569            )
570        })
571        .and_then(|at| at.checked_add_signed(chrono::TimeDelta::minutes(IST_OFFSET_MINUTES)));
572    match ist {
573        Some(at) => format!("{} IST (in {})", at.format("%H:%M"), wait_text(reset_in)),
574        None => format!("in {}", wait_text(reset_in)),
575    }
576}
577
578/// A wait as the two largest units that describe it ("1h 5m", "3m 20s").
579fn wait_text(wait: Duration) -> String {
580    let (hours, minutes, seconds) =
581        (wait.as_secs() / 3600, wait.as_secs() % 3600 / 60, wait.as_secs() % 60);
582    if hours > 0 {
583        format!("{hours}h {minutes}m")
584    } else if minutes > 0 {
585        format!("{minutes}m {seconds}s")
586    } else {
587        format!("{seconds}s")
588    }
589}
590
591#[cfg(any(test, feature = "testing"))]
592impl Workspace {
593    /// Register a durable cron directly, bypassing the MCP create path.
594    /// Cross-crate test access to the otherwise `pub(crate)` cron store
595    /// so a view can exercise the Inspector's `refresh_forge_crons`
596    /// resolution against a seeded cron.
597    ///
598    /// The store write without the announcement: a fixture seeds state, it
599    /// does not perform the write a view is owed a frame for.
600    pub fn seed_test_cron(&self, entry: forge_primitives::CronEntry) {
601        self.with_crons_mut(|crons| {
602            self.persist_cron(&entry);
603            crons.push(entry);
604        });
605    }
606}
607
608#[cfg(test)]
609mod tests {
610    use super::*;
611
612    use tempfile::tempdir;
613
614    use crate::SessionSlot;
615    use crate::protocol::{Command, SessionUpdate};
616    use crate::workspace::PooledAgent;
617    use forge_gateway::AccountKey;
618
619    /// The cron prompts parked for `(project, label)`, under whichever org
620    /// the seeded project belongs to.
621    fn parked_crons(ws: &crate::Workspace, project: &str, label: Option<&str>) -> Vec<String> {
622        let org =
623            ws.list_projects().into_iter().find(|v| v.name == project).expect("seeded project").org;
624        ws.parked_by_slot
625            .lock()
626            .get(&crate::SessionSlot::for_label(&org, project, label))
627            .map(|parked| parked.cron.iter().map(|p| p.text.clone()).collect())
628            .unwrap_or_default()
629    }
630
631    /// Persist a worker row for `label` in `project_key`, failing loudly
632    /// when it did not land. The key must resolve to a configured
633    /// project, since the row lands in the `(org, project name, label)`
634    /// keyed `sessions` table.
635    fn seed_worker_row(ws: &crate::Workspace, project_key: &ProjectKey, label: &str) {
636        ws.seed_test_worker_row(project_key, label);
637        assert!(
638            ws.stored_worker_row(project_key, label).expect("read the seeded row").is_some(),
639            "the worker row for {label} did not land; does {project_key:?} resolve to a project?",
640        );
641    }
642
643    /// Seed the `proj` fixture with a project path that exists, and
644    /// return its key. A non-git worker runs in that directory, so the
645    /// wave checks it is still there before re-spawning the worker - a
646    /// fixture path that cannot exist is not a shape production has.
647    /// A workspace whose `proj` sits at a real directory, with one worker
648    /// row for `label` recorded as git-backed and no worktree on disk -
649    /// the stranded-owner shape every fire test in this module is built on.
650    /// Both tempdirs must outlive the caller.
651    fn stranded_owner_fixture(
652        label: &str,
653    ) -> (Arc<Workspace>, ProjectKey, tempfile::TempDir, tempfile::TempDir) {
654        let (ws, _rx) = Workspace::testing_stub();
655        let dir = tempdir().expect("tempdir");
656        ws.install_db_for_test(
657            crate::store::Db::open(&dir.path().join("db.redb")).expect("open db"),
658        );
659        let project_dir = tempdir().expect("project dir");
660        let key = seed_project_with_a_real_root(&ws, &project_dir);
661        ws.record_worker_row(
662            &key,
663            label,
664            &format!("{label}-uuid"),
665            "c",
666            None,
667            None,
668            false,
669            crate::mcp::workers::types::WorkerTree { is_git_repo: true, cwd: None },
670            None,
671        )
672        .expect("seed the stranded worker's row");
673        (ws, key, dir, project_dir)
674    }
675
676    fn seed_project_with_a_real_root(ws: &crate::Workspace, dir: &tempfile::TempDir) -> ProjectKey {
677        let root = dir.path().join("proj-root");
678        std::fs::create_dir_all(&root).expect("create the project dir");
679        ws.seed_test_project("proj", &root.to_string_lossy());
680        ws.project_key_for_name("proj").expect("seeded project")
681    }
682
683    fn live_worker_entry(project: &str, label: &str) -> crate::mcp::workers::types::WorkerEntry {
684        crate::mcp::workers::types::WorkerEntry {
685            label: label.to_owned(),
686            charter: "c".to_owned(),
687            slot: SessionSlot::worker("TestOrg", project, label),
688            session_id: None,
689            status: forge_primitives::WorkerLiveness::Running,
690            spawned_at: std::time::SystemTime::UNIX_EPOCH,
691            spawned_by: SessionSlot::lead("TestOrg", project),
692            needs_tag: false,
693            is_git_repo_at_spawn: false,
694            cwd: None,
695            diagnostic: None,
696            kick: None,
697        }
698    }
699
700    /// Drain every currently-queued `SessionUpdate` from the test rx.
701    fn drain_updates(
702        rx: &mut tokio::sync::mpsc::UnboundedReceiver<crate::SeatUpdate>,
703    ) -> Vec<crate::protocol::SessionUpdate> {
704        let mut out = Vec::new();
705        while let Ok(frame) = rx.try_recv() {
706            out.push(std::sync::Arc::unwrap_or_clone(frame.update));
707        }
708        out
709    }
710
711    fn project_expanded_path(workspace: &Workspace, name: &str) -> String {
712        workspace.list_projects().into_iter().find(|p| p.name == name).map_or_else(
713            || panic!("project '{name}' missing from workspace"),
714            |p| p.path.to_string_lossy().into_owned(),
715        )
716    }
717
718    /// One daily schedule, as a test's write leaves it.
719    fn schedule(
720        id: &str,
721        project: &str,
722        prompt: &str,
723        owner: Option<&str>,
724    ) -> forge_primitives::cron::CronEntry {
725        use forge_primitives::cron::{CronEntry, CronId, CronKind};
726        CronEntry {
727            id: CronId::from(id),
728            project_name: project.to_owned(),
729            kind: CronKind::Recurring("0 9 * * *".to_owned()),
730            prompt: prompt.to_owned(),
731            created_at: std::time::SystemTime::UNIX_EPOCH,
732            description: None,
733            last_fire: None,
734            next_fire: std::time::SystemTime::UNIX_EPOCH,
735            team_role: owner.map(str::to_owned),
736            wake: true,
737        }
738    }
739
740    /// One delivery as the fire router hands it to `deliver_cron_prompt`: a
741    /// cron entry carrying the project, owner and prompt the delivery
742    /// routes and reads. The id is fixed - these tests are about the route.
743    fn deliver_cron(
744        ws: &Arc<Workspace>,
745        project: &str,
746        owner: Option<&str>,
747        prompt: &str,
748        missed: bool,
749    ) -> crate::spawn::CronFireOutcome {
750        crate::spawn::deliver_cron_prompt(
751            ws,
752            &schedule("delivered-cron", project, prompt, owner),
753            missed,
754        )
755    }
756
757    /// The one `CronSchedulesChanged` on a test's update stream, as its key
758    /// and the set it announces.
759    fn next_crons_changed(
760        rx: &mut tokio::sync::mpsc::UnboundedReceiver<crate::SeatUpdate>,
761    ) -> (SessionSlot, Vec<forge_primitives::cron::CronEntry>) {
762        match rx.try_recv().map(|frame| std::sync::Arc::unwrap_or_clone(frame.update)) {
763            Ok(SessionUpdate::CronSchedulesChanged { key, crons }) => (key, crons),
764            other => panic!("expected a CronSchedulesChanged on the stream, got {other:?}"),
765        }
766    }
767
768    /// A cron create and a cron delete each announce the project's whole
769    /// schedule set on its lead seat, so the inspector's SCHEDULES section
770    /// pops on the frame that moved it rather than on the next unrelated
771    /// read. Mutants: drop either emission (the drain is empty); announce a
772    /// set taken before the write.
773    #[test]
774    fn a_cron_write_announces_the_projects_schedules() {
775        use forge_primitives::cron::CronId;
776        let dir = tempdir().expect("tempdir");
777        let (ws, mut rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
778        ws.seed_test_project("proj", "/tmp/tp-crons");
779        let lead = SessionSlot::lead("TestOrg", "proj");
780
781        ws.push_cron(schedule("c1", "proj", "stand-up", None));
782        let (key, crons) = next_crons_changed(&mut rx);
783        assert_eq!(key, lead, "a create routes on the project's lead seat");
784        assert_eq!(
785            crons.iter().map(|cron| cron.prompt.clone()).collect::<Vec<_>>(),
786            ["stand-up"],
787            "and carries the set the core now holds",
788        );
789
790        assert!(
791            ws.remove_cron_owned_by("proj", &CronId::from("c1"), None).is_some(),
792            "precondition: the lead deletes its own cron",
793        );
794        let (_, crons) = next_crons_changed(&mut rx);
795        assert!(crons.is_empty(), "a delete announces the set it left behind");
796    }
797
798    /// A cron write that changed nothing announces nothing: an id no entry
799    /// carries, a delete refused for ownership, a fire for an id that is
800    /// gone, and a teardown matching no worker cron are all no-ops.
801    ///
802    /// Mutant: emit unconditionally, where each of these is news the section
803    /// never moved for.
804    #[test]
805    fn a_cron_write_that_changed_nothing_announces_nothing() {
806        use forge_primitives::cron::CronId;
807        let dir = tempdir().expect("tempdir");
808        let (ws, mut rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
809        ws.seed_test_project("proj", "/tmp/tp-crons-noop");
810        let key = ws.project_key_for_name("proj").expect("seeded project");
811
812        assert!(
813            !ws.remove_cron("proj", &CronId::from("ghost")),
814            "precondition: no entry carries the id",
815        );
816        assert!(
817            ws.remove_cron_owned_by("proj", &CronId::from("ghost"), None).is_none(),
818            "precondition: no entry carries the id for that owner either",
819        );
820        ws.seed_test_cron(schedule("c1", "proj", "stand-up", Some("reviewer")));
821        assert!(
822            ws.remove_cron_owned_by("proj", &CronId::from("c1"), None).is_none(),
823            "precondition: the entry belongs to the worker, so the lead's delete is refused",
824        );
825        ws.delete_crons_for_worker(&key, "nobody");
826        ws.advance_or_remove_cron(&CronId::from("ghost"), std::time::SystemTime::now());
827
828        let announced = rx.try_recv();
829        assert!(
830            announced.is_err(),
831            "no write here changed a schedule, so nothing is announced, and this was: {announced:?}",
832        );
833    }
834
835    /// The writes no tool call reaches announce as well - a fire that
836    /// advances or removes an entry, the fire router's dead-slot removal, and
837    /// a worker teardown that takes its schedules with it - because the
838    /// section draws `next_fire` and the set itself, not only what the tools
839    /// edit.
840    ///
841    /// Mutant: wire only the tool-driven writes, where a fired schedule's
842    /// countdown, a vanished owner's cron and a despawned worker's schedules
843    /// all stop moving.
844    #[test]
845    fn a_fired_schedule_and_a_teardown_announce_what_they_left() {
846        use forge_primitives::cron::{CronId, CronKind};
847        let dir = tempdir().expect("tempdir");
848        let (ws, mut rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
849        ws.seed_test_project("proj", "/tmp/tp-crons-fire");
850        let key = ws.project_key_for_name("proj").expect("seeded project");
851        let lead = SessionSlot::lead("TestOrg", "proj");
852        let now = std::time::SystemTime::now();
853
854        ws.seed_test_cron(schedule("r", "proj", "stand-up", None));
855        ws.advance_or_remove_cron(&CronId::from("r"), now);
856        let (announced_key, crons) = next_crons_changed(&mut rx);
857        assert_eq!(announced_key, lead, "a fire routes on the project's lead seat");
858        assert_eq!(crons.len(), 1, "a fired recurring keeps its entry");
859        assert!(
860            crons[0].next_fire > now,
861            "and the announcement carries the schedule it advanced to"
862        );
863        assert_eq!(crons[0].last_fire, Some(now), "with the fire it recorded");
864
865        // A run-once leaves the set with its fire, so the announcement is
866        // what is left rather than the entry that is gone.
867        ws.seed_test_cron(forge_primitives::cron::CronEntry {
868            kind: CronKind::Once(now),
869            ..schedule("o", "proj", "deploy", None)
870        });
871        ws.advance_or_remove_cron(&CronId::from("o"), now);
872        let (_, crons) = next_crons_changed(&mut rx);
873        assert!(
874            crons.iter().all(|cron| cron.id != CronId::from("o")),
875            "the fired run-once is announced as gone",
876        );
877
878        // The fire router's dead-slot removal - the cron whose owner left
879        // forge.toml - takes the entry and announces the set it left, or a
880        // cron the section no longer holds keeps its row.
881        ws.seed_test_cron(schedule("orphan", "proj", "no owner", None));
882        assert!(
883            ws.remove_cron("proj", &CronId::from("orphan")),
884            "precondition: the router's removal finds the entry",
885        );
886        let (announced_key, crons) = next_crons_changed(&mut rx);
887        assert_eq!(announced_key, lead, "a dead-slot removal routes the same way");
888        assert!(
889            crons.iter().all(|cron| cron.id != CronId::from("orphan")),
890            "and announces the set it left: {crons:?}",
891        );
892
893        // A worker's teardown announces the schedules it left behind.
894        ws.seed_test_cron(schedule("w", "proj", "review", Some("reviewer")));
895        ws.delete_crons_for_worker(&key, "reviewer");
896        let (announced_key, crons) = next_crons_changed(&mut rx);
897        assert_eq!(announced_key, lead, "a teardown routes on the project's lead seat");
898        assert!(
899            crons.iter().all(|cron| cron.team_role.as_deref() != Some("reviewer")),
900            "and announces what the teardown left: {crons:?}",
901        );
902    }
903
904    #[test]
905    fn cron_methods_push_list_remove_and_persist() {
906        use forge_primitives::cron::{CronEntry, CronId, CronKind};
907        let dir = tempdir().expect("tempdir");
908        let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
909        let db = crate::store::Db::open(&dir.path().join("db.redb")).expect("open db");
910        ws.install_db_for_test(db);
911        let persisted = || {
912            crate::store::cron::list(ws.db.lock().as_ref().expect("db installed")).expect("list")
913        };
914
915        let entry = CronEntry {
916            id: CronId::from("c1"),
917            project_name: "forge".to_owned(),
918            kind: CronKind::Recurring("0 9 * * *".to_owned()),
919            prompt: "stand-up".to_owned(),
920            created_at: std::time::SystemTime::UNIX_EPOCH,
921            description: None,
922            last_fire: None,
923            next_fire: std::time::SystemTime::UNIX_EPOCH,
924            team_role: None,
925            wake: true,
926        };
927
928        ws.push_cron(entry.clone());
929        assert_eq!(ws.crons_for_project("forge"), vec![entry.clone()], "listed for its project");
930        assert!(ws.crons_for_project("other").is_empty(), "scoped by project name");
931        assert_eq!(persisted(), vec![entry.clone()], "push persisted to the store");
932
933        // A different project cannot delete another project's cron.
934        assert!(!ws.remove_cron("other", &entry.id), "delete is scoped to the owning project");
935        assert_eq!(ws.crons_for_project("forge").len(), 1);
936
937        assert!(ws.remove_cron("forge", &entry.id), "the owning project removes it");
938        assert!(ws.crons_for_project("forge").is_empty());
939        assert!(persisted().is_empty(), "removal persisted");
940
941        assert!(!ws.remove_cron("forge", &CronId::from("c1")), "removing a gone id reports false");
942    }
943
944    /// The failed-boot-read state: the store holds durable crons, the
945    /// in-memory set is EMPTY (a `list` failure at boot warned and started
946    /// with none), and the db handle is live. Every cron mutation writes
947    /// only what it changed - a door that changed nothing writes nothing,
948    /// and a door that changed something writes its own row - so no write
949    /// after such a boot can erase the rows the failed read never loaded.
950    ///
951    /// Both kinds of door are walked on purpose: a whole-set write that
952    /// only fires on a successful mutation would slip past a pin that ran
953    /// the no-op doors alone.
954    ///
955    /// Mutant: have any mutation site (or the shared mutation helper) write
956    /// the whole in-memory set, at a door that changed nothing or one that
957    /// changed something, where either write takes both seeded rows with it.
958    #[test]
959    fn cron_mutations_over_a_failed_boot_read_do_not_erase_the_store() {
960        use forge_primitives::cron::{CronEntry, CronId, CronKind};
961        let dir = tempdir().expect("tempdir");
962        let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
963        ws.seed_test_project("proj", "/tmp/tp-crons-failed-load");
964        let db = crate::store::Db::open(&dir.path().join("db.redb")).expect("open db");
965        // The rows the failed boot read left out of memory: in the store only.
966        crate::store::cron::upsert(&db, &schedule("seed-1", "proj", "one", None)).expect("seed 1");
967        crate::store::cron::upsert(&db, &schedule("seed-2", "proj", "two", None)).expect("seed 2");
968        ws.install_db_for_test(db);
969        let persisted = || {
970            crate::store::cron::list(ws.db.lock().as_ref().expect("db installed")).expect("list")
971        };
972        let seeded = |rows: &[forge_primitives::cron::CronEntry]| {
973            rows.iter().filter(|c| c.id.as_str().starts_with("seed-")).count()
974        };
975        // The one assertion every door below makes: it wrote its own row at
976        // most, never the whole in-memory set.
977        let still_seeded = |door: &str| {
978            assert_eq!(
979                seeded(&persisted()),
980                2,
981                "after {door} the rows the failed read never loaded must still be there",
982            );
983        };
984        let key = ws.project_key_for_name("proj").expect("seeded project");
985        let now = std::time::SystemTime::now();
986
987        ws.push_cron(schedule("created", "proj", "new", None));
988        still_seeded("a create");
989
990        // Doors that change nothing: a write here is news about a row that
991        // never moved.
992        assert!(
993            !ws.remove_cron("proj", &CronId::from("seed-1")),
994            "precondition: the empty in-memory set holds no such row",
995        );
996        still_seeded("a delete that removed nothing");
997
998        assert!(
999            ws.remove_cron_owned_by("proj", &CronId::from("seed-1"), None).is_none(),
1000            "precondition: the empty in-memory set holds no such row for that owner either",
1001        );
1002        still_seeded("an owner-scoped delete that matched nothing");
1003
1004        ws.delete_crons_for_worker(&key, "worker");
1005        still_seeded("a worker teardown with no matching crons");
1006
1007        ws.advance_or_remove_cron(&CronId::from("seed-2"), now);
1008        still_seeded("a fire-advance over an unloaded row");
1009
1010        // Doors that DO change something, each on a loaded row: a whole-set
1011        // write gated behind the change takes the unloaded rows with it.
1012        assert!(
1013            ws.remove_cron("proj", &CronId::from("created")),
1014            "precondition: the loaded row is removed",
1015        );
1016        still_seeded("a delete of a loaded row");
1017
1018        ws.push_cron(schedule("owned", "proj", "own", None));
1019        assert!(
1020            ws.remove_cron_owned_by("proj", &CronId::from("owned"), None).is_some(),
1021            "precondition: the loaded row is removed through the owner's door",
1022        );
1023        still_seeded("an owner-scoped delete of a loaded row");
1024
1025        ws.push_cron(schedule("worker-cron", "proj", "w", Some("worker")));
1026        ws.delete_crons_for_worker(&key, "worker");
1027        still_seeded("a worker teardown that took a cron");
1028
1029        ws.push_cron(CronEntry {
1030            kind: CronKind::Once(now),
1031            ..schedule("once", "proj", "fire", None)
1032        });
1033        ws.advance_or_remove_cron(&CronId::from("once"), now);
1034        still_seeded("a fire that removed a run-once");
1035
1036        ws.push_cron(schedule("recurring", "proj", "again", None));
1037        ws.advance_or_remove_cron(&CronId::from("recurring"), now);
1038        still_seeded("a fire that advanced a recurring");
1039
1040        ws.push_cron(CronEntry {
1041            kind: CronKind::Recurring("0 0 30 2 *".to_owned()),
1042            ..schedule("never", "proj", "never", None)
1043        });
1044        ws.advance_or_remove_cron(&CronId::from("never"), now);
1045        still_seeded("a fire that removed a never-occurring recurring");
1046    }
1047
1048    #[test]
1049    fn advance_or_remove_cron_recurring_advances_run_once_removes() {
1050        use forge_primitives::cron::{CronEntry, CronId, CronKind};
1051        let dir = tempdir().expect("tempdir");
1052        let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
1053        let fired = std::time::SystemTime::now();
1054
1055        ws.push_cron(CronEntry {
1056            id: CronId::from("r"),
1057            project_name: "forge".to_owned(),
1058            kind: CronKind::Recurring("*/5 * * * *".to_owned()),
1059            prompt: "p".to_owned(),
1060            created_at: fired,
1061            description: None,
1062            last_fire: None,
1063            next_fire: std::time::SystemTime::UNIX_EPOCH,
1064            team_role: None,
1065            wake: true,
1066        });
1067        ws.advance_or_remove_cron(&CronId::from("r"), fired);
1068        let after = ws.crons_for_project("forge");
1069        assert_eq!(after.len(), 1, "a recurring cron stays after firing");
1070        assert!(after[0].next_fire > fired, "next_fire advanced to a future slot");
1071        assert_eq!(after[0].last_fire, Some(fired), "last_fire recorded");
1072
1073        ws.push_cron(CronEntry {
1074            id: CronId::from("o"),
1075            project_name: "forge".to_owned(),
1076            kind: CronKind::Once(std::time::SystemTime::UNIX_EPOCH),
1077            prompt: "p".to_owned(),
1078            created_at: fired,
1079            description: None,
1080            last_fire: None,
1081            next_fire: std::time::SystemTime::UNIX_EPOCH,
1082            team_role: None,
1083            wake: true,
1084        });
1085        ws.advance_or_remove_cron(&CronId::from("o"), fired);
1086        assert!(
1087            ws.crons_for_project("forge").iter().all(|c| c.id != CronId::from("o")),
1088            "a run-once is removed after firing",
1089        );
1090
1091        // A missing id is a no-op, not a panic.
1092        ws.advance_or_remove_cron(&CronId::from("ghost"), fired);
1093    }
1094
1095    #[test]
1096    fn fire_due_crons_spawns_asleep_project_buffers_prompt_and_skips_future() {
1097        use forge_primitives::cron::{CronEntry, CronId, CronKind};
1098        let dir = tempdir().expect("tempdir");
1099        let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
1100        ws.seed_test_project("forge", "/tmp/forge");
1101
1102        let now = std::time::SystemTime::now();
1103        let past = std::time::SystemTime::UNIX_EPOCH;
1104        let far_future = now + std::time::Duration::from_secs(86_400);
1105
1106        ws.push_cron(CronEntry {
1107            id: CronId::from("due"),
1108            project_name: "forge".to_owned(),
1109            kind: CronKind::Recurring("*/5 * * * *".to_owned()),
1110            prompt: "morning".to_owned(),
1111            created_at: past,
1112            description: None,
1113            last_fire: None,
1114            next_fire: past,
1115            team_role: None,
1116            wake: true,
1117        });
1118        ws.push_cron(CronEntry {
1119            id: CronId::from("later"),
1120            project_name: "forge".to_owned(),
1121            kind: CronKind::Recurring("*/5 * * * *".to_owned()),
1122            prompt: "later".to_owned(),
1123            created_at: past,
1124            description: None,
1125            last_fire: None,
1126            next_fire: far_future,
1127            team_role: None,
1128            wake: true,
1129        });
1130
1131        ws.enable_test_dispatch_intercept();
1132        ws.fire_due_crons(now);
1133        let dispatched = ws.drain_test_dispatch_buffer();
1134
1135        // The asleep project got exactly one SpawnProject - for the due
1136        // cron, not the future one.
1137        let spawns = dispatched
1138            .iter()
1139            .filter(|c| {
1140                matches!(c, crate::protocol::Command::SpawnProject { project_name, .. }
1141                    if project_name == "forge")
1142            })
1143            .count();
1144        assert_eq!(spawns, 1, "one spawn for the single due cron");
1145
1146        // The due cron's prompt is buffered for its slot (the lead) for
1147        // delivery once the session reaches Connected.
1148        let buffered = parked_crons(&ws, "forge", None);
1149        assert_eq!(buffered, vec!["morning".to_owned()], "the due cron's prompt was buffered");
1150
1151        // The due cron advanced past now; the future cron is untouched.
1152        let crons = ws.crons_for_project("forge");
1153        let due = crons.iter().find(|c| c.id == CronId::from("due")).expect("due present");
1154        assert!(due.next_fire > now, "the fired cron advanced past now");
1155        let later = crons.iter().find(|c| c.id == CronId::from("later")).expect("later present");
1156        assert_eq!(later.next_fire, far_future, "the not-yet-due cron is untouched");
1157    }
1158
1159    #[test]
1160    fn boot_catch_up_fires_overdue_once_advances_persists_and_does_not_refire() {
1161        use forge_primitives::cron::{CronEntry, CronId, CronKind};
1162        let dir = tempdir().expect("tempdir");
1163        let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
1164        let db = crate::store::Db::open(&dir.path().join("db.redb")).expect("open db");
1165        ws.install_db_for_test(db);
1166        ws.seed_test_project("forge", "/tmp/forge-catchup");
1167
1168        let now = std::time::SystemTime::now();
1169        let past = std::time::SystemTime::UNIX_EPOCH;
1170        let far_future = now + std::time::Duration::from_secs(86_400);
1171
1172        // An overdue recurring + overdue run-once + a not-yet-due one, all
1173        // for the seeded "forge" project. Boot catch-up is the single
1174        // fire_due_crons call main.rs makes at startup.
1175        for entry in [
1176            CronEntry {
1177                id: CronId::from("rec"),
1178                project_name: "forge".to_owned(),
1179                kind: CronKind::Recurring("*/5 * * * *".to_owned()),
1180                prompt: "morning".to_owned(),
1181                created_at: past,
1182                description: None,
1183                last_fire: None,
1184                next_fire: past,
1185                team_role: None,
1186                wake: true,
1187            },
1188            CronEntry {
1189                id: CronId::from("once"),
1190                project_name: "forge".to_owned(),
1191                kind: CronKind::Once(past),
1192                prompt: "deploy".to_owned(),
1193                created_at: past,
1194                description: None,
1195                last_fire: None,
1196                next_fire: past,
1197                team_role: None,
1198                wake: true,
1199            },
1200            CronEntry {
1201                id: CronId::from("future"),
1202                project_name: "forge".to_owned(),
1203                kind: CronKind::Recurring("*/5 * * * *".to_owned()),
1204                prompt: "later".to_owned(),
1205                created_at: past,
1206                description: None,
1207                last_fire: None,
1208                next_fire: far_future,
1209                team_role: None,
1210                wake: true,
1211            },
1212        ] {
1213            ws.push_cron(entry);
1214        }
1215        assert_eq!(ws.crons_for_project("forge").len(), 3, "all three crons loaded");
1216
1217        // Boot catch-up: the one call main.rs makes at startup.
1218        ws.enable_test_dispatch_intercept();
1219        ws.fire_due_crons(now);
1220        let first = ws.drain_test_dispatch_buffer();
1221        let spawns = first
1222            .iter()
1223            .filter(|c| {
1224                matches!(c, crate::protocol::Command::SpawnProject { project_name, .. }
1225                    if project_name == "forge")
1226            })
1227            .count();
1228        assert_eq!(spawns, 2, "boot fires the two overdue crons, not the future one");
1229
1230        // Overdue recurring advanced past now + recorded last_fire; overdue
1231        // run-once removed; future untouched.
1232        let crons = ws.crons_for_project("forge");
1233        assert!(crons.iter().all(|c| c.id != CronId::from("once")), "overdue run-once removed");
1234        let rec = crons.iter().find(|c| c.id == CronId::from("rec")).expect("recurring present");
1235        assert!(rec.next_fire > now, "overdue recurring advanced past now");
1236        assert_eq!(rec.last_fire, Some(now), "the fired recurring recorded last_fire");
1237        let fut = crons.iter().find(|c| c.id == CronId::from("future")).expect("future present");
1238        assert_eq!(fut.next_fire, far_future, "the future cron is untouched");
1239
1240        // The advance persisted to the store - this is what stops a double-fire.
1241        let persisted =
1242            crate::store::cron::list(ws.db.lock().as_ref().expect("db installed")).expect("list");
1243        assert!(persisted.iter().all(|c| c.id != CronId::from("once")), "removal persisted");
1244        assert!(
1245            persisted
1246                .iter()
1247                .find(|c| c.id == CronId::from("rec"))
1248                .expect("rec persisted")
1249                .next_fire
1250                > now,
1251            "advance persisted",
1252        );
1253
1254        // The next tick does NOT re-fire: the advanced crons aren't due again.
1255        ws.fire_due_crons(now);
1256        let refires = ws
1257            .drain_test_dispatch_buffer()
1258            .iter()
1259            .filter(|c| matches!(c, crate::protocol::Command::SpawnProject { .. }))
1260            .count();
1261        assert_eq!(refires, 0, "advanced crons are not due again; no double-fire");
1262    }
1263
1264    /// A cron fired into a running lead delivers the raw prompt as a plain
1265    /// user turn AND echoes a `CronPromptAppended` so the chat shows a cron
1266    /// block (mirrors the gotify running-target echo). Reproduce-first: the
1267    /// echo is absent until `deliver_cron_prompt` calls
1268    /// `push_cron_prompt_into_chat`.
1269    #[test]
1270    fn deliver_cron_prompt_to_running_lead_emits_cron_prompt_appended() {
1271        let dir = tempdir().expect("tempdir");
1272        let (ws, mut rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
1273        // The fire's schedule pairing is a store write, so this test needs
1274        // the durable layer the shipped workspace always has.
1275        ws.install_db_for_test(
1276            crate::store::Db::open(&dir.path().join("db.redb")).expect("open db"),
1277        );
1278        ws.seed_test_project("cronlead", "/tmp/cron-lead");
1279        // Seed the catalog + pool so the project has a running (open) lead:
1280        // list_projects derives `is_open` from pool membership.
1281        let cwd = project_expanded_path(&ws, "cronlead");
1282        ws.record_connected_session(&cwd, "lead-uuid", None);
1283        let lead_key = SessionSlot::lead("TestOrg", "cronlead");
1284        let (handle, _agent_rx) = Workspace::testing_stub_handle();
1285        ws.pool.lock().insert(
1286            lead_key.clone(),
1287            PooledAgent {
1288                handle: Arc::new(handle),
1289                account: AccountKey("test".to_owned()),
1290                permission_mode: None,
1291                registration: None,
1292                session_id: "pooled-session".to_owned(),
1293            },
1294        );
1295
1296        ws.mark_session_connected_for_test(&lead_key, "lead-uuid");
1297        ws.enable_test_dispatch_intercept();
1298        let mut fired = schedule("c1", "cronlead", "morning", None);
1299        fired.description = Some("Morning summary".to_owned());
1300        let outcome = crate::spawn::deliver_cron_prompt(&ws, &fired, false);
1301        assert!(matches!(outcome, crate::spawn::CronFireOutcome::Delivered));
1302
1303        // The running lead receives the raw cron prompt as a plain user turn.
1304        let dispatched = ws.drain_test_dispatch_buffer();
1305        let wired_id = dispatched
1306            .iter()
1307            .find_map(|c| match c {
1308                Command::PromptUnder { key, text, uuid, .. }
1309                    if key == &lead_key && text == "morning" =>
1310                {
1311                    Some(uuid.clone())
1312                }
1313                _ => None,
1314            })
1315            .expect("the running lead receives the fired cron prompt verbatim, wired under an id");
1316
1317        // AND the delivery echoes a CronPromptAppended so the chat shows a
1318        // block - under the SAME id the prompt runs under, which is what lets
1319        // a view hold the block's row while the prompt waits and pair it with
1320        // the page's copy later. The echo names the entry that fired: a view
1321        // reading only the prompt cannot say which schedule produced it, and
1322        // a run-once entry is gone from the store by the time the row draws.
1323        let echoed = drain_updates(&mut rx).into_iter().find_map(|u| match u {
1324            SessionUpdate::CronPromptAppended { key, text, uuid, cron_id, description }
1325                if key == lead_key && text == "morning" =>
1326            {
1327                Some((uuid, cron_id, description))
1328            }
1329            _ => None,
1330        });
1331        let (echoed_id, cron_id, description) =
1332            echoed.expect("the delivery echoes a cron block for the fired prompt");
1333        assert_eq!(
1334            echoed_id.as_str(),
1335            wired_id.as_str(),
1336            "the block and the fired prompt are one thing by id",
1337        );
1338        assert!(!wired_id.is_empty(), "and the id is real, not a blank");
1339        assert_eq!(cron_id, "c1", "the echo names the entry that fired");
1340        assert_eq!(description.as_deref(), Some("Morning summary"), "and its description");
1341        // **And the pairing is kept for the read half.** The frame names the
1342        // schedule once; a row replayed from a transcript afterwards has no
1343        // frame to read it from, so the fire is remembered under the prompt's
1344        // own id - the id the transcript row carries (#1815).
1345        let stored = crate::store::cron_names::for_uuids(
1346            ws.db.lock().as_ref().expect("db installed"),
1347            &[wired_id.as_str()],
1348        )
1349        .expect("read the stored pairing");
1350        assert_eq!(
1351            stored.get(wired_id.as_str()),
1352            Some(&forge_primitives::cron::CronName {
1353                cron_id: "c1".to_owned(),
1354                description: Some("Morning summary".to_owned()),
1355            }),
1356            "the fired prompt's schedule is readable after the frame is gone"
1357        );
1358    }
1359
1360    #[test]
1361    fn deliver_worker_cron_to_a_live_worker_prompts_the_worker() {
1362        let (ws, _rx) = Workspace::testing_stub();
1363        ws.seed_test_project("proj", "/tmp/wc-live");
1364        let key = ws.list_projects().into_iter().find(|v| v.name == "proj").expect("view").key;
1365        ws.insert_live_worker(&key, live_worker_entry("proj", "reviewer"));
1366        let worker_key = SessionSlot::worker("TestOrg", "proj", "reviewer");
1367        ws.mark_session_connected_for_test(&worker_key, "worker-uuid");
1368
1369        ws.enable_test_dispatch_intercept();
1370        let outcome = deliver_cron(&ws, "proj", Some("reviewer"), "review the diff", false);
1371        assert!(matches!(outcome, crate::spawn::CronFireOutcome::Delivered));
1372        let dispatched = ws.drain_test_dispatch_buffer();
1373        assert!(
1374            dispatched.iter().any(|c| matches!(
1375                c, Command::Prompt { key, text, .. } | Command::PromptUnder { key, text, .. }
1376                    if key == &worker_key && text == "review the diff"
1377            )),
1378            "a live worker's cron fires straight into the worker",
1379        );
1380    }
1381
1382    /// One captured tracing record: its level, its target and its fields.
1383    #[derive(Clone, Default)]
1384    struct Caught(std::sync::Arc<std::sync::Mutex<Vec<(tracing::Level, String, String)>>>);
1385
1386    #[derive(Default)]
1387    struct CaughtFields(String);
1388
1389    impl tracing::field::Visit for CaughtFields {
1390        fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
1391            if !self.0.is_empty() {
1392                self.0.push(' ');
1393            }
1394            let _ = std::fmt::write(&mut self.0, format_args!("{}={value:?}", field.name()));
1395        }
1396    }
1397
1398    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for Caught {
1399        fn on_event(
1400            &self,
1401            event: &tracing::Event<'_>,
1402            _ctx: tracing_subscriber::layer::Context<'_, S>,
1403        ) {
1404            let mut fields = CaughtFields::default();
1405            event.record(&mut fields);
1406            self.0.lock().expect("capture").push((
1407                *event.metadata().level(),
1408                event.metadata().target().to_owned(),
1409                fields.0,
1410            ));
1411        }
1412    }
1413
1414    /// The records a pass emitted for the cooling event at `level`, as
1415    /// their captured fields.
1416    fn cooling_records(caught: &Caught, level: tracing::Level) -> Vec<String> {
1417        caught
1418            .0
1419            .lock()
1420            .expect("capture")
1421            .iter()
1422            .filter(|(recorded, target, fields)| {
1423                *recorded == level
1424                    && target == "forge_workspace::crons"
1425                    && fields.contains("cron_deferred_cooling")
1426            })
1427            .map(|(_, _, fields)| fields.clone())
1428            .collect()
1429    }
1430
1431    /// Every `ServiceStatus` a pass sent, which is what a client draws.
1432    fn service_notices(
1433        rx: &mut tokio::sync::mpsc::UnboundedReceiver<crate::SeatUpdate>,
1434    ) -> Vec<String> {
1435        drain_updates(rx)
1436            .into_iter()
1437            .filter_map(|update| match update {
1438                SessionUpdate::ServiceStatus { message, .. } => Some(message),
1439                _ => None,
1440            })
1441            .collect()
1442    }
1443
1444    fn unix_now() -> u64 {
1445        std::time::SystemTime::now()
1446            .duration_since(std::time::SystemTime::UNIX_EPOCH)
1447            .expect("clock after the epoch")
1448            .as_secs()
1449    }
1450
1451    /// Hold the fixture's one account out of the walk for `wait`, the
1452    /// write the usage probe's cap makes.
1453    fn cool_for(ws: &Workspace, wait: std::time::Duration) {
1454        cool_until(ws, unix_now() + wait.as_secs());
1455    }
1456
1457    /// Hold the fixture's one account out of the walk until `reset_epoch`.
1458    fn cool_until(ws: &Workspace, reset_epoch: u64) {
1459        ws.gateway.report_probe_limit(&AccountKey("acct-a".to_owned()), Some(reset_epoch));
1460    }
1461
1462    /// `wait` from `now`, nudged to sit half a minute into a minute, so a
1463    /// wall clock read for it cannot straddle a minute boundary between
1464    /// the test's clock and the scheduler's own read.
1465    fn reset_mid_minute(now: std::time::SystemTime, wait: std::time::Duration) -> u64 {
1466        let target = now
1467            .duration_since(std::time::SystemTime::UNIX_EPOCH)
1468            .expect("clock after the epoch")
1469            .as_secs()
1470            + wait.as_secs();
1471        target - target % 60 + 30
1472    }
1473
1474    /// The IST wall clock a notice prints for `unix_seconds`.
1475    fn ist_clock(unix_seconds: u64) -> String {
1476        let at = chrono::DateTime::<chrono::Utc>::from(
1477            std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(unix_seconds),
1478        );
1479        (at + chrono::TimeDelta::minutes(5 * 60 + 30)).format("%H:%M").to_string()
1480    }
1481
1482    /// End the account's cooldown, so the walk finds it again.
1483    fn clear_cooling(ws: &Workspace) {
1484        ws.gateway.report_probe_limit(&AccountKey("acct-a".to_owned()), Some(unix_now() - 60));
1485    }
1486
1487    /// A cron deferred by a cooling walk stays due and says why once - the
1488    /// model, the org and the reset - with the repeats at debug until the
1489    /// 30-minute floor re-warns.
1490    ///
1491    /// Mutants: warn on every tick (the second fire emits another record
1492    /// and notice); drop the cooling detail from either line (it names no
1493    /// model, org or reset); silence the repeat instead of recording it;
1494    /// print the tick's own clock instead of the reset's; reset the
1495    /// re-warn marker on every pass, where the floor counts from the
1496    /// repeat rather than from the warning.
1497    #[tokio::test]
1498    async fn a_cooling_walk_defers_the_cron_and_warns_once_per_episode() {
1499        use tracing_subscriber::layer::SubscriberExt as _;
1500
1501        let (ws, _dir) = workspace_with_one_unsettled_account();
1502        ws.seed_test_ready_account("acct-a");
1503        let mut rx = ws.subscribe_answerer();
1504        ws.seed_test_cron(schedule("cool", "proj", "stand-up", None));
1505        let now = std::time::SystemTime::now();
1506        let reset_epoch = reset_mid_minute(now, std::time::Duration::from_secs(2 * 60 * 60));
1507        cool_until(&ws, reset_epoch);
1508
1509        let caught = Caught::default();
1510        let subscriber = tracing_subscriber::registry().with(caught.clone());
1511        let _guard = tracing::subscriber::set_default(subscriber);
1512
1513        ws.fire_due_crons(now);
1514        let warned = cooling_records(&caught, tracing::Level::WARN);
1515        assert_eq!(warned.len(), 1, "the first cooling tick warns once");
1516        let fields = &warned[0];
1517        let clock = ist_clock(reset_epoch);
1518        assert!(fields.contains("org=TestOrg"), "the warning names the org: {fields}");
1519        assert!(fields.contains("model=claude-sonnet-5"), "the warning names the model: {fields}");
1520        assert!(
1521            fields.contains(&format!("resets_at={clock} IST")),
1522            "the warning names when the reset lands: {fields}",
1523        );
1524        assert!(
1525            fields.contains("cooling_accounts=1"),
1526            "the warning counts the org's cooling accounts: {fields}",
1527        );
1528        let notices = service_notices(&mut rx);
1529        assert_eq!(notices.len(), 1, "and sends the client one notice: {notices:?}");
1530        let notice = &notices[0];
1531        assert!(notice.contains("claude-sonnet-5"), "the notice names the model: {notice}");
1532        assert!(notice.contains("TestOrg"), "the notice names the org: {notice}");
1533        assert!(
1534            notice.contains(&format!("{clock} IST")),
1535            "the notice names when the reset lands: {notice}",
1536        );
1537        assert!(
1538            notice.contains("1 account(s) cooling in the org"),
1539            "and counts the accounts the walk skipped: {notice}",
1540        );
1541
1542        ws.fire_due_crons(now + std::time::Duration::from_secs(61));
1543        assert_eq!(
1544            cooling_records(&caught, tracing::Level::WARN).len(),
1545            1,
1546            "the repeat tick is debug, not another warning",
1547        );
1548        assert_eq!(
1549            cooling_records(&caught, tracing::Level::DEBUG).len(),
1550            1,
1551            "and says so at debug rather than staying silent",
1552        );
1553        assert!(service_notices(&mut rx).is_empty(), "and sends the client nothing");
1554
1555        // 1860 s after the warning and 1799 s after the repeat: a floor
1556        // counted from the repeat would stay silent here.
1557        ws.fire_due_crons(now + std::time::Duration::from_secs(30 * 60 + 60));
1558        assert_eq!(
1559            cooling_records(&caught, tracing::Level::WARN).len(),
1560            2,
1561            "the 30-minute floor re-warns while it lasts",
1562        );
1563
1564        let entry = ws
1565            .crons_for_project("proj")
1566            .into_iter()
1567            .find(|cron| cron.id == forge_primitives::CronId::from("cool"))
1568            .expect("a deferral keeps the entry");
1569        assert_eq!(
1570            entry.next_fire,
1571            std::time::SystemTime::UNIX_EPOCH,
1572            "and leaves it due rather than advancing",
1573        );
1574    }
1575
1576    /// A cooling episode that clears drops its marker, so the next episode
1577    /// warns again rather than reading as a repeat of the last one.
1578    ///
1579    /// Mutant: keep the marker across a pass that did not find the cron
1580    /// cooling, where the second episode stays silent.
1581    #[tokio::test]
1582    async fn a_cooling_episode_that_clears_warns_again_next_time() {
1583        let (ws, _dir) = workspace_with_one_unsettled_account();
1584        ws.seed_test_ready_account("acct-a");
1585        let mut rx = ws.subscribe_answerer();
1586        ws.seed_test_cron(schedule("cool", "proj", "stand-up", None));
1587        let now = std::time::SystemTime::now();
1588
1589        cool_for(&ws, std::time::Duration::from_secs(2 * 60 * 60));
1590        ws.fire_due_crons(now);
1591        assert_eq!(service_notices(&mut rx).len(), 1, "the first episode warns once");
1592
1593        // The cooldown ends: this pass finds the walk healthy and the fire
1594        // lands, its dispatch intercepted rather than spawning a child.
1595        ws.enable_test_dispatch_intercept();
1596        clear_cooling(&ws);
1597        ws.fire_due_crons(now + std::time::Duration::from_secs(61));
1598        assert!(service_notices(&mut rx).is_empty(), "a recovered walk has nothing to warn about");
1599        assert!(
1600            ws.drain_test_dispatch_buffer().iter().any(|command| matches!(
1601                command,
1602                Command::SpawnProject { project_name, .. } if project_name == "proj"
1603            )),
1604            "precondition: the recovered fire landed rather than deferring again",
1605        );
1606
1607        // Cooling again, with the same entry due again, is a new episode.
1608        cool_for(&ws, std::time::Duration::from_secs(2 * 60 * 60));
1609        ws.with_crons_mut(|crons| {
1610            let entry = crons
1611                .iter_mut()
1612                .find(|cron| cron.id == forge_primitives::CronId::from("cool"))
1613                .expect("the recovered fire kept the recurring entry");
1614            entry.next_fire = now + std::time::Duration::from_secs(120);
1615        });
1616        ws.fire_due_crons(now + std::time::Duration::from_secs(120));
1617        assert_eq!(service_notices(&mut rx).len(), 1, "a new episode warns again");
1618    }
1619
1620    /// A reset no clock can spell still names the wait, rather than
1621    /// panicking the scheduler that prints it: one past the end of
1622    /// `SystemTime`, and one whose IST offset runs past the last instant
1623    /// chrono can hold.
1624    #[test]
1625    fn an_unformattable_reset_falls_back_to_the_wait() {
1626        let last_second = chrono::DateTime::<chrono::Utc>::MAX_UTC.timestamp();
1627        let near_the_end = std::time::SystemTime::UNIX_EPOCH
1628            + std::time::Duration::from_secs(
1629                u64::try_from(last_second - 60).expect("a positive second count"),
1630            );
1631        for (reset_in, now) in
1632            [(Duration::MAX, std::time::SystemTime::now()), (Duration::from_secs(60), near_the_end)]
1633        {
1634            let text = cooling_reset_text(reset_in, now);
1635            assert!(text.starts_with("in "), "the wait survives without a clock: {text}");
1636            assert!(!text.contains("IST"), "and no wall clock is claimed: {text}");
1637        }
1638    }
1639
1640    #[test]
1641    fn deliver_asleep_worker_cron_buffers_and_wakes_the_project() {
1642        let (ws, _rx) = Workspace::testing_stub();
1643        let dir = tempdir().expect("tempdir");
1644        ws.install_db_for_test(
1645            crate::store::Db::open(&dir.path().join("db.redb")).expect("open db"),
1646        );
1647        // The row is what gives the slot a session while it is asleep; without it
1648        // the fire router collects the cron instead.
1649        let key = seed_project_with_a_real_root(&ws, &dir);
1650        seed_worker_row(&ws, &key, "reviewer");
1651
1652        ws.enable_test_dispatch_intercept();
1653        let outcome = deliver_cron(&ws, "proj", Some("reviewer"), "nightly", false);
1654        assert!(matches!(outcome, crate::spawn::CronFireOutcome::Delivered));
1655        let dispatched = ws.drain_test_dispatch_buffer();
1656        assert!(
1657            dispatched.iter().any(|c| matches!(
1658                c, Command::SpawnProject { project_name, .. } if project_name == "proj"
1659            )),
1660            "an asleep worker cron wakes the whole project via SpawnProject",
1661        );
1662        let buffered = parked_crons(&ws, "proj", Some("reviewer"));
1663        assert_eq!(buffered, vec!["nightly".to_owned()], "buffered for the worker owner");
1664    }
1665
1666    /// A cron for a worker that's a live entry but still Spawning
1667    /// (session_id None) must NOT dispatch a bare Command::Prompt
1668    /// (dropped) - the gate in `live_cron_slot` routes it to the
1669    /// owner-keyed buffer, drained on the worker's own Connected.
1670    #[test]
1671    fn deliver_cron_to_spawning_worker_buffers_via_owner() {
1672        let (ws, _rx) = Workspace::testing_stub();
1673        let dir = tempdir().expect("tempdir");
1674        ws.install_db_for_test(
1675            crate::store::Db::open(&dir.path().join("db.redb")).expect("open db"),
1676        );
1677        let key = seed_project_with_a_real_root(&ws, &dir);
1678        seed_worker_row(&ws, &key, "reviewer");
1679        let worker_key = SessionSlot::worker("TestOrg", "proj", "reviewer");
1680        ws.insert_live_worker(&key, live_worker_entry("proj", "reviewer"));
1681        // Registered but not connected: session_id stays None.
1682        ws.register_domain_session(worker_key.clone(), None);
1683
1684        ws.enable_test_dispatch_intercept();
1685        let outcome = deliver_cron(&ws, "proj", Some("reviewer"), "nightly", false);
1686        assert!(matches!(outcome, crate::spawn::CronFireOutcome::Delivered));
1687        let dispatched = ws.drain_test_dispatch_buffer();
1688        assert!(
1689            !dispatched.iter().any(|c| matches!(
1690                c, Command::Prompt { key, .. } | Command::PromptUnder { key, .. }
1691                    if key == &worker_key
1692            )),
1693            "no bare Prompt to the still-spawning worker (would be dropped)",
1694        );
1695        let buffered = parked_crons(&ws, "proj", Some("reviewer"));
1696        assert_eq!(
1697            buffered,
1698            vec!["nightly".to_owned()],
1699            "buffered for the worker's Connected drain"
1700        );
1701    }
1702
1703    /// A worker spawned this second is live before it connects, and a git
1704    /// worker's worktree is created during that same window. Judged by its
1705    /// row alone it looks unwakeable, and the fire would be dropped; it is
1706    /// only not there YET, so the prompt parks for its own Connected drain.
1707    #[test]
1708    fn deliver_worker_cron_parks_for_an_owner_that_is_still_spawning() {
1709        // A git worker whose worktree claude has not created yet: the row
1710        // alone says this one cannot start.
1711        let (ws, key, _db, _project) = stranded_owner_fixture("reviewer");
1712        ws.insert_live_worker(&key, live_worker_entry("proj", "reviewer"));
1713
1714        ws.enable_test_dispatch_intercept();
1715        let outcome = deliver_cron(&ws, "proj", Some("reviewer"), "nightly", false);
1716        assert!(
1717            matches!(outcome, crate::spawn::CronFireOutcome::Delivered),
1718            "a worker still spawning is not unwakeable - its worktree is being created \
1719             along with it; got {}",
1720            outcome_name(&outcome),
1721        );
1722        assert_eq!(
1723            parked_crons(&ws, "proj", Some("reviewer")),
1724            vec!["nightly".to_owned()],
1725            "the prompt parks for the spawning worker's Connected drain",
1726        );
1727    }
1728
1729    /// A `Failed` entry is not a live one. `transition_worker_to_failed`
1730    /// keeps it visible on purpose, but the boot wave never starts it, so
1731    /// treating it as wakeable parks the fire in a bucket nothing drains -
1732    /// the unbounded park this whole arm exists to keep empty.
1733    #[test]
1734    fn deliver_worker_cron_does_not_park_for_a_failed_entry() {
1735        let (ws, key, _db, _project) = stranded_owner_fixture("reviewer");
1736        let mut failed = live_worker_entry("proj", "reviewer");
1737        failed.status = forge_primitives::WorkerLiveness::Failed;
1738        ws.insert_live_worker(&key, failed);
1739
1740        ws.enable_test_dispatch_intercept();
1741        let outcome = deliver_cron(&ws, "proj", Some("reviewer"), "nightly", false);
1742        assert!(
1743            matches!(outcome, crate::spawn::CronFireOutcome::TargetCannotBeWoken { .. }),
1744            "a Failed entry is not a worker the wave will start, so its fire must not \
1745             park; got {}",
1746            outcome_name(&outcome),
1747        );
1748        assert_eq!(
1749            parked_crons(&ws, "proj", Some("reviewer")).len(),
1750            0,
1751            "nothing parks for a worker that will never connect",
1752        );
1753    }
1754
1755    /// A `Failed` entry's slot can still read as connected when the fire comes;
1756    /// it is not live, so the fire is refused, not dispatched.
1757    #[test]
1758    fn deliver_worker_cron_does_not_dispatch_to_a_connected_failed_entry() {
1759        let (ws, key, _db, _project) = stranded_owner_fixture("reviewer");
1760        let mut failed = live_worker_entry("proj", "reviewer");
1761        failed.status = forge_primitives::WorkerLiveness::Failed;
1762        ws.insert_live_worker(&key, failed.clone());
1763        ws.mark_session_connected_for_test(&failed.slot, "reviewer-uuid");
1764
1765        ws.enable_test_dispatch_intercept();
1766        let outcome = deliver_cron(&ws, "proj", Some("reviewer"), "nightly", false);
1767        assert!(
1768            matches!(outcome, crate::spawn::CronFireOutcome::TargetCannotBeWoken { .. }),
1769            "a failed worker's slot is not a dispatch target, however connected it still \
1770             reads; got {}",
1771            outcome_name(&outcome),
1772        );
1773        let dispatched = ws.drain_test_dispatch_buffer();
1774        assert!(
1775            !dispatched.iter().any(|c| matches!(
1776                c,
1777                Command::Prompt { key, .. } | Command::PromptUnder { key, .. }
1778                    if key == &failed.slot
1779            )),
1780            "and no prompt was dispatched into the failed worker: {dispatched:?}",
1781        );
1782    }
1783
1784    #[tokio::test]
1785    async fn deliver_asleep_dynamic_worker_cron_buffers_and_wakes_the_project() {
1786        let (ws, _rx) = Workspace::testing_stub();
1787        let dir = tempdir().expect("tempdir");
1788        ws.install_db_for_test(
1789            crate::store::Db::open(&dir.path().join("db.redb")).expect("open db"),
1790        );
1791        let key = seed_project_with_a_real_root(&ws, &dir);
1792        // "scratch" exists only via its persisted worker row.
1793        seed_worker_row(&ws, &key, "scratch");
1794
1795        ws.enable_test_dispatch_intercept();
1796        let outcome = deliver_cron(&ws, "proj", Some("scratch"), "hourly", false);
1797        assert!(matches!(outcome, crate::spawn::CronFireOutcome::Delivered));
1798        let dispatched = ws.drain_test_dispatch_buffer();
1799        assert!(
1800            dispatched.iter().any(|c| matches!(
1801                c, Command::SpawnProject { project_name, .. } if project_name == "proj"
1802            )),
1803            "an asleep dynamic worker cron wakes the project too",
1804        );
1805        assert_eq!(
1806            parked_crons(&ws, "proj", Some("scratch")).len(),
1807            1,
1808            "buffered for the dynamic worker owner"
1809        );
1810    }
1811
1812    /// A standing cron that may not wake its owner (#2423/C): while the
1813    /// owner sleeps the fire is skipped and the entry stays due, so the
1814    /// next session start (or a tick while it is awake) runs it. The same
1815    /// entry with `wake: true` spawns, which is the control.
1816    #[test]
1817    fn a_no_wake_cron_skips_a_sleeping_owner_and_stays_due() {
1818        let (ws, _rx) = Workspace::testing_stub();
1819        ws.seed_test_project("proj", "/tmp/wc-nowake");
1820        ws.enable_test_dispatch_intercept();
1821        let now = std::time::SystemTime::now();
1822
1823        let mut sweep = schedule("sweep", "proj", "sweep", None);
1824        sweep.next_fire = std::time::SystemTime::UNIX_EPOCH;
1825        sweep.wake = false;
1826        ws.push_cron(sweep);
1827        ws.fire_due_crons(now);
1828
1829        let stored = ws.crons_for_project("proj");
1830        assert_eq!(stored.len(), 1, "the skipped fire left the cron in place");
1831        assert_eq!(stored[0].last_fire, None, "and did not count as a fire");
1832        assert_eq!(
1833            stored[0].next_fire,
1834            std::time::SystemTime::UNIX_EPOCH,
1835            "the entry stays due, so a session start runs it",
1836        );
1837        assert!(ws.drain_test_dispatch_buffer().is_empty(), "no session was woken");
1838        assert!(parked_crons(&ws, "proj", None).is_empty(), "and nothing was parked");
1839
1840        // Control: the same entry with wake on spawns its sleeping owner.
1841        let mut wakeful = schedule("wakeful", "proj", "stand-up", None);
1842        wakeful.next_fire = std::time::SystemTime::UNIX_EPOCH;
1843        ws.push_cron(wakeful);
1844        ws.fire_due_crons(now);
1845        assert!(
1846            !ws.drain_test_dispatch_buffer().is_empty(),
1847            "a wakeful cron still wakes its owner",
1848        );
1849        assert_eq!(
1850            ws.crons_for_project("proj")
1851                .iter()
1852                .find(|cron| cron.id == forge_primitives::cron::CronId::from("wakeful"))
1853                .and_then(|cron| cron.last_fire),
1854            Some(now),
1855            "and its fire advances the schedule",
1856        );
1857    }
1858
1859    /// A no-wake cron is skipped only while the owner SLEEPS: with a live
1860    /// lead the fire lands and the schedule advances, which is the
1861    /// standing cadence (#2423/C).
1862    #[test]
1863    fn a_no_wake_cron_fires_normally_while_the_owner_is_awake() {
1864        let dir = tempdir().expect("tempdir");
1865        let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
1866        ws.seed_test_project("proj", "/tmp/wc-nowake-awake");
1867        let cwd = project_expanded_path(&ws, "proj");
1868        ws.record_connected_session(&cwd, "lead-uuid", None);
1869        let lead = SessionSlot::lead("TestOrg", "proj");
1870        let (handle, _agent_rx) = Workspace::testing_stub_handle();
1871        ws.pool.lock().insert(
1872            lead.clone(),
1873            PooledAgent {
1874                handle: Arc::new(handle),
1875                account: AccountKey("test".to_owned()),
1876                permission_mode: None,
1877                registration: None,
1878                session_id: "pooled-session".to_owned(),
1879            },
1880        );
1881        ws.mark_session_connected_for_test(&lead, "lead-uuid");
1882
1883        let now = std::time::SystemTime::now();
1884        let mut sweep = schedule("sweep", "proj", "sweep", None);
1885        sweep.next_fire = std::time::SystemTime::UNIX_EPOCH;
1886        sweep.wake = false;
1887        ws.push_cron(sweep);
1888        ws.enable_test_dispatch_intercept();
1889
1890        ws.fire_due_crons(now);
1891        let texts: Vec<String> = ws
1892            .drain_test_dispatch_buffer()
1893            .iter()
1894            .filter_map(|command| match command {
1895                Command::Prompt { text, .. } | Command::PromptUnder { text, .. } => {
1896                    Some(text.clone())
1897                }
1898                _ => None,
1899            })
1900            .collect();
1901        assert!(
1902            texts.iter().any(|text| text.contains("sweep")),
1903            "the awake owner is given its sweep: {texts:?}",
1904        );
1905        assert_eq!(
1906            ws.crons_for_project("proj")
1907                .iter()
1908                .find(|cron| cron.id == forge_primitives::cron::CronId::from("sweep"))
1909                .and_then(|cron| cron.last_fire),
1910            Some(now),
1911            "and the fire advances the schedule",
1912        );
1913    }
1914
1915    #[test]
1916    fn deliver_worker_cron_with_owner_conclusively_gone_is_target_gone() {
1917        let (ws, _rx) = Workspace::testing_stub();
1918        let dir = tempdir().expect("tempdir");
1919        ws.install_db_for_test(
1920            crate::store::Db::open(&dir.path().join("db.redb")).expect("open db"),
1921        );
1922        // Db open, empty session store: the row is the only thing that
1923        // could bring this owner back, so "ghost" is conclusively absent
1924        // and its cron must be collected rather than buffered into a
1925        // bucket nothing will ever drain.
1926        ws.seed_test_project("proj", "/tmp/wc-gone");
1927        let outcome = deliver_cron(&ws, "proj", Some("ghost"), "x", false);
1928        assert!(
1929            matches!(outcome, crate::spawn::CronFireOutcome::TargetGone),
1930            "a label with no row in the session store is conclusively gone",
1931        );
1932    }
1933
1934    /// A cron's owner can have a row and still be unwakeable: the row
1935    /// says it runs in a worktree, and the worktree is gone, so the wave
1936    /// skips it and nothing is left to drain a parked prompt. Buffering
1937    /// would grow the bucket once per fire until process exit, and
1938    /// reporting Delivered would advance the watermark past a prompt that
1939    /// never reached anyone.
1940    ///
1941    /// The control half is the same fire once the worktree stands: it is
1942    /// delivered and parked, so the refusal above is the missing
1943    /// directory and not the fixture.
1944    #[test]
1945    fn deliver_worker_cron_with_an_unwakeable_owner_is_neither_parked_nor_delivered() {
1946        let (ws, _rx) = Workspace::testing_stub();
1947        let dir = tempdir().expect("tempdir");
1948        ws.install_db_for_test(
1949            crate::store::Db::open(&dir.path().join("db.redb")).expect("open db"),
1950        );
1951        let project_dir = tempdir().expect("project dir");
1952        ws.seed_test_project("proj", &project_dir.path().to_string_lossy());
1953        let key = ws.project_key_for_name("proj").expect("seeded project");
1954        ws.record_worker_row(
1955            &key,
1956            "steward",
1957            "steward-uuid",
1958            "c",
1959            None,
1960            None,
1961            false,
1962            crate::mcp::workers::types::WorkerTree { is_git_repo: true, cwd: None },
1963            None,
1964        )
1965        .expect("seed the worker's row");
1966
1967        let outcome = deliver_cron(&ws, "proj", Some("steward"), "x", false);
1968        let crate::spawn::CronFireOutcome::TargetCannotBeWoken { directory } = outcome else {
1969            panic!("an owner whose worktree is gone cannot be woken, so its fire is not delivered");
1970        };
1971        assert_eq!(
1972            directory,
1973            project_dir.path().join(".claude").join("worktrees").join("steward"),
1974            "the outcome has to carry the directory the fire was waiting for: it is the one \
1975             thing an operator can restore",
1976        );
1977        assert_eq!(
1978            parked_crons(&ws, "proj", Some("steward")).len(),
1979            0,
1980            "nothing parks for an owner that can never drain the bucket",
1981        );
1982
1983        std::fs::create_dir_all(
1984            project_dir.path().join(".claude").join("worktrees").join("steward"),
1985        )
1986        .expect("restore the worktree");
1987        let delivered = deliver_cron(&ws, "proj", Some("steward"), "x", false);
1988        assert!(
1989            matches!(delivered, crate::spawn::CronFireOutcome::Delivered),
1990            "the same fire is delivered once the worktree is back, so the refusal above \
1991             was the missing directory and not the fixture",
1992        );
1993        assert_eq!(
1994            parked_crons(&ws, "proj", Some("steward")).len(),
1995            1,
1996            "and its prompt is parked for the owner to drain on connect",
1997        );
1998    }
1999
2000    /// The directory this fire waits on is the ROW's tree, not the
2001    /// project's: a worker the spawn pointed at another repository starts
2002    /// in THAT repository's worktree, so an operator has to be told to
2003    /// restore that one. The project's own worktree stands here, so a
2004    /// wake composing the project path would call the owner wakeable and
2005    /// park a prompt nothing would drain.
2006    #[test]
2007    fn a_cron_wake_reads_the_rows_own_tree() {
2008        let (ws, _rx) = Workspace::testing_stub();
2009        let dir = tempdir().expect("tempdir");
2010        ws.install_db_for_test(
2011            crate::store::Db::open(&dir.path().join("db.redb")).expect("open db"),
2012        );
2013        let project_dir = tempdir().expect("project dir");
2014        ws.seed_test_project("proj", &project_dir.path().to_string_lossy());
2015        let key = ws.project_key_for_name("proj").expect("seeded project");
2016        // The project's own worktree stands: anything composing the
2017        // project path answers "the owner can start".
2018        std::fs::create_dir_all(
2019            project_dir.path().join(".claude").join("worktrees").join("steward"),
2020        )
2021        .expect("create the project worktree");
2022        // The row's own tree, with no worktree under it.
2023        let worker_tree = tempdir().expect("worker tree");
2024        let root = std::fs::canonicalize(worker_tree.path()).expect("canonical tree");
2025        ws.seed_test_worker_row_at(&key, "steward", &root.to_string_lossy(), true);
2026
2027        let outcome = deliver_cron(&ws, "proj", Some("steward"), "x", false);
2028        let crate::spawn::CronFireOutcome::TargetCannotBeWoken { directory } = outcome else {
2029            panic!("an owner whose own tree has no worktree cannot be woken");
2030        };
2031        assert_eq!(
2032            directory,
2033            root.join(".claude").join("worktrees").join("steward"),
2034            "the outcome names the tree an operator has to restore - the worker's own",
2035        );
2036    }
2037
2038    #[test]
2039    fn deliver_worker_cron_leaves_it_when_the_owner_check_cannot_read() {
2040        // No db installed, so the durable-worker lookup fails: absence can't
2041        // be confirmed, so the cron must be left (retried next tick), not
2042        // deleted as owner-gone.
2043        let (ws, _rx) = Workspace::testing_stub();
2044        ws.seed_test_project("proj", "/tmp/wc-unknown");
2045        let outcome = deliver_cron(&ws, "proj", Some("scratch"), "x", false);
2046        assert!(
2047            matches!(outcome, crate::spawn::CronFireOutcome::DispatchFailed),
2048            "a failed owner check leaves the cron for the next tick, not TargetGone",
2049        );
2050    }
2051
2052    /// A spawn needs the account map settled, and one half of that is the
2053    /// gateway listener, which the boot catch-up can outrun. That refusal
2054    /// is transient, so the fire stays unconsumed for the next tick
2055    /// rather than advancing past a prompt that never landed.
2056    #[test]
2057    fn deliver_cron_with_the_listener_unbound_leaves_the_fire_for_the_next_tick() {
2058        let (ws, _rx) = Workspace::testing_stub();
2059        ws.seed_test_project("proj", "/tmp/wc-gate");
2060        ws.enable_test_dispatch_intercept();
2061
2062        // Control: with the listener bound this same fire is delivered, so
2063        // the refusal below is the map and not the owner check.
2064        let bound = deliver_cron(&ws, "proj", None, "x", false);
2065        assert!(
2066            matches!(bound, crate::spawn::CronFireOutcome::Delivered),
2067            "the same fire with the listener bound is delivered",
2068        );
2069        ws.drain_test_dispatch_buffer();
2070
2071        ws.seed_test_gateway_ready(false);
2072        let unbound = deliver_cron(&ws, "proj", None, "x", false);
2073        assert!(
2074            matches!(unbound, crate::spawn::CronFireOutcome::DispatchFailed),
2075            "an unbound listener is a transient refusal, not a delivered fire",
2076        );
2077        assert_eq!(
2078            parked_crons(&ws, "proj", None).len(),
2079            1,
2080            "the deferred fire parked nothing, so the retry parks it once rather than twice",
2081        );
2082    }
2083
2084    /// The other half of an unsettled map is the accounts themselves: the
2085    /// walk skips every account still `Loading`, so a project whose only
2086    /// account has not settled is refused there. Also pins the boundary -
2087    /// `Bailed` is terminal, so it settles the map and the walk falls back
2088    /// to it rather than refusing.
2089    #[test]
2090    fn deliver_cron_with_an_unsettled_account_map_leaves_the_fire_for_the_next_tick() {
2091        let (ws, _dir) = workspace_with_one_unsettled_account();
2092        ws.enable_test_dispatch_intercept();
2093
2094        // Nothing runs the account loader here, so the one account starts
2095        // `Loading`: the fire is left rather than parked.
2096        let unsettled = deliver_cron(&ws, "proj", None, "x", false);
2097        assert_eq!(
2098            outcome_name(&unsettled),
2099            "DispatchFailed",
2100            "an account still loading is a transient refusal, not a delivered fire",
2101        );
2102        assert!(
2103            parked_crons(&ws, "proj", None).is_empty(),
2104            "and nothing is parked, so the retry parks it once rather than twice",
2105        );
2106
2107        // Controls: the same fire is delivered once the map settles, so
2108        // the refusal above is the account map and not the owner check.
2109        ws.seed_test_ready_account("acct-a");
2110        let ready = deliver_cron(&ws, "proj", None, "x", false);
2111        assert_eq!(
2112            outcome_name(&ready),
2113            "Delivered",
2114            "the same fire with the account settled is delivered",
2115        );
2116
2117        // `Bailed` is terminal too, so it settles the map as well, and the
2118        // walk falls back to it as the last resort rather than refusing.
2119        ws.seed_test_account_state("acct-a", forge_gateway::LoadingState::Bailed);
2120        let bailed = deliver_cron(&ws, "proj", None, "x", false);
2121        assert_eq!(
2122            outcome_name(&bailed),
2123            "Delivered",
2124            "a Bailed account settles the map and the walk falls back to it",
2125        );
2126        assert_eq!(
2127            parked_crons(&ws, "proj", None).len(),
2128            2,
2129            "both settled fires parked their prompt",
2130        );
2131    }
2132
2133    /// A cooldown empties the project's walk until its reset, which is
2134    /// transient the same way an unsettled map is, so the fire stays due
2135    /// rather than parking a prompt the refusal would expire.
2136    #[test]
2137    fn deliver_cron_during_a_cooldown_leaves_the_fire_for_the_next_tick() {
2138        let (ws, _dir) = workspace_with_one_unsettled_account();
2139        ws.seed_test_ready_account("acct-a");
2140        ws.enable_test_dispatch_intercept();
2141
2142        // Control: with the one account serving and not cooling, the same
2143        // fire is delivered.
2144        let served = deliver_cron(&ws, "proj", None, "x", false);
2145        assert_eq!(
2146            outcome_name(&served),
2147            "Delivered",
2148            "the same fire with the account serving is delivered",
2149        );
2150
2151        // The usage probe's own verdict is the public way to cool an
2152        // account, and it takes epoch seconds.
2153        let reset_at = std::time::SystemTime::now()
2154            .duration_since(std::time::UNIX_EPOCH)
2155            .expect("epoch")
2156            .as_secs()
2157            + 60;
2158        ws.gateway
2159            .report_probe_limit(&forge_gateway::AccountKey("acct-a".to_owned()), Some(reset_at));
2160        let cooling = deliver_cron(&ws, "proj", None, "x", false);
2161        assert_eq!(
2162            outcome_name(&cooling),
2163            "DeferredCooling",
2164            "a cooling walk is a transient refusal, not a delivered fire",
2165        );
2166        assert_eq!(
2167            parked_crons(&ws, "proj", None).len(),
2168            1,
2169            "the deferred fire parked nothing, so the retry parks it once rather than twice",
2170        );
2171    }
2172
2173    /// The outcome's own name, so a failure says which variant it got
2174    /// rather than only that it was not the expected one.
2175    fn outcome_name(outcome: &crate::spawn::CronFireOutcome) -> &'static str {
2176        match outcome {
2177            crate::spawn::CronFireOutcome::Delivered => "Delivered",
2178            crate::spawn::CronFireOutcome::TargetGone => "TargetGone",
2179            crate::spawn::CronFireOutcome::TargetCannotBeWoken { .. } => "TargetCannotBeWoken",
2180            crate::spawn::CronFireOutcome::DeferredCooling { .. } => "DeferredCooling",
2181            crate::spawn::CronFireOutcome::DispatchFailed => "DispatchFailed",
2182            crate::spawn::CronFireOutcome::SkippedAsleep => "SkippedAsleep",
2183        }
2184    }
2185
2186    /// A workspace over a forge.toml declaring one account and one project
2187    /// with the model that account serves, so the account walk is the
2188    /// thing a spawn would reach. Nothing runs the account loader in a
2189    /// test, so the account map starts unsettled. The tempdir must
2190    /// outlive the caller.
2191    ///
2192    /// `new_for_test` rather than a stubbed workspace on purpose: the
2193    /// stubs carry an EMPTY account pool, where `all_loaded` is vacuously
2194    /// true and `set_loading` is a no-op, so they cannot tell the two
2195    /// halves of the predicate apart.
2196    fn workspace_with_one_unsettled_account()
2197    -> (std::sync::Arc<crate::Workspace>, tempfile::TempDir) {
2198        let dir = tempdir().expect("tempdir");
2199        let forge_dir = crate::config::ensure_forge_data_dir(dir.path()).expect("forge dir");
2200        std::fs::write(
2201            forge_dir.join("forge.toml"),
2202            r#"
2203[[orgs]]
2204name = "TestOrg"
2205accounts = ["acct-a"]
2206
2207[[orgs.projects]]
2208name = "proj"
2209path = "/tmp/wc-unsettled"
2210model = "claude-sonnet-5"
2211
2212[[accounts]]
2213display_name = "acct-a"
2214token = "t"
2215models = ["claude-sonnet-5"]
2216provider = "anthropic"
2217"#,
2218        )
2219        .expect("write forge.toml");
2220        let ws = std::sync::Arc::new(
2221            crate::Workspace::new_for_test(dir.path().to_owned()).expect("boot from the fixture"),
2222        );
2223        (ws, dir)
2224    }
2225
2226    #[test]
2227    fn deliver_cron_marks_an_overdue_fire_as_missed() {
2228        let dir = tempdir().expect("tempdir");
2229        let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
2230        ws.seed_test_project("proj", "/tmp/wc-missed");
2231        let cwd = project_expanded_path(&ws, "proj");
2232        ws.record_connected_session(&cwd, "lead-uuid", None);
2233        let lead_key = SessionSlot::lead("TestOrg", "proj");
2234        let (handle, _agent_rx) = Workspace::testing_stub_handle();
2235        ws.pool.lock().insert(
2236            lead_key.clone(),
2237            PooledAgent {
2238                handle: Arc::new(handle),
2239                account: AccountKey("test".to_owned()),
2240                permission_mode: None,
2241                registration: None,
2242                session_id: "pooled-session".to_owned(),
2243            },
2244        );
2245
2246        ws.mark_session_connected_for_test(&lead_key, "lead-uuid");
2247        ws.enable_test_dispatch_intercept();
2248        deliver_cron(&ws, "proj", None, "standup", true);
2249        deliver_cron(&ws, "proj", None, "standup", false);
2250        let dispatched = ws.drain_test_dispatch_buffer();
2251        let texts: Vec<String> = dispatched
2252            .iter()
2253            .filter_map(|c| match c {
2254                Command::Prompt { text, .. } | Command::PromptUnder { text, .. } => {
2255                    Some(text.clone())
2256                }
2257                _ => None,
2258            })
2259            .collect();
2260        assert!(
2261            texts.contains(&"[missed cron] standup".to_owned()),
2262            "an overdue fire is delivered with the missed marker",
2263        );
2264        assert!(texts.contains(&"standup".to_owned()), "an on-time fire has no marker");
2265    }
2266
2267    #[test]
2268    fn fire_due_crons_missed_threshold_is_two_ticks() {
2269        use forge_primitives::cron::{CronEntry, CronId, CronKind};
2270        let dir = tempdir().expect("tempdir");
2271        let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
2272        ws.seed_test_project("proj", "/tmp/wc-thresh");
2273        let cwd = project_expanded_path(&ws, "proj");
2274        ws.record_connected_session(&cwd, "lead-uuid", None);
2275        let lead_key = SessionSlot::lead("TestOrg", "proj");
2276        let (handle, _agent_rx) = Workspace::testing_stub_handle();
2277        ws.pool.lock().insert(
2278            lead_key.clone(),
2279            PooledAgent {
2280                handle: Arc::new(handle),
2281                account: AccountKey("test".to_owned()),
2282                permission_mode: None,
2283                registration: None,
2284                session_id: "pooled-session".to_owned(),
2285            },
2286        );
2287        ws.mark_session_connected_for_test(&lead_key, "lead-uuid");
2288
2289        let now = std::time::SystemTime::now();
2290        let cron = |id: &str, prompt: &str, next_fire| CronEntry {
2291            id: CronId::from(id),
2292            project_name: "proj".to_owned(),
2293            kind: CronKind::Recurring("*/5 * * * *".to_owned()),
2294            prompt: prompt.to_owned(),
2295            created_at: std::time::SystemTime::UNIX_EPOCH,
2296            description: None,
2297            last_fire: None,
2298            next_fire,
2299            team_role: None,
2300            wake: true,
2301        };
2302        // Due one tick ago: within the jitter window, on-time. Due three
2303        // ticks ago: a genuine catch-up, missed.
2304        ws.push_cron(cron("recent", "recent", now - std::time::Duration::from_secs(60)));
2305        ws.push_cron(cron("stale", "stale", now - std::time::Duration::from_secs(180)));
2306
2307        ws.enable_test_dispatch_intercept();
2308        ws.fire_due_crons(now);
2309        let texts: Vec<String> = ws
2310            .drain_test_dispatch_buffer()
2311            .iter()
2312            .filter_map(|c| match c {
2313                Command::Prompt { text, .. } | Command::PromptUnder { text, .. } => {
2314                    Some(text.clone())
2315                }
2316                _ => None,
2317            })
2318            .collect();
2319        assert!(texts.contains(&"recent".to_owned()), "a one-tick-late fire is on-time");
2320        assert!(
2321            texts.contains(&"[missed cron] stale".to_owned()),
2322            "a three-tick-late fire is marked missed",
2323        );
2324    }
2325
2326    #[test]
2327    fn fire_due_crons_removes_a_cron_whose_project_is_gone() {
2328        use forge_primitives::cron::{CronEntry, CronId, CronKind};
2329        // No db installed, so push_cron stays in memory - no config dir needed.
2330        let (ws, _rx) = Workspace::testing_stub();
2331        let now = std::time::SystemTime::now();
2332
2333        ws.push_cron(CronEntry {
2334            id: CronId::from("orphan"),
2335            project_name: "deleted-project".to_owned(),
2336            kind: CronKind::Recurring("*/5 * * * *".to_owned()),
2337            prompt: "p".to_owned(),
2338            created_at: std::time::SystemTime::UNIX_EPOCH,
2339            description: None,
2340            last_fire: None,
2341            next_fire: std::time::SystemTime::UNIX_EPOCH, // overdue -> due now
2342            team_role: None,
2343            wake: true,
2344        });
2345
2346        ws.enable_test_dispatch_intercept();
2347        ws.fire_due_crons(now);
2348        let dispatched = ws.drain_test_dispatch_buffer();
2349
2350        assert!(
2351            dispatched.iter().all(|c| !matches!(c, crate::protocol::Command::SpawnProject { .. })),
2352            "a cron whose project is gone gets no spawn",
2353        );
2354        assert!(
2355            ws.crons_for_project("deleted-project").is_empty(),
2356            "an overdue cron whose project left forge.toml is removed, not advanced forever",
2357        );
2358    }
2359
2360    /// The router's fate for a fire whose owner cannot be woken. A
2361    /// recurring cron has another slot coming, so this fire is dropped and
2362    /// the schedule advances. A one-shot has no later slot and advancing
2363    /// removes it, so it stays due: dropping it would throw away the only
2364    /// prompt that cron will ever carry.
2365    ///
2366    /// Neither may park - that bucket has nothing to drain it.
2367    #[test]
2368    fn fire_due_crons_advances_recurring_and_keeps_a_one_shot_when_the_owner_cannot_be_woken() {
2369        use forge_primitives::cron::{CronEntry, CronId, CronKind};
2370        let (ws, _key, _db, _project) = stranded_owner_fixture("steward");
2371
2372        let now = std::time::SystemTime::now();
2373        let past = std::time::SystemTime::UNIX_EPOCH;
2374        for (id, kind) in [
2375            ("recurring", CronKind::Recurring("*/5 * * * *".to_owned())),
2376            ("one-shot", CronKind::Once(past)),
2377        ] {
2378            ws.push_cron(CronEntry {
2379                id: CronId::from(id),
2380                project_name: "proj".to_owned(),
2381                kind,
2382                prompt: "p".to_owned(),
2383                created_at: past,
2384                description: None,
2385                last_fire: None,
2386                next_fire: past,
2387                team_role: Some("steward".to_owned()),
2388                wake: true,
2389            });
2390        }
2391
2392        ws.enable_test_dispatch_intercept();
2393        ws.fire_due_crons(now);
2394
2395        let after = ws.crons_for_project("proj");
2396        let ids: Vec<String> = after.iter().map(|c| c.id.0.clone()).collect();
2397        let recurring = after.iter().find(|c| c.id == CronId::from("recurring"));
2398        assert!(
2399            recurring.is_some_and(|c| c.last_fire.is_some()),
2400            "a recurring cron's fire is dropped and its schedule advances, rather than \
2401             the entry being removed; entries left {ids:?}",
2402        );
2403        assert!(
2404            after.iter().any(|c| c.id == CronId::from("one-shot")),
2405            "a one-shot has no later slot, so it stays due rather than being removed \
2406             along with its prompt; entries left {ids:?}",
2407        );
2408        assert_eq!(
2409            parked_crons(&ws, "proj", Some("steward")).len(),
2410            0,
2411            "nothing parks for an owner that can never drain the bucket",
2412        );
2413    }
2414
2415    #[test]
2416    fn advance_or_remove_cron_removes_never_occurring_recurring() {
2417        use forge_primitives::cron::{CronEntry, CronId, CronKind};
2418        let dir = tempdir().expect("tempdir");
2419        let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
2420        // "0 0 30 2 *" (Feb 30) has no upcoming occurrence, so
2421        // next_fire_after returns None and the entry is removed (with a
2422        // warn) rather than left stuck at a stale next_fire.
2423        ws.push_cron(CronEntry {
2424            id: CronId::from("impossible"),
2425            project_name: "forge".to_owned(),
2426            kind: CronKind::Recurring("0 0 30 2 *".to_owned()),
2427            prompt: "p".to_owned(),
2428            created_at: std::time::SystemTime::UNIX_EPOCH,
2429            description: None,
2430            last_fire: None,
2431            next_fire: std::time::SystemTime::UNIX_EPOCH,
2432            team_role: None,
2433            wake: true,
2434        });
2435        ws.advance_or_remove_cron(&CronId::from("impossible"), std::time::SystemTime::now());
2436        assert!(
2437            ws.crons_for_project("forge").is_empty(),
2438            "a recurring cron with no upcoming occurrence is removed, not left stuck",
2439        );
2440    }
2441
2442    /// The registry's idempotency: the first `start_cron_scheduler`
2443    /// registers the member and runs exactly one tick loop, and a second
2444    /// call is a no-op. The due cron is the observable that the start
2445    /// opens - a start that never runs fires nothing.
2446    #[tokio::test(start_paused = true)]
2447    async fn start_cron_scheduler_fires_a_due_cron_once_despite_a_second_start() {
2448        use forge_primitives::cron::{CronEntry, CronId, CronKind};
2449        let dir = tempdir().expect("tempdir");
2450        let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
2451        ws.seed_test_project("proj", "/tmp/wc-scheduler-once");
2452
2453        ws.push_cron(CronEntry {
2454            id: CronId::from("due"),
2455            project_name: "proj".to_owned(),
2456            kind: CronKind::Recurring("*/5 * * * *".to_owned()),
2457            prompt: "tick".to_owned(),
2458            created_at: std::time::SystemTime::UNIX_EPOCH,
2459            description: None,
2460            last_fire: None,
2461            next_fire: std::time::SystemTime::UNIX_EPOCH, // overdue -> due at the first tick
2462            team_role: None,
2463            wake: true,
2464        });
2465
2466        ws.enable_test_dispatch_intercept();
2467        ws.start_cron_scheduler();
2468        assert!(
2469            ws.supervisor().snapshot().iter().any(|member| member.name == "cron_scheduler"),
2470            "the first start registers the scheduler",
2471        );
2472
2473        // Let the spawned task poll once so its interval is registered
2474        // at t=0 (first tick at +60s) before the clock moves.
2475        tokio::task::yield_now().await;
2476        tokio::task::yield_now().await;
2477        tokio::time::advance(CRON_TICK_INTERVAL).await;
2478        tokio::task::yield_now().await;
2479        tokio::task::yield_now().await;
2480        let first_tick = count_proj_spawns(&ws);
2481        assert_eq!(
2482            first_tick, 1,
2483            "the first start runs the scheduler; a guard that never opens fires nothing",
2484        );
2485
2486        // A second start must not spawn a second tick loop. Fires
2487        // cannot tell two loops apart - ticks run sequentially here and
2488        // the first loop's advance hides every due cron from the second
2489        // (on the multi-thread runtime their ticks race, which is what
2490        // the guard prevents) - so its only trace is another task-held
2491        // Weak.
2492        let weak_before = Arc::downgrade(&ws).weak_count();
2493        ws.start_cron_scheduler();
2494        let weak_after = Arc::downgrade(&ws).weak_count();
2495        assert_eq!(weak_after, weak_before, "a second start must not spawn a second tick loop");
2496    }
2497
2498    fn count_proj_spawns(ws: &Workspace) -> usize {
2499        ws.drain_test_dispatch_buffer()
2500            .iter()
2501            .filter(|c| {
2502                matches!(c, crate::protocol::Command::SpawnProject { project_name, .. }
2503                    if project_name == "proj")
2504            })
2505            .count()
2506    }
2507
2508    /// The tick loop holds the workspace only weakly: once the caller's
2509    /// `Arc` is gone the workspace drops even though the task is parked
2510    /// on its next tick. Advancing past a tick drives the
2511    /// upgrade-failure exit; a task that moved a strong `Arc` in would
2512    /// keep the workspace alive and fail the upgrade assert.
2513    #[tokio::test(start_paused = true)]
2514    async fn the_scheduler_task_does_not_hold_the_workspace_alive() {
2515        let dir = tempdir().expect("tempdir");
2516        let weak = {
2517            let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
2518            ws.start_cron_scheduler();
2519            // Poll the task once so its interval is registered and it
2520            // sits parked on the next tick, holding only the Weak.
2521            tokio::task::yield_now().await;
2522            tokio::task::yield_now().await;
2523            Arc::downgrade(&ws)
2524        };
2525        tokio::time::advance(CRON_TICK_INTERVAL * 2).await;
2526        tokio::task::yield_now().await;
2527        tokio::task::yield_now().await;
2528        assert!(weak.upgrade().is_none(), "the scheduler task must hold only a Weak");
2529    }
2530}