Skip to main content

forge_workspace/
chase.rs

1//! The chase sweep: the pass that makes the board drive instead of
2//! report.
3//!
4//! Every pass derives crossings from the board's own marks and the chase
5//! log: a row past its estimate asks its owner once per estimate, a lead
6//! hears at the rung's class cadence while its state holds, and a row
7//! that died, moved and died again is a new occurrence, named at once.
8//! The cadences are the settled two: five minutes for a state burning a
9//! resource (a drifting worker, a row at 1.5x its estimate), thirty for
10//! one parked and stuck (a dead row, a stalled queue, a stuck wait, a
11//! retro or a close owed). **The chase log is the whole slate**: every
12//! naming is deduped by it, so a restart re-reads it rather than
13//! re-firing, and a state that clears has its naming forgotten - off the
14//! log, where a fresh process reads - so a recurrence is named at once,
15//! whatever happened in between.
16//!
17//! Delivery degrades, never wakes: a live owner is asked; with a dead
18//! owner the lead is; with every seat asleep the mark alone stands - except
19//! the user-wait nudge, whose one shot per row is kept unspent until a live
20//! lead takes it. A chase NEVER spawns a session - the board already shows
21//! the state, and waking a project is a cost the chase does not get to
22//! spend.
23
24use std::sync::Arc;
25use std::time::SystemTime;
26
27use forge_primitives::tasks::{
28    By, ChaseRung, TaskChase, TaskId, TaskStatus, TaskTransition, WaitingKind,
29};
30use tracing::Instrument;
31
32use crate::SessionSlot;
33use crate::board::BoardRow;
34use crate::tasks::parked_on_user;
35use crate::workspace::Workspace;
36
37/// How long between sweeps. The same number bounds how stale a
38/// time-derived mark can be in a held client snapshot.
39pub const CHASE_INTERVAL_SECS: u64 = 30;
40
41/// How long after a wait resolves its message is still worth sending.
42const WAIT_RESOLVED_WINDOW_SECS: u64 = 600;
43
44/// The cadence of a rung whose state is burning a resource: a drifting
45/// worker, a row past 1.5x its estimate.
46const CADENCE_BURNING_SECS: u64 = 5 * 60;
47
48/// The cadence of a rung whose state is parked and stuck: a dead row, a
49/// stalled queue, a stuck wait, a retro or a close owed.
50const CADENCE_STUCK_SECS: u64 = 30 * 60;
51
52/// What a delivery attempt found.
53#[derive(Debug, Clone, PartialEq, Eq)]
54pub enum Delivery {
55    Delivered,
56    /// The dispatch failed, carrying the error it returned: a caller that
57    /// names why a delivery did not land reads it from here.
58    NoSession(String),
59}
60
61/// Where a chase message goes. A trait so tests can watch the ladder
62/// without a live session; the production impl injects a prompt.
63pub trait Deliver: Send + Sync {
64    fn to_seat(&self, slot: &SessionSlot, text: &str) -> Delivery;
65}
66
67/// One message the sweep decided to send.
68#[derive(Debug, Clone, PartialEq, Eq)]
69pub struct Crossing {
70    pub rung: ChaseRung,
71    pub task_id: Option<TaskId>,
72    /// The rung's own subject where a row id does not identify it: the
73    /// worker a seat-level rung names, or the estimate a nudge fired
74    /// against.
75    pub subject: Option<String>,
76    /// The seat the message prefers: the row's owner for a nudge, the
77    /// lead for everything else. The ladder falls back to the lead.
78    pub seat: Option<SessionSlot>,
79    pub text: String,
80}
81
82/// The log's most recent naming of `label` by the seat-level rung.
83fn durable_last_named(fired: &[TaskChase], label: &str) -> Option<SystemTime> {
84    fired
85        .iter()
86        .filter(|chase| {
87            chase.rung == ChaseRung::Unaccounted && chase.subject.as_deref() == Some(label)
88        })
89        .map(|chase| chase.at)
90        .max()
91}
92
93/// How long `now` is after `at`, saturating: an `at` in the future reads
94/// as the longest age, so a skewed stamp never silences a rung.
95fn since_secs(at: SystemTime, now: SystemTime) -> u64 {
96    now.duration_since(at).map_or(u64::MAX, |d| d.as_secs())
97}
98
99/// Whether `rung` already fired for `id` and this subject - a restated
100/// estimate changes the subject, which arms the nudge again.
101fn fired_already(fired: &[TaskChase], rung: ChaseRung, id: &TaskId, subject: Option<&str>) -> bool {
102    fired.iter().any(|chase| {
103        chase.rung == rung
104            && chase.task_id.as_ref() == Some(id)
105            && chase.subject.as_deref() == subject
106    })
107}
108
109/// Whether `rung` already fired for `id` at or after `at` - the shape a
110/// rung recurring with a new occurrence needs.
111fn fired_since(fired: &[TaskChase], rung: ChaseRung, id: &TaskId, at: SystemTime) -> bool {
112    fired
113        .iter()
114        .any(|chase| chase.rung == rung && chase.task_id.as_ref() == Some(id) && chase.at >= at)
115}
116
117/// Whether a recurring rung has waited out its cadence: it is due when
118/// nothing fired since `since` (the instant its state appeared, where
119/// the history says), or when the newest firing since then is at least
120/// `cadence` seconds old.
121fn due(
122    fired: &[TaskChase],
123    rung: ChaseRung,
124    id: &TaskId,
125    since: Option<SystemTime>,
126    cadence: u64,
127    now: SystemTime,
128) -> bool {
129    let last = fired
130        .iter()
131        .filter(|chase| {
132            chase.rung == rung
133                && chase.task_id.as_ref() == Some(id)
134                && since.is_none_or(|at| chase.at >= at)
135        })
136        .map(|chase| chase.at)
137        .max();
138    last.is_none_or(|at| since_secs(at, now) >= cadence)
139}
140
141/// When a row last died, if the history says: a row that moved and died
142/// again is a new occurrence.
143fn failed_at(history: &[TaskTransition], id: &TaskId) -> Option<SystemTime> {
144    history
145        .iter()
146        .filter(|transition| transition.task_id == *id && transition.to == TaskStatus::Failed)
147        .map(|transition| transition.at)
148        .max()
149}
150
151/// When the row entered waiting, if the history says: the newest transition
152/// into waiting is the episode a user nudge is deduped on.
153fn waiting_since(history: &[TaskTransition], id: &TaskId) -> Option<SystemTime> {
154    history
155        .iter()
156        .filter(|transition| transition.task_id == *id && transition.to == TaskStatus::Waiting)
157        .map(|transition| transition.at)
158        .max()
159}
160
161/// When a released wait landed, if one did: a transition out of waiting
162/// into pending that the core drove.
163fn released_at(history: &[TaskTransition], id: &TaskId) -> Option<SystemTime> {
164    history
165        .iter()
166        .filter(|transition| {
167            transition.task_id == *id
168                && transition.from == Some(TaskStatus::Waiting)
169                && transition.to == TaskStatus::Pending
170                && transition.by == By::System
171        })
172        .map(|transition| transition.at)
173        .max()
174}
175
176fn fmt_secs(secs: u64) -> String {
177    crate::board::fmt_secs(secs)
178}
179
180/// The crossings one project's board implies, given what already fired.
181pub(crate) fn crossings(
182    lead: &SessionSlot,
183    rows: &[BoardRow],
184    history: &[TaskTransition],
185    unaccounted: &[String],
186    fired: &[TaskChase],
187    now: SystemTime,
188) -> Vec<Crossing> {
189    let mut out = Vec::new();
190    for row in rows {
191        let id = &row.task.id;
192        let estimate = row.task.estimate.as_ref().filter(|e| e.secs > 0);
193        let running = row.task.status == TaskStatus::InProgress;
194
195        if running
196            && let Some(estimate) = estimate
197            && row.worked_secs > estimate.secs
198            && !fired_already(fired, ChaseRung::EstimateNudge, id, Some(&estimate.secs.to_string()))
199        {
200            out.push(Crossing {
201                rung: ChaseRung::EstimateNudge,
202                task_id: Some(id.clone()),
203                subject: Some(estimate.secs.to_string()),
204                seat: row.task.owner.clone(),
205                text: format!(
206                    "task board: \"{}\" has worked {} against its estimate of {} - status? A new \
207                     estimate, completing it, or parking it clears this.",
208                    row.task.subject,
209                    fmt_secs(row.worked_secs),
210                    estimate.words,
211                ),
212            });
213        }
214        if running
215            && let Some(estimate) = estimate
216            && row.worked_secs > estimate.secs + estimate.secs / 2
217            && due(fired, ChaseRung::Escalate, id, None, CADENCE_BURNING_SECS, now)
218        {
219            out.push(Crossing {
220                rung: ChaseRung::Escalate,
221                task_id: Some(id.clone()),
222                subject: None,
223                seat: Some(lead.clone()),
224                text: format!(
225                    "task board: \"{}\" is at {} against an estimate of {} - rescope, reassign, \
226                     take over, or restate the expectation.",
227                    row.task.subject,
228                    fmt_secs(row.worked_secs),
229                    estimate.words,
230                ),
231            });
232        }
233        if row.marks.ready
234            && row.task.owner.is_none()
235            && row.marks.no_movement
236            && due(fired, ChaseRung::QueueStall, id, None, CADENCE_STUCK_SECS, now)
237        {
238            out.push(Crossing {
239                rung: ChaseRung::QueueStall,
240                task_id: Some(id.clone()),
241                subject: None,
242                seat: Some(lead.clone()),
243                text: format!(
244                    "task board: \"{}\" has been ready and unowned for {} - the queue is \
245                     stalling; dispatch it or rank it.",
246                    row.task.subject,
247                    fmt_secs(row.updated_secs_ago),
248                ),
249            });
250        }
251        let wait_kind = row.task.waiting_on.as_ref().and_then(|wait| wait.kind);
252        if row.marks.waiting_too_long
253            && matches!(wait_kind, Some(WaitingKind::Dependency | WaitingKind::Resource))
254            && due(fired, ChaseRung::WaitStall, id, None, CADENCE_STUCK_SECS, now)
255        {
256            out.push(Crossing {
257                rung: ChaseRung::WaitStall,
258                task_id: Some(id.clone()),
259                subject: None,
260                seat: Some(lead.clone()),
261                text: format!(
262                    "task board: \"{}\" has waited {} on a {} - unblock it or change its plan.",
263                    row.task.subject,
264                    fmt_secs(row.updated_secs_ago),
265                    match wait_kind {
266                        Some(WaitingKind::Dependency) => "dependency",
267                        Some(WaitingKind::Resource) => "resource",
268                        _ => "wait",
269                    },
270                ),
271            });
272        }
273        let died = row.task.status == TaskStatus::Failed;
274        let stranded = running && row.marks.stale;
275        // The death repeats at the stuck cadence, counted from the death
276        // itself: a row that moved and died again is a new occurrence.
277        if (died || stranded)
278            && due(fired, ChaseRung::Death, id, failed_at(history, id), CADENCE_STUCK_SECS, now)
279        {
280            out.push(Crossing {
281                rung: ChaseRung::Death,
282                task_id: Some(id.clone()),
283                subject: None,
284                seat: Some(lead.clone()),
285                text: format!(
286                    "task board: \"{}\" {} - re-dispatch it or cancel it.",
287                    row.task.subject,
288                    if died { "failed" } else { "has no live owner" },
289                ),
290            });
291        }
292        if let Some(at) = released_at(history, id)
293            && since_secs(at, now) <= WAIT_RESOLVED_WINDOW_SECS
294            && !fired_since(fired, ChaseRung::WaitResolved, id, at)
295        {
296            out.push(Crossing {
297                rung: ChaseRung::WaitResolved,
298                task_id: Some(id.clone()),
299                subject: None,
300                seat: row.task.owner.clone(),
301                text: format!(
302                    "task board: \"{}\" was waiting and the thing it waited on has ended - \
303                     resume it, re-plan it, or wait again.",
304                    row.task.subject,
305                ),
306            });
307        }
308        if row.marks.to_close && due(fired, ChaseRung::EpicClose, id, None, CADENCE_STUCK_SECS, now)
309        {
310            out.push(Crossing {
311                rung: ChaseRung::EpicClose,
312                task_id: Some(id.clone()),
313                subject: None,
314                seat: Some(lead.clone()),
315                text: format!(
316                    "task board: \"{}\" has every child terminal and its retro run - close it \
317                     when you agree: completing the root row archives the tree.",
318                    row.task.subject,
319                ),
320            });
321        }
322    }
323
324    // The retro: an epic whose children have all stopped and which has no
325    // retro yet asks its owner for one.
326    for row in rows {
327        if row.task.parent.is_none()
328            && row.rollup.is_some()
329            && !row.task.links.iter().any(|l| l.label.as_deref() == Some("retro"))
330            && due(fired, ChaseRung::Retro, &row.task.id, None, CADENCE_STUCK_SECS, now)
331            && rows.iter().any(|child| child.task.parent.as_ref() == Some(&row.task.id))
332            && rows.iter().filter(|child| child.task.parent.as_ref() == Some(&row.task.id)).all(
333                |child| {
334                    matches!(
335                        child.task.status,
336                        TaskStatus::Completed | TaskStatus::Failed | TaskStatus::Canceled
337                    )
338                },
339            )
340        {
341            out.push(Crossing {
342                rung: ChaseRung::Retro,
343                task_id: Some(row.task.id.clone()),
344                subject: None,
345                seat: Some(lead.clone()),
346                text: retro_prompt(row, rows),
347            });
348        }
349    }
350
351    // The rows parked on the user: ONE nudge per row per wait episode, so
352    // a row the lead has acted on (attested, withdrawn, or explicitly
353    // parked) drops out and is never named again. The set is the board's
354    // own derivation (`parked_on_user`), so a verification park a row left
355    // behind under an inherited gate is not chased as a user wait (#2423),
356    // and a wait the lead parked by design is left alone (#2423/D). Rows
357    // bounded by a dependency or a resource are not this list's either.
358    for row in rows.iter().filter(|row| {
359        row.task.status == TaskStatus::Waiting
360            && parked_on_user(&row.task)
361            && row.task.waiting_on.as_ref().is_some_and(|wait| !wait.parked)
362            && row.updated_secs_ago > CADENCE_STUCK_SECS
363    }) {
364        let since = waiting_since(history, &row.task.id).unwrap_or(SystemTime::UNIX_EPOCH);
365        if fired_since(fired, ChaseRung::UserWaitDigest, &row.task.id, since) {
366            continue;
367        }
368        let what = if row.task.waiting_on.as_ref().is_some_and(|wait| wait.verification) {
369            "a look - approve it on the board, or complete it with tasks__approve and the \
370             approval quoted"
371        } else {
372            "a decision - answer it on the board"
373        };
374        out.push(Crossing {
375            rung: ChaseRung::UserWaitDigest,
376            task_id: Some(row.task.id.clone()),
377            subject: None,
378            seat: Some(lead.clone()),
379            text: format!(
380                "task board: \"{}\" ({}) is waiting on the user: {what}. Confirm it's really \
381                 theirs, or re-plan.",
382                row.task.subject,
383                fmt_secs(row.updated_secs_ago),
384            ),
385        });
386    }
387
388    for label in unaccounted {
389        // **The log is the whole slate**, this rung's included: the newest
390        // naming for the worker, and the cadence is what a repeat waits. A
391        // worker whose state cleared had its naming forgotten - the sweep
392        // does that, durably - so its next drift is named at once, across a
393        // restart too.
394        if durable_last_named(fired, label)
395            .is_none_or(|at| since_secs(at, now) >= CADENCE_BURNING_SECS)
396        {
397            out.push(Crossing {
398                rung: ChaseRung::Unaccounted,
399                task_id: None,
400                subject: Some(label.clone()),
401                seat: Some(lead.clone()),
402                text: format!("task board: worker {label} holds no row - feed it or despawn it."),
403            });
404        }
405    }
406    out
407}
408
409/// What an epic's owner is asked when its children have all stopped: the
410/// numbers, computed here and authoritative, and the ask.
411fn retro_prompt(epic: &BoardRow, rows: &[BoardRow]) -> String {
412    let children: Vec<&BoardRow> =
413        rows.iter().filter(|row| row.task.parent.as_ref() == Some(&epic.task.id)).collect();
414    let worked: u64 = children.iter().map(|child| child.worked_secs).sum();
415    let estimated: u64 = children
416        .iter()
417        .filter_map(|child| child.task.estimate.as_ref())
418        .map(|estimate| estimate.secs)
419        .sum();
420    let mut reopened = 0usize;
421    let mut failed = 0usize;
422    for child in &children {
423        if child.task.attempt > 1 {
424            reopened += 1;
425        }
426        if child.task.status == TaskStatus::Failed {
427            failed += 1;
428        }
429    }
430    format!(
431        "task board: epic \"{}\" has every child terminal. Numbers (authoritative - do not \
432         recompute): {} tasks, worked {} against {} estimated, {reopened} reopened, {failed} \
433         failed. Narrate two to four findings and at most three recommendations, each one a \
434         task under this epic with tasks__create; record it as a link labelled `retro` on the \
435         epic.",
436        epic.task.subject,
437        children.len(),
438        fmt_secs(worked),
439        fmt_secs(estimated),
440    )
441}
442
443/// The delivery ladder: the seat the crossing prefers when it is live,
444/// else the lead when it is, else nobody - and nobody means the mark
445/// alone stands.
446fn ladder(
447    crossing: &Crossing,
448    lead: &SessionSlot,
449    live: impl Fn(&SessionSlot) -> bool,
450) -> Option<SessionSlot> {
451    if let Some(seat) = &crossing.seat
452        && live(seat)
453    {
454        return Some(seat.clone());
455    }
456    live(lead).then(|| lead.clone())
457}
458
459impl Workspace {
460    /// Every chase already fired for `project`.
461    fn chases_for_project(&self, project: &str) -> Vec<TaskChase> {
462        let guard = self.db.lock();
463        match guard.as_ref() {
464            Some(db) => {
465                crate::store::task_chases::list_for_project(db, project).unwrap_or_default()
466            }
467            None => Vec::new(),
468        }
469    }
470
471    /// `project`'s transitions, for the rung that reads when a wait
472    /// resolved.
473    fn history_for_project(&self, project: &str) -> Vec<TaskTransition> {
474        let guard = self.db.lock();
475        match guard.as_ref() {
476            Some(db) => {
477                crate::store::task_history::list_for_project(db, project).unwrap_or_default()
478            }
479            None => Vec::new(),
480        }
481    }
482
483    /// Record one fired chase, durably, so the rung never repeats.
484    fn record_chase(&self, chase: &TaskChase) {
485        let guard = self.db.lock();
486        let Some(db) = guard.as_ref() else { return };
487        if let Err(error) = crate::store::task_chases::append(db, std::slice::from_ref(chase)) {
488            tracing::error!(
489                target: "forge_workspace::chase",
490                %error,
491                "persisting a fired chase failed; the rung may fire again next pass",
492            );
493        }
494    }
495
496    /// Forget one worker's naming: its slate cleared, so the record goes
497    /// with the state and the next drift is named at once.
498    fn forget_drift(&self, project: &str, label: &str) {
499        let guard = self.db.lock();
500        let Some(db) = guard.as_ref() else { return };
501        if let Err(error) = crate::store::task_chases::forget(
502            db,
503            project,
504            ChaseRung::Unaccounted,
505            None,
506            Some(label),
507        ) {
508            tracing::error!(
509                target: "forge_workspace::chase",
510                event_name = "chase_slate_forget_failed",
511                %project,
512                %label,
513                %error,
514                "forgetting a cleared worker's naming failed; its next drift may wait out \
515                 the cadence",
516            );
517        }
518    }
519
520    /// One pass of the chase over every project: derive the crossings,
521    /// walk the ladder for each, deliver, and record what fired. Returns
522    /// how many messages reached a session.
523    pub fn chase_sweep(&self, deliver: &dyn Deliver, now: SystemTime) -> usize {
524        let mut delivered = 0;
525        for view in self.list_projects() {
526            let project = view.name.clone();
527            let lead = SessionSlot::lead(&view.org, &project);
528            let rows = self.board_rows(&project, now, crate::board::DEFAULT_STALE_SECS);
529            let history = self.history_for_project(&project);
530            let unaccounted = self.unaccounted(&project);
531            let fired = self.chases_for_project(&project);
532            // A worker that accounted clears its slate here and not in some
533            // process's memory: the naming goes durably, so a recurrence
534            // after a restart is named at once rather than waiting out the
535            // cadence.
536            for chase in &fired {
537                if chase.rung == ChaseRung::Unaccounted
538                    && let Some(label) = chase.subject.as_deref()
539                    && !unaccounted.iter().any(|held| held == label)
540                {
541                    self.forget_drift(&project, label);
542                }
543            }
544            let crossings = crossings(&lead, &rows, &history, &unaccounted, &fired, now);
545            for crossing in crossings {
546                let target = ladder(&crossing, &lead, |seat| self.seat_is_live(seat));
547                let mut landed = false;
548                if let Some(target) = &target
549                    && deliver.to_seat(target, &crossing.text) == Delivery::Delivered
550                {
551                    delivered += 1;
552                    landed = true;
553                }
554                // A user nudge is one shot per row: a tick that found every
555                // seat asleep must not consume it, or the row's only nudge is
556                // lost. Every other rung keeps its mark-alone behaviour.
557                if crossing.rung == ChaseRung::UserWaitDigest && !landed {
558                    continue;
559                }
560                self.record_chase(&TaskChase {
561                    task_id: crossing.task_id.clone(),
562                    subject: crossing.subject.clone(),
563                    project_name: project.clone(),
564                    rung: crossing.rung,
565                    at: now,
566                });
567            }
568        }
569        delivered
570    }
571
572    /// Spawn the chase sweep: a background pass every
573    /// [`CHASE_INTERVAL_SECS`] that delivers the crossings. The registry
574    /// makes a second call a no-op; started once at boot. Mirrors
575    /// [`Workspace::start_cron_scheduler`].
576    pub fn start_chase_sweep(self: &Arc<Self>) {
577        let weak = Arc::downgrade(self);
578        let span = tracing::info_span!("chase_sweep");
579        self.supervisor.spawn("chase_sweep", forge_supervise::Policy::backoff(), move || {
580            let weak = weak.clone();
581            let span = span.clone();
582            async move {
583                let period = std::time::Duration::from_secs(CHASE_INTERVAL_SECS);
584                let mut interval =
585                    tokio::time::interval_at(tokio::time::Instant::now() + period, period);
586                interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
587                loop {
588                    interval.tick().await;
589                    let Some(workspace) = weak.upgrade() else {
590                        return;
591                    };
592                    let delivered = workspace.chase_sweep(
593                        &PromptDeliver { workspace: Arc::clone(&workspace) },
594                        SystemTime::now(),
595                    );
596                    if delivered > 0 {
597                        tracing::debug!(
598                            target: "forge_workspace::chase",
599                            delivered,
600                            "chase sweep delivered",
601                        );
602                    }
603                }
604            }
605            .instrument(span)
606        });
607    }
608}
609
610/// The production delivery: a prompt into a live session, and the turn
611/// its words draw as.
612///
613/// **A chase carries no envelope of its own** - no delivery update draws
614/// it and the CLI echoes no stdin prompt back - so the row is forged here
615/// rather than left to `delivery_turn`, which has no arm for it. The seat
616/// is already known live by the ladder, so a failed dispatch here is a
617/// race, not a reason to spawn.
618pub(crate) struct PromptDeliver {
619    pub(crate) workspace: Arc<Workspace>,
620}
621
622impl Deliver for PromptDeliver {
623    fn to_seat(&self, slot: &SessionSlot, text: &str) -> Delivery {
624        let uuid = forge_sdk::request_id::next_forged_prompt_id();
625        match self.workspace.dispatch_forged_prompt_under(slot, text.to_owned(), uuid) {
626            Ok(()) => Delivery::Delivered,
627            Err(error) => Delivery::NoSession(error.to_string()),
628        }
629    }
630}
631
632#[cfg(test)]
633mod tests {
634    use parking_lot::Mutex;
635    use std::time::Duration;
636
637    use super::*;
638    use crate::protocol::SessionUpdate;
639    use forge_primitives::Message;
640    use forge_primitives::tasks::{
641        By, Estimate, LinkKind, Task, TaskId, TaskLink, TaskStatus, TaskTransition, Verify, Waiting,
642    };
643    use tempfile::tempdir;
644
645    const PROJECT: &str = "proj";
646    const WINDOW: u64 = 4 * 3600;
647
648    fn epoch(secs: u64) -> SystemTime {
649        SystemTime::UNIX_EPOCH + Duration::from_secs(secs)
650    }
651
652    fn fixture(dir: &std::path::Path) -> Arc<Workspace> {
653        let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.to_owned());
654        ws.seed_test_project(PROJECT, "/tmp/tp-chase");
655        let db = crate::store::Db::open(&dir.join("db.redb")).expect("open db");
656        ws.install_db_for_test(db);
657        ws
658    }
659
660    fn task(id: &str, status: TaskStatus) -> Task {
661        Task {
662            id: TaskId::from(id),
663            project_name: PROJECT.to_owned(),
664            subject: format!("subject {id}"),
665            active_form: None,
666            detail: None,
667            status,
668            owner: None,
669            parent: None,
670            waiting_on: None,
671            estimate: None,
672            rank: None,
673            verify: None,
674            attested: None,
675            reserved: None,
676            links: Vec::new(),
677            attempt: 0,
678            archived_at: None,
679            created_at: epoch(0),
680            updated_at: epoch(0),
681        }
682    }
683
684    fn running_over_estimate(ws: &Workspace, id: &str, owner: Option<&str>) {
685        let mut row = task(id, TaskStatus::InProgress);
686        row.estimate = Some(Estimate { words: "10s".to_owned(), secs: 10 });
687        row.owner = owner.map(|label| SessionSlot::worker("TestOrg", PROJECT, label));
688        row.updated_at = epoch(100);
689        ws.seed_test_task(row);
690        let guard = ws.db.lock();
691        crate::store::task_history::append(
692            guard.as_ref().expect("db"),
693            &[TaskTransition {
694                task_id: TaskId::from(id),
695                project_name: PROJECT.to_owned(),
696                from: Some(TaskStatus::Pending),
697                to: TaskStatus::InProgress,
698                at: epoch(10),
699                by: By::System,
700            }],
701        )
702        .expect("history");
703    }
704
705    #[derive(Default)]
706    struct Recording {
707        seen: Mutex<Vec<(SessionSlot, String)>>,
708    }
709
710    impl Deliver for Recording {
711        fn to_seat(&self, slot: &SessionSlot, text: &str) -> Delivery {
712            self.seen.lock().push((slot.clone(), text.to_owned()));
713            Delivery::Delivered
714        }
715    }
716
717    fn sweep(ws: &Arc<Workspace>, deliver: &dyn Deliver, now: u64) -> usize {
718        ws.chase_sweep(deliver, epoch(now))
719    }
720
721    /// A lead seat behind a live session, so lead-addressed rungs have
722    /// somewhere to land. The returned receiver must be held for the
723    /// stub to stay.
724    fn live_lead(
725        ws: &Workspace,
726    ) -> tokio::sync::mpsc::UnboundedReceiver<forge_primitives::AgentCommand> {
727        ws.install_testing_stub(&SessionSlot::lead("TestOrg", PROJECT))
728    }
729
730    /// A live worker in `project` holding no row: the shape the
731    /// unaccounted rung names.
732    fn worker_holding_no_row(ws: &Workspace, project: &str, label: &str) {
733        let key = project_key(ws, project);
734        ws.insert_live_worker(
735            &key,
736            crate::WorkerEntry {
737                label: label.to_owned(),
738                charter: "charter".to_owned(),
739                slot: SessionSlot::worker("TestOrg", project, label),
740                session_id: None,
741                status: forge_primitives::WorkerLiveness::Running,
742                spawned_at: epoch(0),
743                spawned_by: SessionSlot::lead("TestOrg", project),
744                needs_tag: false,
745                is_git_repo_at_spawn: false,
746                cwd: None,
747                diagnostic: None,
748                kick: None,
749            },
750        );
751    }
752
753    fn project_key(ws: &Workspace, project: &str) -> crate::ProjectKey {
754        ws.list_projects().into_iter().find(|v| v.name == project).expect("view").key.clone()
755    }
756
757    /// One status transition, durably logged, at a synthetic instant.
758    fn append_transition(ws: &Workspace, id: &str, from: TaskStatus, to: TaskStatus, at: u64) {
759        let guard = ws.db.lock();
760        crate::store::task_history::append(
761            guard.as_ref().expect("db"),
762            &[TaskTransition {
763                task_id: TaskId::from(id),
764                project_name: PROJECT.to_owned(),
765                from: Some(from),
766                to,
767                at: epoch(at),
768                by: By::System,
769            }],
770        )
771        .expect("history");
772    }
773
774    /// How many delivered messages carry `needle`.
775    fn says(deliver: &Recording, needle: &str) -> usize {
776        deliver.seen.lock().iter().filter(|(_, text)| text.contains(needle)).count()
777    }
778
779    /// The unaccounted rung is per project: one project's pass never
780    /// makes another's namings fresh again. Regression for the sweep
781    /// that re-fired every worker of every project on every pass.
782    #[test]
783    fn two_projects_through_one_sweep_are_named_once_each() {
784        let dir = tempdir().expect("tempdir");
785        let ws = fixture(dir.path());
786        ws.seed_test_project("other", "/tmp/tp-chase-other");
787        let _lead_rx = live_lead(&ws);
788        let _other_rx = ws.install_testing_stub(&SessionSlot::lead("TestOrg", "other"));
789        worker_holding_no_row(&ws, PROJECT, "w-1");
790        worker_holding_no_row(&ws, "other", "w-2");
791        let deliver = Recording::default();
792
793        assert_eq!(sweep(&ws, &deliver, 100), 2, "one nudge per project");
794        let seen = deliver.seen.lock();
795        assert!(
796            seen.iter().any(|(_, text)| text.contains("worker w-1 holds no row")),
797            "proj's worker is named: {seen:?}",
798        );
799        assert!(
800            seen.iter().any(|(_, text)| text.contains("worker w-2 holds no row")),
801            "other's worker is named: {seen:?}",
802        );
803        drop(seen);
804
805        assert_eq!(sweep(&ws, &deliver, 140), 0, "and neither repeats");
806        assert_eq!(deliver.seen.lock().len(), 2, "still exactly the two namings");
807    }
808
809    /// An empty project - one holding no unaccounted worker - leaves
810    /// the other projects' namings standing. Regression for the pass
811    /// that wiped the whole memory whenever any project was empty.
812    #[test]
813    fn an_empty_project_does_not_wipe_another_projects_namings() {
814        let dir = tempdir().expect("tempdir");
815        let ws = fixture(dir.path());
816        ws.seed_test_project("empty", "/tmp/tp-chase-empty");
817        let _lead_rx = live_lead(&ws);
818        worker_holding_no_row(&ws, PROJECT, "w-1");
819        let deliver = Recording::default();
820
821        assert_eq!(sweep(&ws, &deliver, 100), 1, "the worker is named");
822        assert_eq!(sweep(&ws, &deliver, 140), 0, "the empty pass says nothing");
823        assert_eq!(deliver.seen.lock().len(), 1, "and the naming stands");
824    }
825
826    /// The drifted worker's naming repeats at the 5-minute cadence while
827    /// it stays drifted.
828    #[test]
829    fn the_unaccounted_naming_repeats_at_its_cadence() {
830        let dir = tempdir().expect("tempdir");
831        let ws = fixture(dir.path());
832        let _lead_rx = live_lead(&ws);
833        worker_holding_no_row(&ws, PROJECT, "w-1");
834        let deliver = Recording::default();
835
836        assert_eq!(sweep(&ws, &deliver, 100), 1, "named on appearance");
837        assert_eq!(sweep(&ws, &deliver, 399), 0, "quiet inside the cadence");
838        assert_eq!(sweep(&ws, &deliver, 400), 1, "named again at it");
839        assert_eq!(sweep(&ws, &deliver, 500), 0, "quiet again");
840        assert_eq!(sweep(&ws, &deliver, 701), 1, "and again at the next one");
841    }
842
843    /// A worker that accounts resets its slate, durably: the naming goes
844    /// off the log with the state, so a recurrence inside the old cadence
845    /// is named at once - after a restart included, since the log is the
846    /// only slate there is.
847    #[test]
848    fn a_cleared_unaccounted_worker_is_named_at_once_on_its_next_drift() {
849        let dir = tempdir().expect("tempdir");
850        let ws = fixture(dir.path());
851        let _lead_rx = live_lead(&ws);
852        worker_holding_no_row(&ws, PROJECT, "w-1");
853        let deliver = Recording::default();
854
855        assert_eq!(sweep(&ws, &deliver, 100), 1, "named on appearance");
856        // The worker despawns: the state cleared, and the clearing pass
857        // takes its naming off the log rather than off a process's memory.
858        ws.remove_latest_worker(&project_key(&ws, PROJECT), "w-1");
859        assert_eq!(sweep(&ws, &deliver, 150), 0, "nothing drifted");
860        assert!(
861            !ws.chases_for_project(PROJECT)
862                .iter()
863                .any(|chase| chase.rung == ChaseRung::Unaccounted),
864            "the cleared worker's naming is gone from the log",
865        );
866        // It drifts again, inside the old cadence.
867        worker_holding_no_row(&ws, PROJECT, "w-1");
868        assert_eq!(sweep(&ws, &deliver, 200), 1, "the clear reset the slate");
869    }
870
871    /// The naming is durable: it lives on the log alone, so any pass - a
872    /// fresh process's first included - reads it rather than re-firing
873    /// inside the cadence.
874    #[test]
875    fn a_restart_inside_the_cadence_does_not_rename_the_worker() {
876        let dir = tempdir().expect("tempdir");
877        let ws = fixture(dir.path());
878        let _lead_rx = live_lead(&ws);
879        worker_holding_no_row(&ws, PROJECT, "w-1");
880        let deliver = Recording::default();
881
882        assert_eq!(sweep(&ws, &deliver, 100), 1, "named");
883        assert!(
884            ws.chases_for_project(PROJECT).iter().any(|chase| chase.rung == ChaseRung::Unaccounted
885                && chase.subject.as_deref() == Some("w-1")),
886            "and the naming is on the log, where a fresh process would read it",
887        );
888        assert_eq!(sweep(&ws, &deliver, 200), 0, "no pass re-names it inside the cadence");
889        assert_eq!(sweep(&ws, &deliver, 401), 1, "the cadence runs off the logged naming");
890    }
891
892    /// The 30-minute class on a rung with no floor: a stalled queue
893    /// repeats at the cadence while it stays stalled.
894    #[test]
895    fn the_queue_stall_repeats_every_thirty_minutes() {
896        let dir = tempdir().expect("tempdir");
897        let ws = fixture(dir.path());
898        let _lead_rx = live_lead(&ws);
899        let mut row = task("t-1", TaskStatus::Pending);
900        row.updated_at = epoch(0);
901        ws.seed_test_task(row);
902        let deliver = Recording::default();
903
904        assert_eq!(sweep(&ws, &deliver, WINDOW + 1), 1, "stalled: named");
905        assert_eq!(sweep(&ws, &deliver, WINDOW + 1001), 0, "quiet inside the cadence");
906        assert_eq!(sweep(&ws, &deliver, WINDOW + 1801), 1, "named again at it");
907    }
908
909    /// A failed row is named at the cadence while it stays failed, and a
910    /// re-dispatch that fails again is a new occurrence - named at once,
911    /// not held back by the window.
912    #[test]
913    fn a_failed_row_is_named_at_its_cadence_and_a_refail_is_a_new_occurrence() {
914        let dir = tempdir().expect("tempdir");
915        let ws = fixture(dir.path());
916        let _lead_rx = live_lead(&ws);
917        let mut row = task("t-1", TaskStatus::Failed);
918        row.updated_at = epoch(0);
919        ws.seed_test_task(row);
920        append_transition(&ws, "t-1", TaskStatus::Pending, TaskStatus::Failed, 100);
921        let deliver = Recording::default();
922
923        assert_eq!(sweep(&ws, &deliver, 100), 1, "the death is named");
924        assert_eq!(sweep(&ws, &deliver, 1000), 0, "quiet inside the cadence");
925        assert_eq!(sweep(&ws, &deliver, 1901), 1, "named again at it");
926        // Re-dispatched and failed again: a new occurrence.
927        append_transition(&ws, "t-1", TaskStatus::Pending, TaskStatus::InProgress, 5000);
928        append_transition(&ws, "t-1", TaskStatus::InProgress, TaskStatus::Failed, 5000);
929        assert_eq!(sweep(&ws, &deliver, 5010), 1, "the new failure is named at once");
930    }
931
932    /// The 1x overrun nudge is once per estimate: a restated estimate
933    /// arms it again.
934    #[test]
935    fn a_restated_estimate_arms_the_overrun_nudge_again() {
936        let dir = tempdir().expect("tempdir");
937        let ws = fixture(dir.path());
938        let _lead_rx = live_lead(&ws);
939        running_over_estimate(&ws, "t-1", None);
940        let deliver = Recording::default();
941
942        sweep(&ws, &deliver, 200);
943        assert_eq!(says(&deliver, "has worked"), 1, "named once: {:?}", deliver.seen.lock());
944        sweep(&ws, &deliver, 400);
945        assert_eq!(says(&deliver, "has worked"), 1, "never twice for one estimate");
946        // The owner restates the estimate; the row is still over it.
947        ws.update_task(PROJECT, &TaskId::from("t-1"), By::System, |task| {
948            task.estimate = Some(Estimate { words: "1m".to_owned(), secs: 60 });
949        })
950        .expect("no refusal")
951        .expect("the row is there");
952        sweep(&ws, &deliver, 500);
953        assert_eq!(
954            says(&deliver, "has worked"),
955            2,
956            "the restated estimate is a new occurrence: {:?}",
957            deliver.seen.lock(),
958        );
959        sweep(&ws, &deliver, 700);
960        assert_eq!(says(&deliver, "has worked"), 2, "once per estimate, not per sweep");
961    }
962
963    /// The 1.5x escalation repeats every five minutes while the row
964    /// stays over.
965    #[test]
966    fn the_escalation_repeats_at_its_cadence() {
967        let dir = tempdir().expect("tempdir");
968        let ws = fixture(dir.path());
969        let _lead_rx = live_lead(&ws);
970        running_over_estimate(&ws, "t-1", None);
971        let deliver = Recording::default();
972
973        sweep(&ws, &deliver, 200);
974        assert_eq!(says(&deliver, "is at"), 1, "escalated once: {:?}", deliver.seen.lock());
975        sweep(&ws, &deliver, 400);
976        assert_eq!(says(&deliver, "is at"), 1, "quiet inside the cadence");
977        sweep(&ws, &deliver, 501);
978        assert_eq!(says(&deliver, "is at"), 2, "escalated again at it");
979    }
980
981    /// A row parked on the user - a decision wait, or the look's own
982    /// verification park - untouched since `updated`. A verification park
983    /// belongs to a row carrying its own gate (#2423).
984    fn waiting_on_user(ws: &Workspace, id: &str, updated: u64, verification: bool) {
985        let mut row = task(id, TaskStatus::Waiting);
986        if verification {
987            row.verify = Some(Verify::User);
988        }
989        row.waiting_on = Some(Waiting {
990            kind: Some(WaitingKind::Decision),
991            detail: None,
992            on: None,
993            verification,
994            parked: false,
995            since: None,
996        });
997        row.updated_at = epoch(updated);
998        ws.seed_test_task(row);
999    }
1000
1001    /// Every row parked on the user gets ONE nudge of its own, and none
1002    /// repeats on the cadence.
1003    #[test]
1004    fn every_user_wait_takes_one_nudge_and_none_repeats() {
1005        let dir = tempdir().expect("tempdir");
1006        let ws = fixture(dir.path());
1007        let _lead_rx = live_lead(&ws);
1008        for id in ["t-1", "t-2", "t-3"] {
1009            waiting_on_user(&ws, id, 0, false);
1010        }
1011        let deliver = Recording::default();
1012
1013        let now = WINDOW + 1;
1014        assert_eq!(sweep(&ws, &deliver, now), 3, "one message per row, not one digest");
1015        let seen = deliver.seen.lock();
1016        assert_eq!(seen.len(), 3, "three rows, three nudges");
1017        for id in ["t-1", "t-2", "t-3"] {
1018            assert!(
1019                seen.iter().any(|(_, text)| text.contains(&format!("\"subject {id}\""))),
1020                "no nudge names {id}: {seen:?}",
1021            );
1022        }
1023        assert!(
1024            seen.iter().all(|(seat, _)| *seat == SessionSlot::lead("TestOrg", PROJECT)),
1025            "every nudge goes to the lead",
1026        );
1027        drop(seen);
1028
1029        assert_eq!(sweep(&ws, &deliver, now + 1799), 0, "a row is nudged once");
1030        assert_eq!(sweep(&ws, &deliver, now + 1800), 0, "and not again on the cadence");
1031    }
1032
1033    /// A verification park takes its own nudge, once; a row that starts
1034    /// waiting after the sweep gets one of its own.
1035    #[test]
1036    fn a_verify_park_takes_its_own_nudge_once() {
1037        let dir = tempdir().expect("tempdir");
1038        let ws = fixture(dir.path());
1039        let _lead_rx = live_lead(&ws);
1040        waiting_on_user(&ws, "t-1", 0, false);
1041        waiting_on_user(&ws, "t-2", 0, true);
1042        let deliver = Recording::default();
1043
1044        let now = WINDOW + 1;
1045        assert_eq!(sweep(&ws, &deliver, now), 2, "a decision and a park, one nudge each");
1046        let seen = deliver.seen.lock();
1047        let park = seen.iter().find(|(_, text)| text.contains("subject t-2")).expect("t-2 named");
1048        assert!(park.1.contains("a look"), "the park's nudge names the look: {}", park.1);
1049        drop(seen);
1050        assert_eq!(sweep(&ws, &deliver, now + 1800), 0, "neither is nudged twice");
1051
1052        // A row that starts waiting after this sweep is a new episode.
1053        waiting_on_user(&ws, "t-3", 0, false);
1054        assert_eq!(sweep(&ws, &deliver, now + 3600), 1, "the new wait gets its own nudge");
1055        assert!(deliver.seen.lock().last().expect("a nudge").1.contains("subject t-3"));
1056    }
1057
1058    /// A wait the lead parked by design - the user's own deferral - is left
1059    /// alone: no nudge, no repeat (#2423/D).
1060    #[test]
1061    fn a_parked_wait_is_left_alone() {
1062        let dir = tempdir().expect("tempdir");
1063        let ws = fixture(dir.path());
1064        let _lead_rx = live_lead(&ws);
1065        let mut row = task("t-1", TaskStatus::Waiting);
1066        row.waiting_on = Some(Waiting {
1067            kind: Some(WaitingKind::Decision),
1068            detail: Some("backlog at Ved's word".to_owned()),
1069            on: None,
1070            verification: false,
1071            parked: true,
1072            since: None,
1073        });
1074        row.updated_at = epoch(0);
1075        ws.seed_test_task(row);
1076        let deliver = Recording::default();
1077
1078        assert_eq!(sweep(&ws, &deliver, WINDOW + 1), 0, "a parked wait is not the user's chase");
1079        assert!(deliver.seen.lock().is_empty());
1080        assert!(
1081            ws.chases_for_project(PROJECT)
1082                .iter()
1083                .all(|chase| chase.rung != ChaseRung::UserWaitDigest),
1084            "and no mark is left for it",
1085        );
1086    }
1087
1088    /// A resource wait parked by design is left alone by the stall chase too;
1089    /// un-parked, its real age arms the stall and it fires at once (#2472).
1090    #[test]
1091    fn a_parked_wait_is_not_stall_chased() {
1092        let dir = tempdir().expect("tempdir");
1093        let ws = fixture(dir.path());
1094        let _lead_rx = live_lead(&ws);
1095        let mut row = task("t-1", TaskStatus::Waiting);
1096        row.waiting_on = Some(Waiting {
1097            kind: Some(WaitingKind::Resource),
1098            detail: Some("needs the owner's dashboard".to_owned()),
1099            on: None,
1100            verification: false,
1101            parked: true,
1102            since: None,
1103        });
1104        row.updated_at = epoch(0);
1105        ws.seed_test_task(row);
1106        let deliver = Recording::default();
1107
1108        assert_eq!(
1109            sweep(&ws, &deliver, WINDOW + 1),
1110            0,
1111            "a parked wait is not a stall: {:?}",
1112            deliver.seen.lock(),
1113        );
1114        assert!(
1115            ws.chases_for_project(PROJECT).iter().all(|chase| chase.rung != ChaseRung::WaitStall),
1116            "and no WaitStall mark is left for it",
1117        );
1118
1119        ws.update_task(PROJECT, &TaskId::from("t-1"), By::System, |task| {
1120            if let Some(wait) = task.waiting_on.as_mut() {
1121                wait.parked = false;
1122            }
1123        })
1124        .expect("no refusal")
1125        .expect("the row is there");
1126        let updated_at = ws
1127            .tasks_for_project(PROJECT)
1128            .into_iter()
1129            .find(|task| task.id == TaskId::from("t-1"))
1130            .expect("the row survived")
1131            .updated_at;
1132        let later = updated_at + Duration::from_secs(WINDOW + 2);
1133        let later_secs =
1134            later.duration_since(SystemTime::UNIX_EPOCH).expect("the epoch is before it").as_secs();
1135        let rows = ws.board_rows(PROJECT, epoch(later_secs), WINDOW);
1136        let moved = rows.iter().find(|row| row.task.id == TaskId::from("t-1")).expect("the row");
1137        assert!(!moved.task.waiting_on.as_ref().expect("the wait holds").parked, "un-parked");
1138        assert!(
1139            moved.marks.waiting_too_long,
1140            "its real age is the stall again: status={:?} age={} updated={:?} now={:?} wait={:?}",
1141            moved.task.status,
1142            moved.updated_secs_ago,
1143            moved.task.updated_at,
1144            epoch(later_secs),
1145            moved.task.waiting_on,
1146        );
1147        assert_eq!(sweep(&ws, &deliver, later_secs), 1, "un-parked, the stall fires");
1148        let seen = deliver.seen.lock();
1149        assert_eq!(seen.len(), 1, "one WaitStall: {seen:?}");
1150        assert_eq!(
1151            seen[0].1,
1152            "task board: \"subject t-1\" has waited 4h on a resource - unblock it or change its plan.",
1153            "the WaitStall names the row, its age and the kind it waits on",
1154        );
1155    }
1156
1157    /// A verification park whose row owns no gate - the shape an inherited
1158    /// gate left behind - is not a user wait (#2423): the digest does not
1159    /// name it, and no one is asked to confirm it.
1160    #[test]
1161    fn a_verification_park_without_an_own_gate_stays_out_of_the_digest() {
1162        let dir = tempdir().expect("tempdir");
1163        let ws = fixture(dir.path());
1164        let _lead_rx = live_lead(&ws);
1165        waiting_on_user(&ws, "t-1", 0, true);
1166        // The same wait on a row whose gate was never its own: the epic
1167        // carried it, so the park is not an ask on the user.
1168        let mut inherited = task("t-2", TaskStatus::Waiting);
1169        inherited.waiting_on = Some(Waiting {
1170            kind: Some(WaitingKind::Decision),
1171            detail: None,
1172            on: None,
1173            verification: true,
1174            parked: false,
1175            since: None,
1176        });
1177        inherited.updated_at = epoch(0);
1178        ws.seed_test_task(inherited);
1179        let deliver = Recording::default();
1180
1181        assert_eq!(sweep(&ws, &deliver, WINDOW + 1), 1, "only the genuine park is nudged");
1182        let seen = deliver.seen.lock();
1183        assert_eq!(seen.len(), 1, "one nudge, for the row that owns the gate");
1184        let text = &seen[0].1;
1185        assert!(text.contains("\"subject t-1\""), "the own-gate park joins: {text}");
1186        assert!(!text.contains("\"subject t-2\""), "the inherited park stays out: {text}");
1187    }
1188
1189    /// The user nudge never wakes a sleeping lead, and a tick that found it
1190    /// asleep does not consume the row's one nudge (#2423/D): nothing is
1191    /// delivered and nothing is marked, so the first tick with a live lead
1192    /// delivers it. Every other rung keeps its mark-alone behaviour.
1193    #[test]
1194    fn a_user_wait_nudge_waits_for_a_live_lead() {
1195        let dir = tempdir().expect("tempdir");
1196        let ws = fixture(dir.path());
1197        waiting_on_user(&ws, "t-1", 0, false);
1198        let deliver = Recording::default();
1199
1200        assert_eq!(sweep(&ws, &deliver, WINDOW + 1), 0, "nothing delivered");
1201        assert!(deliver.seen.lock().is_empty(), "no seat was reached");
1202        assert!(
1203            !ws.chases_for_project(PROJECT)
1204                .iter()
1205                .any(|chase| chase.rung == ChaseRung::UserWaitDigest),
1206            "a tick that found the lead asleep must not spend the row's one nudge",
1207        );
1208
1209        let _lead_rx = live_lead(&ws);
1210        assert_eq!(sweep(&ws, &deliver, WINDOW + 2), 1, "the first awake tick delivers it");
1211        assert!(
1212            ws.chases_for_project(PROJECT)
1213                .iter()
1214                .any(|chase| chase.rung == ChaseRung::UserWaitDigest),
1215            "and the landed nudge is on the log",
1216        );
1217        assert_eq!(sweep(&ws, &deliver, WINDOW + 1800), 0, "then it is spent");
1218    }
1219
1220    /// The estimate crossing asks the owner once, ever - a later sweep
1221    /// with the same facts stays quiet because the rung is on the log.
1222    #[test]
1223    fn the_estimate_rung_fires_once_ever_per_row() {
1224        let dir = tempdir().expect("tempdir");
1225        let ws = fixture(dir.path());
1226        ws.seed_test_project("proj", "/tmp/tp-chase"); // already seeded; harmless
1227        let key =
1228            ws.list_projects().into_iter().find(|v| v.name == PROJECT).expect("view").key.clone();
1229        ws.insert_live_worker(
1230            &key,
1231            crate::WorkerEntry {
1232                label: "w-1".to_owned(),
1233                charter: "charter".to_owned(),
1234                slot: SessionSlot::worker("TestOrg", PROJECT, "w-1"),
1235                session_id: None,
1236                status: forge_primitives::WorkerLiveness::Running,
1237                spawned_at: epoch(0),
1238                spawned_by: SessionSlot::lead("TestOrg", PROJECT),
1239                needs_tag: false,
1240                is_git_repo_at_spawn: false,
1241                cwd: None,
1242                diagnostic: None,
1243                kick: None,
1244            },
1245        );
1246        running_over_estimate(&ws, "t-1", Some("w-1"));
1247        let deliver = Recording::default();
1248
1249        assert_eq!(sweep(&ws, &deliver, 100), 1, "one nudge");
1250        let seen = deliver.seen.lock();
1251        assert_eq!(seen.len(), 1, "exactly one message: {seen:?}");
1252        assert_eq!(seen[0].0, SessionSlot::worker("TestOrg", PROJECT, "w-1"));
1253        assert!(seen[0].1.contains("subject t-1"), "the message names the row: {}", seen[0].1);
1254        assert!(
1255            seen[0].1.contains("A new estimate, completing it, or parking it"),
1256            "the ask names what actually clears the overrun, not an update: {}",
1257            seen[0].1,
1258        );
1259        drop(seen);
1260
1261        assert_eq!(sweep(&ws, &deliver, 140), 0, "the rung never repeats");
1262        assert_eq!(deliver.seen.lock().len(), 1, "and nothing more was said");
1263    }
1264
1265    /// A dead owner does not swallow the nudge: it lands with the lead.
1266    #[test]
1267    fn a_dead_owners_nudge_goes_to_the_lead() {
1268        let dir = tempdir().expect("tempdir");
1269        let ws = fixture(dir.path());
1270        let _lead_rx = live_lead(&ws);
1271        running_over_estimate(&ws, "t-1", Some("gone"));
1272        let deliver = Recording::default();
1273
1274        sweep(&ws, &deliver, 100);
1275        let seen = deliver.seen.lock();
1276        assert!(
1277            seen.iter().any(|(seat, _)| seat == &SessionSlot::lead("TestOrg", PROJECT)),
1278            "the lead hears it: {seen:?}",
1279        );
1280    }
1281
1282    /// Every seat asleep: the rungs still fire into the log - the mark
1283    /// alone stands - and NOTHING is delivered and nothing is spawned.
1284    #[test]
1285    fn with_every_seat_asleep_the_rung_is_mark_only_and_nothing_spawns() {
1286        let dir = tempdir().expect("tempdir");
1287        let ws = fixture(dir.path());
1288        running_over_estimate(&ws, "t-1", Some("gone"));
1289        let deliver = Recording::default();
1290
1291        assert_eq!(sweep(&ws, &deliver, 100), 0, "nothing delivered");
1292        assert!(deliver.seen.lock().is_empty(), "no seat was reached");
1293        assert!(
1294            !ws.chases_for_project(PROJECT).is_empty(),
1295            "and the rung is still on the log, so it fired once and never repeats",
1296        );
1297    }
1298
1299    /// One nudge per wait EPISODE: the row's own newest transition into
1300    /// waiting is the episode key, so a row that left waiting and came back
1301    /// takes a fresh nudge rather than inheriting the spent one (#2423/D).
1302    #[test]
1303    fn a_wait_re_entering_after_a_move_is_a_new_nudge() {
1304        let dir = tempdir().expect("tempdir");
1305        let ws = fixture(dir.path());
1306        let _lead_rx = live_lead(&ws);
1307        let mut row = task("t-1", TaskStatus::Waiting);
1308        row.waiting_on = Some(Waiting {
1309            kind: Some(WaitingKind::Decision),
1310            detail: None,
1311            on: None,
1312            verification: false,
1313            parked: false,
1314            since: None,
1315        });
1316        row.updated_at = epoch(100);
1317        ws.seed_test_task(row);
1318        let append = |from: Option<TaskStatus>, to: TaskStatus, at: u64| {
1319            let guard = ws.db.lock();
1320            crate::store::task_history::append(
1321                guard.as_ref().expect("db"),
1322                &[TaskTransition {
1323                    task_id: TaskId::from("t-1"),
1324                    project_name: PROJECT.to_owned(),
1325                    from,
1326                    to,
1327                    at: epoch(at),
1328                    by: By::System,
1329                }],
1330            )
1331            .expect("history");
1332        };
1333        append(Some(TaskStatus::Pending), TaskStatus::Waiting, 100);
1334        let deliver = Recording::default();
1335
1336        assert_eq!(sweep(&ws, &deliver, 3000), 1, "the episode's first nudge");
1337        // Off waiting and back: a new episode at 4100.
1338        append(Some(TaskStatus::Waiting), TaskStatus::InProgress, 4000);
1339        append(Some(TaskStatus::InProgress), TaskStatus::Waiting, 4100);
1340        assert_eq!(sweep(&ws, &deliver, 6000), 1, "the new episode is nudged");
1341        assert_eq!(sweep(&ws, &deliver, 9000), 0, "and that nudge is spent in turn");
1342    }
1343
1344    /// A dependency or a resource wait is not the user's: neither is nudged
1345    /// as a user wait however the rows around it are gated (#2415).
1346    #[test]
1347    fn resource_and_dependency_waits_are_not_the_users() {
1348        let dir = tempdir().expect("tempdir");
1349        let ws = fixture(dir.path());
1350        let _lead_rx = live_lead(&ws);
1351        for (id, kind) in [("t-1", WaitingKind::Dependency), ("t-2", WaitingKind::Resource)] {
1352            let mut row = task(id, TaskStatus::Waiting);
1353            row.waiting_on = Some(Waiting {
1354                kind: Some(kind),
1355                detail: None,
1356                on: None,
1357                verification: false,
1358                parked: false,
1359                since: None,
1360            });
1361            row.updated_at = epoch(0);
1362            ws.seed_test_task(row);
1363        }
1364        let deliver = Recording::default();
1365
1366        sweep(&ws, &deliver, WINDOW + 1);
1367        // The stuck-wait rung may name them to the LEAD ("unblock it"), which
1368        // is the task's business; what neither may be is the user's.
1369        assert!(
1370            deliver.seen.lock().iter().all(|(_, text)| !text.contains("waiting on the user")),
1371            "a dependency or resource wait was listed as the user's: {:?}",
1372            deliver.seen.lock(),
1373        );
1374        assert!(
1375            ws.chases_for_project(PROJECT)
1376                .iter()
1377                .all(|chase| chase.rung != ChaseRung::UserWaitDigest),
1378            "and neither leaves a user-wait mark",
1379        );
1380    }
1381
1382    /// A waiting row is never estimate-chased: its clock is stopped, and
1383    /// its wait's age is the signal - which has its own rung.
1384    #[test]
1385    fn a_waiting_row_is_never_estimate_chased() {
1386        let dir = tempdir().expect("tempdir");
1387        let ws = fixture(dir.path());
1388        let _lead_rx = live_lead(&ws);
1389        let mut row = task("t-1", TaskStatus::Waiting);
1390        row.estimate = Some(Estimate { words: "10s".to_owned(), secs: 10 });
1391        row.waiting_on = Some(Waiting {
1392            kind: Some(WaitingKind::Resource),
1393            detail: None,
1394            on: None,
1395            verification: false,
1396            parked: false,
1397            since: None,
1398        });
1399        row.updated_at = epoch(0);
1400        ws.seed_test_task(row);
1401        let deliver = Recording::default();
1402
1403        sweep(&ws, &deliver, WINDOW + 1);
1404        let seen = deliver.seen.lock();
1405        assert!(
1406            !seen.iter().any(|(_, text)| text.contains("estimate")),
1407            "no estimate nudge for a waiting row: {seen:?}",
1408        );
1409        assert!(
1410            seen.iter().any(|(_, text)| text.contains("resource")),
1411            "the wait's own rung carries it: {seen:?}",
1412        );
1413    }
1414
1415    /// A wait that resolved: the waiter's owner hears once, and only once
1416    /// for that resolution.
1417    #[test]
1418    fn a_resolved_wait_messages_the_waiter_once() {
1419        let dir = tempdir().expect("tempdir");
1420        let ws = fixture(dir.path());
1421        let key =
1422            ws.list_projects().into_iter().find(|v| v.name == PROJECT).expect("view").key.clone();
1423        ws.insert_live_worker(
1424            &key,
1425            crate::WorkerEntry {
1426                label: "w-1".to_owned(),
1427                charter: "charter".to_owned(),
1428                slot: SessionSlot::worker("TestOrg", PROJECT, "w-1"),
1429                session_id: None,
1430                status: forge_primitives::WorkerLiveness::Running,
1431                spawned_at: epoch(0),
1432                spawned_by: SessionSlot::lead("TestOrg", PROJECT),
1433                needs_tag: false,
1434                is_git_repo_at_spawn: false,
1435                cwd: None,
1436                diagnostic: None,
1437                kick: None,
1438            },
1439        );
1440        let mut row = task("t-1", TaskStatus::Pending);
1441        row.owner = Some(SessionSlot::worker("TestOrg", PROJECT, "w-1"));
1442        row.updated_at = epoch(100);
1443        ws.seed_test_task(row);
1444        {
1445            let guard = ws.db.lock();
1446            crate::store::task_history::append(
1447                guard.as_ref().expect("db"),
1448                &[TaskTransition {
1449                    task_id: TaskId::from("t-1"),
1450                    project_name: PROJECT.to_owned(),
1451                    from: Some(TaskStatus::Waiting),
1452                    to: TaskStatus::Pending,
1453                    at: epoch(90),
1454                    by: By::System,
1455                }],
1456            )
1457            .expect("history");
1458        }
1459        let deliver = Recording::default();
1460
1461        sweep(&ws, &deliver, 100);
1462        let seen = deliver.seen.lock();
1463        assert!(
1464            seen.iter().any(|(seat, text)| seat == &SessionSlot::worker("TestOrg", PROJECT, "w-1")
1465                && text.contains("was waiting")),
1466            "the waiter's owner hears the release: {seen:?}",
1467        );
1468        let before = seen.len();
1469        drop(seen);
1470        sweep(&ws, &deliver, 130);
1471        assert_eq!(deliver.seen.lock().len(), before, "and it never repeats");
1472    }
1473
1474    /// A stalled queue is the lead's failure and is named once.
1475    #[test]
1476    fn a_stalled_queue_row_messages_the_lead_once() {
1477        let dir = tempdir().expect("tempdir");
1478        let ws = fixture(dir.path());
1479        let _lead_rx = live_lead(&ws);
1480        let mut row = task("t-1", TaskStatus::Pending);
1481        row.updated_at = epoch(0);
1482        ws.seed_test_task(row);
1483        let deliver = Recording::default();
1484
1485        sweep(&ws, &deliver, WINDOW + 1);
1486        let seen = deliver.seen.lock();
1487        assert_eq!(seen.len(), 1, "one message: {seen:?}");
1488        assert!(seen[0].1.contains("queue is stalling"), "{}", seen[0].1);
1489        drop(seen);
1490        sweep(&ws, &deliver, WINDOW + 10);
1491        assert_eq!(deliver.seen.lock().len(), 1, "and never twice");
1492    }
1493
1494    /// An epic with every child terminal and no retro asks for one, and
1495    /// once the retro link is on, it asks for the close instead.
1496    #[test]
1497    fn the_retro_fires_once_and_then_the_close_nudge_fires_once() {
1498        let dir = tempdir().expect("tempdir");
1499        let ws = fixture(dir.path());
1500        let _lead_rx = live_lead(&ws);
1501        ws.seed_test_task(task("epic", TaskStatus::InProgress));
1502        let mut child = task("sub-a", TaskStatus::Completed);
1503        child.parent = Some(TaskId::from("epic"));
1504        ws.seed_test_task(child);
1505        let deliver = Recording::default();
1506
1507        sweep(&ws, &deliver, 100);
1508        let seen = deliver.seen.lock();
1509        assert!(
1510            seen.iter().any(|(_, text)| {
1511                text.contains("every child terminal") && text.contains("Numbers")
1512            }),
1513            "the retro prompt carries the numbers: {seen:?}",
1514        );
1515        drop(seen);
1516
1517        // The retro runs: the link lands, and the next pass asks for the
1518        // close - once.
1519        ws.update_task(
1520            PROJECT,
1521            &TaskId::from("epic"),
1522            By::Seat(SessionSlot::lead("TestOrg", PROJECT)),
1523            |task| {
1524                task.links.push(TaskLink {
1525                    kind: LinkKind::Path,
1526                    label: Some("retro".to_owned()),
1527                    target: "docs/retro.md".to_owned(),
1528                    state: None,
1529                    added_at: epoch(0),
1530                });
1531            },
1532        )
1533        .expect("no refusal")
1534        .expect("the epic is there");
1535        sweep(&ws, &deliver, 140);
1536        let seen = deliver.seen.lock();
1537        assert!(
1538            seen.iter().any(|(_, text)| text.contains("close it when you agree")),
1539            "the close nudge fires: {seen:?}",
1540        );
1541        let before = seen.len();
1542        drop(seen);
1543        sweep(&ws, &deliver, 180);
1544        assert_eq!(deliver.seen.lock().len(), before, "neither rung repeats");
1545    }
1546
1547    /// The delivery forges the chased words as the turn they reached the
1548    /// model as.
1549    ///
1550    /// **A chase has no envelope of its own**: no delivery update draws it,
1551    /// and the CLI echoes no stdin prompt back, so this frame is the only
1552    /// carrier the words have in a live view. Without it the turn opens on
1553    /// its first tool row, with nothing saying what started it.
1554    #[test]
1555    fn the_delivery_forges_the_chased_words_as_their_turn() {
1556        let dir = tempdir().expect("tempdir");
1557        let (ws, mut rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
1558        ws.seed_test_project(PROJECT, "/tmp/tp-chase");
1559        let key = SessionSlot::lead("TestOrg", PROJECT);
1560        ws.enable_test_dispatch_intercept();
1561        let deliver = PromptDeliver { workspace: Arc::clone(&ws) };
1562
1563        let text = "task board: worker w-1 holds no row - feed it or despawn it.";
1564        assert_eq!(deliver.to_seat(&key, text), Delivery::Delivered);
1565
1566        // The wired prompt and the forged frame are one thing by id: the
1567        // lifecycle frames name only the id, and a row minted apart from the
1568        // wire is one nothing can settle.
1569        let wired = match ws.drain_test_dispatch_buffer().as_slice() {
1570            [crate::protocol::Command::PromptUnder { uuid, text: wired, .. }] => {
1571                assert_eq!(wired, text, "the model receives the chase's words");
1572                assert!(
1573                    uuid.starts_with("forge_nudge_"),
1574                    "the id carries the machinery's prefix, which is what a reloaded row keys \
1575                     on: {uuid}",
1576                );
1577                uuid.clone()
1578            }
1579            other => panic!("the delivery wires one prompt; got {other:?}"),
1580        };
1581        let drawn: Vec<SessionUpdate> = std::iter::from_fn(|| rx.try_recv().ok())
1582            .map(|frame| std::sync::Arc::unwrap_or_clone(frame.update))
1583            .collect();
1584        let frame = drawn.iter().find_map(|update| match update {
1585            SessionUpdate::ChatAppended { msg, .. } => Some(msg),
1586            _ => None,
1587        });
1588        let Some(Message::User { message, uuid, .. }) = frame else {
1589            panic!("the chase's delivery forges its words as the turn; got {drawn:?}")
1590        };
1591        assert_eq!(
1592            uuid.as_deref(),
1593            Some(wired.as_str()),
1594            "the frame is stamped with the id the prompt runs under",
1595        );
1596        assert!(
1597            matches!(
1598                message.content.first(),
1599                Some(forge_primitives::ContentBlock::Text { text: said, .. }) if said == text
1600            ),
1601            "the frame carries the prose the model received: {message:?}",
1602        );
1603    }
1604
1605    /// **The chase no longer announces the board on a no-op pass** (#2540).
1606    /// The client's 30 s board beat moves the time-derived marks; a
1607    /// `TasksChanged` for an unchanged set made every board and home
1608    /// subscriber re-read its whole subject. A pass with nothing to deliver
1609    /// must put no task set on the stream.
1610    #[tokio::test(start_paused = true)]
1611    async fn a_noop_chase_pass_announces_no_tasks_changed() {
1612        let dir = tempdir().expect("tempdir");
1613        let (ws, mut rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
1614        ws.seed_test_project(PROJECT, "/tmp/tp-chase");
1615        ws.start_chase_sweep();
1616        tokio::time::advance(Duration::from_secs(CHASE_INTERVAL_SECS + 1)).await;
1617        // The spawned sweep runs on the runtime; give it the tick and the
1618        // updates it would have put on the stream.
1619        for _ in 0..10 {
1620            tokio::task::yield_now().await;
1621        }
1622        while let Ok(frame) = rx.try_recv() {
1623            assert!(
1624                !matches!(&*frame.update, SessionUpdate::TasksChanged { .. }),
1625                "a no-op chase pass announced a task set: {:?}",
1626                frame.update,
1627            );
1628        }
1629    }
1630}