Skip to main content

forge_workspace/
spawn.rs

1//! App-level spawn command handlers. Called from
2//! [`crate::Workspace::dispatch`] when an App-level
3//! `Command::SpawnProject` arrives.
4//! Each handler resolves the id the session will run under, emits
5//! `SessionUpdate::Spawning` under it, and kicks off the agent spawn.
6//! The matching `Connected` arrives later under the same key.
7
8use std::sync::Arc;
9use std::time::{Duration, SystemTime};
10
11use forge_agent::client::SessionLaunchSettings;
12
13use crate::mcp::gotify::types::GotifyNotification;
14use crate::mcp::peers::types::WrappedPrompt;
15use crate::protocol::{
16    Command, PromptSource, SessionUpdate, WorkerSpawnReply, WorkerStatusAction, WorktreeDisposition,
17};
18use crate::target::ProjectKey;
19use crate::workspace::DespawnCleanupPending;
20use crate::workspace::LiveWorkerRefusal;
21use crate::workspace::Workspace;
22use crate::{SessionSlot, SessionTarget};
23use std::fmt::Write as _;
24
25use forge_primitives::slack::SlackMessage;
26
27/// A failed prompt dispatch to a running target unwinds the echo's
28/// already-opened turn: without this its bar counts forever and the
29/// bucket spinner never drops (mirrors the typed-submit
30/// compensation). Per-site warns stay at the call sites.
31pub(crate) fn send_dispatch_turn_error(
32    workspace: &Workspace,
33    key: SessionSlot,
34    err: &crate::protocol::DispatchError,
35) {
36    let _ = workspace.update_sender().send(SessionUpdate::TurnError {
37        key,
38        message: err.to_string(),
39        class: None,
40        terminal_reason: None,
41    });
42}
43
44/// The `claude` CLI's tools forge has replaced with its own surfaces: the
45/// task quartet (`mcp__forge__tasks__*` over forge's own store), the cron
46/// trio (forge's durable crons), `SendMessage` / `ListAgents` (the
47/// `agents__*` group), `Workflow` (forge's workers) and `RemoteTrigger`
48/// (cloud routines, which forge has no equivalent for and nothing here
49/// reaches for).
50const REPLACED_CLI_TOOLS: [&str; 11] = [
51    "CronCreate",
52    "CronDelete",
53    "CronList",
54    "TaskCreate",
55    "TaskGet",
56    "TaskList",
57    "TaskUpdate",
58    "SendMessage",
59    "ListAgents",
60    "Workflow",
61    "RemoteTrigger",
62];
63
64/// The `--disallowedTools` value for a session, comma-separated - the form
65/// the CLI's variadic `<tools...>` parser accepts.
66///
67/// Every session loses the tools forge has replaced, so nothing depends on
68/// a distinction a caller cannot see. A worker additionally loses the two
69/// worktree-hop tools, because it is pinned to its spawn location, and a
70/// session that cannot put a question to the user loses
71/// `AskUserQuestion` - that one is a property of the spawn, not of the
72/// kind.
73fn blocked_tools_for(kind: crate::mcp::SessionKind, interactive: bool) -> String {
74    let mut blocked: Vec<&str> = REPLACED_CLI_TOOLS.to_vec();
75    if matches!(kind, crate::mcp::SessionKind::Worker) {
76        blocked.extend(["EnterWorktree", "ExitWorktree"]);
77    }
78    if !interactive {
79        blocked.push("AskUserQuestion");
80    }
81    blocked.join(",")
82}
83
84/// Deny every session - lead, resumed or worker - the tools forge has
85/// replaced, merging into a `--disallowedTools` value already there, so the
86/// flag is emitted once with every name in it rather than twice. A worker's
87/// spawn is the case that carries one: its own denials already hold that
88/// entry. Called from the one spawn path every session goes through and
89/// from the respawn stamp, so no launch can be assembled without it.
90///
91/// `interactive` decides the question denial, and each caller answers it
92/// from the only place it can. A worker's spawn hands it the flag it was
93/// given; a respawn reads the worker's stored row, which is where the flag
94/// outlives the launch settings a view rebuilds. The one spawn path in
95/// `get_agent_handle_at_key` passes `true` because it cannot know: a
96/// worker's [`build_worker_extra_args`] has already written that name into
97/// the same entry, and the merge leaves it there.
98pub(crate) fn apply_blocked_tools(
99    settings: &mut SessionLaunchSettings,
100    kind: crate::mcp::SessionKind,
101    interactive: bool,
102) {
103    let blocked = blocked_tools_for(kind, interactive);
104    let Some((_, value)) =
105        settings.extra_args.iter_mut().find(|(flag, _)| flag == "disallowedTools")
106    else {
107        settings.extra_args.push(("disallowedTools".to_owned(), Some(blocked)));
108        return;
109    };
110    let existing = value.get_or_insert_with(String::new);
111    for tool in blocked.split(',') {
112        if existing.split(',').any(|name| name == tool) {
113            continue;
114        }
115        if !existing.is_empty() {
116            existing.push(',');
117        }
118        existing.push_str(tool);
119    }
120}
121
122/// Build the list of `(flag, value)` extra CLI args specific to a
123/// worker spawn. When the project is a git repo, append
124/// `("worktree", Some(label))` so the spawned `claude` subprocess
125/// creates a worktree at `<base>/.claude/worktrees/<label>/` - `base`
126/// the repo's canonical root, the main checkout for a project that is
127/// itself a linked worktree - and runs the session inside it. In all
128/// cases, append the worker's
129/// [`blocked_tools_for`] set as its `--disallowedTools` entry: workers are
130/// pinned to their spawn-time location (whether a worktree or the
131/// project cwd) and must not be able to call claude's built-in
132/// worktree-hop tools to escape. Comma-separated value form is
133/// empirically accepted by the CLI's variadic `<tools...>` parser.
134///
135/// Unless `interactive`, `AskUserQuestion` joins that list. A worker's
136/// question renders in its own row, which nobody is usually looking
137/// at, and an answer that does arrive is indistinguishable from a
138/// decision the user made. Denying it at spawn means the capability is
139/// never offered rather than refused after the model has already
140/// decided to ask.
141///
142/// The `is_git_repo` boolean is passed in (already-computed by the
143/// caller) so the git-repo probe runs at most once per spawn even
144/// when the same answer is needed for both the `WorkerEntry.is_git_repo_at_spawn`
145/// field and this argument list.
146fn build_worker_extra_args(
147    is_git_repo: bool,
148    label: &str,
149    interactive: bool,
150) -> Vec<(String, Option<String>)> {
151    let mut args = Vec::new();
152    if is_git_repo {
153        args.push(("worktree".to_owned(), Some(label.to_owned())));
154    }
155    let blocked = blocked_tools_for(crate::mcp::SessionKind::Worker, interactive);
156    args.push(("disallowedTools".to_owned(), Some(blocked)));
157    args
158}
159
160/// Charter every lead session is launched with.
161const DEFAULT_LEAD_CHARTER: &str = include_str!("spawn/lead_charter.md");
162
163/// Stamp [`DEFAULT_LEAD_CHARTER`] onto the launch settings so every lead
164/// session carries one. No-op when a charter is already set - worker
165/// spawns supply their own and we never overwrite it.
166pub(crate) fn apply_lead_charter(settings: &mut SessionLaunchSettings) {
167    if settings.charter.is_some() {
168        return;
169    }
170    settings.charter = Some(DEFAULT_LEAD_CHARTER.to_owned());
171}
172
173/// Stamp the project's `permission_mode` into the launch settings'
174/// `permissions.defaultMode`, where the existing
175/// `applied_permission_mode` arm in `forge_sdk_worker` picks it up;
176/// overrides a launcher-supplied default.
177pub(crate) fn stamp_permission_mode(
178    settings: &mut SessionLaunchSettings,
179    mode: forge_primitives::permission::PermissionMode,
180) {
181    let object =
182        settings.settings.get_or_insert_with(|| serde_json::Value::Object(serde_json::Map::new()));
183    let Some(map) = object.as_object_mut() else {
184        return;
185    };
186    let perms = map
187        .entry(SessionLaunchSettings::PERMISSIONS_KEY.to_owned())
188        .or_insert_with(|| serde_json::Value::Object(serde_json::Map::new()));
189    if let Some(perms) = perms.as_object_mut() {
190        perms.insert(
191            SessionLaunchSettings::PERMISSIONS_DEFAULT_MODE_KEY.to_owned(),
192            serde_json::Value::String(mode.as_wire().to_owned()),
193        );
194    }
195}
196
197/// Emit a `SessionUpdate` and log at debug when no subscriber took it
198/// (the view has not attached yet, or has gone). The send is logically
199/// best-effort - no caller can act on the failure - but visibility
200/// in the log distinguishes "no view took it" from "the emit never
201/// happened" during diagnosis.
202fn try_emit(workspace: &Workspace, label: &'static str, update: SessionUpdate) {
203    // Held before it goes out: a fatal error is an App-level event with no
204    // state behind it, so without this a view that was not subscribed when
205    // it fired can never learn of it at all.
206    if let SessionUpdate::FatalError { error, .. } = &update {
207        workspace.record_fatal_error(error.clone());
208    }
209    if !workspace.update_tx().send(update) {
210        tracing::debug!(
211            target: "forge_workspace::spawn",
212            label,
213            "SessionUpdate not taken - no subscriber attached"
214        );
215    }
216}
217
218/// Resolve the id the project's lead will run under, emit
219/// `SessionUpdate::Spawning` under it, then spawn the agent. The
220/// `Connected` event from the resulting `SessionTask` lands under the
221/// same key, so the announced bucket is the one the child connects
222/// under. A spawn that fails names the reason, which the heal's own
223/// supervisor counts as a failed attempt; a command dispatch drops it.
224pub(crate) fn handle_spawn_project(
225    workspace: &Arc<Workspace>,
226    project_name: &str,
227    launch_settings: SessionLaunchSettings,
228) -> Result<(), String> {
229    spawn_project_inner(workspace, project_name, launch_settings, true)
230}
231
232/// The heal's resume: the same spawn as [`handle_spawn_project`], but it
233/// cannot open a seat the reader closed. A heal recovers a death; it is
234/// never an opening door.
235pub(crate) fn handle_heal_project_spawn(
236    workspace: &Arc<Workspace>,
237    project_name: &str,
238    launch_settings: SessionLaunchSettings,
239) -> Result<(), String> {
240    spawn_project_inner(workspace, project_name, launch_settings, false)
241}
242
243fn spawn_project_inner(
244    workspace: &Arc<Workspace>,
245    project_name: &str,
246    mut launch_settings: SessionLaunchSettings,
247    reopen: bool,
248) -> Result<(), String> {
249    let Some(project) = workspace.find_project_view_by_name(project_name) else {
250        tracing::warn!(
251            target: "forge_workspace::spawn",
252            project = project_name,
253            "Command::SpawnProject for unknown project; ignoring"
254        );
255        try_emit(
256            workspace,
257            "spawn_project::unknown_project",
258            SessionUpdate::ServiceStatus {
259                severity: forge_primitives::cloud::service_status::ServiceSeverity::Warning,
260                message: format!("Unknown project: {project_name}"),
261            },
262        );
263        return Err(format!("Unknown project: {project_name}"));
264    };
265
266    apply_lead_charter(&mut launch_settings);
267
268    // The id this lead will run under, resolved before the spawn so the
269    // bucket announced here is the bucket the child connects under.
270    let session_key = match workspace.resolve_slot(&SessionTarget::Named(project_name.to_owned())) {
271        Ok(key) => key,
272        Err(err) => {
273            tracing::warn!(
274                target: "forge_workspace::spawn",
275                project = project_name,
276                error = %err,
277                "spawn_project: target resolution failed",
278            );
279            // No spawn will happen, so record what this project's lead
280            // has parked rather than leaving it for a connect that never
281            // comes.
282            workspace.expire_parked_for_slot(
283                &crate::SessionSlot::lead(&project.org, &project.name),
284                crate::mcp::peers::types::PeerFailureReason::TargetConnectionFailed,
285            );
286            let message = format!("agent spawn failed: {err}");
287            try_emit(
288                workspace,
289                "spawn_project::ConnectionFailed",
290                SessionUpdate::ConnectionFailed {
291                    key: SessionSlot::lead(&project.org, &project.name),
292                    message: message.clone(),
293                    fatal: false,
294                },
295            );
296            return Err(message);
297        }
298    };
299    try_emit(
300        workspace,
301        "spawn_project::Spawning",
302        SessionUpdate::Spawning {
303            key: session_key.clone(),
304            project_name: project_name.to_owned(),
305            cwd: project.path.to_string_lossy().to_string(),
306            display_name: project.display_path.clone(),
307        },
308    );
309
310    match workspace.get_agent_handle_at_key(
311        SessionTarget::Named(project_name.to_owned()),
312        launch_settings,
313        Some(session_key.clone()),
314        &crate::protocol::SpawnRole::Lead,
315        reopen,
316    ) {
317        Ok(_handle) => {
318            tracing::info!(
319                target: "forge_workspace::spawn",
320                project = project_name,
321                slot = %session_key.display(),
322                // The pool fast path returns a live handle without
323                // building a task, so `forge_sdk_options_built` is the
324                // per-subprocess signal.
325                "spawn dispatched for project"
326            );
327            // The lead's profile comes up with the session rather than
328            // paying the launch on its first browser call.
329            warm_browser_seat(workspace, &session_key);
330            Ok(())
331        }
332        Err(err) => {
333            // **A close that landed mid-spawn is the reader's stop, not a
334            // spawn failure.** Only the heal's non-opening spawn can be
335            // refused this way, and it must leave no failure record or
336            // Failed row for the project the reader just shut.
337            if matches!(
338                err.downcast_ref::<crate::error::WorkspaceError>(),
339                Some(crate::error::WorkspaceError::SeatClosedDuringSpawn { .. })
340            ) {
341                tracing::debug!(
342                    target: "forge_workspace::spawn",
343                    project = project_name,
344                    slot = %session_key.display(),
345                    "the seat was closed while its recovery spawn was in flight; the stop stands"
346                );
347                return Ok(());
348            }
349            tracing::error!(
350                target: "forge_workspace::spawn",
351                project = project_name,
352                error = %err,
353                "spawn_project: get_agent_handle failed"
354            );
355            // No SessionTask exists to run its ConnectionFailed arm, so
356            // record everything parked for this project's lead here -
357            // otherwise the caller's LLM waits on a spawn that never
358            // happened, and a committed delivery is lost unannounced.
359            workspace.expire_parked_for_slot(
360                &crate::SessionSlot::lead(&project.org, &project.name),
361                crate::mcp::peers::types::PeerFailureReason::TargetConnectionFailed,
362            );
363            // The only record of the failure for the same reason: a view
364            // reading the slot afterwards would otherwise see a project
365            // that never started and no way to say why.
366            let message = format!("agent spawn failed: {err}");
367            workspace.record_spawn_failure(&session_key, &message);
368            try_emit(
369                workspace,
370                "spawn_project::ConnectionFailed",
371                SessionUpdate::ConnectionFailed {
372                    key: session_key,
373                    message: message.clone(),
374                    fatal: false,
375                },
376            );
377            Err(message)
378        }
379    }
380}
381
382/// Handle a `Command::DeliverPeerPrompt`. Resolves the target
383/// project to a running SessionTask (deliver immediately) or a
384/// sleeping one (buffer + auto-spawn), then dispatches the wrapped
385/// prompt as a regular `Command::Prompt`.
386///
387/// The answer is this handler's own: `Ok(())` once the prompt was accepted
388/// by the running session or parked for a spawn it kicked off, `Err(reason)`
389/// when nothing could take it. The caller's ack is built from it, so a
390/// delivery that never left forge cannot read as sent (#2342).
391pub(crate) fn handle_deliver_peer_prompt(
392    workspace: &Arc<Workspace>,
393    caller: &SessionSlot,
394    target_project: String,
395    wrapped: WrappedPrompt,
396) -> Result<(), String> {
397    // Find the target project's running lead session (if any). The
398    // `list_projects()` snapshot has `sessions: Vec<SessionView>` per
399    // project; `is_open == true` on a session means an Agent is in
400    // the workspace pool - that's "running." MUST skip worker
401    // sessions: once a worker connects it lands in `view.sessions`
402    // too, and a worker can sit at position 0 / be the first
403    // is_open session. Returning a worker here dispatches the peer
404    // envelope to the worker's chat instead of the lead's, which is
405    // wrong (peers address project leads, not workers). Asking for the
406    // lead slot by name cannot reach a worker: the label is part of the
407    // slot, so the two are distinct keys rather than candidates to
408    // subtract.
409    let target_running_key =
410        workspace.list_projects().into_iter().find(|v| v.name == target_project).and_then(|v| {
411            let slot = SessionSlot::lead(&v.org, &v.name);
412            workspace.session_is_pooled(&slot).then_some(slot)
413        });
414
415    if let Some(target_key) = target_running_key {
416        // Fire the typed peer-envelope echo BEFORE the LLM-side
417        // dispatch so the user-turn block renders in the right
418        // order regardless of which event a view reducer drains
419        // first. The CLI doesn't echo stdin-injected prompts back
420        // on stream-json output (only tool_result-bearing user
421        // envelopes come back), so a view gets no inbound user-turn
422        // signal from the SDK side - `PeerEnvelopeAppended` is how
423        // a view knows to render the peer block.
424        let uuid = forge_sdk::request_id::next_prompt_id();
425        push_peer_user_turn_into_chat(workspace, &target_key, &wrapped, &uuid);
426        let text = wrapped.to_prose();
427        // The peer path keeps the sender: a swap's hold answers that seat if
428        // the handoff never reaches a successor.
429        if let Err(err) = workspace.dispatch_peer_prompt_under(caller, &target_key, text, uuid) {
430            tracing::warn!(
431                target: "forge_workspace::spawn",
432                target_project = %target_project,
433                error = ?err,
434                "DeliverPeerPrompt dispatch to running target failed"
435            );
436            send_dispatch_turn_error(workspace, target_key, &err);
437            return Err(format!("the target's running session did not take the message ({err})"));
438        }
439        return Ok(());
440    }
441
442    // Target is sleeping (or unknown - defensive). If the project
443    // exists in forge.toml, park the envelope for its lead and dispatch
444    // SpawnProject.
445    let Some(target) = workspace.find_project_view_by_name(&target_project) else {
446        tracing::warn!(
447            target: "forge_workspace::spawn",
448            target_project = %target_project,
449            "DeliverPeerPrompt target not in forge.toml; dropping"
450        );
451        return Err(format!(
452            "project '{target_project}' is not in forge.toml any more, so the message went nowhere"
453        ));
454    };
455
456    // The lead's own first `Connected` drains the bucket, so the park
457    // must land BEFORE the spawn. The sender rides beside the payload so a
458    // spawn that never connects can acknowledge the message back to it.
459    let target_key = crate::SessionSlot::lead(&target.org, &target.name);
460    let message_id = wrapped.id.clone();
461    workspace.park_peer_prompt(&target_key, caller, wrapped);
462
463    // Dispatch SpawnProject. Move target_project
464    // into the command rather than cloning (it's the last use).
465    let project_for_log = target_project.clone();
466    if let Err(err) = workspace.dispatch(Command::SpawnProject {
467        project_name: target_project,
468        launch_settings: SessionLaunchSettings::default(),
469    }) {
470        tracing::warn!(
471            target: "forge_workspace::spawn",
472            target_project = %project_for_log,
473            error = ?err,
474            "DeliverPeerPrompt SpawnProject dispatch failed"
475        );
476        return Err(format!(
477            "the target's project could not be woken ({err}); the parked message stays with it"
478        ));
479    }
480    // **The dispatch's Ok is not the wake's.** `handle_spawn_project` runs
481    // inline and can fail, and its own arm expires the bucket it just parked;
482    // a message that is no longer parked was not accepted, whatever the
483    // command returned.
484    if !workspace.peer_prompt_is_parked(&target_key, &message_id) {
485        return Err(format!(
486            "project '{}' could not be woken, so the parked message was returned to you",
487            target.org,
488        ));
489    }
490    Ok(())
491}
492
493/// Outcome of a cron fire's delivery, so `fire_due_crons` can decide the
494/// entry's fate rather than treating every hand-off as a success.
495pub(crate) enum CronFireOutcome {
496    /// Dispatched into a running session, or buffered + a spawn kicked off.
497    /// The caller advances/removes the cron as normal.
498    Delivered,
499    /// The cron's project is no longer in forge.toml. The caller removes
500    /// the entry instead of advancing a dead cron forever.
501    TargetGone,
502    /// The owner still has a row, but the boot wave would skip it, so
503    /// nothing can drain a prompt buffered for it and the prompt cannot
504    /// reach anyone. `directory` is where its session would have started,
505    /// for the warning. Distinct from [`Self::TargetGone`] because the
506    /// row is kept: a restored worktree brings the owner back.
507    ///
508    /// The caller's fate for the entry splits on the kind. A recurring
509    /// cron drops this fire and advances - leaving it due would re-fire
510    /// every tick, and parking it would grow a bucket nothing drains. A
511    /// one-shot stays due instead, because advancing removes a one-shot:
512    /// dropping it would throw the prompt away with no later slot.
513    TargetCannotBeWoken { directory: std::path::PathBuf },
514    /// Every account serving the project's model is cooling until the
515    /// carried reset: the caller leaves the entry due, because a refusal
516    /// right now would expire the parked prompt.
517    DeferredCooling { cooling: crate::workspace::ProjectCooling },
518    /// The Command channel is closed (workspace shutting down). The caller
519    /// leaves the cron due so the next boot catch-up re-fires it.
520    DispatchFailed,
521    /// The owner is asleep and this cron may not wake it (`wake: false`).
522    /// The caller leaves the entry due without advancing, so the next
523    /// session start (or a tick while the owner is awake) delivers it.
524    SkippedAsleep,
525}
526
527/// Text delivered for a cron fire: the raw prompt, prefixed with a plain
528/// marker when the fire is overdue so the owner can tell a catch-up from
529/// an on-time fire.
530pub(crate) fn missed_cron_text(prompt: &str, missed: bool) -> String {
531    if missed { format!("[missed cron] {prompt}") } else { prompt.to_owned() }
532}
533
534/// Deliver a due cron's prompt into its OWNER's session as a plain user
535/// turn AND echo it as a cron block. `team_role` `None` routes to the
536/// project lead, `Some(label)` to that worker. A live owner gets a
537/// `Command::Prompt`; an asleep-but-existing owner is woken by dispatching
538/// `Command::SpawnProject` (which resumes the lead and, through the lead's
539/// respawn, the worker), the prompt buffered by `(project, team_role)`
540/// and drained on the owner's connect. An owner that no longer exists (a
541/// gone project, or a worker label with no persisted row) yields
542/// `TargetGone`.
543pub(crate) fn deliver_cron_prompt(
544    workspace: &Arc<Workspace>,
545    cron: &forge_primitives::CronEntry,
546    missed: bool,
547) -> CronFireOutcome {
548    let project_name = cron.project_name.as_str();
549    let team_role = cron.team_role.as_deref();
550    let Some(view) = workspace.list_projects().into_iter().find(|v| v.name == project_name) else {
551        return CronFireOutcome::TargetGone;
552    };
553
554    if let Some(target_key) = live_cron_slot(workspace, &view, team_role) {
555        // Echo the cron block BEFORE the LLM-side dispatch so it renders in
556        // order regardless of which event a view reducer drains first. The id
557        // is minted here so the block's row and the dispatched prompt's
558        // lifecycle frames are one thing by id.
559        let text = missed_cron_text(&cron.prompt, missed);
560        let uuid = forge_sdk::request_id::next_prompt_id();
561        push_cron_prompt_into_chat(
562            workspace,
563            &target_key,
564            &text,
565            &uuid,
566            cron.id.as_str(),
567            cron.description.as_deref(),
568        );
569        return match workspace.dispatch_workspace_prompt_under(
570            &target_key,
571            text,
572            PromptSource::Cron,
573            uuid,
574        ) {
575            Ok(()) => CronFireOutcome::Delivered,
576            Err(err) => {
577                tracing::warn!(
578                    target: "forge_workspace::spawn",
579                    project = %project_name,
580                    error = ?err,
581                    "cron fire dispatch to live owner failed",
582                );
583                send_dispatch_turn_error(workspace, target_key, &err);
584                CronFireOutcome::DispatchFailed
585            }
586        };
587    }
588
589    // Asleep: only wake an owner we can confirm still exists. A conclusive
590    // absence removes the cron; an owner check that could not read leaves it
591    // for the next tick rather than deleting a real owner's cron on a hiccup.
592    match cron_slot_exists(workspace, &view, team_role) {
593        CronOwnerCheck::Exists => {}
594        CronOwnerCheck::CannotBeWoken { directory } => {
595            if cron.wake {
596                return CronFireOutcome::TargetCannotBeWoken { directory };
597            }
598        }
599        CronOwnerCheck::Absent => return CronFireOutcome::TargetGone,
600        CronOwnerCheck::Unknown => {
601            if cron.wake {
602                return CronFireOutcome::DispatchFailed;
603            }
604        }
605    }
606    // No live slot and no wake: skip the tick and leave the entry due, so
607    // the next session start runs it. A `wake: false` cron asks for this.
608    if !cron.wake {
609        return CronFireOutcome::SkippedAsleep;
610    }
611    // A spawn needs the account map settled: the walk skips every account
612    // still `Loading`, so a project whose only account for its model has
613    // not settled is refused at the walk, and that refusal is transient
614    // where a dead target is not. Deferring the fire is a delay rather
615    // than a drop, because the next tick retries once the map settles;
616    // parking now would have the spawn refusal expire the prompt instead.
617    //
618    // This is NOT a boot gate, and do not simplify it into one. The two
619    // halves are different kinds of thing: `gateway_ready` is bind-time
620    // only and monotonic, while `all_loaded` is defined over the account
621    // map, so it goes false again whenever anything re-enters `Loading`.
622    // A fire deferred by the second half is a delay for the same reason
623    // as the first, not a drop.
624    if !workspace.all_accounts_loaded() {
625        tracing::warn!(
626            target: "forge_workspace::spawn",
627            project = %project_name,
628            "cron fire deferred: the account map has not settled, so a spawn would be refused \
629             and its parked prompt expired",
630        );
631        return CronFireOutcome::DispatchFailed;
632    }
633    // A cooldown empties the walk until its reset, which is transient the
634    // same way an unsettled map is, and a refusal there expires the park
635    // too. A walk empty because no account declares the model is the other
636    // case, and that one is permanent - deferring it would retry a broken
637    // cron forever, so it is left to fail as before.
638    if let Some(cooling) = workspace.project_walk_cooling(&view.key) {
639        return CronFireOutcome::DeferredCooling { cooling };
640    }
641    // Buffer by owner, then wake via resume: SpawnProject resumes the lead,
642    // whose reconnect re-spawns the persisted workers; each drains its own
643    // bucket on connect.
644    let slot = crate::SessionSlot::for_label(&view.org, &view.name, team_role);
645    workspace.park_cron(&slot, cron, missed);
646    match workspace.dispatch(Command::SpawnProject {
647        project_name: project_name.to_owned(),
648        launch_settings: SessionLaunchSettings::default(),
649    }) {
650        Ok(()) => CronFireOutcome::Delivered,
651        Err(err) => {
652            tracing::warn!(
653                target: "forge_workspace::spawn",
654                project = %project_name,
655                error = ?err,
656                "cron fire SpawnProject dispatch failed",
657            );
658            CronFireOutcome::DispatchFailed
659        }
660    }
661}
662
663/// The live session that fills the cron's slot in `view`: for a lead
664/// cron the running lead (the first open session that is not a live
665/// worker); for a worker cron the live worker with that label. `None`
666/// when no session holds the slot.
667fn live_cron_slot(
668    workspace: &Arc<Workspace>,
669    view: &crate::views::ProjectView,
670    team_role: Option<&str>,
671) -> Option<SessionSlot> {
672    let live = workspace.list_live_workers(&view.key);
673    let candidate = if let Some(label) = team_role {
674        // **`live_worker_with_label`, not a bare label match**: a `Failed`
675        // entry is kept by design and is NOT live, so a bare match picks a
676        // slot whose session failed and drops the prompt into it.
677        // `cron_slot_exists` beside this applies the same rule (#1208's
678        // class, one resolver over).
679        crate::mcp::workers::types::live_worker_with_label(&live, label).map(|w| w.slot.clone())
680    } else {
681        let slot = SessionSlot::lead(&view.org, &view.name);
682        workspace.session_is_pooled(&slot).then_some(slot)
683    };
684    // Only treat the owner as a live dispatch target once it has stamped
685    // its session_id. A still-spawning owner (session_id None) would drop
686    // a bare Command::Prompt, so fall through to the buffer-and-wake path
687    // where its own Connected handler drains the owner-keyed buffer.
688    let key = candidate?;
689    workspace.domain_session_for(&key).filter(|d| d.lock().session_id.is_some()).map(|_| key)
690}
691
692/// Outcome of checking whether a cron's owner still exists to be woken.
693enum CronOwnerCheck {
694    /// The owner exists (the lead, or a worker with a persisted row).
695    Exists,
696    /// The owner has a row, but the boot wave would skip it - a resume
697    /// would have no directory to start in - so there is nothing to wake
698    /// and nothing that would drain a parked prompt. Carries that
699    /// directory for the fire's warning.
700    CannotBeWoken { directory: std::path::PathBuf },
701    /// Conclusively gone: the read succeeded and the label has no row in
702    /// the session store.
703    Absent,
704    /// The durable-worker lookup could not read, so absence is unconfirmed.
705    Unknown,
706}
707
708/// Whether a cron's slot still has a session that can be woken. A lead's
709/// does whenever its project does; a worker's does while its label has a
710/// row AND that row can still start, since the boot wave is what
711/// re-spawns it. A read failure yields [`CronOwnerCheck::Unknown`] so the
712/// fire router leaves the cron rather than deleting a live slot's cron on
713/// a transient hiccup.
714fn cron_slot_exists(
715    workspace: &Arc<Workspace>,
716    view: &crate::views::ProjectView,
717    team_role: Option<&str>,
718) -> CronOwnerCheck {
719    let Some(label) = team_role else {
720        return CronOwnerCheck::Exists;
721    };
722    // A worker spawned this second is live before it is connected, and
723    // `live_cron_slot` cannot address it until it stamps a session id on
724    // connect. Its directory is being created along with it, so it is not
725    // unwakeable: the fire parks and the worker's own Connected drains it.
726    //
727    // `live_worker_with_label`, not a bare label match: a `Failed` entry
728    // is kept by design and is NOT live, so counting one would answer
729    // "wakeable" for a worker the wave will never start - parking the fire
730    // in the bucket this path exists to keep empty, with nothing left to
731    // drain it.
732    if crate::mcp::workers::types::live_worker_with_label(
733        &workspace.list_live_workers(&view.key),
734        label,
735    )
736    .is_some()
737    {
738        return CronOwnerCheck::Exists;
739    }
740    match workspace.stored_worker_row(&view.key, label) {
741        Ok(None) => CronOwnerCheck::Absent,
742        Ok(Some(row)) => {
743            // The tree this fire would wake the worker in is the one the
744            // ROW records: a worker pointed at another directory starts
745            // there, so its worktree (and its absence) is read there.
746            let root = row.cwd.as_deref().map_or(view.path.as_path(), std::path::Path::new);
747            let directory = crate::mcp::workers::types::worker_tag_dir(
748                root,
749                &row.label,
750                matches!(row.is_git_repo, Some(true)),
751            );
752            let can_start = crate::mcp::workers::types::worker_row_can_start(
753                root,
754                &row.label,
755                row.is_git_repo,
756                row.session_id.is_some(),
757            );
758            if can_start {
759                CronOwnerCheck::Exists
760            } else {
761                CronOwnerCheck::CannotBeWoken { directory }
762            }
763        }
764        Err(_) => CronOwnerCheck::Unknown,
765    }
766}
767
768/// Deliver a matched Gotify `notification` into `project` as a plain user
769/// turn AND echo it into the target's chat as a notification block. When
770/// `team_role` names a running team worker, deliver straight to it;
771/// otherwise deliver to the project lead - echo + dispatch a
772/// `Command::Prompt` if it's running, else park it for the target's slot
773/// and dispatch `Command::SpawnProject`
774/// (`SessionTask` drains + echoes on Connected via
775/// `deliver_parked_gotify`). A team-worker subscription with NO live entry
776/// falls through to lead delivery (spawning the project brings the team
777/// up); a live-but-not-yet-connected worker parks for its own label
778/// instead. A project no longer in forge.toml is logged and skipped.
779pub(crate) fn deliver_gotify_message(
780    workspace: &Arc<Workspace>,
781    project: &str,
782    team_role: Option<&str>,
783    notification: GotifyNotification,
784) {
785    // A team-worker subscription targets that worker. Once it's a live
786    // entry the notification belongs to it, never the lead - so handle
787    // both the connected case (dispatch now) and the still-spawning case
788    // (buffer on the worker's own DomainSession for its Connected drain)
789    // here, and never fall through to the lead below.
790    if let Some(role) = team_role
791        && let Some(worker_key) = team_worker_key(workspace, project, role)
792    {
793        let connected = workspace
794            .domain_session_for(&worker_key)
795            .is_some_and(|d| d.lock().session_id.is_some());
796        if connected {
797            // Echo the notification block BEFORE the LLM-side dispatch so
798            // it renders in order regardless of which event a view reducer
799            // drains first (mirrors handle_deliver_peer_prompt).
800            let uuid = forge_sdk::request_id::next_prompt_id();
801            push_gotify_notification_into_chat(workspace, &worker_key, &notification, &uuid);
802            if let Err(err) = workspace.dispatch_workspace_prompt_under(
803                &worker_key,
804                notification.to_prose(),
805                PromptSource::Gotify,
806                uuid,
807            ) {
808                tracing::warn!(
809                    target: "forge_workspace::spawn",
810                    project = %project,
811                    role = %role,
812                    error = ?err,
813                    "gotify deliver to running team worker failed",
814                );
815                send_dispatch_turn_error(workspace, worker_key, &err);
816            }
817        } else {
818            // Still spawning: park it for the worker's slot, drained by
819            // its own first `Connected`.
820            workspace.park_gotify(&worker_key, notification);
821        }
822        return;
823    }
824
825    let running_lead =
826        workspace.list_projects().into_iter().find(|v| v.name == project).and_then(|v| {
827            let slot = SessionSlot::lead(&v.org, &v.name);
828            workspace.session_is_pooled(&slot).then_some(slot)
829        });
830
831    if let Some(target_key) = running_lead {
832        let uuid = forge_sdk::request_id::next_prompt_id();
833        push_gotify_notification_into_chat(workspace, &target_key, &notification, &uuid);
834        if let Err(err) = workspace.dispatch_workspace_prompt_under(
835            &target_key,
836            notification.to_prose(),
837            PromptSource::Gotify,
838            uuid,
839        ) {
840            tracing::warn!(
841                target: "forge_workspace::spawn",
842                project = %project,
843                error = ?err,
844                "gotify deliver to running project failed",
845            );
846            send_dispatch_turn_error(workspace, target_key, &err);
847        }
848        return;
849    }
850
851    // Asleep: park it for the project's lead and spawn the project (only
852    // if it's a real forge.toml project).
853    let Some(view) = workspace.find_project_view_by_name(project) else {
854        tracing::warn!(
855            target: "forge_workspace::spawn",
856            project = %project,
857            "gotify delivery target gone from forge.toml; skipping",
858        );
859        return;
860    };
861
862    workspace.park_gotify(&crate::SessionSlot::lead(&view.org, &view.name), notification);
863
864    if let Err(err) = workspace.dispatch(Command::SpawnProject {
865        project_name: project.to_owned(),
866        launch_settings: SessionLaunchSettings::default(),
867    }) {
868        tracing::warn!(
869            target: "forge_workspace::spawn",
870            project = %project,
871            error = ?err,
872            "gotify fire SpawnProject dispatch failed",
873        );
874    }
875}
876
877/// The user-turn prose for one conversation's delivered Slack messages. It
878/// carries the ids a reply needs - conversation, each member's own ts, and
879/// its thread - because the only way an agent can answer in place is to feed
880/// those back to `slack__post` or `slack__edit`.
881/// `forge_server::envelope::detect_inbound` parses this shape into a Slack
882/// envelope and a view renders that as a block,
883/// so the bracketed header, its optional age clause, its member count and the
884/// one-line-per-member body are a contract with the parser.
885pub(crate) fn slack_bundle_to_prose(
886    messages: &[SlackMessage],
887    stale_after: Duration,
888    now: SystemTime,
889) -> String {
890    let Some(head) = messages.first() else { return String::new() };
891    let newest = messages.iter().map(|message| message.ts.as_str()).max().unwrap_or(&head.ts);
892    let mut out = format!(
893        "[Slack - workspace '{}', {}] id {} ts {newest}",
894        head.workspace, head.conversation_label, head.conversation,
895    );
896    if let Some(age) = stale_age_clause(newest, stale_after, now) {
897        out.push_str(&age);
898    }
899    if messages.len() > 1 {
900        let _ = write!(out, " ({} messages)", messages.len());
901    }
902    for message in in_thread_order(messages) {
903        out.push('\n');
904        out.push_str(&member_line(message));
905    }
906    out
907}
908
909const SECONDS_PER_DAY: u64 = 24 * 60 * 60;
910
911/// The `, 8d old` clause a header carries when its newest message is at or
912/// past the workspace's stale age. Zero turns the mark off; a future or
913/// unparseable ts carries none.
914fn stale_age_clause(newest: &str, stale_after: Duration, now: SystemTime) -> Option<String> {
915    if stale_after.is_zero() {
916        return None;
917    }
918    let seconds: u64 =
919        newest.split_once('.').map_or(newest, |(seconds, _)| seconds).parse().ok()?;
920    let now = now.duration_since(SystemTime::UNIX_EPOCH).ok()?.as_secs();
921    let age = now.checked_sub(seconds)?;
922    if age < stale_after.as_secs() {
923        return None;
924    }
925    Some(if age >= 2 * SECONDS_PER_DAY {
926        format!(", {}d old", age / SECONDS_PER_DAY)
927    } else {
928        format!(", {}h old", age / 3_600)
929    })
930}
931
932/// The bundle's members in reading order: oldest first, with a thread's
933/// replies directly under the parent they answer.
934fn in_thread_order(messages: &[SlackMessage]) -> Vec<&SlackMessage> {
935    let mut grouped: Vec<(String, &SlackMessage)> = messages
936        .iter()
937        .map(|message| {
938            // A reply whose parent is not in the bundle keeps its place in
939            // time: the parent ages out of the sweep window while the
940            // thread keeps moving.
941            let key = match &message.thread_ts {
942                Some(parent) if messages.iter().any(|other| &other.ts == parent) => parent.clone(),
943                _ => message.ts.clone(),
944            };
945            (key, message)
946        })
947        .collect();
948    grouped.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.ts.cmp(&b.1.ts)));
949    grouped.into_iter().map(|(_, message)| message).collect()
950}
951
952/// One bundle member. The `<author>: ` clause is what
953/// `forge_server::envelope::detect_inbound` keys on, so a member always
954/// carries one. A resolved name is preferred; the raw id is only a fallback
955/// the block drops, since an id is never a name to print.
956fn member_line(message: &SlackMessage) -> String {
957    let author = message.author.as_deref().or(message.user.as_deref()).unwrap_or("unknown");
958    let mut out = format!("{author}: {} [ts {}", message.text, message.ts);
959    // A parent carries its own ts as `thread_ts`, and the ts above already
960    // answers into that thread, so only a real parent's reply says so.
961    if let Some(thread) = message.thread_ts.as_deref()
962        && thread != message.ts
963    {
964        let _ = write!(out, " in thread {thread}");
965    }
966    // An edited message keeps its ts, so a correction must be marked or it reads as a duplicate.
967    if message.edited.is_some() {
968        out.push_str(" edited");
969    }
970    out.push(']');
971    for file in &message.files {
972        let _ = write!(out, " [file {} {}]", file.id, file.name);
973    }
974    out
975}
976
977/// Deliver one conversation's matched Slack messages to its subscriber's
978/// session: dispatch them now when that session is running, buffer them on
979/// the session's own domain when it is still spawning. Mirrors
980/// [`deliver_gotify_message`], including its rule that a worker-owned
981/// subscription falls through to the lead only when the worker is gone
982/// entirely: the despawn race (teardown removed its subscriptions first),
983/// or a `Failed` entry, which is kept by design and is not live - a bucket
984/// parked for one would never drain. The fall-through commits under the
985/// worker's own dedupe key, so the batch counts as delivered and stays
986/// with the lead: a durable worker that later respawns does not
987/// re-receive it.
988///
989/// Returns whether the batch reached a destination it can be read from:
990/// dispatched, or buffered for one that will. `false` tells the pump the
991/// cursor must not advance past this batch, so a sweep re-runs it - which is
992/// why the dedupe entries commit only after a successful hand-off.
993pub(crate) fn deliver_slack_message(
994    workspace: &Arc<Workspace>,
995    project: &str,
996    team_role: Option<&str>,
997    messages: Vec<SlackMessage>,
998) -> bool {
999    // A sweep re-runs a batch after a 429, a failed watermark write or a
1000    // crash; the re-run must drop what was already handed over. "Already
1001    // delivered" is success for the caller: the cursor may advance.
1002    let Some(head) = messages.first() else { return true };
1003    let conversation = head.conversation.clone();
1004    let workspace_label = head.workspace.clone();
1005    let pending: Vec<SlackMessage> = messages
1006        .into_iter()
1007        .filter(|message| !workspace.slack_delivery_seen(project, team_role, message))
1008        .collect();
1009    if pending.is_empty() {
1010        tracing::debug!(
1011            target: "forge_workspace::spawn",
1012            project = %project,
1013            conversation = %conversation,
1014            "every message in this slack batch was already delivered; dropping the re-run",
1015        );
1016        return true;
1017    }
1018    let prose = slack_bundle_to_prose(
1019        &pending,
1020        workspace.slack_stale_after(&workspace_label),
1021        SystemTime::now(),
1022    );
1023
1024    if let Some(role) = team_role
1025        && let Some(worker_key) = team_worker_key(workspace, project, role)
1026    {
1027        let connected = workspace
1028            .domain_session_for(&worker_key)
1029            .is_some_and(|d| d.lock().session_id.is_some());
1030        if connected {
1031            let uuid = forge_sdk::request_id::next_prompt_id();
1032            if let Err(err) = workspace.dispatch_workspace_prompt_under(
1033                &worker_key,
1034                prose.clone(),
1035                PromptSource::Slack,
1036                uuid.clone(),
1037            ) {
1038                tracing::warn!(
1039                    target: "forge_workspace::spawn",
1040                    project = %project,
1041                    role = %role,
1042                    error = ?err,
1043                    "slack deliver to running team worker failed",
1044                );
1045                send_dispatch_turn_error(workspace, worker_key, &err);
1046                return false;
1047            }
1048            // Echo only once the dispatch lands: a failed one returns false so
1049            // the sweep re-runs the batch, and an echo pushed before it would
1050            // paint the block twice for a turn the LLM sees once.
1051            push_slack_message_into_chat(workspace, &worker_key, &prose, &uuid);
1052            workspace.slack_delivery_commit(project, team_role, &pending);
1053        } else {
1054            // Still spawning: commit the dedupe (the sweep must not
1055            // re-run it) and park it for the worker's slot, drained by
1056            // its own first `Connected`. The slot carries the worker's
1057            // own org and project, so a project dropped from forge.toml
1058            // since its spawn still keys correctly.
1059            workspace.slack_delivery_commit(project, team_role, &pending);
1060            workspace.park_slack(&worker_key, pending);
1061        }
1062        return true;
1063    }
1064
1065    let running_lead =
1066        workspace.list_projects().into_iter().find(|v| v.name == project).and_then(|v| {
1067            let slot = SessionSlot::lead(&v.org, &v.name);
1068            workspace.session_is_pooled(&slot).then_some(slot)
1069        });
1070
1071    if let Some(target_key) = running_lead {
1072        let uuid = forge_sdk::request_id::next_prompt_id();
1073        if let Err(err) = workspace.dispatch_workspace_prompt_under(
1074            &target_key,
1075            prose.clone(),
1076            PromptSource::Slack,
1077            uuid.clone(),
1078        ) {
1079            tracing::warn!(
1080                target: "forge_workspace::spawn",
1081                project = %project,
1082                error = ?err,
1083                "slack deliver to running project failed",
1084            );
1085            send_dispatch_turn_error(workspace, target_key, &err);
1086            return false;
1087        }
1088        // Echo after the dispatch lands, so the sweep's re-run of a failed
1089        // delivery does not paint a second block for the same batch.
1090        push_slack_message_into_chat(workspace, &target_key, &prose, &uuid);
1091        workspace.slack_delivery_commit(project, team_role, &pending);
1092        return true;
1093    }
1094
1095    // Asleep: park it for the project's lead and spawn the project (only
1096    // if it's a real forge.toml project). A target missing from
1097    // forge.toml returns false uncommitted: the sweep re-runs it, and
1098    // delivery resumes if the project returns or the subscription is
1099    // removed - the pump sees a decision, not a silent drop.
1100    let Some(view) = workspace.find_project_view_by_name(project) else {
1101        tracing::warn!(
1102            target: "forge_workspace::spawn",
1103            project = %project,
1104            "slack delivery target gone from forge.toml; leaving it for the next sweep",
1105        );
1106        return false;
1107    };
1108
1109    workspace.slack_delivery_commit(project, team_role, &pending);
1110    workspace.park_slack(&crate::SessionSlot::lead(&view.org, &view.name), pending);
1111
1112    if let Err(err) = workspace.dispatch(Command::SpawnProject {
1113        project_name: project.to_owned(),
1114        launch_settings: SessionLaunchSettings::default(),
1115    }) {
1116        tracing::warn!(
1117            target: "forge_workspace::spawn",
1118            project = %project,
1119            error = ?err,
1120            "slack fire SpawnProject dispatch failed",
1121        );
1122    }
1123    true
1124}
1125
1126/// The team worker labelled `label` in `project`, if a LIVE entry
1127/// exists - regardless of whether it has finished connecting. The
1128/// caller checks connectedness to decide dispatch-vs-buffer.
1129///
1130/// A `Failed` entry is kept by design (its tombstone row stays
1131/// visible) but is not live: the worker has no session to drain a
1132/// bucket and no spawn coming to make one, so it is not a delivery
1133/// target - parking for its slot is the bucket nothing drains, whose
1134/// contents land as fresh turns whenever the worker is next opened.
1135fn team_worker_key(workspace: &Arc<Workspace>, project: &str, label: &str) -> Option<SessionSlot> {
1136    let view = workspace.list_projects().into_iter().find(|v| v.name == project)?;
1137    crate::mcp::workers::types::live_worker_with_label(
1138        &workspace.list_live_workers(&view.key),
1139        label,
1140    )
1141    .map(|w| w.slot.clone())
1142}
1143
1144/// Emit a typed `PeerEnvelopeAppended` so the target session's
1145/// chat buffer shows the inbound peer user-turn. A view's reducer
1146/// builds the chat-side echo directly from the `WrappedPrompt`'s
1147/// typed fields - workspace no longer forges an SDK `Message::User`
1148/// frame (audit I11).
1149///
1150/// Note: this only affects the visible chat echo. The
1151/// recipient's `claude` subprocess still receives the prose via a
1152/// separate `Command::Prompt` dispatch - the CLI's input channel
1153/// is text-shaped and stays that way.
1154pub(crate) fn push_peer_user_turn_into_chat(
1155    workspace: &Workspace,
1156    target_key: &SessionSlot,
1157    wrapped: &WrappedPrompt,
1158    uuid: &str,
1159) {
1160    let _ = workspace.update_sender().send(SessionUpdate::PeerEnvelopeAppended {
1161        key: target_key.clone(),
1162        wrapped: wrapped.clone(),
1163        uuid: uuid.to_owned(),
1164    });
1165}
1166
1167/// Emit a typed `GotifyNotificationAppended` so the target session's
1168/// chat buffer shows the inbound notification block. Mirrors
1169/// [`push_peer_user_turn_into_chat`] - the target's `claude` subprocess
1170/// still receives the prose via a separate `Command::Prompt` dispatch;
1171/// this only drives the visible chat echo.
1172pub(crate) fn push_gotify_notification_into_chat(
1173    workspace: &Workspace,
1174    target_key: &SessionSlot,
1175    notification: &GotifyNotification,
1176    uuid: &str,
1177) {
1178    let _ = workspace.update_sender().send(SessionUpdate::GotifyNotificationAppended {
1179        key: target_key.clone(),
1180        notification: notification.clone(),
1181        uuid: uuid.to_owned(),
1182    });
1183}
1184
1185/// Emit a typed `CronPromptAppended` so the target session's chat
1186/// buffer shows a cron block for the fired prompt. Mirrors
1187/// [`push_gotify_notification_into_chat`] - the target's `claude`
1188/// subprocess still receives the raw prompt via a separate
1189/// `Command::Prompt`; this only drives the visible chat echo.
1190pub(crate) fn push_cron_prompt_into_chat(
1191    workspace: &Workspace,
1192    target_key: &SessionSlot,
1193    text: &str,
1194    uuid: &str,
1195    cron_id: &str,
1196    description: Option<&str>,
1197) {
1198    // The frame names the schedule once, and the row outlives it: keep the
1199    // pairing so the read path can title the same row after a reload.
1200    workspace.remember_cron_name(target_key, uuid, cron_id, description);
1201    let _ = workspace.update_sender().send(SessionUpdate::CronPromptAppended {
1202        key: target_key.clone(),
1203        text: text.to_owned(),
1204        uuid: uuid.to_owned(),
1205        cron_id: cron_id.to_owned(),
1206        description: description.map(str::to_owned),
1207    });
1208}
1209
1210/// Emit a typed `SlackMessageAppended` so the target session's chat buffer
1211/// shows the inbound Slack block. Mirrors
1212/// [`push_gotify_notification_into_chat`] - the target's `claude` subprocess
1213/// still receives the prose via a separate `Command::Prompt` dispatch; this
1214/// only drives the visible chat echo.
1215pub(crate) fn push_slack_message_into_chat(
1216    workspace: &Workspace,
1217    target_key: &SessionSlot,
1218    prose: &str,
1219    uuid: &str,
1220) {
1221    let _ = workspace.update_sender().send(SessionUpdate::SlackMessageAppended {
1222        key: target_key.clone(),
1223        prose: prose.to_owned(),
1224        uuid: uuid.to_owned(),
1225    });
1226}
1227
1228/// The caller-facing refusal for an at-cap worker spawn. One source so
1229/// the classifier pin in the facade tests tracks the real text.
1230pub(crate) fn worker_limit_reached_message(project: &str, live: usize, cap: usize) -> String {
1231    let workers = if live == 1 { "worker" } else { "workers" };
1232    format!(
1233        "worker limit reached: project '{project}' has {live} {workers} live and its cap is {cap} (the project's max_workers in forge.toml, default {}); despawn one first, or raise/remove max_workers",
1234        crate::config::DEFAULT_MAX_WORKERS_PER_PROJECT
1235    )
1236}
1237
1238/// The refusal for a spawn whose tree resolves to a project `forge.toml`
1239/// declares (D3). One source: the facade refuses with it before it
1240/// ensures anything, and the handler refuses with it as the shared core
1241/// both dispatch sources pass through.
1242///
1243/// The sentence interpolates a project NAME, which is whatever a project
1244/// is called - "worktree" included - and the classifier scans this text
1245/// for exactly that word. What keeps it a dispatch failure is the
1246/// classifier's own guard on the phrase "which forge.toml declares",
1247/// which sits ahead of the name in both arms.
1248pub(crate) fn known_project_refusal(known: &str, own: &str) -> String {
1249    if known == own {
1250        format!(
1251            "this worker's tree is inside '{known}', which forge.toml declares - your own \
1252             project, which a worker of it already runs in by default; omit `cwd`"
1253        )
1254    } else {
1255        format!(
1256            "this worker's tree is inside the project '{known}', which forge.toml declares; a \
1257             declared project is driven by its own agent - message that project's agent \
1258             instead of pointing a worker at its tree"
1259        )
1260    }
1261}
1262
1263/// The per-worker half of a spawn request, bundled so a caller cannot
1264/// transpose `kick` with `resume_kick` - both are `Option<String>` and
1265/// the mistake is silent.
1266pub(crate) struct WorkerSpawnArgs {
1267    pub label: String,
1268    pub charter: String,
1269    pub kick: Option<String>,
1270    pub resume_kick: Option<String>,
1271    pub interactive: bool,
1272    /// The validated MCP-family allowlist from `agents__spawn`; `None`
1273    /// or empty means every family.
1274    pub mcp_families: Option<Vec<String>>,
1275    /// The tree this spawn runs the worker in, resolved to its repo root
1276    /// by the facade (or, on a boot re-spawn, taken from the row): `None`
1277    /// means the project's own path.
1278    pub cwd: Option<String>,
1279}
1280
1281/// The charter a spawned worker's prompt carries: the stored text plus
1282/// the withheld-family lines for the EFFECTIVE set - the same config
1283/// gate the tool build applies, so the prompt and the tools cannot
1284/// disagree about a family.
1285fn spawn_prompt_charter(
1286    charter: &str,
1287    families: Option<&[String]>,
1288    systemone_available: bool,
1289    voice_available: bool,
1290    vault_available: bool,
1291) -> String {
1292    charter_with_withheld_families(
1293        charter,
1294        &crate::mcp::effective_mcp_families(
1295            families,
1296            systemone_available,
1297            voice_available,
1298            vault_available,
1299        ),
1300    )
1301}
1302
1303/// The worker's prompt charter: the stored text plus one line per
1304/// withheld MCP family. The row keeps the RAW charter - these lines are
1305/// composed per spawn from the current selection, so they can neither
1306/// duplicate across re-spawns nor outlive a revision.
1307fn charter_with_withheld_families(
1308    charter: &str,
1309    families: &std::collections::BTreeSet<crate::mcp::McpFamily>,
1310) -> String {
1311    let lines = crate::mcp::withheld_family_lines(families);
1312    if lines.is_empty() {
1313        return charter.to_owned();
1314    }
1315    let mut text = String::with_capacity(
1316        charter.len() + lines.iter().map(|line| line.len() + 1).sum::<usize>(),
1317    );
1318    text.push_str(charter);
1319    for line in lines {
1320        text.push('\n');
1321        text.push_str(line);
1322    }
1323    text
1324}
1325
1326/// Handle a `Command::SpawnWorker`: insert a `Spawning` worker entry
1327/// in `live_workers[project_key]`, dispatch a spawn for the id the
1328/// worker will run under (or the id being resumed) with the charter
1329/// threaded onto `SessionLaunchSettings`, then reply on `return_to`
1330/// with that id and the tag value. The Connected handler in
1331/// `session_task::translate_event` writes the actual JSONL tag row
1332/// and transitions the entry from Spawning to Running (or rolls back
1333/// on tag-write failure).
1334///
1335/// The reply's session id is informational: the LLM's `agents__spawn`
1336/// caller addresses the worker by label, not by id, and logs the id to
1337/// have a stable handle on the row.
1338pub(crate) fn handle_spawn_worker(
1339    workspace: &Arc<Workspace>,
1340    project_key: ProjectKey,
1341    args: WorkerSpawnArgs,
1342    spawned_by: SessionSlot,
1343    resume_existing: Option<&str>,
1344    from_boot_respawn: bool,
1345    return_to: tokio::sync::oneshot::Sender<Result<WorkerSpawnReply, String>>,
1346) {
1347    let WorkerSpawnArgs { label, charter, kick, resume_kick, interactive, mcp_families, cwd } =
1348        args;
1349    let label = label.as_str();
1350    let resume_kick = resume_kick.as_deref();
1351    // **The label rules belong to this handler, not only to the tool.** The
1352    // command bus and the socket pass-through reach it without the MCP
1353    // facade's own validation, and a `lead` label there collides with the
1354    // lead's slot while a slash desynchronises forge's worktree path and
1355    // branch from claude's rewritten pair (#1682, #1603). Nothing is minted
1356    // or probed before this refusal.
1357    if let Err(refusal) = crate::mcp::workers::facade::validate_worker_label(label) {
1358        let _ = return_to.send(Err(refusal.message()));
1359        return;
1360    }
1361    // Verify the project exists before minting the worker's id. A worker
1362    // that already has a row takes its git-repo-ness from that row: it is
1363    // the flag the launchpad and the boot wave read to decide whether the
1364    // directory this spawn would enter is there, so probing again here
1365    // could compose a different cwd than the one they cleared. Only a
1366    // first spawn probes the project path - a blocking FS call,
1367    // deliberately BEFORE the live_workers critical section below so it
1368    // never widens the dedup window. Either way the result feeds both the
1369    // WorkerEntry flag and the `--worktree` extra-arg threading below.
1370    let projects = workspace.list_projects();
1371    let Some(view) = projects.iter().find(|v| v.key == project_key) else {
1372        let _ = return_to.send(Err(format!("project not found: {}", project_key.as_str())));
1373        return;
1374    };
1375    // The tree this spawn runs the worker in: the resolved root the facade
1376    // resolved (or the boot wave took off the row), else the project's own
1377    // path. Every probe below and every path composed from it reads the
1378    // same root, so the entry, the row and the CLI cannot disagree about
1379    // where the worker lives.
1380    let root = cwd.as_deref().map_or_else(|| view.path.clone(), std::path::PathBuf::from);
1381    // D3: a tree inside a project `forge.toml` declares is driven by that
1382    // project's own agent, so the spawn is refused here - before the label
1383    // is reserved, before the obstruction probe and before the row is
1384    // written, so nothing at all is minted for a spawn that cannot run.
1385    // The sentence names the project as the door; the classifier reads it
1386    // as a dispatch failure by its own phrase guard, which see.
1387    if cwd.is_some()
1388        && let Some(known) = workspace.project_name_for_resolved_path(&root.to_string_lossy())
1389    {
1390        let reason = known_project_refusal(&known, &view.name);
1391        tracing::warn!(
1392            target: "forge_workspace::spawn",
1393            event_name = "worker_spawn_refused_known_project",
1394            project = %project_key.as_str(),
1395            label = %label,
1396            tree = %root.display(),
1397            known_project = %known,
1398            "the worker's tree resolves to a declared project, so the spawn is refused",
1399        );
1400        let _ = return_to.send(Err(reason));
1401        return;
1402    }
1403    let is_git = if cwd.is_some() {
1404        // A stated tree answers for itself: the row's recorded gitness
1405        // belongs to the directory the row names, which is not this one.
1406        forge_agent::env::worktree::is_git_repo(&root)
1407    } else {
1408        workspace
1409            .recorded_worker_is_git_repo(&project_key, label)
1410            .unwrap_or_else(|error| {
1411                tracing::warn!(
1412                    target: "forge_workspace::spawn",
1413                    event_name = "worker_row_gitness_unreadable",
1414                    project = %project_key.as_str(),
1415                    label = %label,
1416                    %error,
1417                    "reading the worker's recorded gitness failed; probing the project path, \
1418                     which may compose a different directory than the row names",
1419                );
1420                None
1421            })
1422            .unwrap_or_else(|| forge_agent::env::worktree::is_git_repo(&root))
1423    };
1424
1425    // The tree this worker runs in: the CLI's location under the base
1426    // checkout.
1427    let worktree_path =
1428        is_git.then(|| crate::mcp::workers::types::worker_tag_dir(&root, label, true));
1429
1430    // Hold the composed path for the whole spawn, so the claim taken and
1431    // the registry insert cannot be separated by another project's spawn
1432    // or by a despawn removing the tree this one is entering. Re-entrant
1433    // with a resume facade's pre-ensure claim.
1434    let _claim = match worktree_path.as_ref() {
1435        Some(path) => match workspace.claim_worktree(
1436            path,
1437            &project_key,
1438            label,
1439            crate::workspace::WorktreeOp::Spawn,
1440        ) {
1441            Ok(claim) => Some(claim),
1442            Err(reason) => {
1443                tracing::warn!(
1444                    target: "forge_workspace::spawn",
1445                    event_name = "worker_spawn_refused_worktree_claimed",
1446                    project = %project_key.as_str(),
1447                    label = %label,
1448                    tree = %root.display(),
1449                    reason = %reason,
1450                    "the worker's worktree is in use, so the spawn is refused",
1451                );
1452                let _ = return_to.send(Err(format!(
1453                    "this worker's worktree cannot be shared: {reason}; spawn under a different \
1454                     label"
1455                )));
1456                return;
1457            }
1458        },
1459        None => None,
1460    };
1461
1462    // The composed worktree is repository-global, so a label can name a
1463    // tree that already belongs to another project - its own checkout, or
1464    // another project's worker's worktree. Reusing it would put two
1465    // sessions in one tree and hand the later despawn the power to delete
1466    // it; refuse before anything is minted.
1467    if let Some(collision) = worktree_path.as_ref().and_then(|dir| {
1468        workspace.worktree_collision(dir, &project_key, label, crate::workspace::WorktreeOp::Spawn)
1469    }) {
1470        tracing::warn!(
1471            target: "forge_workspace::spawn",
1472            event_name = "worker_spawn_refused_shared_worktree",
1473            project = %project_key.as_str(),
1474            label = %label,
1475            tree = %root.display(),
1476            reason = %collision,
1477            "the worker's worktree is shared with another project, so the spawn is refused",
1478        );
1479        let _ = return_to.send(Err(format!(
1480            "this worker's worktree cannot be shared: {collision}; spawn under a different label"
1481        )));
1482        return;
1483    }
1484
1485    // The pool key a fresh worker spawns under: an id minted here and
1486    // recorded under the worker's slot before the child starts, so the
1487    // pool, the registry entry, the gateway binding and the CLI's own
1488    // `--session-id` all name the same string and nothing has to move on
1489    // `Connected`. A resume keys on the id being resumed instead.
1490    // The worker's slot, and the id it runs under. A fresh spawn mints
1491    // and records the id here; a resume adopts the one being resumed, so
1492    // the row names the occupant the child will actually run as.
1493    let is_resume = resume_existing.is_some();
1494    let slot = SessionSlot::worker(&view.org, &view.name, label);
1495    let session_id = match resume_existing {
1496        Some(resuming) => forge_primitives::SessionId::new(resuming),
1497        None => forge_primitives::SessionId::new(uuid::Uuid::new_v4().to_string()),
1498    };
1499    let tag = forge_primitives::worker_tag(label);
1500
1501    // Insert WorkerEntry as Spawning BEFORE the agent spawn so the
1502    // Connected handler can find the entry via worker_lookup_for_session
1503    // when it fires (worker_lookup_for_session reads live_workers).
1504    //
1505    // On the resume path `needs_tag` is false - the tag is already on
1506    // disk in the JSONL (the lead Connected hook only resumes sessions
1507    // whose tag matches `forge:worker:<label>` so this invariant is
1508    // guaranteed by the caller).
1509    //
1510    // The entry is keyed by the worker's slot, and carries the id it
1511    // runs under: a fresh one was minted above and recorded below, once
1512    // the guard admits the spawn, so the pool, the registry and the
1513    // child's `--session-id` agree from the first instant and `Connected`
1514    // has nothing to move.
1515    let entry = crate::mcp::workers::types::WorkerEntry {
1516        label: label.to_owned(),
1517        charter: charter.clone(),
1518        slot: slot.clone(),
1519        session_id: Some(session_id.clone()),
1520        status: forge_primitives::WorkerLiveness::Spawning,
1521        spawned_at: std::time::SystemTime::now(),
1522        spawned_by,
1523        needs_tag: !is_resume,
1524        is_git_repo_at_spawn: is_git,
1525        cwd: cwd.as_deref().map(std::path::PathBuf::from),
1526        diagnostic: None,
1527        kick,
1528    };
1529    // Label uniqueness AND the project's worker cap, enforced atomically
1530    // at this shared core so neither dispatch source - the boot
1531    // re-spawn or an MCP `agents__spawn` - can double-insert and fork
1532    // two subprocesses onto one worktree, or overshoot the cap on
1533    // genuinely-concurrent dispatches. The cap is per project: the
1534    // project's `max_workers` override, else the
1535    // default. Boot re-spawns pass no cap: they restore persisted
1536    // workers the user already had, and their spawn reply is dropped,
1537    // so a refusal there could never reach a caller.
1538    let cap = (!from_boot_respawn).then(|| {
1539        workspace
1540            .project_for_key(&project_key)
1541            .and_then(|project| project.max_workers)
1542            .unwrap_or(crate::config::DEFAULT_MAX_WORKERS_PER_PROJECT)
1543    });
1544    if let Err(refusal) =
1545        workspace.insert_live_worker_if_label_absent(&project_key, entry.clone(), cap)
1546    {
1547        match refusal {
1548            LiveWorkerRefusal::LabelLive(existing) => {
1549                let existing_session = existing.display();
1550                tracing::debug!(
1551                    target: "forge_workspace::spawn",
1552                    project = %project_key.as_str(),
1553                    label = %label,
1554                    %existing_session,
1555                    "spawn_worker: label already live; skipping duplicate spawn",
1556                );
1557                let _ = return_to.send(Err(format!(
1558                    "a worker labeled '{label}' is already live (session {existing_session}); message it with agents__send_message or close it first (one live worker per label)"
1559                )));
1560            }
1561            LiveWorkerRefusal::CleanupPending => {
1562                tracing::info!(
1563                    target: "forge_workspace::spawn",
1564                    project = %project_key.as_str(),
1565                    label = %label,
1566                    "spawn_worker: refused, the label's previous despawn is still cleaning up",
1567                );
1568                // No "worktree" in this sentence, and no interpolated
1569                // label either (a label may carry the word): the facade's
1570                // failure classifier routes that word to a
1571                // worktree-creation failure, which is not what happened.
1572                let _ = return_to.send(Err(
1573                    "this label's previous despawn is still cleaning up; retry once it finishes"
1574                        .to_owned(),
1575                ));
1576            }
1577            LiveWorkerRefusal::AtCap { live, cap } => {
1578                tracing::info!(
1579                    target: "forge_workspace::spawn",
1580                    project = %project_key.as_str(),
1581                    label = %label,
1582                    live,
1583                    cap,
1584                    "spawn_worker: refused, at the concurrent worker cap",
1585                );
1586                let _ = return_to.send(Err(worker_limit_reached_message(
1587                    project_key.as_str(),
1588                    live,
1589                    cap,
1590                )));
1591            }
1592        }
1593        return;
1594    }
1595    // The worker's directory is not optional, and claude's `--worktree
1596    // <label>` creation is not a check: when it does nothing (measured:
1597    // the derived branch is already checked out in another worktree),
1598    // the session starts anyway and the worker reads live with nowhere to
1599    // work. Ask git what creation would be refused for, and refuse the
1600    // spawn here instead - before the row is written and before any
1601    // subprocess exists - leaving claude to create the worktree as it
1602    // always has.
1603    if is_git
1604        && !is_resume
1605        && let Some(obstacle) = forge_agent::env::worktree::worktree_creation_obstacle(
1606            &root,
1607            label,
1608            &crate::mcp::workers::types::worker_tag_dir(&root, label, true),
1609        )
1610    {
1611        tracing::warn!(
1612            target: "forge_workspace::spawn",
1613            event_name = "worker_spawn_refused_no_worktree",
1614            project = %project_key.as_str(),
1615            label = %label,
1616            %obstacle,
1617            "a worktree cannot be created for this worker where it would have to run, so the \
1618             spawn is refused rather than started with nowhere to work",
1619        );
1620        // A boot re-spawn's reply is dropped, so rolling the entry back
1621        // would leave the row listed and nothing running with the reason
1622        // nowhere on screen. It keeps the entry as the visible failure the
1623        // async spawn failures leave instead; every caller that DOES hear
1624        // the reply gets the rollback, so nothing half-live is left behind.
1625        if from_boot_respawn {
1626            crate::workspace::transition_worker_to_failed(
1627                workspace,
1628                &project_key,
1629                &slot,
1630                Some(obstacle.clone()),
1631            );
1632        } else {
1633            let _ = workspace.remove_latest_worker(&project_key, label);
1634        }
1635        // The obstacle text reads as one sentence, so the MCP tool's
1636        // "worktree creation failed:" label lands on top of it as one
1637        // statement rather than two.
1638        let _ = return_to.send(Err(obstacle));
1639        return;
1640    }
1641    // The row is the whole registry entry a boot re-spawns from, so it
1642    // carries the spawn args alongside the id rather than leaving them in
1643    // memory. One path writes it for both spawns and re-spawns; what a
1644    // re-spawn must not restate is the `kick` handled just below.
1645    //
1646    // AFTER the guard above, which is the point of the order: both
1647    // refusals return without spawning, and a row written first would
1648    // outlive them. A refused duplicate would have overwritten the
1649    // RUNNING worker's row with a fresh id and the new args, so the next
1650    // boot would resume an id no session ever ran under; an at-cap
1651    // refusal would leave a row the boot re-spawn wave picks up with no
1652    // cap, bringing back a worker the caller was told does not exist.
1653    // The row's `kick` is the worker's first turn, which only a first
1654    // spawn states. A resume's live kick is the restart note (or the row's
1655    // `resume_kick`) when the seat's turn was cut off - decided upstream
1656    // from the pre-spawn row - and writing that here would make it the
1657    // worker's opening turn on every later `--new` re-spawn.
1658    let kick_field = if is_resume { None } else { entry.kick.as_deref() };
1659    // The stored-selection contract, mirrored exactly by `families`
1660    // below so the reply, the running surface and the row cannot
1661    // disagree: a re-spawn stating nothing adopts the stored value; a
1662    // stated list is a revision; a fresh spawn stating nothing means
1663    // every family, which writes as absence.
1664    let resume_like = is_resume || from_boot_respawn;
1665    let families_field: Option<&[String]> = if resume_like && mcp_families.is_none() {
1666        None
1667    } else {
1668        Some(mcp_families.as_deref().unwrap_or(&[]))
1669    };
1670    let durability_warning = match workspace.record_worker_row(
1671        &project_key,
1672        label,
1673        session_id.as_str(),
1674        &charter,
1675        kick_field,
1676        resume_kick,
1677        interactive,
1678        crate::mcp::workers::types::WorkerTree {
1679            is_git_repo: is_git,
1680            // What the spawn resolved, written verbatim: a stated tree is
1681            // the row's durable handle to where the JSONL lives, and a
1682            // spawn that stated none records none.
1683            cwd: cwd.clone(),
1684        },
1685        families_field,
1686    ) {
1687        Ok(()) => None,
1688        Err(error) => {
1689            tracing::warn!(
1690                target: "forge_workspace::spawn",
1691                project = %project_key.as_str(),
1692                label = %label,
1693                %error,
1694                "recording the worker's row failed; it will not survive a restart",
1695            );
1696            Some(format!(
1697                "recording this worker for durability failed ({error}); it will not survive a forge restart"
1698            ))
1699        }
1700    };
1701    try_emit(
1702        workspace,
1703        "spawn_worker::WorkerStatusChanged::Added",
1704        SessionUpdate::WorkerStatusChanged {
1705            project_key: project_key.clone(),
1706            action: WorkerStatusAction::Added,
1707            status: entry.to_status(),
1708            worktree: WorktreeDisposition::untouched(entry.is_git_repo_at_spawn),
1709        },
1710    );
1711
1712    // Spawn the fresh session under the picked account. The charter
1713    // is threaded onto SessionLaunchSettings.charter; the spawn path
1714    // (forge_sdk_worker::build_options_with_callback) appends it to
1715    // the system prompt via --append-system-prompt. `extra_args`
1716    // carries `("worktree", Some(label))` for git-repo projects so
1717    // claude forks a worktree at `<base>/.claude/worktrees/<label>/`,
1718    // `base` being the repo's canonical root (#2460).
1719    //
1720    // Resume path: `SessionTarget::Session` reads the original cwd
1721    // from the catalog (Workspace::session_cwd_for) so claude lands
1722    // back in the worktree it was first spawned in. If the worktree
1723    // dir was removed out-of-band, claude's spawn fails and the
1724    // existing ConnectionFailed surface (workspace.rs:570) reports
1725    // it - we don't silently fall back to fresh-spawn (that would
1726    // lose state without warning).
1727    // The selection this spawn composes from: a stated argument wins (a
1728    // resume stating one is a revision); otherwise a resume or boot
1729    // re-spawn adopts the row; a fresh spawn with nothing stated means
1730    // every family.
1731    let families = if mcp_families.is_some() {
1732        mcp_families.clone()
1733    } else if resume_like {
1734        workspace.recorded_worker_mcp_families(&project_key, label).unwrap_or_else(|error| {
1735            tracing::warn!(
1736                target: "forge_workspace::spawn",
1737                event_name = "worker_row_families_unreadable",
1738                project = %project_key.as_str(),
1739                label = %label,
1740                %error,
1741                "reading the worker's recorded mcp families failed; \
1742                 composing the full surface",
1743            );
1744            None
1745        })
1746    } else {
1747        None
1748    };
1749    let settings = SessionLaunchSettings {
1750        charter: Some(spawn_prompt_charter(
1751            &charter,
1752            families.as_deref(),
1753            workspace.systemone.read().is_some(),
1754            workspace.voice_enabled(),
1755            workspace.vault.read().is_some(),
1756        )),
1757        extra_args: build_worker_extra_args(is_git, label, interactive),
1758        ..Default::default()
1759    };
1760    let target = if resume_existing.is_some() {
1761        SessionTarget::Session(slot.clone())
1762    } else {
1763        SessionTarget::FreshInProject {
1764            slot: slot.clone(),
1765            // The same resolved root the row and the entry carry, so the
1766            // child starts where they say it runs.
1767            cwd: cwd.as_deref().map(std::path::PathBuf::from),
1768        }
1769    };
1770    match workspace.get_agent_handle_at_key(
1771        target,
1772        settings,
1773        None,
1774        &crate::protocol::SpawnRole::Worker {
1775            label: label.to_owned(),
1776            // The same predicate the synchronous rollback below deletes
1777            // under: a resume and a boot re-spawn adopt a row that was
1778            // already there, so only a spawn that minted this one may
1779            // take it away.
1780            wrote_row: !is_resume && !from_boot_respawn,
1781            mcp_families: families.clone(),
1782        },
1783        // A worker spawn is an opening door like the lead's: it starts the
1784        // seat it names.
1785        true,
1786    ) {
1787        Ok(handle) => {
1788            tracing::info!(
1789                target: "forge_workspace::spawn",
1790                project = %project_key.as_str(),
1791                label = %label,
1792                slot = %slot.display(),
1793                "spawn dispatched for worker"
1794            );
1795            // Best-effort, like the lead's: the profile is up before the
1796            // worker's first browser call rather than because of it.
1797            warm_browser_seat(workspace, &slot);
1798            // The walk lands on a saturated or bailed account only when
1799            // nothing else in the pin declares the project's model, so
1800            // the lead hears about it at spawn rather than when the
1801            // worker stalls on a 429.
1802            let rate_limited_account =
1803                handle.display_name().and_then(|name| workspace.degraded_account_name(&name));
1804            // Reply to the LLM with the id the worker runs under. The
1805            // LLM addresses subsequent calls by label; the session_id
1806            // field is informational, and it is no longer a placeholder
1807            // the rekey machinery has to correct.
1808            let _ = return_to.send(Ok(WorkerSpawnReply {
1809                session_id: session_id.as_str().to_owned(),
1810                tag,
1811                // The stored selection, canonicalised; `None` means every
1812                // family. The lead sees exactly what this worker got.
1813                mcp_families: families,
1814                rate_limited_account,
1815                // Set when the row could not be written above; the boot
1816                // re-spawn paths drop the reply, so they read the warn
1817                // instead.
1818                durability_warning,
1819                // Where the worker runs, which this handler decided above:
1820                // a git project's worker gets the CLI's
1821                // `<base>/.claude/worktrees/<label>`, and everything else
1822                // runs in the project root and has no worktree to name.
1823                worktree: worktree_path.as_ref().map(|dir| dir.display().to_string()),
1824                // What the arguments say, which is all this handler
1825                // knows: a `resume_session` spawn whose lookup found
1826                // nothing arrives here as a fresh spawn like any other,
1827                // and the facade that resolved it restates this.
1828                session_choice: if is_resume {
1829                    crate::protocol::SessionChoice::Resumed
1830                } else {
1831                    crate::protocol::SessionChoice::Fresh
1832                },
1833            }));
1834        }
1835        Err(err) => {
1836            let message = format!("agent spawn failed: {err}");
1837            tracing::error!(
1838                target: "forge_workspace::spawn",
1839                project = %project_key.as_str(),
1840                label = %label,
1841                error = %err,
1842                "spawn_worker: get_agent_handle failed"
1843            );
1844            // **The reason a view reads.** The lead's arm records the same
1845            // failure for its slot, and without this one the rollback's
1846            // `Removed` event is all a view has: it names no cause, and the
1847            // chat draws it as a close.
1848            workspace.record_spawn_failure(&slot, &message);
1849            // A fresh spawn the caller asked for creates its row above, so
1850            // a failure here has to take the row with it: the caller is
1851            // told the spawn failed and the live entry is rolled back, and
1852            // a row left behind brings the worker back on the next boot.
1853            // A resume and a boot re-spawn are not that case - their row
1854            // pre-existed and is the only handle on the id being resumed,
1855            // so deleting it would lose the worker rather than let it
1856            // retry.
1857            if !is_resume && !from_boot_respawn {
1858                let _ = workspace.delete_worker_row(&project_key, label);
1859            }
1860            // Roll back the live_workers entry we just inserted.
1861            let removed = workspace.remove_latest_worker(&project_key, label);
1862            if let Some(mut rolled) = removed {
1863                // The last-known snapshot is carried for symmetry, so the
1864                // reason rides it too: a view that renders the event's
1865                // status has the cause even though the entry is gone.
1866                rolled.diagnostic = Some(message.clone());
1867                try_emit(
1868                    workspace,
1869                    "spawn_worker::WorkerStatusChanged::Removed",
1870                    SessionUpdate::WorkerStatusChanged {
1871                        project_key,
1872                        action: WorkerStatusAction::Removed,
1873                        status: rolled.to_status(),
1874                        // The rollback beat the subprocess, so
1875                        // `--worktree <label>` never reached one and
1876                        // there is no worktree to point the user at.
1877                        worktree: WorktreeDisposition::Absent,
1878                    },
1879                );
1880            }
1881            // The caller's error is the same sentence the slot records, so
1882            // the two cannot name different failures.
1883            let _ = return_to.send(Err(message));
1884        }
1885    }
1886}
1887
1888/// Shared worker teardown used by both `handle_close_worker` (a view's
1889/// X-button) and `handle_despawn_worker` (the `agents__despawn` MCP
1890/// tool): remove the latest-spawned worker matching `label` from
1891/// `live_workers[project_key]` and release its session (terminates the
1892/// claude subprocess on drop). Returns
1893/// the removed `WorkerEntry`, or `None` when no live worker matched.
1894/// JSONL on disk is NOT deleted - teardown only removes the in-memory
1895/// live state.
1896///
1897/// The `Removed` event is the caller's to emit via
1898/// [`emit_worker_removed`]: the despawn path only learns what became
1899/// of the worktree after this returns.
1900fn teardown_worker(
1901    workspace: &Arc<Workspace>,
1902    project_key: &ProjectKey,
1903    label: &str,
1904) -> Option<crate::mcp::workers::types::WorkerEntry> {
1905    let entry = workspace.remove_latest_worker(project_key, label)?;
1906    // Both entry points into this routine (the Projects-pane close and
1907    // the `agents__despawn` MCP tool) delete the persisted worker row so
1908    // it never re-spawns. Cancel and the lead-close cascade go through
1909    // other paths and deliberately leave the row intact.
1910    let _ = workspace.delete_worker_row(project_key, label);
1911    // The row is gone, so nothing re-spawns this label: its durable state
1912    // has no owner left to wake and goes with it.
1913    workspace.remove_gotify_subscriptions_for_worker(project_key, label);
1914    // Same for its Slack subscriptions: a despawned worker cannot strand
1915    // records that would reload at boot, and a delivery for it must not
1916    // fall through to the lead.
1917    workspace.remove_slack_subscriptions_for_worker(project_key, label);
1918    workspace.stop_slack_subsystem_if_idle();
1919    workspace.delete_crons_for_worker(project_key, label);
1920    // Closes exactly one worker, so the non-cascading primitive. The
1921    // cascading variant would reach the same result: it cascades only
1922    // from a lead slot, and this one carries the worker's own label.
1923    // Per-row close must only affect the worker being closed, so the
1924    // narrower call is the one to read here.
1925    workspace.release_session(&entry.slot);
1926    // A payload parked for this label while it was still spawning has no
1927    // session left to drain it.
1928    workspace.expire_parked_for_slot(
1929        &entry.slot,
1930        crate::mcp::peers::types::PeerFailureReason::TargetConnectionFailed,
1931    );
1932    Some(entry)
1933}
1934
1935/// Announce a torn-down worker to the views, carrying what became of its
1936/// worktree.
1937fn emit_worker_removed(
1938    workspace: &Arc<Workspace>,
1939    project_key: &ProjectKey,
1940    entry: &crate::mcp::workers::types::WorkerEntry,
1941    worktree: WorktreeDisposition,
1942) {
1943    let _ = workspace.update_tx().send(SessionUpdate::WorkerStatusChanged {
1944        project_key: project_key.clone(),
1945        action: WorkerStatusAction::Removed,
1946        status: entry.to_status(),
1947        worktree,
1948    });
1949}
1950
1951/// Handle a `Command::CloseWorker` (a view's per-row X-button): tear
1952/// the worker down via [`teardown_worker`]. Does NOT touch the git
1953/// worktree - that's the `agents__despawn` path's job; the X-button's
1954/// behavior is intentionally unchanged.
1955pub(crate) fn handle_close_worker(
1956    workspace: &Arc<Workspace>,
1957    project_key: &ProjectKey,
1958    label: &str,
1959) {
1960    // Announced before the teardown, and only where a worker is really here:
1961    // other views' rows say where the seat is going for the seconds the close
1962    // takes, and a label with no live worker has no row to mark (#1930). The
1963    // peek is the despawn handler's own shape, for the same reason - the
1964    // single-threaded command loop means nothing moves between it and the
1965    // teardown below.
1966    let live = workspace
1967        .list_live_workers(project_key)
1968        .into_iter()
1969        .rev()
1970        .find(|worker| worker.label == label);
1971    // **A failed spawn is a seat to dismiss, not a session to tear down.**
1972    // Its entry is a tombstone (or was rolled back), so there is nothing
1973    // running to kill and the persisted row is what lets the label start
1974    // again - [`teardown_worker`] would delete it.
1975    if live
1976        .as_ref()
1977        .is_some_and(|entry| matches!(entry.status, forge_primitives::WorkerLiveness::Failed))
1978    {
1979        dismiss_failed_worker(workspace, project_key, label, live.as_ref());
1980        return;
1981    }
1982    if let Some(entry) = live.as_ref() {
1983        workspace.announce_release(&entry.slot);
1984    }
1985    let Some(entry) = teardown_worker(workspace, project_key, label) else {
1986        // No live entry: the seat may still be here as a persisted row with
1987        // a spawn failure that rolled its entry back. The close dismisses
1988        // that trace; with none, there is nothing to close.
1989        if dismiss_failed_worker(workspace, project_key, label, None) {
1990            return;
1991        }
1992        tracing::warn!(
1993            target: "forge_workspace::spawn",
1994            project = %project_key.as_str(),
1995            label = %label,
1996            "handle_close_worker: no matching live worker"
1997        );
1998        return;
1999    };
2000    let worktree = WorktreeDisposition::untouched(entry.is_git_repo_at_spawn);
2001    emit_worker_removed(workspace, project_key, &entry, worktree);
2002    sweep_browser_seats(workspace);
2003}
2004
2005/// Close a failed worker, or one whose failure is all that is left of it:
2006/// take the tombstone entry when one survives, clear the slot's failure
2007/// record, and announce the release so views re-read the row without it.
2008///
2009/// **Not [`teardown_worker`]**: the persisted row stays, because it is what
2010/// lets the label start again once whatever refused the spawn is fixed. The
2011/// worker's subscriptions and schedules stay with it for the same reason.
2012/// Answers whether anything was there to dismiss.
2013fn dismiss_failed_worker(
2014    workspace: &Arc<Workspace>,
2015    project_key: &ProjectKey,
2016    label: &str,
2017    entry: Option<&crate::mcp::workers::types::WorkerEntry>,
2018) -> bool {
2019    let slot = if let Some(entry) = entry {
2020        entry.slot.clone()
2021    } else {
2022        // The slot is the identity the failure is recorded under; a project
2023        // no longer in the config has no seat left to address.
2024        let Some(view) =
2025            workspace.list_projects().into_iter().find(|view| view.key == *project_key)
2026        else {
2027            return false;
2028        };
2029        SessionSlot::worker(&view.org, &view.name, label)
2030    };
2031    if entry.is_none() && workspace.spawn_failure(&slot).is_none() {
2032        return false;
2033    }
2034    if let Some(entry) = entry {
2035        let _ = workspace.remove_worker_by_session_key(&entry.slot);
2036    }
2037    workspace.announce_release(&slot);
2038    workspace.expire_parked_for_slot(
2039        &slot,
2040        crate::mcp::peers::types::PeerFailureReason::TargetConnectionFailed,
2041    );
2042    if let Some(entry) = entry {
2043        let worktree = WorktreeDisposition::untouched(entry.is_git_repo_at_spawn);
2044        emit_worker_removed(workspace, project_key, entry, worktree);
2045    }
2046    sweep_browser_seats(workspace);
2047    true
2048}
2049
2050/// Tell the browser fleet which seats remain: **the seat is down, so its
2051/// tabs go with it**, except any a parked hand-off holds. The sweep runs off
2052/// the teardown path, which is reaping a child and can take minutes; a fleet
2053/// that is not installed is a workspace with no browsers to sweep. The
2054/// snapshot carries its capture generation, so the fleet drops a sweep that
2055/// arrives after a newer capture.
2056fn sweep_browser_seats(workspace: &Arc<Workspace>) {
2057    let (Some(host), Some(snapshot)) = (workspace.browser_host(), workspace.live_seats()) else {
2058        return;
2059    };
2060    tokio::spawn(async move { host.sweep_sessions(&snapshot.seats, snapshot.generation).await });
2061}
2062
2063/// Warm the seat's browser profile: create the entry and start its host, with
2064/// no tab. **Best-effort**, off the spawn's own path: a profile whose host
2065/// cannot start is logged by the fleet rather than failing the spawn.
2066fn warm_browser_seat(workspace: &Arc<Workspace>, seat: &forge_primitives::SessionSlot) {
2067    let Some(host) = workspace.browser_host() else { return };
2068    let seat = seat.clone();
2069    tokio::spawn(async move { host.warm(&seat).await });
2070}
2071
2072/// Handle a `Command::DespawnWorker` (the `agents__despawn` MCP
2073/// tool): the lead's clean-close gesture. Unlike `handle_close_worker`
2074/// it also cleans up the worker's git worktree.
2075///
2076/// Order matters: the worktree dirty-check runs BEFORE any teardown,
2077/// so a dirty worker (uncommitted/untracked or unpushed) is blocked
2078/// without being killed (unless `force`). Teardown then releases the
2079/// session, which drops the worker's command sender; the disconnect
2080/// itself runs asynchronously on the worker's own task and can take
2081/// up to the 5s close-wait budget to reap the child. The worktree
2082/// removal runs on the blocking pool (see [`DespawnCleanup`]) only after
2083/// that task has finished dying - its exit path reaps the session's own
2084/// child (waits it out, then SIGKILLs), so that child is dead before the
2085/// removal takes the tree (#1767; descendants that outlive a SIGKILLed
2086/// child are #2337) - without holding the caller's runtime, and bounded
2087/// by [`TEARDOWN_WAIT`] so a teardown that never answers cannot wedge the
2088/// reply. A post-teardown worktree-removal failure is surfaced as a
2089/// warning in the
2090/// [`DespawnResult`](crate::protocol::DespawnResult) but never rolls back
2091/// the kill - teardown and worktree cleanup are independent.
2092pub(crate) fn handle_despawn_worker(
2093    workspace: &Arc<Workspace>,
2094    project_key: &ProjectKey,
2095    label: &str,
2096    force: bool,
2097    respond: tokio::sync::oneshot::Sender<crate::protocol::DespawnResult>,
2098) {
2099    handle_despawn_worker_within(workspace, project_key, label, force, TEARDOWN_WAIT, respond);
2100}
2101
2102/// How long the despawn holds its worktree removal for the session task's
2103/// teardown signal before giving up and removing anyway: the SDK's
2104/// close-wait budget (the child's reap) plus a margin for the task to
2105/// observe its closed channel, disconnect and drop. An expiry is logged -
2106/// it is the one shape where the removal goes ahead unprotected.
2107pub(crate) const TEARDOWN_WAIT: std::time::Duration =
2108    forge_sdk::CLOSE_WAIT_TIMEOUT.saturating_add(std::time::Duration::from_secs(5));
2109
2110/// [`handle_despawn_worker`] with the teardown wait injected, so a case
2111/// can drive the expiry without sitting out the production budget.
2112pub(crate) fn handle_despawn_worker_within(
2113    workspace: &Arc<Workspace>,
2114    project_key: &ProjectKey,
2115    label: &str,
2116    force: bool,
2117    teardown_wait: std::time::Duration,
2118    respond: tokio::sync::oneshot::Sender<crate::protocol::DespawnResult>,
2119) {
2120    use crate::protocol::DespawnResult;
2121
2122    // `lead` is not a worker label: it is the project lead's own row, the
2123    // stored id an ordinary boot resumes the lead from, and the spawn path
2124    // reserves the label so no worker can hold it. Clearing that row and
2125    // reporting a despawn would orphan the lead's conversation.
2126    if label == crate::store::sessions::LEAD_LABEL {
2127        let _ = respond.send(DespawnResult::NotFound);
2128        return;
2129    }
2130
2131    // Peek the latest-spawned matching worker WITHOUT removing it, so a
2132    // blocked despawn leaves it live.
2133    let live =
2134        workspace.list_live_workers(project_key).into_iter().rev().find(|w| w.label == label);
2135
2136    // The gitness this despawn acts on, and whether there is a worker here
2137    // at all. A live entry stamped its gitness at spawn; with no entry the
2138    // durable row answers both, and the two questions are separate: a row
2139    // whose `is_git_repo` is absent has a worker but no known worktree,
2140    // which is how `worker_tag_dir` and the boot wave read the same field.
2141    // Asking one question for both is what makes a row the migration wrote
2142    // - it records no gitness before the first spawn - look like no row,
2143    // and that row is exactly the one #1142 exists to reach.
2144    //
2145    // The tree is read in the same breath for the same reason: a worker
2146    // the spawn pointed at another directory keeps its worktree THERE, so
2147    // the probe, the removal and the branch reap all run against that
2148    // tree - composing them against the project path would probe (and
2149    // block on) a directory the worker never ran in.
2150    let (is_git_repo, recorded_cwd) = match live.as_ref() {
2151        Some(entry) => (Some(entry.is_git_repo_at_spawn), entry.cwd.clone()),
2152        None => match workspace.stored_worker_row(project_key, label) {
2153            Ok(Some(row)) => {
2154                (Some(matches!(row.is_git_repo, Some(true))), row.cwd.map(std::path::PathBuf::from))
2155            }
2156            Ok(None) => (None, None),
2157            // A row that cannot be read is not an absent one, and reading
2158            // it is how this call decides whether there is anything here.
2159            Err(error) => {
2160                let _ = respond.send(DespawnResult::Failed {
2161                    reason: format!(
2162                        "could not read the durable row for '{label}': {error}; whether one \
2163                         exists is unknown"
2164                    ),
2165                });
2166                return;
2167            }
2168        },
2169    };
2170    let Some(is_git_repo) = is_git_repo else {
2171        let _ = respond.send(DespawnResult::NotFound);
2172        return;
2173    };
2174
2175    // Resolve the worktree path for git-repo workers (claude's
2176    // `<base>/.claude/worktrees/<label>/`, `base` being the CLI's
2177    // canonical root, so a project that is itself a linked worktree
2178    // resolves to the main checkout - which is where the CLI created the
2179    // worktree). Non-git workers have no worktree to clean.
2180    let project_view = workspace.list_projects().into_iter().find(|v| v.key == *project_key);
2181    let spawn_root = recorded_cwd.or_else(|| project_view.as_ref().map(|v| v.path.clone()));
2182    let worktree_path = if is_git_repo {
2183        spawn_root
2184            .as_ref()
2185            .map(|root| crate::mcp::workers::types::worker_tag_dir(root, label, true))
2186    } else {
2187        None
2188    };
2189    // Every worktree operation below runs against the checkout whose
2190    // `.claude/worktrees` holds the tree - the CLI's base, which is the
2191    // main checkout for a linked-worktree project.
2192    let worktree_root = if is_git_repo {
2193        spawn_root.map(|root| forge_agent::env::worktree::worktree_base(&root))
2194    } else {
2195        spawn_root
2196    };
2197
2198    // The path is repository-global: it may be another project's checkout
2199    // or another project's worker's tree. Removing it would delete that,
2200    // so refuse before the probe; a force cannot override this.
2201    if let Some(path) = worktree_path.as_ref()
2202        && let Some(collision) = workspace.worktree_collision(
2203            path,
2204            project_key,
2205            label,
2206            crate::workspace::WorktreeOp::Despawn,
2207        )
2208    {
2209        tracing::warn!(
2210            target: "forge_workspace::spawn",
2211            event_name = "despawn_refused_shared_worktree",
2212            project = %project_key.as_str(),
2213            label = %label,
2214            worktree = %path.display(),
2215            reason = %collision,
2216            "the worker's worktree is shared with another project, so the despawn is refused",
2217        );
2218        let _ = respond.send(DespawnResult::Failed {
2219            reason: format!(
2220                "refusing to despawn '{label}': {collision}; if it is not that project's tree, \
2221                 remove it by hand"
2222            ),
2223        });
2224        return;
2225    }
2226
2227    // Hold the path through the teardown and the removal: teardown drops
2228    // the durable row before the removal runs, so another project's spawn
2229    // would otherwise be admitted into a tree the cleanup is deleting.
2230    // Released when the cleanup ends.
2231    let claim = match worktree_path.as_ref() {
2232        Some(path) => match workspace.claim_worktree(
2233            path,
2234            project_key,
2235            label,
2236            crate::workspace::WorktreeOp::Despawn,
2237        ) {
2238            Ok(claim) => Some(claim),
2239            Err(reason) => {
2240                let _ = respond.send(DespawnResult::Failed {
2241                    reason: format!("refusing to despawn '{label}': {reason}"),
2242                });
2243                return;
2244            }
2245        },
2246        None => None,
2247    };
2248
2249    // A worker's worktree carries its own build tree, so the removal
2250    // below deletes hundreds of thousands of files and can run for
2251    // minutes. Nothing else on this path is slow, so these phase records
2252    // are what tells a long close apart from a hung one (#1113).
2253    let worktree_display =
2254        worktree_path.as_ref().map_or_else(|| "none".to_owned(), |path| path.display().to_string());
2255    tracing::info!(
2256        target: "forge_workspace::spawn",
2257        project = %project_key.as_str(),
2258        label = %label,
2259        force,
2260        worktree = %worktree_display,
2261        "despawn: starting",
2262    );
2263
2264    // A stranded row whose worktree is already gone is the shape the boot
2265    // wave skips: nothing is on disk to remove and nothing in it to lose,
2266    // so the checks that guard live work stand down for it. A live
2267    // worker's worktree vanishing is the opposite case, and keeps them.
2268    let worktree_already_gone =
2269        live.is_none() && worktree_path.as_ref().is_some_and(|path| !path.exists());
2270
2271    // Dirty-check BEFORE teardown: block (nothing torn down) when the
2272    // worktree is dirty and `force` is not set. `force` skips the probe
2273    // rather than ignoring its verdict, and so does a worktree that is not
2274    // there to probe - `worktree_dirty_reason` cannot read a directory
2275    // that does not exist, and blocking on that would refuse a despawn
2276    // over work there is no worktree to hold.
2277    let dirty_reason = if force || worktree_already_gone {
2278        None
2279    } else {
2280        worktree_path.as_ref().and_then(|path| worktree_dirty_verdict(path))
2281    };
2282    tracing::info!(
2283        target: "forge_workspace::spawn",
2284        project = %project_key.as_str(),
2285        label = %label,
2286        dirty = dirty_reason.is_some(),
2287        reason = dirty_reason.as_deref().unwrap_or("none"),
2288        "despawn: dirty verdict",
2289    );
2290    if let Some(reason) = dirty_reason {
2291        let _ = respond.send(DespawnResult::Blocked { reason });
2292        return;
2293    }
2294
2295    // The session's teardown signal, taken before the teardown releases
2296    // it, so the cleanup below can hold the worktree removal until the
2297    // worker's session has finished dying (#1767). Only a git worker has
2298    // a worktree to order against, so only it takes one.
2299    //
2300    // A git worker with a live entry but no signal yet is MID-SPAWN - the
2301    // entry goes in before the session task is minted - so the removal
2302    // would race a child that is only starting: #1767's own class of
2303    // overlap, for that window. Refuse rather than report Despawned over
2304    // a live spawn; the caller retries once the worker is up.
2305    let mut teardown_done = None;
2306    if is_git_repo && let Some(entry) = live.as_ref() {
2307        teardown_done = workspace.session_teardown(&entry.slot);
2308        if teardown_done.is_none()
2309            && matches!(entry.status, forge_primitives::WorkerLiveness::Spawning)
2310        {
2311            let _ = respond.send(DespawnResult::Blocked {
2312                reason: format!(
2313                    "'{label}' is still spawning, so its session's teardown cannot be waited \
2314                     for; retry once it is up"
2315                ),
2316            });
2317            return;
2318        }
2319    }
2320
2321    // The label stays reserved for the whole cleanup: marked here, before
2322    // the teardown frees the entry, so no spawn can take the label in
2323    // between and be handed the worktree this despawn is about to delete.
2324    // The guard rides into the cleanup below and releases the label when
2325    // the cleanup ends (or unwinds). Every refusal path above this point
2326    // returns before the mark, so a blocked or absent despawn frees
2327    // nothing it did not take.
2328    let cleanup_pending = workspace.mark_despawn_cleanup_pending(project_key, label);
2329
2330    // Teardown. A live worker goes through `teardown_worker`, which kills
2331    // the subprocess on drop, removes the entry, deletes the row and
2332    // clears the records and payloads addressed to it.
2333    //
2334    // A stranded row has no entry to tear down: the row is the whole of
2335    // it, so it goes here with the records and payloads its label still
2336    // owns. Without that, a despawn would report a worker gone while
2337    // leaving exactly what the live path exists to clear (#1142).
2338    if live.is_some() {
2339        // Announced here rather than at the removal below: this despawn's
2340        // own frame only goes out once the worktree cleanup finishes, and
2341        // that can run for minutes - so other views' rows read the seat as
2342        // working for the whole cleanup without this (#1930). Every refusal
2343        // path returned above, so a blocked or absent despawn marks nothing.
2344        if let Some(entry) = live.as_ref() {
2345            workspace.announce_release(&entry.slot);
2346        }
2347        // The single-threaded command loop means nothing mutated
2348        // `live_workers` between the peek above and here, but re-checking
2349        // the removal is defensive.
2350        if teardown_worker(workspace, project_key, label).is_none() {
2351            let _ = respond.send(DespawnResult::NotFound);
2352            return;
2353        }
2354    } else {
2355        match workspace.delete_worker_row(project_key, label) {
2356            Ok(true) => {}
2357            Ok(false) => {
2358                let _ = respond.send(DespawnResult::NotFound);
2359                return;
2360            }
2361            // A store that could not be written is not an absent row:
2362            // reporting NotFound would send the caller looking for a
2363            // worker that is right there. `delete_worker_row` has already
2364            // logged why, so the caller gets the reason and the log has
2365            // the site.
2366            Err(error) => {
2367                let _ = respond.send(DespawnResult::Failed {
2368                    reason: format!(
2369                        "could not clear the durable row for '{label}': {error}; whether one \
2370                         exists is unknown, so nothing was reported as removed"
2371                    ),
2372                });
2373                return;
2374            }
2375        }
2376        workspace.remove_gotify_subscriptions_for_worker(project_key, label);
2377        workspace.remove_slack_subscriptions_for_worker(project_key, label);
2378        workspace.stop_slack_subsystem_if_idle();
2379        workspace.delete_crons_for_worker(project_key, label);
2380        if let Some(view) = project_view.as_ref() {
2381            workspace.expire_parked_for_slot(
2382                &SessionSlot::worker(&view.org, &view.name, label),
2383                crate::mcp::peers::types::PeerFailureReason::TargetConnectionFailed,
2384            );
2385        }
2386        tracing::info!(
2387            target: "forge_workspace::spawn",
2388            event_name = "despawn_cleared_stranded_worker",
2389            project = %project_key.as_str(),
2390            label = %label,
2391            "despawn: no live worker matched; cleared its stranded durable row and records",
2392        );
2393    }
2394
2395    // **The seat is down, so its browser profiles go with it** - here, after
2396    // teardown and before the worktree cleanup, which can run for minutes.
2397    sweep_browser_seats(workspace);
2398
2399    // Everything from here runs `git` or walks the tree, and the removal
2400    // alone can take minutes: inline on a runtime worker it holds the
2401    // socket's own work for the whole time (#1633), so it goes to the
2402    // blocking pool, as the subagent worktree reap does. The reply, the
2403    // pane's event and any warning still go out when the cleanup finishes.
2404    let log_project = project_key.clone();
2405    let log_label = label.to_owned();
2406    let cleanup = DespawnCleanup {
2407        workspace: Arc::clone(workspace),
2408        project_key: project_key.clone(),
2409        label: label.to_owned(),
2410        force,
2411        worktree_path,
2412        worktree_display,
2413        worktree_already_gone,
2414        claim,
2415        worktree_root,
2416        project_view,
2417        live,
2418        is_git_repo,
2419        respond,
2420    };
2421    if let Ok(handle) = tokio::runtime::Handle::try_current() {
2422        // A wrapper task holds the removal until the worker's session
2423        // has finished dying, then awaits the blocking handle, so a
2424        // panic in the cleanup names its site in the log instead of
2425        // vanishing with the JoinHandle; the caller still sees the
2426        // dropped reply.
2427        //
2428        // The wait is the point: the session's exit path is what reaps
2429        // the claude child (waits it out, then SIGKILLs), and a
2430        // removal that overlapped those final seconds is how a
2431        // runaway wrote back into the path the removal had already
2432        // taken (#1767). `Err` means the task went away without
2433        // firing - nothing is left to wait for. The wait is BOUNDED:
2434        // the close-wait budget bounds the reap, not the whole path to
2435        // the task's drop, and an unbounded wait would leave the reply
2436        // never arriving and the label cleanup-pending forever.
2437        handle.spawn(async move {
2438            if let Some(mut done) = teardown_done
2439                && tokio::time::timeout(teardown_wait, async {
2440                    if *done.borrow_and_update() {
2441                        return;
2442                    }
2443                    let _ = done.changed().await;
2444                })
2445                .await
2446                .is_err()
2447            {
2448                warn_teardown_wait_expired(&log_project, &log_label, teardown_wait);
2449            }
2450            let task = tokio::task::spawn_blocking(move || cleanup.run(cleanup_pending));
2451            if let Err(join_err) = task.await {
2452                tracing::warn!(
2453                    target: "forge_workspace::spawn",
2454                    event_name = "despawn_cleanup_panicked",
2455                    project = %log_project.as_str(),
2456                    label = %log_label,
2457                    error = %join_err,
2458                    "despawn: the worktree cleanup panicked part-way through",
2459                );
2460            }
2461        });
2462    } else {
2463        // No runtime to hand it to - a caller outside one: the cleanup runs
2464        // where it is, the way this path always did. The wait still
2465        // happens, bounded: the value is polled on a deadline, since the
2466        // watch's change notification is async-only.
2467        if let Some(mut done) = teardown_done {
2468            let deadline = std::time::Instant::now() + teardown_wait;
2469            while !*done.borrow_and_update() && std::time::Instant::now() < deadline {
2470                std::thread::sleep(std::time::Duration::from_millis(25));
2471            }
2472            if !*done.borrow() {
2473                warn_teardown_wait_expired(project_key, label, teardown_wait);
2474            }
2475        }
2476        cleanup.run(cleanup_pending);
2477    }
2478}
2479
2480/// The removal went ahead without the ordering: the session's teardown did
2481/// not answer within the wait. Warned rather than silent - it is the one
2482/// shape where the despawn's protection is absent.
2483fn warn_teardown_wait_expired(project_key: &ProjectKey, label: &str, wait: std::time::Duration) {
2484    tracing::warn!(
2485        target: "forge_workspace::spawn",
2486        event_name = "despawn_teardown_wait_expired",
2487        project = %project_key.as_str(),
2488        label = %label,
2489        ?wait,
2490        "despawn: the worker's session did not finish dying within the wait; removing the \
2491         worktree without it",
2492    );
2493}
2494
2495/// The despawn's dirty verdict, taken off the runtime's worker: `git status`
2496/// walks the tracked tree, and the verdict gates the teardown, so the caller
2497/// cannot await a `spawn_blocking` handle. `block_in_place` hands the worker's
2498/// tasks to another thread for the call; a current-thread runtime has no other
2499/// thread, and stays inline. A `LocalSet` is not detected - tokio still reports
2500/// `MultiThread` there, where `block_in_place` panics - and the workspace runs
2501/// none.
2502fn worktree_dirty_verdict(path: &std::path::Path) -> Option<String> {
2503    let on_runtime_worker = tokio::runtime::Handle::try_current()
2504        .is_ok_and(|handle| handle.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread);
2505    if on_runtime_worker {
2506        return tokio::task::block_in_place(|| {
2507            forge_agent::env::worktree::worktree_dirty_reason(path)
2508        });
2509    }
2510    forge_agent::env::worktree::worktree_dirty_reason(path)
2511}
2512
2513/// The despawn's filesystem half, handed to the blocking pool.
2514///
2515/// Split out of [`handle_despawn_worker`] so the peek, the dirty verdict's
2516/// ordering and the teardown stay on the calling task, while the probe and the
2517/// cleanup's own `git` calls - the `git worktree remove` above all - run off
2518/// the runtime.
2519struct DespawnCleanup {
2520    workspace: Arc<Workspace>,
2521    project_key: ProjectKey,
2522    label: String,
2523    force: bool,
2524    worktree_path: Option<std::path::PathBuf>,
2525    worktree_display: String,
2526    worktree_already_gone: bool,
2527    /// The path's claim, held until the cleanup ends so no spawn in
2528    /// another project can enter the tree while it is removed.
2529    claim: Option<crate::workspace::WorktreeClaimGuard>,
2530    /// The checkout whose `.claude/worktrees` holds the worker's tree -
2531    /// the repo every `git` call in the cleanup anchors on. The main
2532    /// checkout for a worker the CLI forked under it. `None` when the
2533    /// project is no longer loaded and the row recorded no tree of its
2534    /// own.
2535    worktree_root: Option<std::path::PathBuf>,
2536    project_view: Option<crate::views::ProjectView>,
2537    live: Option<crate::mcp::workers::types::WorkerEntry>,
2538    is_git_repo: bool,
2539    respond: tokio::sync::oneshot::Sender<crate::protocol::DespawnResult>,
2540}
2541
2542impl DespawnCleanup {
2543    /// Run the cleanup, holding `pending` - the label's in-flight mark -
2544    /// until the end, so the label is reserved for every step that touches
2545    /// the worktree and released only once the removal is done and its
2546    /// outcome has been announced.
2547    fn run(self, pending: DespawnCleanupPending) {
2548        use crate::protocol::DespawnResult;
2549
2550        let Self {
2551            workspace,
2552            project_key,
2553            label,
2554            force,
2555            worktree_path,
2556            worktree_display,
2557            worktree_already_gone,
2558            // Held for the cleanup's whole span; released on drop.
2559            claim: _claim,
2560            worktree_root,
2561            project_view,
2562            live,
2563            is_git_repo,
2564            respond,
2565        } = self;
2566        let label = label.as_str();
2567        // Resolve the torn-down worktree's `(project name, branch)` while
2568        // the path still exists, so its persisted review state can be judged
2569        // once the worktree is gone. Keyed by the forge.toml project NAME to
2570        // match what the diff overlay saved under.
2571        let review_key = worktree_path.as_ref().and_then(|path| {
2572            let Some(branch) = forge_agent::env::worktree::worktree_branch(path) else {
2573                tracing::warn!(
2574                    target: "forge_workspace::spawn",
2575                    project = %project_key.as_str(),
2576                    label = %label,
2577                    "despawn: could not resolve the worktree branch (detached HEAD or git error); its review threads are not cleaned up and may resurrect on a later worktree reusing the branch",
2578                );
2579                return None;
2580            };
2581            let Some(name) = project_view.as_ref().map(|v| v.name.clone()) else {
2582                tracing::warn!(
2583                    target: "forge_workspace::spawn",
2584                    project = %project_key.as_str(),
2585                    label = %label,
2586                    branch = %branch,
2587                    "despawn: could not resolve the project name; the branch's review threads are not cleaned up",
2588                );
2589                return None;
2590            };
2591            Some((name, branch))
2592        });
2593
2594        // Worktree cleanup runs AFTER teardown on a verified-clean (or
2595        // forced) worktree. A failure here is reported but never rolls
2596        // back the already-completed teardown.
2597        let mut branch_cleanup_warning = None;
2598        let worktree_cleanup_warning = match worktree_path.as_ref() {
2599            // A stranded row whose worktree is already gone is the definition
2600            // of that state - it is why the boot wave skips the row - so there
2601            // is nothing to remove and git's failure over an untracked path
2602            // would be a warning about nothing. A live worker's worktree
2603            // vanishing is the opposite: anomalous, and the live shape below
2604            // keeps reporting what git said.
2605            //
2606            // The BRANCH is a different matter, and it is still there: `git
2607            // worktree remove` never deletes one. With the directory gone the
2608            // branch is certainly not checked out, so this is the one place
2609            // reaping it is unambiguously safe.
2610            Some(path) if worktree_already_gone => {
2611                tracing::debug!(
2612                    target: "forge_workspace::spawn",
2613                    event_name = "despawn_stranded_worktree_already_gone",
2614                    project = %project_key.as_str(),
2615                    label = %label,
2616                    worktree = %worktree_display,
2617                    "despawn: the stranded row's worktree is already gone; nothing to remove",
2618                );
2619                // The directory is gone, so any earlier admission for this
2620                // label is spent; retract it rather than let it authorize
2621                // deleting a directory that later takes its place.
2622                if let Some(root) = worktree_root.as_deref() {
2623                    forge_agent::env::worktree::record_worktree_gone(root, label, path);
2624                }
2625                branch_cleanup_warning = reap_branch_and_reviews(
2626                    &workspace,
2627                    worktree_root.as_deref(),
2628                    review_key.as_ref(),
2629                    label,
2630                );
2631                None
2632            }
2633            Some(path) => {
2634                // git refuses a worktree holding initialized submodules
2635                // however clean it is (git-worktree(1)), so its refusal is
2636                // not a verdict on the disk and cannot stand as one. Re-judge
2637                // it against the worktree as it is now: `--force` discards
2638                // uncommitted work, so it stands in only for a verdict taken
2639                // here rather than one taken before the worker was torn down.
2640                // The admitted removal is recorded before git runs, so a
2641                // removal that dies part-way leaves the record the
2642                // leftover clear deletes on. `worktree_root` is Some
2643                // whenever `worktree_path` is (the path is derived from
2644                // the root above), so the plain-removal arm is a fallback
2645                // the flow cannot reach today.
2646                let remove = |force: bool| match worktree_root.as_deref() {
2647                    Some(root) => {
2648                        forge_agent::env::worktree::remove_owned_worktree(root, label, path, force)
2649                    }
2650                    None => forge_agent::env::worktree::remove_worktree(path, force),
2651                };
2652                let removal = match remove(force) {
2653                    Err(err)
2654                        if !force
2655                            && forge_agent::env::worktree::worktree_dirty_reason(path)
2656                                .is_none() =>
2657                    {
2658                        tracing::debug!(
2659                            target: "forge_workspace::spawn",
2660                            event_name = "despawn_worktree_removal_forced",
2661                            project = %project_key.as_str(),
2662                            label = %label,
2663                            worktree = %path.display(),
2664                            error = %err,
2665                            "despawn: git refused the removal of a clean worktree; retrying with --force",
2666                        );
2667                        remove(true)
2668                    }
2669                    other => other,
2670                };
2671                // git deregisters a worktree before it walks the tree, so
2672                // a delete that failed part-way (a build still writing
2673                // into `target/`) leaves the directory with no
2674                // registration: the re-check above can no longer read the
2675                // tree and every later `git worktree remove` refuses it.
2676                // With the tree unreadable, the fallback re-checks the
2677                // path's shape, that git no longer lists it, and the
2678                // removal record written above; the cleanliness behind
2679                // the delete is the verdict taken before teardown (or the
2680                // caller's `force`), since a deregistered tree cannot be
2681                // asked again.
2682                let removal = match removal {
2683                    Ok(()) => Ok(()),
2684                    Err(git_error) => match worktree_root.as_deref() {
2685                        None => Err(git_error.to_string()),
2686                        Some(root) => {
2687                            match forge_agent::env::worktree::remove_orphaned_worktree_dir(
2688                                root, path,
2689                            ) {
2690                                Ok(()) => {
2691                                    tracing::info!(
2692                                        target: "forge_workspace::spawn",
2693                                        event_name = "despawn_worktree_leftover_removed",
2694                                        project = %project_key.as_str(),
2695                                        label = %label,
2696                                        worktree = %path.display(),
2697                                        "despawn: removed the leftover directory git could no \
2698                                         longer remove",
2699                                    );
2700                                    Ok(())
2701                                }
2702                                Err(
2703                                    forge_agent::env::worktree::WorktreeError::LeftoverUndeletable {
2704                                        error,
2705                                        ..
2706                                    },
2707                                ) => Err(format!(
2708                                    "{git_error}; and its leftover directory could not be \
2709                                     deleted: {error}"
2710                                )),
2711                                // A refusal means the directory was kept -
2712                                // a registered worktree, a foreign one, or
2713                                // one with no admitted removal. Carry its
2714                                // detail so the operator sees why, and the
2715                                // remedy, rather than git's bare error.
2716                                Err(refusal) => {
2717                                    tracing::debug!(
2718                                        target: "forge_workspace::spawn",
2719                                        event_name = "despawn_worktree_leftover_kept",
2720                                        project = %project_key.as_str(),
2721                                        label = %label,
2722                                        worktree = %path.display(),
2723                                        error = %refusal,
2724                                        "despawn: the leftover path was left in place",
2725                                    );
2726                                    Err(format!(
2727                                        "{git_error}; and the leftover directory was kept: \
2728                                         {refusal}"
2729                                    ))
2730                                }
2731                            }
2732                        }
2733                    },
2734                };
2735                match removal {
2736                    Ok(()) => {
2737                        tracing::info!(
2738                            target: "forge_workspace::spawn",
2739                            project = %project_key.as_str(),
2740                            label = %label,
2741                            worktree = %path.display(),
2742                            "despawn: worktree removed",
2743                        );
2744                        // Only after a successful removal: while the worktree
2745                        // stands it holds the branch checked out, and git refuses
2746                        // to delete a checked-out branch.
2747                        branch_cleanup_warning = reap_branch_and_reviews(
2748                            &workspace,
2749                            worktree_root.as_deref(),
2750                            review_key.as_ref(),
2751                            label,
2752                        );
2753                        None
2754                    }
2755                    Err(err) => {
2756                        tracing::warn!(
2757                            target: "forge_workspace::spawn",
2758                            project = %project_key.as_str(),
2759                            label = %label,
2760                            error = %err,
2761                            "despawn: worker torn down but worktree cleanup failed"
2762                        );
2763                        Some(err)
2764                    }
2765                }
2766            }
2767            None => None,
2768        };
2769
2770        // Ground-truthed against the directory rather than git's exit code:
2771        // git errors for a path it no longer tracks whether or not the
2772        // directory survives, and the toast's only claim is what is on disk.
2773        let worktree = match (worktree_path.as_ref(), worktree_cleanup_warning.as_ref()) {
2774            (None, _) => WorktreeDisposition::untouched(is_git_repo),
2775            (Some(path), Some(_)) if path.exists() => WorktreeDisposition::RemovalFailed,
2776            (Some(_), _) => WorktreeDisposition::Removed,
2777        };
2778        // Only a torn-down live worker has a pane row to remove; a stranded
2779        // row's label reaches the launchpad by reading the store per frame, so
2780        // it needs no event.
2781        //
2782        // The event is keyed by label and lands here, after the removal, so
2783        // it evicts whichever occupant holds the seat when a viewer reads it.
2784        // The in-flight mark is load-bearing for that: it keeps a same-label
2785        // spawn from putting a new occupant under the seat this event is
2786        // about to clear, and it is held until the guard below drops.
2787        if let Some(entry) = live.as_ref() {
2788            emit_worker_removed(&workspace, &project_key, entry, worktree);
2789        }
2790
2791        // The label is free from here, before the reply: the worktree this
2792        // held it for is gone (or the failure to remove it was reported),
2793        // and a caller that re-admits the label the moment its reply lands
2794        // must find it free by ordering, not by luck.
2795        drop(pending);
2796        let _ = respond
2797            .send(DespawnResult::Despawned { worktree_cleanup_warning, branch_cleanup_warning });
2798    }
2799}
2800
2801/// Reap the worker's branch and, once that is gone, the review state keyed
2802/// to it. Returns the warning a kept branch produces, which the caller's
2803/// result carries.
2804///
2805/// The branch outlives the worktree - `git worktree remove` does not delete
2806/// one - so this runs after a removal, and also for a worktree that was
2807/// already gone: gone means the branch is no longer checked out, which is
2808/// exactly when the delete is safe.
2809fn reap_branch_and_reviews(
2810    workspace: &Arc<Workspace>,
2811    repo: Option<&std::path::Path>,
2812    review_key: Option<&(String, String)>,
2813    label: &str,
2814) -> Option<String> {
2815    let warning = repo.and_then(|repo| reap_worker_branch(repo, label));
2816    // Threads are only orphaned once their branch is gone, and the reap
2817    // above can be what removes it - a worker that made no branch of its
2818    // own sits on `worktree-<label>`. The review state stays keyed by the
2819    // project's name: that is where the diff overlay saved it, even when
2820    // the worker ran in another tree.
2821    if let Some((project, branch)) = review_key
2822        && let Some(repo) = repo
2823        && !forge_agent::env::worktree::branch_ref_exists(repo, branch)
2824    {
2825        workspace.delete_branch_review_state(project, branch);
2826    }
2827    warning
2828}
2829
2830/// Reap the `worktree-<label>` branch claude creates for a worker's
2831/// worktree, which outlives the worktree itself. Returns a warning when
2832/// the branch was left in place, `None` when it was deleted or never
2833/// existed.
2834///
2835/// The branch is named by convention, never taken from whatever the
2836/// worktree had checked out: a worker that made its own feature branch
2837/// has real work on that one, and this must not aim at it.
2838fn reap_worker_branch(repo: &std::path::Path, label: &str) -> Option<String> {
2839    use forge_agent::env::worktree::BranchReapOutcome;
2840
2841    let branch = format!("worktree-{label}");
2842    let warning = match forge_agent::env::worktree::reap_worktree_branch(repo, &branch) {
2843        BranchReapOutcome::Reaped => None,
2844        BranchReapOutcome::NotPresent => {
2845            tracing::debug!(
2846                target: "forge_workspace::spawn",
2847                label = %label,
2848                branch = %branch,
2849                "despawn: no worktree branch to reap"
2850            );
2851            None
2852        }
2853        BranchReapOutcome::Kept { count, tip } => {
2854            let plural = if count == 1 { "" } else { "s" };
2855            Some(format!(
2856                "branch '{branch}' kept: {count} commit{plural} reachable from no other ref \
2857                 (tip {tip}). Inspect with 'git log -{count} {tip}', then 'git branch -D \
2858                 {branch}' once it has landed."
2859            ))
2860        }
2861        BranchReapOutcome::KeptOnError { reason } => Some(format!(
2862            "branch '{branch}' kept: could not verify it holds no unique commits ({reason}). \
2863             Check 'git log {branch}' and delete it by hand."
2864        )),
2865        BranchReapOutcome::DeleteFailed { reason } => Some(format!(
2866            "branch '{branch}' holds no unique commits, but the delete failed ({reason}). \
2867             Retry with 'git branch -D {branch}'."
2868        )),
2869    };
2870    // The oneshot reply reaches an LLM's tool result and nothing else, so
2871    // without this the operator's log has no record of a kept branch.
2872    if let Some(warning) = &warning {
2873        tracing::warn!(
2874            target: "forge_workspace::spawn",
2875            label = %label,
2876            branch = %branch,
2877            warning = %warning,
2878            "despawn: worktree branch kept"
2879        );
2880    }
2881    warning
2882}
2883
2884/// Handle a `Command::DeliverWorkerPrompt`: route a wrapped peer-style
2885/// envelope to the worker matching `target_label` in the caller's
2886/// project. Latest-spawned-wins on duplicate labels. The typed
2887/// PeerEnvelopeAppended echo follows the same pattern as
2888/// `handle_deliver_peer_prompt` - workers reuse the peer envelope
2889/// verbatim so a view's chat render is identical between the two
2890/// paths.
2891///
2892/// Unlike `handle_deliver_peer_prompt`, this handler never buffers +
2893/// auto-spawns: workers are only addressable while live. If the
2894/// target label vanished between the dispatch and the handler firing
2895/// (worker closed, lead cascade fired), the prompt is dropped with a
2896/// warn log.
2897/// Buffer `wrapped` on `target_key`'s `DomainSession` when the target
2898/// hasn't finished its Connected handshake (no `session_id` yet). Returns
2899/// `Some(wrapped)` when the target IS connected (caller delivers now), or
2900/// `None` when it parked (the target's Connected handler drains the
2901/// bucket, doing the render + dispatch). Mirrors the sleeping-peer
2902/// buffering in `handle_deliver_peer_prompt`, sender included so an
2903/// expired bucket can still acknowledge the message.
2904fn buffer_prompt_until_connected(
2905    workspace: &Arc<Workspace>,
2906    sender: &crate::SessionSlot,
2907    target_key: &SessionSlot,
2908    wrapped: WrappedPrompt,
2909) -> Option<WrappedPrompt> {
2910    let Some(domain) = workspace.domain_session_for(target_key) else {
2911        // No DomainSession to park against (target vanished); let the
2912        // caller proceed - its dispatch warns on the missing target.
2913        return Some(wrapped);
2914    };
2915    if domain.lock().session_id.is_some() {
2916        return Some(wrapped);
2917    }
2918    workspace.park_peer_prompt(target_key, sender, wrapped);
2919    None
2920}
2921
2922pub(crate) fn handle_deliver_worker_prompt(
2923    workspace: &Arc<Workspace>,
2924    caller: &SessionSlot,
2925    project_key: &ProjectKey,
2926    target_label: &str,
2927    wrapped: WrappedPrompt,
2928    respond: Option<tokio::sync::oneshot::Sender<Result<(), String>>>,
2929) {
2930    // Latest-spawned matching label wins (mirrors the addressing rule
2931    // in agents__send_message). Failed entries are skipped, the same rule
2932    // the spawn gate applies: a crashed worker keeps its row for the label
2933    // to be respawnable, and a prompt addressed there would go nowhere.
2934    let Some(entry) = workspace
2935        .list_live_workers(project_key)
2936        .into_iter()
2937        .rev()
2938        .find(|w| w.label == target_label && w.is_live())
2939    else {
2940        tracing::warn!(
2941            target: "forge_workspace::spawn",
2942            project = %project_key.as_str(),
2943            label = %target_label,
2944            "deliver_worker_prompt: no matching live worker (target gone since dispatch)"
2945        );
2946        answer_delivery(
2947            respond,
2948            Err(format!(
2949                "no live worker with label '{target_label}' in '{}' any more",
2950                project_key.as_str()
2951            )),
2952        );
2953        return;
2954    };
2955    let target_key = entry.slot.clone();
2956
2957    // A worker addressed before it finishes its Connected handshake has
2958    // no session_id yet, so a bare Command::Prompt would be dropped by
2959    // execute_command_via_handle. Park it for the worker's label instead -
2960    // its Connected handler drains the bucket (render + dispatch)
2961    // exactly like the sleeping-peer path. Skips the tag retry / dispatch
2962    // below.
2963    let Some(wrapped) = buffer_prompt_until_connected(workspace, caller, &target_key, wrapped)
2964    else {
2965        // Parked for the worker's connect, whose drain delivers it.
2966        answer_delivery(respond, Ok(()));
2967        return;
2968    };
2969
2970    // A delivery that lands while the seat is mid-turn is folded into the
2971    // running turn as a queued_command attachment with no standing of its
2972    // own - the follow-up a working worker reads past (measured on
2973    // 2.1.280). Hold it and deliver it at turn end, so it starts a turn.
2974    let held = {
2975        let domain = workspace.domain_session_for(&target_key);
2976        workspace.hold_peer_prompt_unless_turn_ended(&target_key, caller, wrapped, || {
2977            domain.as_ref().is_some_and(|domain| delivery_hold_signal(&domain.lock()))
2978        })
2979    };
2980    let Some(wrapped) = held else {
2981        answer_delivery(respond, Ok(()));
2982        return;
2983    };
2984
2985    // Opportunistic tag-write retry: if this worker was spawned idle
2986    // (no initial_prompt), the JSONL didn't exist at Connected and
2987    // the tag-write at that point exhausted into a deferred state.
2988    // claude is about to process this turn, which means it's about
2989    // to write the JSONL - kick off a fire-and-forget retry now.
2990    if entry.needs_tag
2991        && let Some(session_id) = entry.session_id.clone()
2992    {
2993        // The retry composes the tag directory from the worker's OWN
2994        // tree - the same rule every read-back follows - with the
2995        // project's path as the fallback for a worker that has none: a
2996        // worker the spawn pointed at another repository keeps its JSONL
2997        // under that repository's worktree, so a retry aimed at the
2998        // project's path never finds it and the worker stays untagged.
2999        let root = entry.cwd.clone().or_else(|| {
3000            workspace
3001                .list_projects()
3002                .into_iter()
3003                .find(|view| view.key == *project_key)
3004                .map(|view| view.path)
3005        });
3006        if let Some(root) = root {
3007            workspace.retry_worker_tag_opportunistic(
3008                project_key,
3009                &target_key,
3010                session_id.as_str(),
3011                target_label,
3012                &root.to_string_lossy(),
3013                entry.is_git_repo_at_spawn,
3014            );
3015        } else {
3016            tracing::debug!(
3017                target: "forge_workspace::spawn",
3018                project = %project_key.as_str(),
3019                label = %target_label,
3020                "deliver_worker_prompt: project view missing; skipping tag retry"
3021            );
3022        }
3023    }
3024
3025    // Fire the typed peer-envelope echo BEFORE the LLM-side dispatch
3026    // so the user-turn block renders in the right order regardless
3027    // of which event a view reducer drains first. Compute the prose
3028    // body BEFORE the move into PeerEnvelopeAppended so we consume
3029    // `wrapped` exactly once (the push_peer_user_turn_into_chat helper
3030    // takes `&WrappedPrompt` and clones internally).
3031    let text = wrapped.to_prose();
3032    let uuid = forge_sdk::request_id::next_prompt_id();
3033    push_peer_user_turn_into_chat(workspace, &target_key, &wrapped, &uuid);
3034    drop(wrapped);
3035    if let Err(err) =
3036        workspace.dispatch_workspace_prompt_under(&target_key, text, PromptSource::Peer, uuid)
3037    {
3038        tracing::warn!(
3039            target: "forge_workspace::spawn",
3040            project = %project_key.as_str(),
3041            label = %target_label,
3042            error = ?err,
3043            "DeliverWorkerPrompt dispatch to worker failed"
3044        );
3045        send_dispatch_turn_error(workspace, target_key, &err);
3046        answer_delivery(respond, Err(format!("the worker's session did not take it ({err:?})")));
3047        return;
3048    }
3049    answer_delivery(respond, Ok(()));
3050}
3051
3052/// The turn signal a delivery is held on: only what a `Result` clears.
3053///
3054/// Stricter than
3055/// [`turn_in_flight`](crate::domain_session::DomainSession::turn_in_flight),
3056/// which ORs in `runtime_state` for the in-flight guards - no `Result`
3057/// clears that one, so a gate reading it could hold every later delivery
3058/// with nothing left to release it. Both signals here are down once a
3059/// Result has landed, which is what lets the hold and the turn-end drain
3060/// pair up.
3061fn delivery_hold_signal(domain: &crate::domain_session::DomainSession) -> bool {
3062    domain.turn_pending || domain.turn_open
3063}
3064
3065/// Answer a delivery's `respond` channel, if the caller asked for one:
3066/// `Ok(())` once the prompt was accepted - handed to the target's session
3067/// queue, held for its turn end, or parked for its connect - and
3068/// `Err(reason)` when nothing could take it. The sender's ack is built
3069/// from this answer alone, so it never claims a delivery that did not
3070/// happen.
3071fn answer_delivery(
3072    respond: Option<tokio::sync::oneshot::Sender<Result<(), String>>>,
3073    outcome: Result<(), String>,
3074) {
3075    if let Some(tx) = respond {
3076        let _ = tx.send(outcome);
3077    }
3078}
3079
3080/// Deliver everything held for `slot` while a turn was in flight, in the
3081/// order it arrived. Called where the turn's Result lands, so each held
3082/// prompt starts a turn of its own instead of folding into the one that
3083/// was running. A prompt the seat cannot take now - the session died
3084/// between the hold and the turn end - is told to its sender rather than
3085/// dropped, which is the whole point of holding it.
3086///
3087/// **The boundary's own settle is the session task's**, since it knows which
3088/// turn ended; this drain clears no liveness. The take shares its lock with a
3089/// concurrent hold's check-and-enqueue, so a delivery decides on
3090/// [`delivery_hold_signal`] either before this drain (and is delivered by it)
3091/// or after (and is delivered directly).
3092pub(crate) fn drain_turn_held_for_slot(workspace: &Arc<Workspace>, slot: &SessionSlot) {
3093    for crate::parked::ParkedPeer { sender, wrapped } in workspace.take_turn_held_for_slot(slot) {
3094        let text = wrapped.to_prose();
3095        let uuid = forge_sdk::request_id::next_prompt_id();
3096        push_peer_user_turn_into_chat(workspace, slot, &wrapped, &uuid);
3097        if let Err(err) =
3098            workspace.dispatch_workspace_prompt_under(slot, text, PromptSource::Peer, uuid)
3099        {
3100            tracing::warn!(
3101                target: "forge_workspace::spawn",
3102                slot = %slot.display(),
3103                error = ?err,
3104                "a held prompt could not be delivered when the turn ended"
3105            );
3106            workspace.notice_undelivered_message(
3107                &sender,
3108                slot,
3109                crate::mcp::peers::types::PeerFailureReason::TargetConnectionFailed,
3110            );
3111        }
3112    }
3113}
3114
3115/// Handle a `Command::DeliverWorkerPromptToLead`: route a wrapped
3116/// peer-style prompt from a worker back to its lead, addressed by
3117/// the lead's `SessionSlot` (resolved at Tool dispatch from the
3118/// worker's `spawned_by_session_id`). Same wire shape as
3119/// `DeliverWorkerPrompt` - PeerEnvelopeAppended echo + Command::Prompt
3120/// dispatch - so a view renders the lead's message identically to
3121/// a sibling-worker delivery.
3122///
3123/// Drops with a warn log when the target lead session is no longer
3124/// in the pool (lead closed since the worker captured its
3125/// `spawned_by_session_id`). The lead can't be auto-respawned from
3126/// this path: that's a project-level decision the worker isn't
3127/// authorized to make.
3128pub(crate) fn handle_deliver_worker_prompt_to_lead(
3129    workspace: &Arc<Workspace>,
3130    caller: &SessionSlot,
3131    target_lead_key: &SessionSlot,
3132    wrapped: WrappedPrompt,
3133    respond: Option<tokio::sync::oneshot::Sender<Result<(), String>>>,
3134) {
3135    // Defensive: confirm the lead session is still in the pool. If it
3136    // closed since the worker captured its `spawned_by_session_id`,
3137    // drop the prompt with a warn - same shape as `DeliverWorkerPrompt`
3138    // does for a worker that vanished between dispatch and handler.
3139    if !workspace.pool.lock().contains_key(target_lead_key) {
3140        tracing::warn!(
3141            target: "forge_workspace::spawn",
3142            slot = %target_lead_key.display(),
3143            "deliver_worker_prompt_to_lead: lead session not in pool (closed since dispatch)"
3144        );
3145        answer_delivery(respond, Err("the lead's session is no longer running".to_owned()));
3146        return;
3147    }
3148
3149    // Same pre-Connect guard as the sibling-worker path: if the lead
3150    // hasn't stamped its session_id yet, buffer for its Connected drain
3151    // rather than dispatching a Command::Prompt that would be dropped.
3152    let Some(wrapped) = buffer_prompt_until_connected(workspace, caller, target_lead_key, wrapped)
3153    else {
3154        // Parked for the lead's connect, whose drain delivers it.
3155        answer_delivery(respond, Ok(()));
3156        return;
3157    };
3158
3159    // Same hold as the sibling-worker path: a lead mid-turn folds the
3160    // message into its running turn, where it has no standing of its own.
3161    let held = {
3162        let domain = workspace.domain_session_for(target_lead_key);
3163        workspace.hold_peer_prompt_unless_turn_ended(target_lead_key, caller, wrapped, || {
3164            domain.as_ref().is_some_and(|domain| delivery_hold_signal(&domain.lock()))
3165        })
3166    };
3167    let Some(wrapped) = held else {
3168        answer_delivery(respond, Ok(()));
3169        return;
3170    };
3171
3172    let text = wrapped.to_prose();
3173    let uuid = forge_sdk::request_id::next_prompt_id();
3174    push_peer_user_turn_into_chat(workspace, target_lead_key, &wrapped, &uuid);
3175    drop(wrapped);
3176    if let Err(err) =
3177        workspace.dispatch_workspace_prompt_under(target_lead_key, text, PromptSource::Peer, uuid)
3178    {
3179        tracing::warn!(
3180            target: "forge_workspace::spawn",
3181            slot = %target_lead_key.display(),
3182            error = ?err,
3183            "DeliverWorkerPromptToLead dispatch to lead failed"
3184        );
3185        send_dispatch_turn_error(workspace, target_lead_key.clone(), &err);
3186        answer_delivery(respond, Err(format!("the lead's session did not take it ({err:?})")));
3187        return;
3188    }
3189    answer_delivery(respond, Ok(()));
3190}
3191
3192#[cfg(test)]
3193#[allow(clippy::expect_used, clippy::unwrap_used)]
3194mod tests {
3195    use super::*;
3196    use crate::SeatUpdate;
3197    use crate::Workspace;
3198    use std::fs;
3199    use tempfile::tempdir;
3200
3201    /// Ensure `forge/` exists and return the production `forge/forge.toml`
3202    /// path, so tests write where forge reads (not the legacy fallback).
3203    fn forge_toml_path(config_dir: &std::path::Path) -> std::path::PathBuf {
3204        crate::config::ensure_forge_data_dir(config_dir).expect("forge/ dir").join("forge.toml")
3205    }
3206
3207    /// A fatal error is an App-level event with no state behind it, so a
3208    /// view that was not subscribed when it fired could never learn of it -
3209    /// and a client attaching to a running forge would draw a healthy
3210    /// startup that had already failed. The emit holds it; the read answers
3211    /// from what was held.
3212    #[test]
3213    fn a_fatal_error_is_held_where_it_is_emitted() {
3214        let (ws, _rx) = Workspace::testing_stub();
3215        let failed = forge_primitives::error::AppError::ConnectionFailed;
3216
3217        assert!(ws.last_fatal_error().is_none(), "nothing has failed fatally yet");
3218
3219        try_emit(
3220            &ws,
3221            "test",
3222            SessionUpdate::FatalError {
3223                message: failed.user_message().to_owned(),
3224                error: failed.clone(),
3225            },
3226        );
3227
3228        assert_eq!(
3229            ws.last_fatal_error(),
3230            Some(failed),
3231            "the error the emit carried is the error the read answers with",
3232        );
3233    }
3234
3235    /// The project path the shared fixture points at. The maintainer's own
3236    /// checkout, which is a git repo, so the gitness probe answers true for
3237    /// it; a fixture that needs a different answer writes its own.
3238    const FIXTURE_PROJECT_PATH: &str = "~/Projects/forge";
3239
3240    fn write_forge_toml(dir: &std::path::Path, project_path: &str) {
3241        fs::write(
3242            forge_toml_path(dir),
3243            format!(
3244                r#"
3245[[orgs]]
3246name = "Default"
3247accounts = ["Stargate"]
3248
3249[[orgs.projects]]
3250name = "forge"
3251path = "{project_path}"
3252model = "claude-sonnet-5"
3253
3254[[accounts]]
3255display_name = "Stargate"
3256token = "t"
3257models = ["claude-sonnet-5"]
3258provider = "anthropic"
3259"#
3260            ),
3261        )
3262        .expect("write forge.toml");
3263    }
3264
3265    #[test]
3266    fn project_permission_mode_stamps_into_absent_settings() {
3267        let mut settings = SessionLaunchSettings::default();
3268        stamp_permission_mode(
3269            &mut settings,
3270            forge_primitives::permission::PermissionMode::BypassPermissions,
3271        );
3272        let mode = settings
3273            .settings
3274            .as_ref()
3275            .and_then(|s| s.get("permissions"))
3276            .and_then(|p| p.get("defaultMode"))
3277            .and_then(serde_json::Value::as_str);
3278        assert_eq!(
3279            mode,
3280            Some("bypassPermissions"),
3281            "the project mode reaches the settings JSON the worker's applied_permission_mode arm reads",
3282        );
3283    }
3284
3285    #[test]
3286    fn project_permission_mode_overrides_the_launcher_default_and_keeps_siblings() {
3287        let mut settings = SessionLaunchSettings {
3288            settings: Some(serde_json::json!({
3289                "permissions": { "defaultMode": "default" },
3290                "model": "haiku",
3291            })),
3292            ..SessionLaunchSettings::default()
3293        };
3294        stamp_permission_mode(
3295            &mut settings,
3296            forge_primitives::permission::PermissionMode::BypassPermissions,
3297        );
3298        let record = settings.settings.as_ref().expect("settings kept");
3299        assert_eq!(
3300            record
3301                .get("permissions")
3302                .and_then(|p| p.get("defaultMode"))
3303                .and_then(serde_json::Value::as_str),
3304            Some("bypassPermissions"),
3305            "the project mode wins over the launcher's session default",
3306        );
3307        assert_eq!(
3308            record.get("model").and_then(serde_json::Value::as_str),
3309            Some("haiku"),
3310            "sibling settings keys survive the stamp",
3311        );
3312    }
3313
3314    /// `handle_spawn_project` for an unknown project name must not
3315    /// panic, must not emit a `SessionUpdate::Spawning`, and must
3316    /// log a warning. Important for the user-input boundary - a
3317    /// click on a stale row shouldn't crash forge.
3318    #[tokio::test]
3319    async fn spawn_project_unknown_project_emits_no_update() {
3320        let dir = tempdir().expect("tempdir");
3321        write_forge_toml(dir.path(), FIXTURE_PROJECT_PATH);
3322        let workspace = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("new"));
3323        let mut rx = workspace.subscribe_answerer();
3324
3325        assert!(
3326            handle_spawn_project(&workspace, "no-such-project", SessionLaunchSettings::default())
3327                .is_err(),
3328            "an unknown project is refused rather than ignored",
3329        );
3330
3331        // The user sees a typed notice naming the unknown project.
3332        assert!(
3333            matches!(
3334                rx.try_recv(),
3335                Ok(frame)
3336                    if matches!(&*frame.update, SessionUpdate::ServiceStatus { message, .. } if message.contains("no-such-project"))
3337            ),
3338            "unknown project surfaces a ServiceStatus notice"
3339        );
3340        assert!(rx.try_recv().is_err(), "nothing beyond the notice");
3341    }
3342
3343    /// `handle_spawn_project` for a known project must emit a
3344    /// `SessionUpdate::Spawning` under the id the lead will run under,
3345    /// so a view can show a Waking placeholder before the agent reaches
3346    /// Connected and the bucket it draws is the one that connects.
3347    #[tokio::test]
3348    async fn spawn_project_known_project_announces_the_id_it_will_run_under() {
3349        let dir = tempdir().expect("tempdir");
3350        write_forge_toml(dir.path(), FIXTURE_PROJECT_PATH);
3351        let workspace = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("new"));
3352        let mut rx = workspace.subscribe_answerer();
3353
3354        let _ = handle_spawn_project(&workspace, "forge", SessionLaunchSettings::default());
3355
3356        let update = rx.try_recv().expect("Spawning emit");
3357        match std::sync::Arc::unwrap_or_clone(update.update) {
3358            SessionUpdate::Spawning { key, project_name, .. } => {
3359                assert_eq!(
3360                    key,
3361                    SessionSlot::lead("Default", "forge"),
3362                    "the announced key is the project's lead slot",
3363                );
3364                assert_eq!(project_name, "forge");
3365            }
3366            other => panic!("expected Spawning update; got {other:?}"),
3367        }
3368    }
3369
3370    /// A spawn that fails before a `SessionTask` exists records the failure
3371    /// where a view reads it. The `ConnectionFailed` arm that would
3372    /// normally record it runs inside that task, so this path has to do it
3373    /// itself; a view rebuilding later reads the slot, and the record is
3374    /// the only thing left to say why the row reads failed.
3375    #[tokio::test]
3376    async fn spawn_project_records_the_failure_it_announces() {
3377        let dir = tempdir().expect("tempdir");
3378        write_forge_toml(dir.path(), FIXTURE_PROJECT_PATH);
3379        let workspace = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("new"));
3380        // A shut boot gate is the cheapest way to a spawn that never
3381        // builds a task: the slot resolves and is announced, then the
3382        // handle construction refuses.
3383        workspace.seed_test_gateway_ready(false);
3384        let mut rx = workspace.subscribe_answerer();
3385
3386        let _ = handle_spawn_project(&workspace, "forge", SessionLaunchSettings::default());
3387
3388        assert!(
3389            matches!(
3390                rx.try_recv(),
3391                Ok(frame) if matches!(&*frame.update, SessionUpdate::Spawning { .. })
3392            ),
3393            "the spawn is announced before the handle is built",
3394        );
3395        let key = match std::sync::Arc::unwrap_or_clone(
3396            rx.try_recv().expect("ConnectionFailed emit").update,
3397        ) {
3398            SessionUpdate::ConnectionFailed { key, .. } => key,
3399            other => panic!("expected ConnectionFailed; got {other:?}"),
3400        };
3401        assert_eq!(key, SessionSlot::lead("Default", "forge"));
3402        let record = workspace
3403            .spawn_failure(&key)
3404            .expect("the announced failure is recorded against the key the envelope carried");
3405        assert!(
3406            record.contains("agent spawn failed"),
3407            "the record carries the reason the envelope announced: {record}",
3408        );
3409    }
3410
3411    /// The worker counterpart of `spawn_project_records_the_failure_it_announces`:
3412    /// a synchronous worker spawn failure records the reason on the worker's
3413    /// slot and carries it on the rollback's `Removed` event, so a view says
3414    /// why instead of drawing the rollback as a close.
3415    #[tokio::test]
3416    async fn a_synchronous_worker_spawn_failure_names_its_cause() {
3417        let dir = tempdir().expect("tempdir");
3418        write_forge_toml(dir.path(), FIXTURE_PROJECT_PATH);
3419        let workspace = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("new"));
3420        // A shut boot gate is the cheapest way to a spawn that never
3421        // builds a task: the entry is written and then the handle
3422        // construction refuses.
3423        workspace.seed_test_gateway_ready(false);
3424        let project =
3425            workspace.list_projects().into_iter().next().expect("fixture project present").key;
3426        let mut rx = workspace.subscribe_answerer();
3427        let (tx, reply) = tokio::sync::oneshot::channel();
3428
3429        handle_spawn_worker(
3430            &workspace,
3431            project,
3432            WorkerSpawnArgs {
3433                label: "builder".to_owned(),
3434                charter: "charter".to_owned(),
3435                kick: None,
3436                resume_kick: None,
3437                interactive: false,
3438                mcp_families: None,
3439                cwd: None,
3440            },
3441            SessionSlot::from_str_for_test("lead"),
3442            None,
3443            false,
3444            tx,
3445        );
3446
3447        let reason = reply.await.expect("reply channel").expect_err("the shut gate refuses");
3448        assert!(
3449            reason.contains("agent spawn failed") && reason.contains("boot gate is shut"),
3450            "the caller is told the cause, not just that it failed: {reason}",
3451        );
3452        let slot = SessionSlot::worker("Default", "forge", "builder");
3453        let record = workspace.spawn_failure(&slot).expect("the failure is recorded on the slot");
3454        assert!(
3455            record.contains("boot gate is shut"),
3456            "the record carries the cause itself: {record}",
3457        );
3458
3459        let mut removed_diagnostic = None;
3460        while let Ok(frame) = rx.try_recv() {
3461            if let SessionUpdate::WorkerStatusChanged {
3462                action: WorkerStatusAction::Removed,
3463                status,
3464                ..
3465            } = &*frame.update
3466            {
3467                removed_diagnostic = status.diagnostic.clone();
3468            }
3469        }
3470        assert!(
3471            removed_diagnostic
3472                .as_ref()
3473                .is_some_and(|diagnostic| diagnostic.contains("agent spawn failed")),
3474            "the rollback event carries the cause a view shows: {removed_diagnostic:?}",
3475        );
3476    }
3477
3478    /// The label rules are the spawn handler's, not only the MCP tool's:
3479    /// the command bus and the socket pass-through reach this handler
3480    /// directly. A `lead` label there collides with the lead's slot (#1682),
3481    /// and a slash desynchronises forge's worktree path and branch from
3482    /// claude's rewritten pair (#1603). Neither may mint anything.
3483    #[tokio::test]
3484    async fn the_spawn_handler_refuses_a_reserved_or_path_shaped_label() {
3485        let (workspace, _rx) = Workspace::testing_stub();
3486        workspace.seed_test_project("forge", "/tmp/forge");
3487        let project = workspace
3488            .list_projects()
3489            .into_iter()
3490            .find(|view| view.name == "forge")
3491            .expect("seeded project")
3492            .key;
3493
3494        for (label, expected) in [("lead", "reserved"), ("a/b", "slash")] {
3495            let (tx, reply) = tokio::sync::oneshot::channel();
3496            handle_spawn_worker(
3497                &workspace,
3498                project.clone(),
3499                WorkerSpawnArgs {
3500                    label: label.to_owned(),
3501                    charter: "charter".to_owned(),
3502                    kick: None,
3503                    resume_kick: None,
3504                    interactive: false,
3505                    mcp_families: None,
3506                    cwd: None,
3507                },
3508                SessionSlot::from_str_for_test("lead"),
3509                None,
3510                false,
3511                tx,
3512            );
3513            let refusal = reply.await.expect("reply channel").expect_err("the label is refused");
3514            assert!(
3515                refusal.contains(expected),
3516                "the refusal names the rule '{expected}' for '{label}': {refusal}",
3517            );
3518        }
3519        assert!(
3520            workspace.list_live_workers(&project).is_empty(),
3521            "a refused label mints no worker entry",
3522        );
3523    }
3524
3525    /// A pooled handle whose session task is gone is not a session: the
3526    /// spawn fast path handed it back as a success, so no child started,
3527    /// the client read "no session task registered", and every later spawn
3528    /// repeated it (#2683). The spawn must rebuild the task instead.
3529    #[tokio::test]
3530    async fn a_spawn_rebuilds_a_pooled_handle_whose_task_is_gone() {
3531        let dir = tempdir().expect("tempdir");
3532        write_forge_toml(dir.path(), FIXTURE_PROJECT_PATH);
3533        let workspace = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("new"));
3534        workspace.seed_test_ready_account("Stargate");
3535        let view = workspace.list_projects().into_iter().next().expect("fixture project");
3536        let key = SessionSlot::lead(&view.org, &view.name);
3537        // The poison is the shape a dropped task leaves: an ANSWERED
3538        // teardown signal (a presence check passes it) and a command sender
3539        // whose task has gone.
3540        let (stale, _stale_commands) = Workspace::testing_stub_handle();
3541        let stale = Arc::new(stale);
3542        let teardown = workspace.mint_session_teardown(&key).expect("a teardown for the dead task");
3543        teardown.send_replace(true);
3544        let (stale_tx, mut stale_rx) = tokio::sync::mpsc::unbounded_channel();
3545        workspace.command_senders.lock().insert(key.clone(), stale_tx);
3546        workspace.pool.lock().insert(
3547            key.clone(),
3548            crate::workspace::PooledAgent {
3549                handle: Arc::clone(&stale),
3550                account: forge_gateway::AccountKey("Stargate".to_owned()),
3551                permission_mode: None,
3552                registration: None,
3553                session_id: "stale-id".to_owned(),
3554            },
3555        );
3556        // The cold path's stand-in, so the rebuild launches no real child.
3557        let (fresh, _fresh_commands) = Workspace::testing_stub_handle();
3558        workspace.install_test_spawn_handle(fresh);
3559
3560        let handle = workspace
3561            .get_agent_handle_at_key(
3562                crate::target::SessionTarget::Named(view.name.clone()),
3563                SessionLaunchSettings::default(),
3564                Some(key.clone()),
3565                &crate::protocol::SpawnRole::Lead,
3566                true,
3567            )
3568            .expect("the spawn rebuilds rather than handing back a dead handle");
3569
3570        assert!(!Arc::ptr_eq(&handle, &stale), "the stale handle is not the answer");
3571        assert!(
3572            workspace.session_is_live(&key),
3573            "and a session task now consumes the slot the stale handle held",
3574        );
3575        assert!(
3576            matches!(
3577                stale_rx.try_recv(),
3578                Err(tokio::sync::mpsc::error::TryRecvError::Disconnected)
3579            ),
3580            "the stale sender was replaced, so the command channel its old task reads is closed",
3581        );
3582    }
3583
3584    fn fixture_wrapped() -> WrappedPrompt {
3585        WrappedPrompt {
3586            id: crate::mcp::peers::types::MessageId::mint(),
3587            kind: crate::mcp::peers::types::WrappedKind::Message,
3588            sender_name: "forge".to_owned(),
3589            sender_org: "Default".to_owned(),
3590            body: "fyi".to_owned(),
3591        }
3592    }
3593
3594    /// I3 - `handle_deliver_peer_prompt` for an unknown target must
3595    /// not panic and must not emit a fatal envelope, and its answer must be
3596    /// an error naming the target: the tool's ack is this handler's own, so
3597    /// a delivery that went nowhere cannot read as sent (#2342).
3598    #[tokio::test]
3599    async fn handle_deliver_peer_prompt_unknown_target_is_an_error() {
3600        let dir = tempdir().expect("tempdir");
3601        write_forge_toml(dir.path(), FIXTURE_PROJECT_PATH);
3602        let workspace = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("new"));
3603        let mut rx = workspace.subscribe_answerer();
3604
3605        let caller = SessionSlot::from_str_for_test("caller-1");
3606        let outcome = handle_deliver_peer_prompt(
3607            &workspace,
3608            &caller,
3609            "no-such-project".to_owned(),
3610            fixture_wrapped(),
3611        );
3612
3613        let refusal = outcome.expect_err("an unknown target is not a delivery");
3614        assert!(
3615            refusal.contains("no-such-project"),
3616            "the refusal names the target that went nowhere: {refusal}",
3617        );
3618        while let Ok(frame) = rx.try_recv() {
3619            assert!(
3620                !matches!(&*frame.update, SessionUpdate::FatalError { .. }),
3621                "unknown target must not emit FatalError"
3622            );
3623        }
3624    }
3625
3626    /// A wake whose spawn fails inline is not an accepted delivery: the
3627    /// project handler's own arm expires the parked message, and the ack
3628    /// says so rather than reading the dispatch's Ok as the wake working
3629    /// (#2342).
3630    #[tokio::test]
3631    async fn a_delivery_whose_wake_fails_is_an_error() {
3632        let dir = tempdir().expect("tempdir");
3633        write_forge_toml(dir.path(), FIXTURE_PROJECT_PATH);
3634        let workspace = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("new"));
3635        // A shut boot gate is the cheapest wake that fails with no child.
3636        workspace.seed_test_gateway_ready(false);
3637        let caller = SessionSlot::from_str_for_test("caller-wake");
3638        let wrapped = fixture_wrapped();
3639        let message_id = wrapped.id.clone();
3640        let lead = SessionSlot::lead("Default", "forge");
3641
3642        let outcome = handle_deliver_peer_prompt(&workspace, &caller, "forge".to_owned(), wrapped);
3643
3644        let refusal = outcome.expect_err("a wake that never started is not a delivery");
3645        assert!(
3646            refusal.contains("parked message was returned to you"),
3647            "the refusal is the parked-bucket read, not the dispatch error arm: {refusal}",
3648        );
3649        assert!(
3650            !workspace.peer_prompt_is_parked(&lead, &message_id),
3651            "the failed wake's arm expired the parked message",
3652        );
3653    }
3654
3655    /// The running branch's own acceptance: a live lead takes the prompt and
3656    /// the handler answers Ok, routing a `PromptUnder` to its session. A
3657    /// branch that answered Err unconditionally would read every peer
3658    /// message to a live lead as failed.
3659    #[tokio::test]
3660    async fn a_delivery_a_running_lead_takes_is_ok() {
3661        let (workspace, _updates) = Workspace::testing_stub();
3662        workspace.seed_test_project("forge", "/tmp/forge");
3663        let lead = SessionSlot::lead("TestOrg", "forge");
3664        // The two facts the running branch resolves on: a pooled handle and
3665        // a live command sender behind it.
3666        let (handle, _commands) = Workspace::testing_stub_handle();
3667        workspace.pool.lock().insert(
3668            lead.clone(),
3669            crate::workspace::PooledAgent {
3670                handle: Arc::new(handle),
3671                account: forge_gateway::AccountKey("acct-a".to_owned()),
3672                permission_mode: None,
3673                registration: None,
3674                session_id: "lead-session".to_owned(),
3675            },
3676        );
3677        let (commands, mut routed) = tokio::sync::mpsc::unbounded_channel();
3678        workspace.command_senders.lock().insert(lead.clone(), commands);
3679
3680        let outcome = handle_deliver_peer_prompt(
3681            &workspace,
3682            &SessionSlot::from_str_for_test("caller-running"),
3683            "forge".to_owned(),
3684            fixture_wrapped(),
3685        );
3686
3687        assert!(outcome.is_ok(), "a live session takes the prompt: {outcome:?}");
3688        match routed.try_recv().expect("the prompt is routed to the lead's session") {
3689            Command::PromptUnder { text, .. } => {
3690                assert!(text.contains("fyi"), "the routed prose is the wrapped body: {text}");
3691            }
3692            other => panic!("expected PromptUnder; got {other:?}"),
3693        }
3694    }
3695
3696    /// I3 - `handle_deliver_peer_prompt` against a sleeping known
3697    /// project parks the prompt for that project's lead and triggers a
3698    /// SpawnProject. The bucket is addressed by `(org, project, label)`,
3699    /// so no synthetic key is anywhere in the path; the drain side is
3700    /// pinned in `session_task`, which delivers a parked prompt on the
3701    /// owner's first `Connected`.
3702    #[tokio::test]
3703    async fn handle_deliver_peer_prompt_sleeping_target_parks_for_the_projects_lead() {
3704        let dir = tempdir().expect("tempdir");
3705        fs::write(
3706            forge_toml_path(dir.path()),
3707            r#"
3708[[orgs]]
3709name = "Default"
3710accounts = ["Stargate"]
3711
3712[[orgs.projects]]
3713name = "forge"
3714path = "~/Projects/forge"
3715
3716[[orgs.projects]]
3717name = "gateway-backend"
3718path = "~/Projects/gateway-backend"
3719
3720[[accounts]]
3721display_name = "Stargate"
3722token = "t"
3723models = ["claude-sonnet-5"]
3724provider = "anthropic"
3725"#,
3726        )
3727        .expect("write forge.toml");
3728        let workspace = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("new"));
3729        // Intercept dispatch so the spawn this delivery triggers does not
3730        // run inline and expire the park it just made: with no `claude`
3731        // binary the spawn's failure arm releases the slot, which is the
3732        // production behaviour but hides the parking under test.
3733        workspace.enable_test_dispatch_intercept();
3734        let caller = SessionSlot::from_str_for_test("caller-sleep");
3735        let w = fixture_wrapped();
3736
3737        handle_deliver_peer_prompt(&workspace, &caller, "gateway-backend".to_owned(), w.clone())
3738            .expect("a sleeping target's parked message is an accepted delivery");
3739
3740        // The sleeping branch parks the envelope for the project's lead
3741        // under `(org, project, None)`. EXACTLY ONE bucket holds our
3742        // wrapped prompt - assert on the id as well, so a mis-keyed
3743        // parking cannot pass by parking twice.
3744        let parked = workspace.parked_by_slot.lock();
3745        let bucket = parked
3746            .get(&crate::SessionSlot::lead("Default", "gateway-backend"))
3747            .expect("the project's lead bucket");
3748        assert_eq!(bucket.peer.len(), 1, "the project's lead bucket holds the wrapped prompt");
3749        assert_eq!(
3750            bucket.peer[0].wrapped.id, w.id,
3751            "and it is the payload that was handed to the delivery",
3752        );
3753        assert_eq!(
3754            bucket.peer[0].sender, caller,
3755            "with the SENDER beside it, which is the seat the ack goes back to when the spawn \
3756             fails - the bucket's own key is the target, so a mutant that parked the target \
3757             there would pass every other assertion here",
3758        );
3759    }
3760
3761    fn fake_worker_entry(label: &str, key: &str) -> crate::mcp::workers::types::WorkerEntry {
3762        crate::mcp::workers::types::WorkerEntry {
3763            label: label.into(),
3764            charter: "c".into(),
3765            slot: SessionSlot::from_str_for_test(key),
3766            session_id: None,
3767            status: forge_primitives::WorkerLiveness::Running,
3768            spawned_at: std::time::SystemTime::UNIX_EPOCH,
3769            spawned_by: SessionSlot::from_str_for_test("lead-uuid"),
3770            needs_tag: false,
3771            is_git_repo_at_spawn: false,
3772            cwd: None,
3773            diagnostic: None,
3774            kick: None,
3775        }
3776    }
3777
3778    use forge_primitives::slack::{SlackFile, SlackFileShares};
3779
3780    fn slack_msg(text: &str) -> SlackMessage {
3781        SlackMessage {
3782            workspace: "acme".to_owned(),
3783            conversation: "D1".to_owned(),
3784            conversation_label: "U9".to_owned(),
3785            ts: "100.000001".to_owned(),
3786            thread_ts: None,
3787            user: Some("U9".to_owned()),
3788            author: None,
3789            text: text.to_owned(),
3790            parent_user_id: None,
3791            latest_reply: None,
3792            edited: None,
3793            files: Vec::new(),
3794        }
3795    }
3796
3797    fn slack_msg_at(ts: &str, user: &str, text: &str) -> SlackMessage {
3798        SlackMessage { ts: ts.to_owned(), user: Some(user.to_owned()), ..slack_msg(text) }
3799    }
3800
3801    /// The prose at a fixed moment below the default stale threshold, for the
3802    /// shape tests that do not exercise the age clause.
3803    fn fresh_prose(messages: &[SlackMessage]) -> String {
3804        slack_bundle_to_prose(
3805            messages,
3806            Duration::from_secs(24 * 60 * 60),
3807            SystemTime::UNIX_EPOCH + Duration::from_secs(200),
3808        )
3809    }
3810
3811    #[test]
3812    fn a_slack_message_for_a_worker_targets_the_worker_not_the_lead() {
3813        let (ws, _rx) = Workspace::testing_stub();
3814        ws.seed_test_project("forge", "/tmp/slack-forge");
3815        let key = ws.list_projects().into_iter().find(|v| v.name == "forge").expect("seeded").key;
3816        // Delivery routes by the live entry's slot, so the fixture carries
3817        // the one production derives for this worker.
3818        let worker_key = crate::SessionSlot::worker("TestOrg", "forge", "tester");
3819        let mut entry = fake_worker_entry("tester", "worker-uuid");
3820        entry.slot = worker_key.clone();
3821        ws.insert_live_worker(&key, entry);
3822
3823        deliver_slack_message(&ws, "forge", Some("tester"), vec![slack_msg("hello")]);
3824
3825        let parked =
3826            ws.parked_by_slot.lock().get(&worker_key).map_or(0, |parked| parked.slack.len());
3827        assert_eq!(parked, 1, "the message is parked for the worker that subscribed");
3828        assert_eq!(
3829            ws.parked_by_slot
3830                .lock()
3831                .get(&crate::SessionSlot::lead("TestOrg", "forge"))
3832                .map_or(0, |parked| parked.slack.len()),
3833            0,
3834            "a worker-owned subscription never falls through to the lead",
3835        );
3836    }
3837
3838    /// A `Failed` entry is kept by design (its tombstone row stays visible),
3839    /// but it is NOT live: the worker has no session to drain a bucket and no
3840    /// spawn coming to make one. So it must not be a delivery target - the
3841    /// batch would park forever and land as fresh user turns whenever the
3842    /// worker is next opened, however long that takes (#1208). Routing as if
3843    /// the worker were gone sends the message to the lead, which wakes the
3844    /// project the way the lead path always has.
3845    #[test]
3846    fn a_slack_message_for_a_failed_worker_goes_to_the_lead_not_a_frozen_bucket() {
3847        let dir = tempdir().expect("tempdir");
3848        let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
3849        ws.seed_test_project("forge", "/tmp/slack-failed-worker");
3850        let project_key =
3851            ws.list_projects().into_iter().find(|v| v.name == "forge").expect("seeded").key;
3852        // Delivery resolves the target through the live entry's slot, so the
3853        // fixture carries the one production derives for this worker.
3854        let worker_key = crate::SessionSlot::worker("TestOrg", "forge", "tester");
3855        let mut failed = fake_worker_entry("tester", "worker-uuid");
3856        failed.slot = worker_key.clone();
3857        failed.status = forge_primitives::WorkerLiveness::Failed;
3858        failed.diagnostic = Some("connection closed".to_owned());
3859        ws.insert_live_worker(&project_key, failed);
3860        ws.enable_test_dispatch_intercept();
3861
3862        deliver_slack_message(&ws, "forge", Some("tester"), vec![slack_msg("hello")]);
3863
3864        assert_eq!(
3865            ws.parked_by_slot.lock().get(&worker_key).map_or(0, |parked| parked.slack.len()),
3866            0,
3867            "a failed worker's slot is not a destination: nothing parks for a bucket nothing \
3868             will drain",
3869        );
3870        let dispatched = ws.drain_test_dispatch_buffer();
3871        assert!(
3872            dispatched.iter().any(|c| matches!(
3873                c,
3874                crate::protocol::Command::SpawnProject { project_name, .. }
3875                    if project_name == "forge"
3876            )),
3877            "the message wakes the project: {dispatched:?}",
3878        );
3879        assert_eq!(
3880            ws.parked_by_slot
3881                .lock()
3882                .get(&crate::SessionSlot::lead("TestOrg", "forge"))
3883                .map_or(0, |parked| parked.slack.len()),
3884            1,
3885            "and waits for the lead the spawn brings up",
3886        );
3887    }
3888
3889    /// A live delivery to a connected team worker dispatches the prose AND
3890    /// echoes the block into that worker's chat. The CLI never echoes a
3891    /// stdin-injected prompt back, so without the echo the agent works the
3892    /// message and the user sees nothing; asserting only the echo would let a
3893    /// regression through where the block paints and the agent never receives it.
3894    #[test]
3895    fn deliver_slack_message_to_a_connected_worker_dispatches_and_echoes() {
3896        let (ws, mut update_rx) = Workspace::testing_stub();
3897        ws.seed_test_project("forge", "/tmp/slack-worker-echo");
3898        let key = ws.list_projects().into_iter().find(|v| v.name == "forge").expect("seeded").key;
3899        ws.insert_live_worker(&key, fake_worker_entry("tester", "worker-uuid"));
3900        let worker_key = SessionSlot::from_str_for_test("worker-uuid");
3901        ws.mark_session_connected_for_test(&worker_key, "worker-uuid");
3902        ws.enable_test_dispatch_intercept();
3903
3904        deliver_slack_message(&ws, "forge", Some("tester"), vec![slack_msg("hello")]);
3905
3906        let dispatched = ws.drain_test_dispatch_buffer();
3907        assert!(
3908            dispatched.iter().any(|c| matches!(
3909                c, crate::protocol::Command::Prompt { key, text, .. }
3910                    | crate::protocol::Command::PromptUnder { key, text, .. }
3911                    if *key == worker_key && text.contains("hello")
3912            )),
3913            "the worker receives the message as a prompt: {dispatched:?}",
3914        );
3915
3916        let mut echoed = false;
3917        while let Ok(frame) = update_rx.try_recv() {
3918            if matches!(
3919                &*frame.update,
3920                crate::protocol::SessionUpdate::SlackMessageAppended { key, prose, .. }
3921                    if key == &worker_key
3922                        && prose.starts_with("[Slack")
3923                        && prose.contains("hello")
3924            ) {
3925                echoed = true;
3926            }
3927        }
3928        assert!(
3929            echoed,
3930            "a connected worker's delivery emits a SlackMessageAppended echo with the body",
3931        );
3932    }
3933
3934    /// The delivery path reads the workspace's own threshold, in both
3935    /// directions: a million-hour threshold leaves the 1970 fixture unmarked
3936    /// where a wrong key would fall back to 24 hours and mark it, and a
3937    /// one-hour threshold marks a two-hour-old message where a zero (the
3938    /// feature off) would not.
3939    #[test]
3940    fn a_workspace_threshold_bounds_what_delivery_marks() {
3941        let dir = tempdir().expect("tempdir");
3942        let mut config = crate::config::LoadedConfig::empty_for_test();
3943        config.slack = vec![
3944            forge_primitives::slack::SlackConfig {
3945                workspace: "acme".to_owned(),
3946                token: "xoxp-test".to_owned(),
3947                poll_seconds: 30,
3948                dm_poll_seconds: 150,
3949                thread_idle_days: 14,
3950                stale_after_hours: 1_000_000_000,
3951            },
3952            forge_primitives::slack::SlackConfig {
3953                workspace: "live".to_owned(),
3954                token: "xoxp-test".to_owned(),
3955                poll_seconds: 30,
3956                dm_poll_seconds: 150,
3957                thread_idle_days: 14,
3958                stale_after_hours: 1,
3959            },
3960        ];
3961        let (ws, mut update_rx) =
3962            Workspace::testing_stub_with_config(dir.path().to_path_buf(), config)
3963                .expect("the stub config's [[slack]] entries are well-formed");
3964        ws.seed_test_project("forge", "/tmp/slack-stale-threshold");
3965        let key = ws.list_projects().into_iter().find(|v| v.name == "forge").expect("seeded").key;
3966        ws.insert_live_worker(&key, fake_worker_entry("tester", "worker-uuid"));
3967        let worker_key = SessionSlot::from_str_for_test("worker-uuid");
3968        ws.mark_session_connected_for_test(&worker_key, "worker-uuid");
3969        ws.enable_test_dispatch_intercept();
3970
3971        let now = SystemTime::now()
3972            .duration_since(SystemTime::UNIX_EPOCH)
3973            .expect("the clock is past the epoch")
3974            .as_secs();
3975        let mut recent = slack_msg_at(&format!("{}.000000", now - 2 * 3_600), "U9", "recent");
3976        recent.workspace = "live".to_owned();
3977
3978        deliver_slack_message(&ws, "forge", Some("tester"), vec![slack_msg("hello")]);
3979        deliver_slack_message(&ws, "forge", Some("tester"), vec![recent]);
3980
3981        let mut echoes = Vec::new();
3982        while let Ok(frame) = update_rx.try_recv() {
3983            if let crate::protocol::SessionUpdate::SlackMessageAppended { key, prose, .. } =
3984                std::sync::Arc::unwrap_or_clone(frame.update)
3985                && key == worker_key
3986            {
3987                echoes.push(prose);
3988            }
3989        }
3990        let unmarked = echoes
3991            .iter()
3992            .find(|prose| prose.contains("hello"))
3993            .expect("the huge-threshold delivery echoes its block");
3994        assert!(
3995            !unmarked.contains(" old"),
3996            "a million-hour threshold leaves the header unmarked: {unmarked}",
3997        );
3998        let marked = echoes
3999            .iter()
4000            .find(|prose| prose.contains("recent"))
4001            .expect("the small-threshold delivery echoes its block");
4002        assert!(
4003            marked.contains(", 2h old"),
4004            "a one-hour threshold marks a two-hour-old message: {marked}",
4005        );
4006    }
4007
4008    /// A worker spawn that fails rolls its live entry back, so no live row
4009    /// lingers with no session behind it. The failure is the same
4010    /// `FreshInProject` miss the project-spawn arm hits: the overlay satisfies
4011    /// `list_projects` but not the config lookup inside the spawn.
4012    /// A fresh worker is keyed by the id it will run under: minted and
4013    /// recorded before the child starts, so the registry entry, the pool
4014    /// and the child's `--session-id` all name one string and `Connected`
4015    /// has nothing to move. Verified on a worker rather than a lead,
4016    /// because the worker used to be the path that spawned under a
4017    /// placeholder and rekeyed later.
4018    #[tokio::test]
4019    async fn a_fresh_worker_is_keyed_by_the_id_it_runs_under() {
4020        let dir = tempdir().expect("tempdir");
4021        // A repo of this test's own, because it reads the worktree the reply
4022        // names: the fixture project's gitness must not depend on where the
4023        // suite runs, and a fixed home path is a git repo on one machine and
4024        // a plain directory on a runner.
4025        let repo = tempdir().expect("repo tempdir");
4026        run_git(repo.path(), &["init", "-q"]);
4027        run_git(repo.path(), &["config", "user.email", "t@example.com"]);
4028        run_git(repo.path(), &["config", "user.name", "Test"]);
4029        run_git(repo.path(), &["config", "commit.gpgsign", "false"]);
4030        std::fs::write(repo.path().join("README.md"), "seed").expect("write seed");
4031        run_git(repo.path(), &["add", "."]);
4032        run_git(repo.path(), &["commit", "-q", "-m", "init"]);
4033        write_forge_toml(dir.path(), &repo.path().to_string_lossy());
4034        let ws = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("workspace"));
4035        ws.seed_test_ready_account("Stargate");
4036        ws.seed_test_gateway_ready(true);
4037        let key = ws
4038            .list_projects()
4039            .into_iter()
4040            .find(|v| v.name == "forge")
4041            .expect("fixture project")
4042            .key;
4043        let (tx, rx) = tokio::sync::oneshot::channel();
4044
4045        handle_spawn_worker(
4046            &ws,
4047            key,
4048            WorkerSpawnArgs {
4049                label: "tester".to_owned(),
4050                charter: "charter".to_owned(),
4051                kick: None,
4052                resume_kick: None,
4053                interactive: false,
4054                mcp_families: None,
4055                cwd: None,
4056            },
4057            SessionSlot::from_str_for_test("lead"),
4058            None,
4059            false,
4060            tx,
4061        );
4062
4063        let reply = rx.await.expect("spawn replies").expect("spawn succeeds");
4064        assert!(
4065            uuid::Uuid::parse_str(&reply.session_id).is_ok(),
4066            "the worker is told the id it runs under, not a placeholder: {}",
4067            reply.session_id,
4068        );
4069        // The fixture project is a git repo, so this worker runs in its own
4070        // worktree and the reply names it: the path a lead uses to find the
4071        // worker's work has to be the directory the worker actually runs in.
4072        // Canonical like the CLI's own root: a fixture under `/var` (a
4073        // symlink to `/private/var` on macOS) would otherwise compare a
4074        // spelling claude never uses.
4075        let expected_worktree = std::fs::canonicalize(
4076            ws.list_projects()
4077                .into_iter()
4078                .find(|v| v.name == "forge")
4079                .expect("fixture project")
4080                .path,
4081        )
4082        .expect("canonical fixture project")
4083        .join(".claude/worktrees/tester");
4084        assert_eq!(
4085            reply.worktree.as_deref(),
4086            Some(expected_worktree.to_string_lossy().into_owned().as_str()),
4087            "the reply names the worktree the worker was spawned into",
4088        );
4089        let entry = ws
4090            .list_projects()
4091            .into_iter()
4092            .find(|v| v.name == "forge")
4093            .and_then(|view| ws.list_live_workers(&view.key).into_iter().next())
4094            .expect("the worker is registered");
4095        assert_eq!(
4096            entry.slot,
4097            SessionSlot::worker("Default", "forge", "tester"),
4098            "the registry entry is keyed by the worker's slot",
4099        );
4100        assert!(ws.pool.lock().contains_key(&entry.slot), "and so is the pool");
4101        let stored = {
4102            let db = ws.db.lock();
4103            let db = db.as_ref().expect("db");
4104            crate::store::sessions::get(db, "Default", "forge", "tester")
4105                .expect("read")
4106                .and_then(|row| row.session_id)
4107        };
4108        assert_eq!(stored.as_deref(), Some(reply.session_id.as_str()), "and the store row");
4109    }
4110
4111    /// Drive a spawn that fails at dispatch and return the durable row it
4112    /// left behind. The project is in the test overlay but not in
4113    /// `forge.toml`, and the config lookup inside the spawn is what misses.
4114    ///
4115    /// A row is seeded first, under the same `(org, project, label)` the
4116    /// spawn will write, so a missing row afterwards means one was removed
4117    /// rather than that none was ever written - without it, `is_none()`
4118    /// passes just as well against a `record_worker_row` that never ran.
4119    async fn failed_spawn_leftover_row(
4120        resume_existing: Option<&str>,
4121        from_boot_respawn: bool,
4122        seeded_is_git: Option<bool>,
4123    ) -> Option<crate::store::sessions::SessionRecord> {
4124        let dir = tempdir().expect("tempdir");
4125        write_forge_toml(dir.path(), FIXTURE_PROJECT_PATH);
4126        let ws = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("workspace"));
4127        ws.seed_test_project("overlayonly", "/tmp/slack-worker-rollback");
4128        let key = ws
4129            .list_projects()
4130            .into_iter()
4131            .find(|v| v.name == "overlayonly")
4132            .expect("seeded project present")
4133            .key;
4134        {
4135            let db = ws.db.lock();
4136            crate::store::sessions::put(
4137                db.as_ref().expect("db"),
4138                &crate::store::sessions::SessionRecord {
4139                    org: "TestOrg".to_owned(),
4140                    project: "overlayonly".to_owned(),
4141                    label: "tester".to_owned(),
4142                    session_id: Some("tester-id".to_owned()),
4143                    charter: None,
4144                    kick: None,
4145                    resume_kick: None,
4146                    interactive: None,
4147                    seat_state: None,
4148                    is_git_repo: seeded_is_git,
4149                    mcp_families: None,
4150                    cwd: None,
4151                },
4152            )
4153            .expect("seed the row the spawn will write");
4154        }
4155        let (tx, rx) = tokio::sync::oneshot::channel();
4156        handle_spawn_worker(
4157            &ws,
4158            key.clone(),
4159            WorkerSpawnArgs {
4160                label: "tester".to_owned(),
4161                charter: "charter".to_owned(),
4162                kick: None,
4163                resume_kick: None,
4164                interactive: false,
4165                mcp_families: None,
4166                cwd: None,
4167            },
4168            SessionSlot::from_str_for_test("lead"),
4169            resume_existing,
4170            from_boot_respawn,
4171            tx,
4172        );
4173
4174        assert!(matches!(rx.await, Ok(Err(_))), "the spawn reports the failure to its caller");
4175        assert!(
4176            ws.list_live_workers(&key).is_empty(),
4177            "and the placeholder entry is rolled back, not left live",
4178        );
4179        let db = ws.db.lock();
4180        crate::store::sessions::get(db.as_ref().expect("db"), "TestOrg", "overlayonly", "tester")
4181            .expect("read")
4182    }
4183
4184    #[tokio::test]
4185    async fn a_failed_worker_spawn_rolls_the_live_entry_back() {
4186        let row = failed_spawn_leftover_row(None, false, None).await;
4187        assert!(
4188            row.is_none(),
4189            "the durable row goes with the rollback: left behind, the next boot re-spawns a \
4190             worker the caller was told had failed",
4191        );
4192    }
4193
4194    /// A RESUME whose dispatch fails keeps its row, because that row is the
4195    /// only durable handle on the id being resumed - deleting it loses the
4196    /// worker rather than letting it retry. Nothing else reaches this arm
4197    /// with the flag set, so without this the guard's negative half is
4198    /// pinned by nothing.
4199    #[tokio::test]
4200    async fn a_failed_resume_spawn_keeps_the_row_holding_the_id() {
4201        let row = failed_spawn_leftover_row(Some("tester-id"), false, None).await;
4202        assert_eq!(
4203            row.and_then(|row| row.session_id).as_deref(),
4204            Some("tester-id"),
4205            "the rollback leaves the row that names the id being resumed",
4206        );
4207    }
4208
4209    /// The same for a boot re-spawn, which carries no `resume_existing` when
4210    /// it comes up fresh under `--new` - so the resume flag alone does not
4211    /// separate it from a spawn a caller asked for, and the boot flag is
4212    /// what keeps its row.
4213    ///
4214    /// That path mints a new id and writes it over the row, so what the
4215    /// guard protects here is the row's survival rather than this id: the
4216    /// boot wave owns the worker, and deleting the row on a failure would
4217    /// drop it from every later restart.
4218    #[tokio::test]
4219    async fn a_failed_boot_respawn_keeps_its_row() {
4220        let row = failed_spawn_leftover_row(None, true, None).await;
4221        assert!(row.is_some(), "the boot re-spawn keeps the row a later restart re-spawns from");
4222    }
4223
4224    /// A resume may deliver the restart note (or the row's `resume_kick`)
4225    /// as this connection's kick - the re-spawn decider above hands one
4226    /// only to a seat whose turn was cut off, and this test states one
4227    /// directly to prove it - and that text must not land in the row's
4228    /// `kick`: that field is the worker's original first turn, and it is
4229    /// what a later `--new` re-spawn opens the worker with. Written, the
4230    /// resume text becomes the worker's opening turn for the rest of its
4231    /// life.
4232    #[tokio::test]
4233    async fn a_resume_spawn_leaves_the_stored_kick_alone() {
4234        let dir = tempdir().expect("tempdir");
4235        write_forge_toml(dir.path(), FIXTURE_PROJECT_PATH);
4236        let ws = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("workspace"));
4237        ws.seed_test_ready_account("Stargate");
4238        ws.seed_test_gateway_ready(true);
4239        let key = ws
4240            .list_projects()
4241            .into_iter()
4242            .find(|v| v.name == "forge")
4243            .expect("fixture project")
4244            .key;
4245        // The row a real first spawn writes, non-git so the resume runs in
4246        // the project root rather than a worktree that is not there.
4247        ws.record_worker_row(
4248            &key,
4249            "tester",
4250            "tester-id",
4251            "charter",
4252            Some("original kick"),
4253            None,
4254            false,
4255            crate::mcp::workers::types::WorkerTree { is_git_repo: false, cwd: None },
4256            None,
4257        )
4258        .expect("seed the row the resume re-writes");
4259
4260        let (tx, rx) = tokio::sync::oneshot::channel();
4261        handle_spawn_worker(
4262            &ws,
4263            key.clone(),
4264            WorkerSpawnArgs {
4265                label: "tester".to_owned(),
4266                charter: "charter".to_owned(),
4267                // What `dispatch_worker_respawns` hands a resuming worker.
4268                kick: Some("This session was restarted by forge; continue.".to_owned()),
4269                resume_kick: None,
4270                interactive: false,
4271                mcp_families: None,
4272                cwd: None,
4273            },
4274            SessionSlot::from_str_for_test("lead"),
4275            Some("tester-id"),
4276            true,
4277            tx,
4278        );
4279        let reply = rx.await.expect("spawn replies");
4280        assert!(reply.is_ok(), "the resume spawn succeeds: {reply:?}");
4281
4282        assert_eq!(
4283            ws.list_live_workers(&key).into_iter().next().and_then(|entry| entry.kick).as_deref(),
4284            Some("This session was restarted by forge; continue."),
4285            "the resuming connection is kicked with the resume text",
4286        );
4287        let stored = ws
4288            .worker_rows_for_project(&key)
4289            .into_iter()
4290            .find(|row| row.label == "tester")
4291            .expect("the row survives the resume");
4292        assert_eq!(
4293            stored.kick.as_deref(),
4294            Some("original kick"),
4295            "the row keeps the worker's original first turn; overwriting it makes the restart \
4296             note the worker's opening turn on every later --new re-spawn",
4297        );
4298    }
4299
4300    fn spawn_scaffold() -> (Arc<Workspace>, crate::ProjectKey, tempfile::TempDir) {
4301        let dir = tempdir().expect("tempdir");
4302        write_forge_toml(dir.path(), FIXTURE_PROJECT_PATH);
4303        let ws = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("workspace"));
4304        ws.seed_test_ready_account("Stargate");
4305        ws.seed_test_gateway_ready(true);
4306        let key = ws
4307            .list_projects()
4308            .into_iter()
4309            .find(|v| v.name == "forge")
4310            .expect("fixture project")
4311            .key;
4312        (ws, key, dir)
4313    }
4314
4315    /// A spawn whose tree is inside a project `forge.toml` declares is
4316    /// refused (D3): that project is driven by its own agent, so the
4317    /// door is a message to it. The refusal comes BEFORE the label is
4318    /// reserved, the obstruction probed or the row written, so a refused
4319    /// spawn leaves nothing at all behind.
4320    ///
4321    /// The tree is fed in canonical, the way the facade resolves one,
4322    /// while the declared path is the alias a tempdir hands out - the
4323    /// shape `/tmp` and a symlinked parent both take - so the refusal has
4324    /// to resolve both sides rather than compare strings.
4325    #[tokio::test]
4326    async fn a_cwd_inside_a_declared_project_is_refused() {
4327        let (workspace, (_config, projects_dir)) = stub_with_project_cap(5);
4328        let project = seeded_project(&workspace);
4329        let inside = std::fs::canonicalize(projects_dir.path().join("notes"))
4330            .expect("canonical notes path")
4331            .join("crates");
4332        std::fs::create_dir_all(&inside).expect("create the nested dir");
4333
4334        let (tx, rx) = tokio::sync::oneshot::channel();
4335        handle_spawn_worker(
4336            &workspace,
4337            project.clone(),
4338            WorkerSpawnArgs {
4339                label: "trespasser".to_owned(),
4340                charter: "c".to_owned(),
4341                kick: None,
4342                resume_kick: None,
4343                interactive: false,
4344                mcp_families: None,
4345                cwd: Some(inside.to_string_lossy().into_owned()),
4346            },
4347            SessionSlot::from_str_for_test("lead"),
4348            None,
4349            false,
4350            tx,
4351        );
4352
4353        let error = rx.await.expect("reply").expect_err("a declared project's tree is refused");
4354        assert!(
4355            error.contains("'notes'") && error.contains("which forge.toml declares"),
4356            "the refusal names the project and the reason, not merely a path the fixture \
4357             happens to spell: {error}",
4358        );
4359        assert!(
4360            error.contains("message that project's agent"),
4361            "and points at the door - a different project's agent: {error}",
4362        );
4363        assert!(
4364            !error.contains("worktree"),
4365            "the word routes the spawn failure classifier to a worktree-creation failure, \
4366             which this is not: {error}",
4367        );
4368        assert!(
4369            workspace.list_live_workers(&project).is_empty(),
4370            "a refused spawn leaves no live worker",
4371        );
4372        assert!(
4373            workspace.worker_rows_for_project(&project).is_empty(),
4374            "and no durable row, so nothing comes back on the next boot",
4375        );
4376    }
4377
4378    /// The despawn acts on the tree the ROW records, not the project's:
4379    /// a worker the spawn pointed at another repository keeps its
4380    /// worktree there, so the dirty probe and the removal have to read
4381    /// that tree. Composing them against the project path probes a
4382    /// directory the worker never ran in - today's code would read the
4383    /// project's tree, find it clean, and block on the dirty one under
4384    /// the worker's own root instead.
4385    #[tokio::test]
4386    async fn a_despawn_reads_the_tree_the_row_records() {
4387        let (workspace, project_key, project_wt, repo, _config) = git_despawn_fixture("traveller");
4388        let dir = tempfile::tempdir().expect("store dir");
4389        workspace.install_db_for_test(
4390            crate::store::Db::open(&dir.path().join("db.redb")).expect("open db"),
4391        );
4392
4393        // The worker's own tree: a sibling repo the spawn pointed it at,
4394        // with the worktree claude's `--worktree` would have forked.
4395        let sibling = tempfile::tempdir().expect("sibling repo");
4396        run_git(sibling.path(), &["init", "-q"]);
4397        run_git(sibling.path(), &["config", "user.email", "t@example.com"]);
4398        run_git(sibling.path(), &["config", "user.name", "Test"]);
4399        run_git(sibling.path(), &["config", "commit.gpgsign", "false"]);
4400        std::fs::write(sibling.path().join("README.md"), "seed").expect("write seed");
4401        run_git(sibling.path(), &["add", "."]);
4402        run_git(sibling.path(), &["commit", "-q", "-m", "init"]);
4403        let sibling_root = std::fs::canonicalize(sibling.path()).expect("canonical sibling");
4404        let worker_wt = sibling_root.join(".claude").join("worktrees").join("traveller");
4405        std::fs::create_dir_all(worker_wt.parent().expect("wt parent")).expect("mkdir worktrees");
4406        run_git(
4407            &sibling_root,
4408            &[
4409                "worktree",
4410                "add",
4411                "-q",
4412                "-b",
4413                "worktree-traveller",
4414                worker_wt.to_str().expect("utf8 path"),
4415            ],
4416        );
4417
4418        // What the spawn writes: the row names the sibling tree, and the
4419        // live entry carries the same root.
4420        workspace
4421            .record_worker_row(
4422                &project_key,
4423                "traveller",
4424                "traveller-id",
4425                "charter",
4426                None,
4427                None,
4428                false,
4429                crate::mcp::workers::types::WorkerTree {
4430                    is_git_repo: true,
4431                    cwd: Some(sibling_root.to_string_lossy().into_owned()),
4432                },
4433                None,
4434            )
4435            .expect("record the row the despawn reads");
4436        workspace.remove_latest_worker(&project_key, "traveller");
4437        let mut entry = fake_git_worker_entry("traveller", "worker-1");
4438        entry.cwd = Some(sibling_root.clone());
4439        workspace.insert_live_worker(&project_key, entry);
4440
4441        // Dirty in the WORKER's tree, clean in the project's: the probe
4442        // must block on the worker's.
4443        std::fs::write(worker_wt.join("dirty.txt"), "uncommitted").expect("write dirty file");
4444        let (tx, rx) = tokio::sync::oneshot::channel();
4445        handle_despawn_worker(&workspace, &project_key, "traveller", false, tx);
4446        let blocked = rx.await.expect("despawn result");
4447        let crate::protocol::DespawnResult::Blocked { reason } = blocked else {
4448            panic!("a dirty worktree in the worker's own tree must block the despawn: {blocked:?}");
4449        };
4450        assert!(
4451            reason.contains("uncommitted or untracked changes"),
4452            "the reason is the worker's tree's own verdict, not the project's: {reason}",
4453        );
4454
4455        // Clean: the despawn removes the worktree under the worker's own
4456        // tree, not one under the project.
4457        std::fs::remove_file(worker_wt.join("dirty.txt")).expect("clean the worktree");
4458        let (tx, rx) = tokio::sync::oneshot::channel();
4459        handle_despawn_worker(&workspace, &project_key, "traveller", false, tx);
4460        let result = rx.await.expect("despawn result");
4461        assert!(
4462            matches!(result, crate::protocol::DespawnResult::Despawned { .. }),
4463            "a clean despawn lands: {result:?}",
4464        );
4465        assert!(!worker_wt.exists(), "the worker's own worktree is the one removed");
4466        let reaped = std::process::Command::new("git")
4467            .args(["branch", "--list", "worktree-traveller"])
4468            .current_dir(&sibling_root)
4469            .output()
4470            .expect("git branch");
4471        assert!(
4472            String::from_utf8_lossy(&reaped.stdout).trim().is_empty(),
4473            "the worker's own tree is where the branch reap aimed: {:?}",
4474            String::from_utf8_lossy(&reaped.stdout),
4475        );
4476        let decoy = std::process::Command::new("git")
4477            .args(["branch", "--list", "worktree-traveller"])
4478            .current_dir(repo.path())
4479            .output()
4480            .expect("git branch");
4481        assert!(
4482            String::from_utf8_lossy(&decoy.stdout).contains("worktree-traveller"),
4483            "and the project's same-named branch is untouched: {:?}",
4484            String::from_utf8_lossy(&decoy.stdout),
4485        );
4486        assert!(
4487            project_wt.exists(),
4488            "as is the worktree the project forked for its own worker of that name: the \
4489             despawn acted on the worker's tree, not the project's",
4490        );
4491    }
4492
4493    /// A project that is itself a linked worktree: `--worktree <label>`
4494    /// creates the worktree under the MAIN checkout's `.claude/worktrees`,
4495    /// because the CLI composes it from the repository's canonical root
4496    /// (measured on 2.1.280), not from the project path. Recording it
4497    /// under the project path leaves the despawn's dirty probe reading a
4498    /// directory nothing created, so the probe answers "could not
4499    /// determine" and blocks, and a forced despawn leaves the real
4500    /// worktree and branch behind (#2460).
4501    #[tokio::test]
4502    async fn a_linked_worktree_project_records_its_worker_where_the_cli_creates_it() {
4503        let _binary = MOCK_BINARY.lock().await;
4504        let main = tempdir().expect("main repo tempdir");
4505        run_git(main.path(), &["init", "-q"]);
4506        run_git(main.path(), &["config", "user.email", "t@example.com"]);
4507        run_git(main.path(), &["config", "user.name", "Test"]);
4508        run_git(main.path(), &["config", "commit.gpgsign", "false"]);
4509        fs::write(main.path().join("README.md"), "seed").expect("write seed");
4510        run_git(main.path(), &["add", "."]);
4511        run_git(main.path(), &["commit", "-q", "-m", "init"]);
4512        let main_root = std::fs::canonicalize(main.path()).expect("canonical main");
4513
4514        // The project path is a linked worktree of that repo, the shape the
4515        // dev stack builds.
4516        let holder = tempdir().expect("linked holder");
4517        let linked = holder.path().join("linked");
4518        run_git(
4519            &main_root,
4520            &["worktree", "add", "-q", "-b", "linked-project", linked.to_str().expect("utf8")],
4521        );
4522
4523        let dir = tempdir().expect("config tempdir");
4524        write_forge_toml(dir.path(), &linked.to_string_lossy());
4525        let ws = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("workspace"));
4526        ws.seed_test_ready_account("Stargate");
4527        ws.seed_test_gateway_ready(true);
4528        // **The worker's spawn is a warm too**, not only the lead's: the
4529        // fleet hears the seat before the worker's first browser call.
4530        let (calls, mut heard) = tokio::sync::mpsc::unbounded_channel();
4531        ws.install_browser_host(Arc::new(crate::browser::testing::StubHost { calls }));
4532        let key = ws
4533            .list_projects()
4534            .into_iter()
4535            .find(|v| v.name == "forge")
4536            .expect("fixture project")
4537            .key;
4538
4539        // Where the CLI creates it: the fixture forks it there itself, so
4540        // the assertion below compares forge's composition against the
4541        // CLI's own root rather than against the code under test.
4542        let created = main_root.join(".claude").join("worktrees").join("labourer");
4543        fs::create_dir_all(created.parent().expect("wt parent")).expect("mkdir worktrees");
4544        run_git(
4545            &main_root,
4546            &[
4547                "worktree",
4548                "add",
4549                "-q",
4550                "-b",
4551                "worktree-labourer",
4552                created.to_str().expect("utf8 path"),
4553            ],
4554        );
4555
4556        forge_agent::set_test_sdk_binary(Some(
4557            concat!(env!("CARGO_MANIFEST_DIR"), "/../forge-sdk/tests/fixtures/mock_claude.sh")
4558                .to_owned(),
4559        ));
4560        let mut updates = ws.subscribe_mirror();
4561        let reply = spawn_reply(&ws, &key, "labourer", false)
4562            .await
4563            .expect("reply channel")
4564            .expect("the spawn is admitted");
4565        assert_eq!(
4566            reply.worktree.as_deref(),
4567            Some(created.to_string_lossy().as_ref()),
4568            "the recorded worktree must be the directory the CLI creates",
4569        );
4570        let slot = SessionSlot::worker("Default", "forge", "labourer");
4571        let mut warmed = None;
4572        let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5);
4573        while tokio::time::Instant::now() < deadline {
4574            let left = deadline.saturating_duration_since(tokio::time::Instant::now());
4575            let Ok(Some((what, args))) = tokio::time::timeout(left, heard.recv()).await else {
4576                break;
4577            };
4578            if what == "warm" {
4579                warmed = Some(args);
4580                break;
4581            }
4582        }
4583        assert_eq!(
4584            warmed,
4585            Some(serde_json::json!(slot)),
4586            "the worker's spawn path warms its own seat",
4587        );
4588        assert_eq!(
4589            ws.cwd_for_session(&slot),
4590            Some(created.to_string_lossy().into_owned()),
4591            "and every read-back composes the same directory",
4592        );
4593        // The despawn waits for the worker's child to die, so the session
4594        // has to be up before it is asked.
4595        tokio::time::timeout(spawn_wait(), async {
4596            while let Some(seat) = updates.recv().await {
4597                if matches!(&*seat.update, SessionUpdate::Connected { key, .. } if *key == slot) {
4598                    return;
4599                }
4600            }
4601            panic!("the update stream closed before the worker's session connected");
4602        })
4603        .await
4604        .expect("the worker's session reaches Connected");
4605
4606        // Dirty in the worktree the CLI created: the probe must find it and
4607        // block for the worktree's own reason, not "could not determine".
4608        fs::write(created.join("dirty.txt"), "uncommitted").expect("write dirty file");
4609        let (tx, rx) = tokio::sync::oneshot::channel();
4610        handle_despawn_worker(&ws, &key, "labourer", false, tx);
4611        let blocked = rx.await.expect("despawn result");
4612        let crate::protocol::DespawnResult::Blocked { reason } = blocked else {
4613            panic!("the dirty probe must find the worktree the CLI created: {blocked:?}");
4614        };
4615        assert!(
4616            reason.contains("uncommitted or untracked changes"),
4617            "the verdict is the created worktree's own: {reason}",
4618        );
4619
4620        fs::remove_file(created.join("dirty.txt")).expect("clean the worktree");
4621        let (tx, rx) = tokio::sync::oneshot::channel();
4622        handle_despawn_worker(&ws, &key, "labourer", false, tx);
4623        let result = rx.await.expect("despawn result");
4624        assert!(
4625            matches!(result, crate::protocol::DespawnResult::Despawned { .. }),
4626            "a clean despawn lands: {result:?}",
4627        );
4628        assert!(!created.exists(), "the worktree the CLI created is the one removed");
4629        let branches = std::process::Command::new("git")
4630            .args(["branch", "--list", "worktree-labourer"])
4631            .current_dir(&main_root)
4632            .output()
4633            .expect("git branch");
4634        assert!(
4635            String::from_utf8_lossy(&branches.stdout).trim().is_empty(),
4636            "and the branch behind it is reaped: {:?}",
4637            String::from_utf8_lossy(&branches.stdout),
4638        );
4639        forge_agent::set_test_sdk_binary(None);
4640    }
4641
4642    /// A main repo with one linked worktree per name, each declared as a
4643    /// project of that name. Every one of them composes its workers'
4644    /// worktrees under the main checkout (#2460), so they share one
4645    /// `<main>/.claude/worktrees/<label>` namespace. No project declares a
4646    /// model, so a spawn that gets past the worktree checks dies at the
4647    /// account walk instead of starting a child.
4648    fn linked_projects_fixture(
4649        names: &[&str],
4650    ) -> (Arc<Workspace>, std::path::PathBuf, Vec<tempfile::TempDir>) {
4651        let holder = tempdir().expect("holder");
4652        let main = holder.path().join("main");
4653        fs::create_dir_all(&main).expect("main dir");
4654        run_git(&main, &["init", "-q"]);
4655        run_git(&main, &["config", "user.email", "t@example.com"]);
4656        run_git(&main, &["config", "user.name", "Test"]);
4657        run_git(&main, &["config", "commit.gpgsign", "false"]);
4658        fs::write(main.join("README.md"), "seed").expect("write seed");
4659        run_git(&main, &["add", "."]);
4660        run_git(&main, &["commit", "-q", "-m", "init"]);
4661        let main = std::fs::canonicalize(&main).expect("canonical main");
4662
4663        let mut projects = String::new();
4664        for name in names {
4665            let linked = holder.path().join(name);
4666            run_git(&main, &["worktree", "add", "-q", "-b", name, linked.to_str().expect("utf8")]);
4667            let _ = writeln!(
4668                projects,
4669                "[[orgs.projects]]\nname = \"{name}\"\npath = \"{}\"\n",
4670                linked.display()
4671            );
4672        }
4673        let config = tempdir().expect("config tempdir");
4674        fs::write(
4675            forge_toml_path(config.path()),
4676            format!(
4677                "[[orgs]]\nname = \"Default\"\naccounts = [\"Stargate\"]\n\n{projects}\
4678                 [[accounts]]\ndisplay_name = \"Stargate\"\ntoken = \"t\"\n\
4679                 models = [\"claude-sonnet-5\"]\nprovider = \"anthropic\"\n"
4680            ),
4681        )
4682        .expect("write forge.toml");
4683        let workspace =
4684            Arc::new(Workspace::new_for_test(config.path().to_owned()).expect("workspace"));
4685        (workspace, main, vec![holder, config])
4686    }
4687
4688    /// The base is repository-global, so two projects of one repo compose
4689    /// the same worktree path for one label. The one-live-worker guard is
4690    /// per project, so without this a second spawn would run in the first
4691    /// worker's tree - and its despawn would delete it. A label that names
4692    /// a project parked under `.claude/worktrees` is the same collision
4693    /// against that project's own checkout.
4694    #[tokio::test]
4695    async fn a_spawn_refuses_a_worktree_another_project_holds() {
4696        let (ws, main, _keep) = linked_projects_fixture(&["proj-a", "proj-b"]);
4697        let key_of = |name: &str| {
4698            ws.list_projects().into_iter().find(|v| v.name == name).expect("declared project").key
4699        };
4700        ws.insert_live_worker(&key_of("proj-a"), fake_git_worker_entry("shared", "worker-1"));
4701
4702        let err = spawn_reply(&ws, &key_of("proj-b"), "shared", false)
4703            .await
4704            .expect("reply channel")
4705            .expect_err("the shared worktree is refused");
4706        assert!(err.contains("worker 'shared'"), "the refusal names the holder: {err}");
4707        assert!(err.contains("proj-a"), "and the project it belongs to: {err}");
4708        assert!(
4709            ws.list_live_workers(&key_of("proj-b")).is_empty(),
4710            "the refused spawn mints nothing",
4711        );
4712
4713        // A crashed worker's row still claims its path until the row is
4714        // closed: refusing is the safe direction.
4715        let mut failed = fake_git_worker_entry("ghost", "worker-2");
4716        failed.status = forge_primitives::WorkerLiveness::Failed;
4717        ws.insert_live_worker(&key_of("proj-b"), failed);
4718        let err = spawn_reply(&ws, &key_of("proj-a"), "ghost", false)
4719            .await
4720            .expect("reply channel")
4721            .expect_err("the crashed worker's row still claims its tree");
4722        assert!(
4723            err.contains("worker 'ghost'") && err.contains("proj-b"),
4724            "the refusal names the crashed worker and its project: {err}",
4725        );
4726
4727        // A seat still spawning holds its tree too: every state counts,
4728        // not only Running.
4729        let mut spawning = fake_git_worker_entry("launching", "worker-3");
4730        spawning.status = forge_primitives::WorkerLiveness::Spawning;
4731        ws.insert_live_worker(&key_of("proj-b"), spawning);
4732        let err = spawn_reply(&ws, &key_of("proj-a"), "launching", false)
4733            .await
4734            .expect("reply channel")
4735            .expect_err("a spawning worker's tree is claimed");
4736        assert!(
4737            err.contains("worker 'launching'") && err.contains("proj-b"),
4738            "the refusal names the spawning worker and its project: {err}",
4739        );
4740
4741        // A durable row claims the tree even with no in-memory entry:
4742        // after a restart, before that project's lead connects.
4743        ws.record_worker_row(
4744            &key_of("proj-b"),
4745            "parked",
4746            "550e8400-e29b-41d4-a716-446655440083",
4747            "charter",
4748            None,
4749            None,
4750            false,
4751            crate::mcp::workers::types::WorkerTree { is_git_repo: true, cwd: None },
4752            None,
4753        )
4754        .expect("record proj-b's parked row");
4755        let err = spawn_reply(&ws, &key_of("proj-a"), "parked", false)
4756            .await
4757            .expect("reply channel")
4758            .expect_err("the durable row claims the composed path");
4759        assert!(
4760            err.contains("worker 'parked'") && err.contains("proj-b"),
4761            "the durable row names its project: {err}",
4762        );
4763
4764        // An in-flight claim in another project refuses the spawn even
4765        // with no entry or row yet: the check and the insert are one step.
4766        let owner = ws.project_root_for_key(&key_of("proj-a")).expect("proj-a root");
4767        let claimed = crate::mcp::workers::types::worker_tag_dir(&owner, "held", true);
4768        let claim = ws
4769            .claim_worktree(
4770                &claimed,
4771                &key_of("proj-a"),
4772                "held",
4773                crate::workspace::WorktreeOp::Spawn,
4774            )
4775            .expect("claim");
4776        let err = spawn_reply(&ws, &key_of("proj-b"), "held", false)
4777            .await
4778            .expect("reply channel")
4779            .expect_err("another project's claim refuses the spawn");
4780        assert!(
4781            err.contains("in use by the worker 'held'") && err.contains("proj-a"),
4782            "the refusal names the claim holder: {err}",
4783        );
4784        drop(claim);
4785
4786        // A row from before `is_git_repo` existed still claims its
4787        // base+label path: refusing on that name is the safe direction.
4788        let store = tempdir().expect("store dir");
4789        let db = crate::store::Db::open(&store.path().join("db.redb")).expect("open db");
4790        crate::store::sessions::put(
4791            &db,
4792            &crate::store::sessions::SessionRecord {
4793                org: "Default".to_owned(),
4794                project: "proj-b".to_owned(),
4795                label: "old".to_owned(),
4796                session_id: None,
4797                charter: None,
4798                kick: None,
4799                resume_kick: None,
4800                interactive: None,
4801                seat_state: None,
4802                is_git_repo: None,
4803                mcp_families: None,
4804                cwd: None,
4805            },
4806        )
4807        .expect("write a row with no recorded gitness");
4808        ws.install_db_for_test(db);
4809        let err = spawn_reply(&ws, &key_of("proj-a"), "old", false)
4810            .await
4811            .expect("reply channel")
4812            .expect_err("a gitness-less row still claims the composed path");
4813        assert!(
4814            err.contains("worker 'old'") && err.contains("proj-b"),
4815            "the old row names its project: {err}",
4816        );
4817
4818        // A project declared at the path a worker label would compose is
4819        // its own checkout, not a worktree anyone may reuse or remove.
4820        let checkout = main.join(".claude").join("worktrees").join("dev");
4821        fs::create_dir_all(checkout.parent().expect("worktrees root")).expect("mkdir worktrees");
4822        run_git(
4823            &main,
4824            &["worktree", "add", "-q", "-b", "dev-project", checkout.to_str().expect("utf8")],
4825        );
4826        ws.seed_test_project("proj-dev", &checkout.to_string_lossy());
4827        let err = spawn_reply(&ws, &key_of("proj-b"), "dev", false)
4828            .await
4829            .expect("reply channel")
4830            .expect_err("a project's own checkout is refused");
4831        assert!(err.contains("checkout of project 'proj-dev'"), "the refusal names it: {err}");
4832    }
4833
4834    /// The collision check compares real paths: the shared tree reached
4835    /// through a symlinked spelling is the same tree a live worker holds.
4836    #[test]
4837    fn the_collision_check_reads_a_symlinked_spelling_as_the_same_tree() {
4838        let (ws, main, _keep) = linked_projects_fixture(&["proj-a", "proj-b"]);
4839        let key_of = |name: &str| {
4840            ws.list_projects().into_iter().find(|v| v.name == name).expect("declared project").key
4841        };
4842        let shared = main.join(".claude").join("worktrees").join("shared");
4843        fs::create_dir_all(shared.parent().expect("worktrees root")).expect("mkdir worktrees");
4844        run_git(
4845            &main,
4846            &["worktree", "add", "-q", "-b", "worktree-shared", shared.to_str().expect("utf8")],
4847        );
4848        ws.insert_live_worker(&key_of("proj-a"), fake_git_worker_entry("shared", "worker-1"));
4849        let holder = tempdir().expect("alias holder");
4850        let alias = holder.path().join("shared-alias");
4851        std::os::unix::fs::symlink(&shared, &alias).expect("symlink");
4852
4853        let collision = ws.worktree_collision(
4854            &alias,
4855            &key_of("proj-b"),
4856            "shared",
4857            crate::workspace::WorktreeOp::Spawn,
4858        );
4859
4860        assert!(
4861            collision.as_deref().is_some_and(|reason| reason.contains("worker 'shared'")),
4862            "the canonical comparison finds the holder through the alias: {collision:?}",
4863        );
4864    }
4865
4866    /// The despawn end of the same collision: removing the shared tree
4867    /// would delete the other project's live worker's cwd, so it refuses
4868    /// even when forced and leaves the tree standing.
4869    #[tokio::test]
4870    async fn a_despawn_refuses_a_worktree_another_project_holds() {
4871        let (ws, main, _keep) = linked_projects_fixture(&["proj-a", "proj-b"]);
4872        let key_of = |name: &str| {
4873            ws.list_projects().into_iter().find(|v| v.name == name).expect("declared project").key
4874        };
4875        let shared = main.join(".claude").join("worktrees").join("shared");
4876        fs::create_dir_all(shared.parent().expect("worktrees root")).expect("mkdir worktrees");
4877        run_git(
4878            &main,
4879            &["worktree", "add", "-q", "-b", "worktree-shared", shared.to_str().expect("utf8")],
4880        );
4881        ws.insert_live_worker(&key_of("proj-a"), fake_git_worker_entry("shared", "a-worker"));
4882        ws.insert_live_worker(&key_of("proj-b"), fake_git_worker_entry("shared", "b-worker"));
4883
4884        let (tx, rx) = tokio::sync::oneshot::channel();
4885        handle_despawn_worker(&ws, &key_of("proj-a"), "shared", true, tx);
4886        let result = rx.await.expect("despawn result");
4887        let crate::protocol::DespawnResult::Failed { reason } = result else {
4888            panic!("a shared worktree must refuse the despawn even when forced: {result:?}");
4889        };
4890        assert!(reason.contains("worker 'shared'"), "the reason names the holder: {reason}");
4891        assert!(reason.contains("proj-b"), "and its project: {reason}");
4892        assert!(shared.exists(), "the shared tree was not removed");
4893        assert_eq!(ws.list_live_workers(&key_of("proj-a")).len(), 1, "and the worker stays live");
4894    }
4895
4896    /// A despawn consults the same repository-wide claims: while another
4897    /// project's spawn or cleanup holds the path, it refuses.
4898    #[tokio::test]
4899    async fn a_despawn_refuses_a_worktree_another_project_claims() {
4900        let (ws, main, _keep) = linked_projects_fixture(&["proj-a", "proj-b"]);
4901        let key_of = |name: &str| {
4902            ws.list_projects().into_iter().find(|v| v.name == name).expect("declared project").key
4903        };
4904        let shared = main.join(".claude").join("worktrees").join("shared");
4905        fs::create_dir_all(shared.parent().expect("worktrees root")).expect("mkdir worktrees");
4906        run_git(
4907            &main,
4908            &["worktree", "add", "-q", "-b", "worktree-shared", shared.to_str().expect("utf8")],
4909        );
4910        ws.insert_live_worker(&key_of("proj-b"), fake_git_worker_entry("shared", "b-worker"));
4911        let claim = ws
4912            .claim_worktree(
4913                &shared,
4914                &key_of("proj-a"),
4915                "shared",
4916                crate::workspace::WorktreeOp::Spawn,
4917            )
4918            .expect("proj-a holds the path");
4919
4920        let (tx, rx) = tokio::sync::oneshot::channel();
4921        handle_despawn_worker(&ws, &key_of("proj-b"), "shared", true, tx);
4922        let result = rx.await.expect("despawn result");
4923        let crate::protocol::DespawnResult::Failed { reason } = result else {
4924            panic!("a claimed worktree refuses the despawn: {result:?}");
4925        };
4926        assert!(
4927            reason.contains("in use by the worker 'shared'") && reason.contains("proj-a"),
4928            "the reason names the claim holder: {reason}",
4929        );
4930        assert!(shared.exists(), "the claimed tree was not removed");
4931        drop(claim);
4932    }
4933
4934    /// A linked project's worktree sits under the MAIN checkout, and the
4935    /// despawn-side conversion is what lets the leftover fallback aim
4936    /// there: composing against the project path refuses a directory that
4937    /// is not under its `.claude/worktrees` and leaves it behind.
4938    #[tokio::test]
4939    async fn despawn_removes_a_deregistered_leftover_in_a_linked_project() {
4940        let (ws, main, _keep) = linked_projects_fixture(&["proj-a"]);
4941        let key = ws.list_projects().into_iter().next().expect("declared project").key;
4942        let leftover = main.join(".claude").join("worktrees").join("leftover");
4943        fs::create_dir_all(leftover.parent().expect("worktrees root")).expect("mkdir worktrees");
4944        run_git(
4945            &main,
4946            &["worktree", "add", "-q", "-b", "worktree-leftover", leftover.to_str().expect("utf8")],
4947        );
4948        // The state a failed delete leaves: deregistered, directory on
4949        // disk.
4950        fs::remove_dir_all(main.join(".git").join("worktrees").join("leftover"))
4951            .expect("deregister the worktree");
4952        let refused = std::process::Command::new("git")
4953            .args(["worktree", "remove", "--force", leftover.to_str().expect("utf8 path")])
4954            .current_dir(leftover.parent().expect("wt parent"))
4955            .output()
4956            .expect("spawn git");
4957        assert!(
4958            !refused.status.success(),
4959            "fixture precondition: git refuses the deregistered leftover outright"
4960        );
4961        ws.insert_live_worker(&key, fake_git_worker_entry("leftover", "worker-1"));
4962
4963        let (tx, rx) = tokio::sync::oneshot::channel();
4964        handle_despawn_worker(&ws, &key, "leftover", true, tx);
4965        let result = rx.await.expect("despawn result");
4966        assert!(
4967            matches!(
4968                result,
4969                crate::protocol::DespawnResult::Despawned {
4970                    worktree_cleanup_warning: None,
4971                    branch_cleanup_warning: None,
4972                }
4973            ),
4974            "the leftover is removed under the main checkout: {result:?}"
4975        );
4976        assert!(!leftover.exists(), "the leftover directory is gone");
4977    }
4978
4979    /// The one handle on the process-wide mock-binary slot below: the case
4980    /// that drives it holds this for its span, so no sibling case can spawn
4981    /// against a mock it did not ask for (or against the clearing of one).
4982    static MOCK_BINARY: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
4983
4984    /// How long a case waits for a spawn's first event: the SDK's own
4985    /// handshake budget plus a settle margin - a wait shorter than the
4986    /// budget under it fails a machine that is merely slow.
4987    fn spawn_wait() -> std::time::Duration {
4988        forge_sdk::INIT_HANDSHAKE_TIMEOUT + std::time::Duration::from_secs(5)
4989    }
4990
4991    /// The real mock wrapped so the child's death throes write into the
4992    /// worktree: after its stdin closes (the despawn's first teardown act)
4993    /// it waits `throes`, recreates the path and writes, then exits. The
4994    /// path is baked in because nothing tells the child where its worktree
4995    /// is - the real CLI derives it from `--worktree`, which the mock
4996    /// ignores. The write also lands a marker OUTSIDE the worktree, which
4997    /// is what lets the case prove the throes really ran: inside, the
4998    /// removal erases the evidence along with the write.
4999    fn death_throes_mock(
5000        dir: &std::path::Path,
5001        worktree: &std::path::Path,
5002        throes: std::time::Duration,
5003    ) -> String {
5004        use std::os::unix::fs::PermissionsExt as _;
5005        let real =
5006            concat!(env!("CARGO_MANIFEST_DIR"), "/../forge-sdk/tests/fixtures/mock_claude.sh");
5007        let wrapper = dir.join("death-throes-mock");
5008        fs::write(
5009            &wrapper,
5010            format!(
5011                "#!/bin/bash\nif [[ ${{1:-}} == \"--version\" ]]; then exec {real} \"$@\"; fi\n\
5012                 {real} \"$@\"\n\
5013                 /bin/sleep {}\n\
5014                 mkdir -p {wt} && printf 'wrote while dying\\n' > {wt}/late-write.txt\n\
5015                 printf 'throes ran\\n' > {marker}\n",
5016                throes.as_secs_f64(),
5017                wt = worktree.display(),
5018                marker = death_throes_marker(dir).display(),
5019            ),
5020        )
5021        .expect("write the death-throes mock");
5022        std::fs::set_permissions(&wrapper, std::fs::Permissions::from_mode(0o755)).expect("+x");
5023        wrapper.to_string_lossy().into_owned()
5024    }
5025
5026    /// Where the death-throes mock records that its dying write ran.
5027    fn death_throes_marker(dir: &std::path::Path) -> std::path::PathBuf {
5028        dir.join("death-throes-ran")
5029    }
5030
5031    /// **The despawn must take the worker's session down before the
5032    /// worktree.** The incident (#1767): the worktree came out at 15:47:08
5033    /// while the session's child was still alive for five seconds more,
5034    /// and in that gap the dying vite wrote its cache back into the path
5035    /// the removal had already taken. The despawn used to overlap them -
5036    /// teardown released the session and returned, the disconnect reaped
5037    /// the child asynchronously, and `DespawnCleanup` removed the worktree
5038    /// immediately.
5039    ///
5040    /// This drives exactly that shape: a real worker session over a mock
5041    /// child that, once its stdin closes, waits and then writes into the
5042    /// worktree before it exits. The despawn must not come back - and so
5043    /// must not remove anything - before that child is dead.
5044    #[tokio::test]
5045    async fn a_despawn_holds_the_removal_until_the_worker_child_has_died() {
5046        let _binary = MOCK_BINARY.lock().await;
5047        let dir = tempdir().expect("config tempdir");
5048        let repo = tempdir().expect("repo tempdir");
5049        run_git(repo.path(), &["init", "-q"]);
5050        run_git(repo.path(), &["config", "user.email", "t@example.com"]);
5051        run_git(repo.path(), &["config", "user.name", "Test"]);
5052        run_git(repo.path(), &["config", "commit.gpgsign", "false"]);
5053        fs::write(repo.path().join("README.md"), "seed").expect("write seed");
5054        run_git(repo.path(), &["add", "."]);
5055        run_git(repo.path(), &["commit", "-q", "-m", "init"]);
5056        let repo_root = std::fs::canonicalize(repo.path()).expect("canonical repo");
5057        write_forge_toml(dir.path(), &repo_root.to_string_lossy());
5058        let ws = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("workspace"));
5059        ws.seed_test_ready_account("Stargate");
5060        ws.seed_test_gateway_ready(true);
5061        let key = ws
5062            .list_projects()
5063            .into_iter()
5064            .find(|v| v.name == "forge")
5065            .expect("fixture project")
5066            .key;
5067
5068        // The worktree `--worktree dying` would fork (the mock ignores the
5069        // flag, so the fixture forks it).
5070        let wt = repo_root.join(".claude").join("worktrees").join("dying");
5071        std::fs::create_dir_all(wt.parent().expect("wt parent")).expect("mkdir worktrees");
5072        run_git(
5073            &repo_root,
5074            &["worktree", "add", "-q", "-b", "worktree-dying", wt.to_str().expect("utf8")],
5075        );
5076        // The dying child's write is a build artifact git ignores - the
5077        // incident's vite cache - so the removal it races is the ordinary
5078        // one, not the dirty-worktree refusal.
5079        let excludes = dir.path().join("excludes");
5080        fs::write(&excludes, "late-write.txt\n").expect("write excludes");
5081        run_git(&repo_root, &["config", "core.excludesFile", excludes.to_str().expect("utf8")]);
5082
5083        let throes = std::time::Duration::from_secs(2);
5084        forge_agent::set_test_sdk_binary(Some(death_throes_mock(dir.path(), &wt, throes)));
5085
5086        let mut updates = ws.subscribe_mirror();
5087        let (tx, rx) = tokio::sync::oneshot::channel();
5088        handle_spawn_worker(
5089            &ws,
5090            key.clone(),
5091            WorkerSpawnArgs {
5092                label: "dying".to_owned(),
5093                charter: "charter".to_owned(),
5094                kick: None,
5095                resume_kick: None,
5096                interactive: false,
5097                mcp_families: None,
5098                cwd: None,
5099            },
5100            SessionSlot::from_str_for_test("lead"),
5101            None,
5102            false,
5103            tx,
5104        );
5105        rx.await.expect("spawn replies").expect("spawn succeeds");
5106        // The despawn has an ordering to enforce only once the child is up
5107        // and holding its stdin; Connected is that moment. A stream that
5108        // ends first is a failure too - the wait must not read as settled
5109        // on a closed channel.
5110        let slot = SessionSlot::worker("Default", "forge", "dying");
5111        tokio::time::timeout(spawn_wait(), async {
5112            while let Some(seat) = updates.recv().await {
5113                if matches!(&*seat.update, SessionUpdate::Connected { key, .. } if *key == slot) {
5114                    return;
5115                }
5116            }
5117            panic!("the update stream closed before the worker's session connected");
5118        })
5119        .await
5120        .expect("the worker's session reaches Connected");
5121
5122        let started = std::time::Instant::now();
5123        let (tx, rx) = tokio::sync::oneshot::channel();
5124        handle_despawn_worker(&ws, &key, "dying", false, tx);
5125        let result = rx.await.expect("despawn result");
5126        let elapsed = started.elapsed();
5127
5128        assert!(
5129            matches!(
5130                result,
5131                crate::protocol::DespawnResult::Despawned {
5132                    worktree_cleanup_warning: None,
5133                    branch_cleanup_warning: None,
5134                }
5135            ),
5136            "the despawn lands clean: {result:?}",
5137        );
5138        assert!(
5139            elapsed >= throes,
5140            "the despawn must not report the worker gone before its child is dead: it came back \
5141             after {elapsed:?}, while the child was still alive for {throes:?} of its death",
5142        );
5143        assert!(
5144            death_throes_marker(dir.path()).exists(),
5145            "the child's dying write really ran before the despawn came back",
5146        );
5147        assert!(
5148            !wt.exists(),
5149            "and the removal must come after the child's last write: a write landing in the \
5150             already-removed path is exactly the leftover this ordering exists to prevent",
5151        );
5152        forge_agent::set_test_sdk_binary(None);
5153    }
5154
5155    // ---- auto-heal over the mock child -------------------------------------
5156
5157    /// `forge.toml` with the `[sessions]` backoff the heal tests run on:
5158    /// real seconds would make a three-restart case wait out 5+30+120 s.
5159    fn write_heal_toml(dir: &std::path::Path, project_path: &str) {
5160        fs::write(
5161            forge_toml_path(dir),
5162            format!(
5163                r#"
5164[[orgs]]
5165name = "Default"
5166accounts = ["Stargate"]
5167
5168[[orgs.projects]]
5169name = "forge"
5170path = "{project_path}"
5171model = "claude-sonnet-5"
5172
5173[[accounts]]
5174display_name = "Stargate"
5175token = "t"
5176models = ["claude-sonnet-5"]
5177provider = "anthropic"
5178
5179[sessions]
5180heal_backoff = ["150ms", "200ms", "300ms"]
5181heal_stable = "30s"
5182"#
5183            ),
5184        )
5185        .expect("write forge.toml");
5186    }
5187
5188    /// The real mock wrapped so runs `die_from..=die_through` die of their
5189    /// own accord shortly after the handshake, which is what forge sees as
5190    /// a subprocess that closed mid-run. Every later run stays up. The run
5191    /// count is left on disk, which is how a case proves how many children
5192    /// ever started.
5193    fn heal_mock(dir: &std::path::Path, die_from: u32, die_through: u32) -> String {
5194        use std::os::unix::fs::PermissionsExt as _;
5195        let real =
5196            concat!(env!("CARGO_MANIFEST_DIR"), "/../forge-sdk/tests/fixtures/mock_claude.sh");
5197        let wrapper = dir.join("heal-mock");
5198        fs::write(
5199            &wrapper,
5200            format!(
5201                "#!/bin/bash\n\
5202                 if [[ ${{1:-}} == \"--version\" ]]; then exec {real} \"$@\"; fi\n\
5203                 runs={runs}\n\
5204                 n=$(( $(cat \"$runs\" 2>/dev/null || echo 0) + 1 ))\n\
5205                 echo \"$n\" > \"$runs\"\n\
5206                 {real} \"$@\" <&0 & child=$!\n\
5207                 if [[ \"$n\" -ge {die_from} && \"$n\" -le {die_through} ]]; then\n\
5208                   ( /bin/sleep 0.6; kill -9 \"$child\" 2>/dev/null ) &\n\
5209                 fi\n\
5210                 wait \"$child\"\n",
5211                real = real,
5212                runs = dir.join("heal-mock-runs").display(),
5213                die_from = die_from,
5214                die_through = die_through,
5215            ),
5216        )
5217        .expect("write the heal mock");
5218        std::fs::set_permissions(&wrapper, std::fs::Permissions::from_mode(0o755)).expect("+x");
5219        wrapper.to_string_lossy().into_owned()
5220    }
5221
5222    /// How many mock children have started.
5223    fn mock_runs(dir: &std::path::Path) -> u32 {
5224        fs::read_to_string(dir.join("heal-mock-runs"))
5225            .ok()
5226            .and_then(|raw| raw.trim().parse().ok())
5227            .unwrap_or(0)
5228    }
5229
5230    /// Every update up to and including the one that satisfies `done`, before
5231    /// `spawn_wait` runs out. The frames are kept rather than matched one by
5232    /// one, because a case that waits for two things cannot know which
5233    /// arrives first.
5234    async fn await_updates(
5235        updates: &mut tokio::sync::mpsc::UnboundedReceiver<crate::update_fanout::SeatUpdate>,
5236        what: &str,
5237        mut done: impl FnMut(&[std::sync::Arc<SessionUpdate>]) -> bool,
5238    ) -> Vec<std::sync::Arc<SessionUpdate>> {
5239        let mut seen = Vec::new();
5240        tokio::time::timeout(spawn_wait(), async {
5241            while let Some(seat) = updates.recv().await {
5242                seen.push(std::sync::Arc::clone(&seat.update));
5243                if done(&seen) {
5244                    return;
5245                }
5246            }
5247            panic!("the update stream closed before {what}");
5248        })
5249        .await
5250        .unwrap_or_else(|_| {
5251            let shapes: Vec<String> = seen.iter().map(|update| shape(update)).collect();
5252            panic!("timed out waiting for {what}; saw {shapes:?}")
5253        });
5254        seen
5255    }
5256
5257    /// An update's variant and, where it has one, the seat it names: enough
5258    /// to read a timeout without the payloads.
5259    fn shape(update: &SessionUpdate) -> String {
5260        let debug = format!("{update:?}");
5261        let head = debug.split_whitespace().next().unwrap_or("?");
5262        let at = update.slot().map_or_else(String::new, |slot| format!("@{}", slot.display()));
5263        match update {
5264            SessionUpdate::ChatAppended { msg, .. } => {
5265                let text = user_text(msg);
5266                format!("{head}{at}({}..)", text.chars().take(40).collect::<String>())
5267            }
5268            _ => format!("{head}{at}"),
5269        }
5270    }
5271
5272    /// The next update `wants` accepts, before `spawn_wait` runs out.
5273    async fn await_update(
5274        updates: &mut tokio::sync::mpsc::UnboundedReceiver<crate::update_fanout::SeatUpdate>,
5275        what: &str,
5276        mut wants: impl FnMut(&SessionUpdate) -> bool,
5277    ) -> std::sync::Arc<SessionUpdate> {
5278        await_updates(updates, what, |seen| seen.iter().any(|update| wants(update)))
5279            .await
5280            .pop()
5281            .expect("a matched update")
5282    }
5283
5284    /// **The budget is the bound**: a child that keeps dying gets three
5285    /// restarts and then one escalation, and no fourth child is ever
5286    /// started. Catches a supervisor that spins past its cap.
5287    #[tokio::test]
5288    async fn a_crash_loop_stops_after_three_restarts_and_escalates() {
5289        let _binary = MOCK_BINARY.lock().await;
5290        let (dir, _project, ws) = heal_fixture();
5291        // Run 1 is the lead; runs 2..=5 are the worker's four deaths.
5292        forge_agent::set_test_sdk_binary(Some(heal_mock(dir.path(), 2, 5)));
5293        let mut updates = ws.subscribe_mirror();
5294        let lead = start_lead(&ws, &mut updates).await;
5295        let slot = SessionSlot::worker("Default", "forge", "probe-heal");
5296        spawn_mock_worker(&ws, &lead, "probe-heal").await;
5297
5298        let seen = await_updates(&mut updates, "the escalation", |seen| {
5299            seen.iter().any(|update| {
5300                matches!(
5301                    update.as_ref(),
5302                    SessionUpdate::PeerEnvelopeAppended { key, wrapped, .. }
5303                        if *key == lead && wrapped.body.contains("Left Failed after 3 restarts")
5304                )
5305            })
5306        })
5307        .await;
5308        assert_eq!(mock_runs(dir.path()), 5, "the lead, the first worker and its three resumes");
5309        assert_eq!(connects(&seen, &slot), 4, "the first connect and three restarts");
5310
5311        // Nothing starts a fourth child once the budget is spent.
5312        tokio::time::sleep(std::time::Duration::from_millis(1200)).await;
5313        assert_eq!(mock_runs(dir.path()), 5, "no fourth restart after the escalation");
5314        assert!(ws.heal_progress(&slot).is_none(), "the seat is left failed, not restarting");
5315
5316        // Every death was reported, the escalation last.
5317        let notices: Vec<String> = seen
5318            .iter()
5319            .filter_map(|update| match update.as_ref() {
5320                SessionUpdate::PeerEnvelopeAppended { key, wrapped, .. } if *key == lead => {
5321                    Some(wrapped.body.clone())
5322                }
5323                _ => None,
5324            })
5325            .collect();
5326        assert_eq!(notices.len(), 4, "one notice per death: {notices:?}");
5327        assert!(
5328            notices.last().is_some_and(|body| body.contains("Left Failed after 3 restarts")),
5329            "and the last is the escalation: {notices:?}",
5330        );
5331        forge_agent::set_test_sdk_binary(None);
5332    }
5333
5334    /// **A despawn never heals.** The stop owns the seat, so the death its
5335    /// own teardown causes must not bring the worker back.
5336    #[tokio::test]
5337    async fn a_despawned_worker_does_not_heal() {
5338        let _binary = MOCK_BINARY.lock().await;
5339        let (dir, _project, ws) = heal_fixture();
5340        forge_agent::set_test_sdk_binary(Some(heal_mock(dir.path(), 2, 2)));
5341        let mut updates = ws.subscribe_mirror();
5342        let lead = start_lead(&ws, &mut updates).await;
5343        let key = forge_key(&ws);
5344        let slot = SessionSlot::worker("Default", "forge", "probe-heal");
5345        spawn_mock_worker(&ws, &lead, "probe-heal").await;
5346        await_update(
5347            &mut updates,
5348            "the worker's first connect",
5349            |update| matches!(update, SessionUpdate::Connected { key, .. } if *key == slot),
5350        )
5351        .await;
5352
5353        let (tx, rx) = tokio::sync::oneshot::channel();
5354        handle_despawn_worker(&ws, &key, "probe-heal", false, tx);
5355        rx.await.expect("despawn result");
5356
5357        // The child dies 0.6 s in; a heal that wrongly fired would start a
5358        // new one 150 ms after that.
5359        tokio::time::sleep(std::time::Duration::from_millis(1500)).await;
5360        assert_eq!(mock_runs(dir.path()), 2, "nothing was started for the stop's own death");
5361        assert!(ws.heal_progress(&slot).is_none(), "the seat is not restarting");
5362        forge_agent::set_test_sdk_binary(None);
5363    }
5364
5365    /// **A seat with no durable row is not forge's to bring back**: a row
5366    /// deleted under a live seat leaves a death with nothing a resume could
5367    /// return, and it stays dead until a person starts it.
5368    #[tokio::test]
5369    async fn an_ad_hoc_seats_death_does_not_heal() {
5370        let _binary = MOCK_BINARY.lock().await;
5371        let (dir, _project, ws) = heal_fixture();
5372        forge_agent::set_test_sdk_binary(Some(heal_mock(dir.path(), 2, 2)));
5373        let mut updates = ws.subscribe_mirror();
5374        let lead = start_lead(&ws, &mut updates).await;
5375        let key = forge_key(&ws);
5376        let slot = SessionSlot::worker("Default", "forge", "probe-heal");
5377        spawn_mock_worker(&ws, &lead, "probe-heal").await;
5378        await_update(
5379            &mut updates,
5380            "the worker's first connect",
5381            |update| matches!(update, SessionUpdate::Connected { key, .. } if *key == slot),
5382        )
5383        .await;
5384
5385        assert!(
5386            ws.delete_worker_row(&key, "probe-heal").expect("the row is deleted"),
5387            "precondition: the row existed",
5388        );
5389
5390        tokio::time::sleep(std::time::Duration::from_millis(1500)).await;
5391        assert_eq!(mock_runs(dir.path()), 2, "no child was started for the ad-hoc death");
5392        assert!(ws.heal_progress(&slot).is_none(), "the seat is not restarting");
5393        forge_agent::set_test_sdk_binary(None);
5394    }
5395
5396    /// Wait until `done` holds, polling every 20 ms, up to `spawn_wait`.
5397    async fn wait_until(what: &str, mut done: impl FnMut() -> bool) {
5398        tokio::time::timeout(spawn_wait(), async {
5399            while !done() {
5400                tokio::time::sleep(std::time::Duration::from_millis(20)).await;
5401            }
5402        })
5403        .await
5404        .unwrap_or_else(|_| panic!("timed out waiting for {what}"));
5405    }
5406
5407    /// **A lead whose project tree is gone fails every attempt with the
5408    /// tree's own reason and escalates at the cap**, rather than reporting a
5409    /// generic failure or resting as if it had come back.
5410    #[tokio::test]
5411    async fn a_lead_whose_project_tree_is_gone_escalates_at_the_cap() {
5412        let _binary = MOCK_BINARY.lock().await;
5413        let (dir, project, ws) = heal_fixture();
5414        // Run 1 is the lead, which then dies; its resumes never start.
5415        forge_agent::set_test_sdk_binary(Some(heal_mock(dir.path(), 1, 1)));
5416        let mut updates = ws.subscribe_mirror();
5417        let lead = start_lead(&ws, &mut updates).await;
5418        let root = std::fs::canonicalize(project.path()).expect("canonical tree");
5419        fs::remove_dir_all(&root).expect("remove the tree");
5420
5421        // The lead's child dies of its own accord, the first attempt is
5422        // scheduled, and then every attempt is refused by the gone tree
5423        // until the budget runs out.
5424        wait_until("the first attempt", || ws.heal_progress(&lead).is_some()).await;
5425        wait_until("the budget to run out", || ws.heal_progress(&lead).is_none()).await;
5426
5427        let reason = ws.spawn_failure(&lead).expect("the last attempt's reason stays on the row");
5428        assert!(reason.contains("spawn failed"), "the refusal's own words: {reason}");
5429        assert!(
5430            reason.contains(root.to_string_lossy().as_ref()),
5431            "the tree the attempt could not enter: {reason}",
5432        );
5433        // The first spawn's Spawning frame went with the connect; every
5434        // frame left on the stream is one heal attempt taking the tree on.
5435        let mut attempts = 0;
5436        while let Ok(seat) = updates.try_recv() {
5437            if matches!(&*seat.update, SessionUpdate::Spawning { key, .. } if *key == lead) {
5438                attempts += 1;
5439            }
5440        }
5441        assert_eq!(attempts, 3, "the cap's three attempts were each made, then it escalated");
5442        assert_eq!(mock_runs(dir.path()), 1, "no resume ever started a child");
5443        forge_agent::set_test_sdk_binary(None);
5444    }
5445
5446    /// **A worker whose worktree is gone comes back anyway**: the heal
5447    /// resumes through the boot re-spawn path, which re-creates the tree the
5448    /// worker needs rather than refusing to start. This is the
5449    /// inference-shop death's shape.
5450    #[tokio::test]
5451    async fn a_dead_workers_missing_worktree_is_recreated_by_the_heal() {
5452        let _binary = MOCK_BINARY.lock().await;
5453        let dir = tempdir().expect("config tempdir");
5454        let repo = tempdir().expect("repo tempdir");
5455        run_git(repo.path(), &["init", "-q"]);
5456        run_git(repo.path(), &["config", "user.email", "t@example.com"]);
5457        run_git(repo.path(), &["config", "user.name", "Test"]);
5458        run_git(repo.path(), &["config", "commit.gpgsign", "false"]);
5459        fs::write(repo.path().join("README.md"), "seed").expect("write seed");
5460        run_git(repo.path(), &["add", "."]);
5461        run_git(repo.path(), &["commit", "-q", "-m", "init"]);
5462        let repo_root = std::fs::canonicalize(repo.path()).expect("canonical repo");
5463        // The worktree `--worktree probe-heal` names: the mock ignores the
5464        // flag, so the fixture forks it the way the CLI would.
5465        let worktree = repo_root.join(".claude/worktrees/probe-heal");
5466        std::fs::create_dir_all(worktree.parent().expect("worktrees dir")).expect("mkdir");
5467        run_git(
5468            &repo_root,
5469            &[
5470                "worktree",
5471                "add",
5472                "-q",
5473                "-b",
5474                "worktree-probe-heal",
5475                worktree.to_str().expect("utf8"),
5476            ],
5477        );
5478        write_heal_toml(dir.path(), &repo_root.to_string_lossy());
5479        let ws = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("workspace"));
5480        ws.seed_test_ready_account("Stargate");
5481        ws.seed_test_gateway_ready(true);
5482        ws.start_kick_dispatcher();
5483        forge_agent::set_test_sdk_binary(Some(heal_mock(dir.path(), 2, 2)));
5484
5485        let mut updates = ws.subscribe_mirror();
5486        let lead = start_lead(&ws, &mut updates).await;
5487        let slot = SessionSlot::worker("Default", "forge", "probe-heal");
5488        spawn_mock_worker(&ws, &lead, "probe-heal").await;
5489        await_update(
5490            &mut updates,
5491            "the worker's first connect",
5492            |update| matches!(update, SessionUpdate::Connected { key, .. } if *key == slot),
5493        )
5494        .await;
5495        // Mid-work when the tree goes, so the heal's resume carries the
5496        // restart note this test waits on.
5497        ws.record_seat_state(&slot, crate::store::sessions::SeatState::InTurn);
5498
5499        // The tree the worker needs disappears while it is up; the resume
5500        // is what must put it back. The registration goes with it, so the
5501        // re-create is a fresh `git worktree add` rather than a repair.
5502        fs::remove_dir_all(&worktree).expect("remove the worktree");
5503        run_git(&repo_root, &["worktree", "prune"]);
5504        assert!(!worktree.is_dir(), "precondition: the worktree is gone");
5505
5506        // The scan starts after the first connect, so the restart is one
5507        // more connect on the same seat.
5508        let seen = await_updates(&mut updates, "the restart and its note", |seen| {
5509            connects(seen, &slot) >= 1 && heard(seen, &slot, "This session was restarted by forge")
5510        })
5511        .await;
5512        assert_eq!(connects(&seen, &slot), 1, "the seat came back despite the missing tree");
5513        assert!(worktree.is_dir(), "the resume re-created the worktree the worker needs");
5514        forge_agent::set_test_sdk_binary(None);
5515    }
5516
5517    fn user_text(msg: &forge_primitives::Message) -> String {
5518        let forge_primitives::Message::User { message, .. } = msg else {
5519            return String::new();
5520        };
5521        message
5522            .content
5523            .iter()
5524            .filter_map(|block| match block {
5525                forge_primitives::ContentBlock::Text { text, .. } => Some(text.as_str()),
5526                _ => None,
5527            })
5528            .collect::<Vec<_>>()
5529            .join("\n")
5530    }
5531
5532    /// Whether `seen` holds a chat row on `slot` whose user text contains
5533    /// `needle`.
5534    fn heard(seen: &[std::sync::Arc<SessionUpdate>], slot: &SessionSlot, needle: &str) -> bool {
5535        seen.iter().any(|update| match update.as_ref() {
5536            SessionUpdate::ChatAppended { key, msg, .. } => {
5537                key == slot && user_text(msg).contains(needle)
5538            }
5539            _ => false,
5540        })
5541    }
5542
5543    /// How many connects on `slot` `seen` holds.
5544    fn connects(seen: &[std::sync::Arc<SessionUpdate>], slot: &SessionSlot) -> usize {
5545        seen.iter()
5546            .filter(|update| matches!(update.as_ref(), SessionUpdate::Connected { key, .. } if key == slot))
5547            .count()
5548    }
5549
5550    /// A project the mock can be spawned into, with the heal `[sessions]`
5551    /// table. The project is not a git repo, so its workers run in the
5552    /// project root and forge's first-connect worktree check stays out of
5553    /// the way - a git worker needs the worktree the mock never creates,
5554    /// which the worktree case builds for itself.
5555    fn heal_fixture() -> (tempfile::TempDir, tempfile::TempDir, Arc<Workspace>) {
5556        let dir = tempdir().expect("config tempdir");
5557        let project = tempdir().expect("project tempdir");
5558        let root = std::fs::canonicalize(project.path()).expect("canonical project");
5559        write_heal_toml(dir.path(), &root.to_string_lossy());
5560        let ws = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("workspace"));
5561        ws.seed_test_ready_account("Stargate");
5562        ws.seed_test_gateway_ready(true);
5563        // The server starts this at boot; a kick a worker is meant to
5564        // continue on would otherwise sit in the queue forever.
5565        ws.start_kick_dispatcher();
5566        (dir, project, ws)
5567    }
5568
5569    /// Start the project's lead over the mock and wait for it, so a worker's
5570    /// notice lands in a running chat rather than spawning the lead in the
5571    /// middle of the case.
5572    async fn start_lead(
5573        ws: &Arc<Workspace>,
5574        updates: &mut tokio::sync::mpsc::UnboundedReceiver<crate::update_fanout::SeatUpdate>,
5575    ) -> SessionSlot {
5576        let lead = SessionSlot::lead("Default", "forge");
5577        crate::spawn::handle_spawn_project(
5578            ws,
5579            "forge",
5580            forge_agent::client::SessionLaunchSettings::default(),
5581        )
5582        .expect("the lead spawns");
5583        await_update(
5584            updates,
5585            "the lead's connect",
5586            |update| matches!(update, SessionUpdate::Connected { key, .. } if *key == lead),
5587        )
5588        .await;
5589        lead
5590    }
5591
5592    /// The fixture's project key.
5593    fn forge_key(ws: &Workspace) -> crate::ProjectKey {
5594        ws.list_projects()
5595            .into_iter()
5596            .find(|view| view.name == "forge")
5597            .expect("fixture project")
5598            .key
5599    }
5600
5601    /// Spawn one worker over the mock and answer its spawn reply.
5602    async fn spawn_mock_worker(ws: &Arc<Workspace>, lead: &SessionSlot, label: &str) {
5603        let key = forge_key(ws);
5604        let (tx, rx) = tokio::sync::oneshot::channel();
5605        handle_spawn_worker(
5606            ws,
5607            key,
5608            WorkerSpawnArgs {
5609                label: label.to_owned(),
5610                charter: "charter".to_owned(),
5611                kick: None,
5612                resume_kick: None,
5613                interactive: false,
5614                mcp_families: None,
5615                cwd: None,
5616            },
5617            lead.clone(),
5618            None,
5619            false,
5620            tx,
5621        );
5622        rx.await.expect("spawn replies").expect("spawn succeeds");
5623    }
5624
5625    /// **A worker whose child closes while it is up comes back by itself**,
5626    /// on its own seat, with the restart note, and its project's lead hears
5627    /// the death and the outcome. Catches a death path that records the seat
5628    /// dead and leaves it there.
5629    #[tokio::test]
5630    async fn a_dead_worker_comes_back_and_its_lead_hears() {
5631        let _binary = MOCK_BINARY.lock().await;
5632        let (dir, _project, ws) = heal_fixture();
5633        // Run 1 is the lead, run 2 the worker that dies, run 3 its restart.
5634        forge_agent::set_test_sdk_binary(Some(heal_mock(dir.path(), 2, 2)));
5635
5636        let mut updates = ws.subscribe_mirror();
5637        let lead = start_lead(&ws, &mut updates).await;
5638        let slot = SessionSlot::worker("Default", "forge", "probe-heal");
5639        spawn_mock_worker(&ws, &lead, "probe-heal").await;
5640        // The worker is mid-work when its child dies, which is the state
5641        // that earns the heal's restart note; a worker found parked comes
5642        // back silent.
5643        ws.record_seat_state(&slot, crate::store::sessions::SeatState::InTurn);
5644
5645        // The first connect, then the child dies of its own accord 0.6 s in;
5646        // the supervisor's 150 ms wait brings the same seat back with the
5647        // restart note, and the lead hears the death and the outcome.
5648        let seen =
5649            await_updates(&mut updates, "the restart, its note and the lead's notice", |seen| {
5650                connects(seen, &slot) >= 2
5651                    && heard(seen, &slot, "This session was restarted by forge")
5652                    && seen.iter().any(|update| {
5653                        matches!(
5654                            update.as_ref(),
5655                            SessionUpdate::PeerEnvelopeAppended { key, .. } if *key == lead
5656                        )
5657                    })
5658            })
5659            .await;
5660        assert_eq!(mock_runs(dir.path()), 3, "the lead, the dead worker, and its resume");
5661        assert_eq!(connects(&seen, &slot), 2, "the same seat came back once");
5662
5663        let told = seen
5664            .iter()
5665            .find_map(|update| match update.as_ref() {
5666                SessionUpdate::PeerEnvelopeAppended { key, wrapped, .. } if *key == lead => {
5667                    Some(wrapped.body.clone())
5668                }
5669                _ => None,
5670            })
5671            .expect("the lead's notice");
5672        assert!(told.contains("died at"), "the time: {told}");
5673        assert!(told.contains(" IST ("), "IST first, UTC after: {told}");
5674        assert!(told.contains("probe-heal"), "the seat the lead must act on: {told}");
5675        assert!(told.contains("attempt 1 of 3"), "the attempt and the cap: {told}");
5676        assert!(told.contains("Restarted"), "the outcome, even on success: {told}");
5677        forge_agent::set_test_sdk_binary(None);
5678    }
5679
5680    /// A despawn racing the spawn itself: the live entry is registered
5681    /// before the session task (and its teardown signal) is minted, so a
5682    /// despawn in that window would remove the worktree under a child that
5683    /// is only starting - #1767's own overlap, for that window. It refuses
5684    /// instead, and leaves the entry alone so the caller's retry finds a
5685    /// live worker.
5686    #[tokio::test]
5687    async fn a_despawn_refuses_a_worker_that_is_still_spawning() {
5688        let (workspace, _rx) = Workspace::testing_stub();
5689        let project = ProjectKey::new("forge");
5690        let mut entry = fake_git_worker_entry("mid-spawn", "worker-1");
5691        entry.status = forge_primitives::WorkerLiveness::Spawning;
5692        workspace.insert_live_worker(&project, entry);
5693        // No teardown signal registered: the spawn has not minted it yet.
5694
5695        let (tx, rx) = tokio::sync::oneshot::channel();
5696        handle_despawn_worker(&workspace, &project, "mid-spawn", false, tx);
5697        let result = rx.await.expect("despawn result");
5698
5699        let crate::protocol::DespawnResult::Blocked { reason } = result else {
5700            panic!("a mid-spawn worker must be refused, not despawned: {result:?}");
5701        };
5702        assert!(reason.contains("still spawning"), "the reason says what to wait for: {reason}");
5703        assert_eq!(
5704            workspace.list_live_workers(&project).len(),
5705            1,
5706            "the worker is left alone for the caller's retry",
5707        );
5708    }
5709
5710    /// The same refusal after a re-spawn: the map still holds the previous
5711    /// occupant's answered entry while the new child starts, and handing
5712    /// that stale answer back would let the removal run over a child that
5713    /// is only starting.
5714    #[tokio::test]
5715    async fn a_respawns_mid_spawn_window_is_refused_despite_the_previous_answer() {
5716        let (workspace, _rx) = Workspace::testing_stub();
5717        let project = ProjectKey::new("forge");
5718        let mut entry = fake_git_worker_entry("mid-spawn", "worker-1");
5719        let slot = entry.slot.clone();
5720        // The previous occupant finished: its entry is still there, answered.
5721        let previous = workspace.mint_session_teardown(&slot).expect("a mint before shutdown");
5722        let _ = previous.send_replace(true);
5723        entry.status = forge_primitives::WorkerLiveness::Spawning;
5724        workspace.insert_live_worker(&project, entry);
5725
5726        let (tx, rx) = tokio::sync::oneshot::channel();
5727        handle_despawn_worker(&workspace, &project, "mid-spawn", false, tx);
5728        let result = rx.await.expect("despawn result");
5729
5730        let crate::protocol::DespawnResult::Blocked { reason } = result else {
5731            panic!("a stale answer must not wave the removal through: {result:?}");
5732        };
5733        assert!(reason.contains("still spawning"), "the reason says what to wait for: {reason}");
5734    }
5735
5736    /// A session teardown that never answers must not wedge the despawn:
5737    /// after the wait expires (injected small here) the removal goes ahead
5738    /// and the reply lands, rather than the label staying cleanup-pending
5739    /// forever.
5740    #[tokio::test]
5741    async fn a_despawn_whose_teardown_never_answers_removes_anyway() {
5742        let (workspace, project_key, wt, _repo, _config) = git_despawn_fixture("mute");
5743        // A registered session whose task never drops: the test holds the
5744        // sender and never fires it.
5745        let _mute = workspace.mint_session_teardown(&SessionSlot::from_str_for_test("worker-1"));
5746
5747        let (tx, rx) = tokio::sync::oneshot::channel();
5748        handle_despawn_worker_within(
5749            &workspace,
5750            &project_key,
5751            "mute",
5752            false,
5753            std::time::Duration::from_millis(100),
5754            tx,
5755        );
5756        let result = rx.await.expect("despawn result");
5757
5758        assert!(
5759            matches!(result, crate::protocol::DespawnResult::Despawned { .. }),
5760            "the despawn answers once its wait expires: {result:?}",
5761        );
5762        assert!(!wt.exists(), "and the removal went ahead after the wait expired");
5763    }
5764
5765    /// A non-git worker's despawn removes no worktree, so it waits on no
5766    /// teardown signal: with a signal registered that never fires and a
5767    /// wait budget far larger than the work, the reply must land
5768    /// immediately rather than sit out the wait.
5769    #[tokio::test]
5770    async fn a_non_git_worker_despawn_waits_on_no_teardown_signal() {
5771        let (workspace, _rx) = Workspace::testing_stub();
5772        let project = ProjectKey::new("forge");
5773        workspace.insert_live_worker(&project, fake_worker_entry("plain", "worker-1"));
5774        // A session that never dies: if the despawn waited for it, the
5775        // reply would arrive only when the budget below expired.
5776        let _mute = workspace.mint_session_teardown(&SessionSlot::from_str_for_test("worker-1"));
5777
5778        let started = std::time::Instant::now();
5779        let (tx, rx) = tokio::sync::oneshot::channel();
5780        handle_despawn_worker_within(
5781            &workspace,
5782            &project,
5783            "plain",
5784            false,
5785            std::time::Duration::from_secs(15),
5786            tx,
5787        );
5788        let result = rx.await.expect("despawn result");
5789        let elapsed = started.elapsed();
5790
5791        let despawned = matches!(result, crate::protocol::DespawnResult::Despawned { .. });
5792        assert!(despawned, "{result:?}");
5793        assert!(
5794            elapsed < std::time::Duration::from_secs(5),
5795            "a worker with no worktree has no removal to order, so its despawn must not sit on \
5796             the teardown wait: it took {elapsed:?} against a 15s budget",
5797        );
5798    }
5799
5800    /// The boot wave carries a persisted row's tree into the spawn it
5801    /// dispatches, so a worker pointed at another directory comes back in
5802    /// it after a restart.
5803    #[tokio::test]
5804    async fn the_boot_wave_carries_the_rows_tree() {
5805        let dir = tempdir().expect("tempdir");
5806        let sibling = tempdir().expect("sibling repo");
5807        std::fs::create_dir_all(&sibling).expect("sibling dir");
5808        write_forge_toml(dir.path(), FIXTURE_PROJECT_PATH);
5809        let ws = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("workspace"));
5810        ws.seed_test_ready_account("Stargate");
5811        let key = ws
5812            .list_projects()
5813            .into_iter()
5814            .find(|v| v.name == "forge")
5815            .expect("fixture project")
5816            .key;
5817        let store = tempdir().expect("store dir");
5818        ws.install_db_for_test(
5819            crate::store::Db::open(&store.path().join("db.redb")).expect("open db"),
5820        );
5821        let root = std::fs::canonicalize(sibling.path()).expect("canonical sibling");
5822        ws.seed_test_worker_row_at(&key, "traveller", &root.to_string_lossy(), false);
5823        ws.enable_test_dispatch_intercept();
5824
5825        ws.dispatch_worker_respawns(
5826            &SessionSlot::lead("Default", "forge"),
5827            &key,
5828            &ws.worker_rows_for_project(&key),
5829            false,
5830        );
5831
5832        let dispatched = ws.drain_test_dispatch_buffer();
5833        let spawn = dispatched
5834            .iter()
5835            .find_map(|cmd| match cmd {
5836                crate::protocol::Command::SpawnWorker { cwd, .. } => Some(cwd.clone()),
5837                _ => None,
5838            })
5839            .expect("the wave dispatches the persisted row");
5840        assert_eq!(
5841            spawn.as_deref(),
5842            Some(root.to_string_lossy().as_ref()),
5843            "the re-spawn carries the row's tree, so the worker comes back in it",
5844        );
5845    }
5846
5847    /// A fresh spawn that omits `mcp_families` states "every family", so
5848    /// a stale stored selection must not survive it: reply, running
5849    /// surface and row have to agree.
5850    #[tokio::test]
5851    async fn a_fresh_spawn_without_families_clears_a_stale_selection() {
5852        let (ws, key, _dir) = spawn_scaffold();
5853        ws.record_worker_row(
5854            &key,
5855            "tester",
5856            "tester-id",
5857            "charter",
5858            None,
5859            None,
5860            false,
5861            crate::mcp::workers::types::WorkerTree { is_git_repo: false, cwd: None },
5862            Some(&["cron".to_owned()]),
5863        )
5864        .expect("seed a stale selection");
5865
5866        let (tx, rx) = tokio::sync::oneshot::channel();
5867        handle_spawn_worker(
5868            &ws,
5869            key.clone(),
5870            WorkerSpawnArgs {
5871                label: "tester".to_owned(),
5872                charter: "charter".to_owned(),
5873                kick: None,
5874                resume_kick: None,
5875                interactive: false,
5876                mcp_families: None,
5877                cwd: None,
5878            },
5879            SessionSlot::from_str_for_test("lead"),
5880            None,
5881            false,
5882            tx,
5883        );
5884        let reply = rx.await.expect("spawn replies");
5885        assert!(reply.is_ok(), "the fresh spawn succeeds: {reply:?}");
5886        assert_eq!(reply.expect("ok").mcp_families, None, "the reply says every family");
5887
5888        let stored = ws
5889            .worker_rows_for_project(&key)
5890            .into_iter()
5891            .find(|row| row.label == "tester")
5892            .expect("the row the spawn wrote");
5893        assert_eq!(
5894            stored.mcp_families, None,
5895            "the row cannot keep a selection the running surface does not have; a restart \
5896             would silently narrow the worker",
5897        );
5898    }
5899
5900    /// A spawn that states a tree writes it to the row AND the live
5901    /// entry. The row is the durable handle every read-back composes
5902    /// from - resume, the boot wave, the launchpad, the cron wake and
5903    /// the despawn - and the entry is what a seat read answers from, so
5904    /// a regression writing `None` (or carrying a stale tree forward,
5905    /// the way the row's kick and family fields deliberately do) leaves
5906    /// the feature half-working with the suite green.
5907    #[tokio::test]
5908    async fn a_stated_cwd_is_recorded_on_the_row_and_the_entry() {
5909        let (ws, key, _dir) = spawn_scaffold();
5910        let tree = tempdir().expect("worker tree");
5911        let root = std::fs::canonicalize(tree.path()).expect("canonical tree");
5912
5913        let (tx, rx) = tokio::sync::oneshot::channel();
5914        handle_spawn_worker(
5915            &ws,
5916            key.clone(),
5917            WorkerSpawnArgs {
5918                label: "traveller".to_owned(),
5919                charter: "charter".to_owned(),
5920                kick: None,
5921                resume_kick: None,
5922                interactive: false,
5923                mcp_families: None,
5924                cwd: Some(root.to_string_lossy().into_owned()),
5925            },
5926            SessionSlot::from_str_for_test("lead"),
5927            None,
5928            false,
5929            tx,
5930        );
5931        let reply = rx.await.expect("spawn replies");
5932        assert!(reply.is_ok(), "the spawn succeeds: {reply:?}");
5933
5934        let stored = ws
5935            .worker_rows_for_project(&key)
5936            .into_iter()
5937            .find(|row| row.label == "traveller")
5938            .expect("the row the spawn wrote");
5939        assert_eq!(
5940            stored.cwd.as_deref(),
5941            Some(root.to_string_lossy().as_ref()),
5942            "the row records the tree the spawn resolved",
5943        );
5944        let entry = ws
5945            .list_live_workers(&key)
5946            .into_iter()
5947            .find(|entry| entry.label == "traveller")
5948            .expect("the live entry the spawn inserted");
5949        assert_eq!(
5950            entry.cwd.as_deref(),
5951            Some(root.as_path()),
5952            "and the live entry carries it, which is what a seat read answers from",
5953        );
5954    }
5955
5956    /// A fresh spawn that states no tree clears one an older row
5957    /// recorded: a fresh spawn runs in the project's own path, and the
5958    /// row's other re-spawn fields carry forward only where the contract
5959    /// says so.
5960    #[tokio::test]
5961    async fn a_fresh_spawn_without_a_cwd_clears_a_stale_tree() {
5962        let (ws, key, _dir) = spawn_scaffold();
5963        let tree = tempdir().expect("worker tree");
5964        let root = std::fs::canonicalize(tree.path()).expect("canonical tree");
5965        ws.seed_test_worker_row_at(&key, "tester", &root.to_string_lossy(), true);
5966
5967        let (tx, rx) = tokio::sync::oneshot::channel();
5968        handle_spawn_worker(
5969            &ws,
5970            key.clone(),
5971            WorkerSpawnArgs {
5972                label: "tester".to_owned(),
5973                charter: "charter".to_owned(),
5974                kick: None,
5975                resume_kick: None,
5976                interactive: false,
5977                mcp_families: None,
5978                cwd: None,
5979            },
5980            SessionSlot::from_str_for_test("lead"),
5981            None,
5982            false,
5983            tx,
5984        );
5985        let reply = rx.await.expect("spawn replies");
5986        assert!(reply.is_ok(), "the fresh spawn succeeds: {reply:?}");
5987
5988        let stored = ws
5989            .worker_rows_for_project(&key)
5990            .into_iter()
5991            .find(|row| row.label == "tester")
5992            .expect("the row the spawn wrote");
5993        assert_eq!(
5994            stored.cwd, None,
5995            "a spawn that states no tree records none; carrying the old one forward would \
5996             run the worker somewhere the caller never asked for",
5997        );
5998    }
5999
6000    /// A re-spawn that states no selection adopts the row's; one that
6001    /// states a selection is a revision and both the reply and the row
6002    /// carry it.
6003    #[tokio::test]
6004    async fn a_respawn_adopts_the_rows_selection_or_honours_a_stated_one() {
6005        let (ws, key, _dir) = spawn_scaffold();
6006        ws.record_worker_row(
6007            &key,
6008            "tester",
6009            "tester-id",
6010            "charter",
6011            None,
6012            None,
6013            false,
6014            crate::mcp::workers::types::WorkerTree { is_git_repo: false, cwd: None },
6015            Some(&["cron".to_owned()]),
6016        )
6017        .expect("seed the stored selection");
6018
6019        // Adopt: no argument, boot re-spawn reads the row.
6020        let (tx, rx) = tokio::sync::oneshot::channel();
6021        handle_spawn_worker(
6022            &ws,
6023            key.clone(),
6024            WorkerSpawnArgs {
6025                label: "tester".to_owned(),
6026                charter: "charter".to_owned(),
6027                kick: None,
6028                resume_kick: None,
6029                interactive: false,
6030                mcp_families: None,
6031                cwd: None,
6032            },
6033            SessionSlot::from_str_for_test("lead"),
6034            None,
6035            true,
6036            tx,
6037        );
6038        let reply = rx.await.expect("spawn replies");
6039        assert_eq!(
6040            reply.expect("ok").mcp_families,
6041            Some(vec!["cron".to_owned()]),
6042            "a re-spawn with nothing stated adopts the row's selection",
6043        );
6044
6045        // State one: it wins, and the row is revised. Clear the live
6046        // entry the first re-spawn left, or the one-live-worker guard
6047        // refuses this second spawn for a reason unrelated to families.
6048        ws.remove_latest_worker(&key, "tester");
6049        let (tx, rx) = tokio::sync::oneshot::channel();
6050        handle_spawn_worker(
6051            &ws,
6052            key.clone(),
6053            WorkerSpawnArgs {
6054                label: "tester".to_owned(),
6055                charter: "charter".to_owned(),
6056                kick: None,
6057                resume_kick: None,
6058                interactive: false,
6059                mcp_families: Some(vec!["tasks".to_owned()]),
6060                cwd: None,
6061            },
6062            SessionSlot::from_str_for_test("lead"),
6063            None,
6064            true,
6065            tx,
6066        );
6067        let reply = rx.await.expect("spawn replies");
6068        assert_eq!(
6069            reply.expect("ok").mcp_families,
6070            Some(vec!["tasks".to_owned()]),
6071            "a stated selection wins on a re-spawn",
6072        );
6073        let stored = ws
6074            .worker_rows_for_project(&key)
6075            .into_iter()
6076            .find(|row| row.label == "tester")
6077            .expect("the row");
6078        assert_eq!(stored.mcp_families, Some(vec!["tasks".to_owned()]), "and is written");
6079    }
6080
6081    /// The prompt charter carries one line per withheld family; the ROW
6082    /// keeps the raw charter, so the lines are recomposed per spawn and
6083    /// can neither duplicate nor outlive a revision.
6084    #[tokio::test]
6085    async fn a_narrowed_spawn_keeps_its_rows_charter_raw() {
6086        let dir = tempdir().expect("tempdir");
6087        write_forge_toml(dir.path(), FIXTURE_PROJECT_PATH);
6088        let ws = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("workspace"));
6089        ws.seed_test_ready_account("Stargate");
6090        ws.seed_test_gateway_ready(true);
6091        let key = ws
6092            .list_projects()
6093            .into_iter()
6094            .find(|v| v.name == "forge")
6095            .expect("fixture project")
6096            .key;
6097
6098        let (tx, rx) = tokio::sync::oneshot::channel();
6099        handle_spawn_worker(
6100            &ws,
6101            key.clone(),
6102            WorkerSpawnArgs {
6103                label: "tester".to_owned(),
6104                charter: "charter".to_owned(),
6105                kick: None,
6106                resume_kick: None,
6107                interactive: false,
6108                mcp_families: Some(vec!["slack".to_owned()]),
6109                cwd: None,
6110            },
6111            SessionSlot::from_str_for_test("lead"),
6112            None,
6113            false,
6114            tx,
6115        );
6116        let reply = rx.await.expect("spawn replies");
6117        assert!(reply.is_ok(), "the narrowed spawn succeeds: {reply:?}");
6118
6119        let stored = ws
6120            .worker_rows_for_project(&key)
6121            .into_iter()
6122            .find(|row| row.label == "tester")
6123            .expect("the row the spawn wrote");
6124        assert_eq!(
6125            stored.charter.as_deref(),
6126            Some("charter"),
6127            "the row keeps the raw charter; a later revision compares against this text, \
6128             so composed lines here would compound on every re-spawn",
6129        );
6130        assert_eq!(stored.mcp_families, Some(vec!["slack".to_owned()]));
6131    }
6132
6133    /// The wiring the round-2 axis probed: an argument-less spawn on a
6134    /// systemone-less workspace composes the systemone withheld line,
6135    /// and the same spawn with the section on does not.
6136    #[test]
6137    fn the_spawn_prompt_charter_follows_the_config_gate() {
6138        let line = "This session has no `systemone__*` decision tools. When a decision is material and not obvious, route it to your lead instead of guessing.";
6139        let without = spawn_prompt_charter("charter", None, false, true, true);
6140        assert!(without.contains(line), "[systemone] off composes the line: {without}");
6141        let with = spawn_prompt_charter("charter", None, true, true, true);
6142        assert!(!with.contains(line), "the section on composes none: {with}");
6143    }
6144
6145    /// The composer's contract: nothing withheld means the charter is
6146    /// untouched; a withheld family appends its line exactly once; the
6147    /// present family appends nothing.
6148    #[test]
6149    fn the_prompt_charter_carries_only_the_withheld_lines() {
6150        let all = crate::mcp::McpFamily::all();
6151        assert_eq!(charter_with_withheld_families("charter", &all), "charter");
6152
6153        let slack_only: std::collections::BTreeSet<crate::mcp::McpFamily> =
6154            [crate::mcp::McpFamily::Slack].into_iter().collect();
6155        let text = charter_with_withheld_families("charter", &slack_only);
6156        assert!(text.starts_with("charter\n"), "{text}");
6157        assert!(text.contains("no `systemone__*` decision tools"), "{text}");
6158        assert!(!text.contains("no `slack__*` tools"), "the present family gets no line: {text}");
6159    }
6160
6161    /// A first spawn writes the kick it opened the worker with: the row's
6162    /// `kick` is the worker's first turn, and a later `--new` re-spawn
6163    /// delivers it. A row that never got it opens the worker with nothing.
6164    #[tokio::test]
6165    async fn a_first_spawn_writes_the_kick_it_was_given() {
6166        let dir = tempdir().expect("tempdir");
6167        write_forge_toml(dir.path(), FIXTURE_PROJECT_PATH);
6168        let ws = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("workspace"));
6169        ws.seed_test_ready_account("Stargate");
6170        ws.seed_test_gateway_ready(true);
6171        let key = ws
6172            .list_projects()
6173            .into_iter()
6174            .find(|v| v.name == "forge")
6175            .expect("fixture project")
6176            .key;
6177
6178        let (tx, rx) = tokio::sync::oneshot::channel();
6179        handle_spawn_worker(
6180            &ws,
6181            key.clone(),
6182            WorkerSpawnArgs {
6183                label: "tester".to_owned(),
6184                charter: "charter".to_owned(),
6185                kick: Some("opening turn".to_owned()),
6186                resume_kick: None,
6187                interactive: false,
6188                mcp_families: None,
6189                cwd: None,
6190            },
6191            SessionSlot::from_str_for_test("lead"),
6192            None,
6193            false,
6194            tx,
6195        );
6196        let reply = rx.await.expect("spawn replies");
6197        assert!(reply.is_ok(), "the first spawn succeeds: {reply:?}");
6198
6199        let stored = ws
6200            .worker_rows_for_project(&key)
6201            .into_iter()
6202            .find(|row| row.label == "tester")
6203            .expect("the row the spawn wrote");
6204        assert_eq!(
6205            stored.kick.as_deref(),
6206            Some("opening turn"),
6207            "the row keeps the kick the spawn opened the worker with; dropped, a later --new \
6208             re-spawn opens the worker with no first turn at all",
6209        );
6210    }
6211
6212    /// The spawn composes the working directory from the gitness the ROW
6213    /// records, in preference to probing the project - that is what keeps
6214    /// it on the directory the launchpad already cleared. A fixture whose
6215    /// row agrees with the disk cannot tell the two apart, so this one
6216    /// disagrees: the row says worktree, the project path is not a repo.
6217    #[tokio::test]
6218    async fn a_spawn_keeps_the_rows_recorded_gitness_over_a_probe() {
6219        let row = failed_spawn_leftover_row(Some("tester-id"), false, Some(true)).await;
6220        assert_eq!(
6221            row.and_then(|row| row.is_git_repo),
6222            Some(true),
6223            "the spawn writes back the gitness the row recorded, not what a probe of the \
6224             project path would answer",
6225        );
6226    }
6227
6228    /// The other direction of the row-over-probe preference: with no row to
6229    /// read, the probe is still the answer. A first spawn into a git
6230    /// project that recorded `false` here would pass no `--worktree`, and
6231    /// every later resume, the launchpad and the cron router would read
6232    /// that false record as the directory the worker runs in.
6233    ///
6234    /// The spawn has to succeed for the row to survive: a refused fresh
6235    /// spawn rolls its row back, so there would be nothing left to read.
6236    #[tokio::test]
6237    async fn a_first_spawn_probes_for_gitness_when_there_is_no_row() {
6238        let dir = tempdir().expect("tempdir");
6239        let repo = tempdir().expect("git project dir");
6240        let status = std::process::Command::new("git")
6241            .args(["init", "-q"])
6242            .current_dir(repo.path())
6243            .status()
6244            .expect("spawn git");
6245        assert!(status.success(), "fixture precondition: git init");
6246        write_forge_toml(dir.path(), &repo.path().to_string_lossy());
6247        let ws = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("workspace"));
6248        ws.seed_test_ready_account("Stargate");
6249        ws.seed_test_gateway_ready(true);
6250        let key = ws
6251            .list_projects()
6252            .into_iter()
6253            .find(|v| v.name == "forge")
6254            .expect("fixture project")
6255            .key;
6256        // Deliberately no row: this is the label's first spawn.
6257
6258        let (tx, rx) = tokio::sync::oneshot::channel();
6259        handle_spawn_worker(
6260            &ws,
6261            key,
6262            WorkerSpawnArgs {
6263                label: "tester".to_owned(),
6264                charter: "charter".to_owned(),
6265                kick: None,
6266                resume_kick: None,
6267                interactive: false,
6268                mcp_families: None,
6269                cwd: None,
6270            },
6271            SessionSlot::from_str_for_test("lead"),
6272            None,
6273            false,
6274            tx,
6275        );
6276        let admitted = rx.await.expect("reply").is_ok();
6277        assert!(admitted, "fixture precondition: the spawn is admitted, so its row survives");
6278
6279        let db = ws.db.lock();
6280        let row =
6281            crate::store::sessions::get(db.as_ref().expect("db"), "Default", "forge", "tester")
6282                .expect("read")
6283                .expect("the spawn recorded its row");
6284        drop(db);
6285        assert_eq!(
6286            row.is_git_repo,
6287            Some(true),
6288            "with no row to read the probe is the answer; recording false here would send \
6289             every later resume, the launchpad and the cron router to the project root",
6290        );
6291    }
6292
6293    /// A spawn refused before it reaches the project leaves the caller's
6294    /// parked payload where it is, so the caller's own expiry still reaches
6295    /// it rather than the workspace having consumed it on the way past.
6296    #[tokio::test]
6297    async fn a_refused_spawn_leaves_the_parked_payload_to_the_caller() {
6298        let dir = tempdir().expect("tempdir");
6299        write_forge_toml(dir.path(), FIXTURE_PROJECT_PATH);
6300        let ws = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("workspace"));
6301        ws.seed_test_ready_account("Stargate");
6302        ws.park_slack(
6303            &crate::SessionSlot::lead("Default", "missing"),
6304            vec![slack_msg("parked while asleep")],
6305        );
6306
6307        let result = ws.get_agent_handle_at_key(
6308            crate::target::SessionTarget::FreshInProject {
6309                slot: SessionSlot::worker("TestOrg", "not-in-config", "minted-worker-id"),
6310                cwd: None,
6311            },
6312            forge_agent::client::SessionLaunchSettings::default(),
6313            None,
6314            &crate::protocol::SpawnRole::Worker {
6315                label: "reviewer".to_owned(),
6316                wrote_row: false,
6317                mcp_families: None,
6318            },
6319            true,
6320        );
6321
6322        assert!(result.is_err(), "a target mapping to no project is refused");
6323        let owner = crate::SessionSlot::lead("Default", "missing");
6324        assert_eq!(
6325            ws.parked_by_slot.lock().get(&owner).map_or(0, |parked| parked.slack.len()),
6326            1,
6327            "the refusal left the bucket alone, so the caller's expiry can still reach it",
6328        );
6329
6330        ws.expire_parked_for_slot(
6331            &owner,
6332            crate::mcp::peers::types::PeerFailureReason::TargetConnectionFailed,
6333        );
6334        assert_eq!(
6335            ws.parked_by_slot.lock().get(&owner).map_or(0, |parked| parked.slack.len()),
6336            0,
6337            "and the caller's expiry is what records the loss",
6338        );
6339    }
6340
6341    /// The synchronous spawn-failure arm is reachable: a project present only
6342    /// in the test overlay resolves by name but not by target, so the spawn
6343    /// fails before any `SessionTask` exists. Everything a delivery parked for
6344    /// that project's lead has to be recorded there, because nothing else will
6345    /// drain a bucket the session never reached.
6346    #[test]
6347    fn a_failed_project_spawn_expires_the_parked_payloads() {
6348        let (ws, _rx) = Workspace::testing_stub();
6349        ws.seed_test_project("overlayonly", "/tmp/slack-overlay-only");
6350        ws.park_slack(
6351            &crate::SessionSlot::lead("TestOrg", "overlayonly"),
6352            vec![slack_msg("parked while asleep")],
6353        );
6354
6355        let _ = crate::spawn::handle_spawn_project(
6356            &ws,
6357            "overlayonly",
6358            forge_agent::client::SessionLaunchSettings::default(),
6359        );
6360
6361        assert_eq!(
6362            ws.parked_by_slot
6363                .lock()
6364                .get(&crate::SessionSlot::lead("TestOrg", "overlayonly"))
6365                .map_or(0, |parked| parked.slack.len()),
6366            0,
6367            "the failed spawn records the messages parked for that project's lead",
6368        );
6369    }
6370
6371    /// A delivery whose dispatch fails must not echo: `false` tells the sweep
6372    /// to re-run the message, and an echo pushed first would paint the block a
6373    /// second time on that re-run, for a turn the LLM never received.
6374    #[test]
6375    fn a_failed_worker_dispatch_leaves_no_echo_for_the_sweep_to_duplicate() {
6376        let (ws, mut update_rx) = Workspace::testing_stub();
6377        ws.seed_test_project("forge", "/tmp/slack-worker-doomed");
6378        let key = ws.list_projects().into_iter().find(|v| v.name == "forge").expect("seeded").key;
6379        ws.insert_live_worker(&key, fake_worker_entry("tester", "worker-uuid"));
6380        let worker_key = SessionSlot::from_str_for_test("worker-uuid");
6381        // Connected, but with no pooled handle and no conn: the dispatch fails.
6382        ws.mark_session_connected_for_test(&worker_key, "worker-uuid");
6383
6384        let delivered =
6385            deliver_slack_message(&ws, "forge", Some("tester"), vec![slack_msg("hello")]);
6386
6387        assert!(!delivered, "a failed dispatch tells the sweep to re-run the message");
6388        let echoed = std::iter::from_fn(|| update_rx.try_recv().ok()).any(|frame| {
6389            matches!(&*frame.update, crate::protocol::SessionUpdate::SlackMessageAppended { .. })
6390        });
6391        assert!(!echoed, "a failed worker dispatch must not echo a SlackMessageAppended");
6392    }
6393
6394    /// The ids in the prose are the arguments a reply takes: without them
6395    /// the agent cannot answer in the place the message came from.
6396    #[test]
6397    fn slack_prose_carries_the_ids_a_reply_needs() {
6398        let mut message = slack_msg("please look");
6399        message.thread_ts = Some("100.0".to_owned());
6400        message.files = vec![SlackFile {
6401            id: "F1".to_owned(),
6402            name: "notes.txt".to_owned(),
6403            url_private: "https://files.slack.com/x".to_owned(),
6404            shares: SlackFileShares::default(),
6405        }];
6406
6407        let prose = fresh_prose(&[message]);
6408        assert!(prose.contains("D1"), "the conversation id is present: {prose}");
6409        assert!(prose.contains("100.000001"), "the ts is present: {prose}");
6410        assert!(prose.contains("in thread 100.0"), "the thread is present: {prose}");
6411        assert!(prose.contains("F1"), "the file id is present: {prose}");
6412    }
6413
6414    /// The header and the `<author>: ` member line are a contract with
6415    /// `forge_server::envelope::detect_inbound`, so the whole shape is
6416    /// pinned: a reformat here silently reverts the block to painting nothing.
6417    #[test]
6418    fn slack_prose_names_the_workspace_and_the_author() {
6419        let prose = fresh_prose(&[slack_msg("hello there")]);
6420        assert_eq!(
6421            prose,
6422            "[Slack - workspace 'acme', U9] id D1 ts 100.000001\nU9: hello there [ts 100.000001]",
6423            "the exact prose forge_server::envelope::detect_inbound keys on",
6424        );
6425    }
6426
6427    /// A block that waited, in the parked bucket or across a sweep gap, says
6428    /// how old its newest message is, before the member count. The oldest
6429    /// member is older still, so the label has to be read off the newest.
6430    #[test]
6431    fn a_stale_bundle_carries_its_age_in_the_header() {
6432        let day = 24 * 60 * 60;
6433        // Ten-digit seconds, as Slack sends them: the header takes the newest
6434        // by string order, which is only time order at that width.
6435        let oldest = 1_001_827_200;
6436        let prose = slack_bundle_to_prose(
6437            &[
6438                slack_msg_at("1001827200.0", "alice", "hi"),
6439                slack_msg_at("1002000000.0", "bob", "there"),
6440            ],
6441            Duration::from_secs(day),
6442            SystemTime::UNIX_EPOCH + Duration::from_secs(oldest + 10 * day),
6443        );
6444        assert!(
6445            prose.starts_with(
6446                "[Slack - workspace 'acme', U9] id D1 ts 1002000000.0, 8d old (2 messages)\n"
6447            ),
6448            "the header carries the newest message's age: {prose}",
6449        );
6450    }
6451
6452    /// A single message is marked the same way a bundle is: the clause is
6453    /// written by the header, not by the count branch.
6454    #[test]
6455    fn a_stale_single_message_carries_its_age() {
6456        let prose = slack_bundle_to_prose(
6457            &[slack_msg("hi")],
6458            Duration::from_secs(3_600),
6459            SystemTime::UNIX_EPOCH + Duration::from_secs(100 + 3 * 3_600),
6460        );
6461        assert!(
6462            prose.starts_with("[Slack - workspace 'acme', U9] id D1 ts 100.000001, 3h old\n"),
6463            "a one-message block carries its age too: {prose}",
6464        );
6465    }
6466
6467    /// The threshold is a floor: below it the header is untouched, at it the
6468    /// block is marked, and the label reads in hours up to two days and days
6469    /// from there. A future or unparseable ts carries nothing.
6470    #[test]
6471    fn a_stale_age_reads_hours_below_two_days_and_days_from_there() {
6472        let day = 24 * 60 * 60;
6473        let at = |age: u64| {
6474            stale_age_clause(
6475                "100.0",
6476                Duration::from_secs(day),
6477                SystemTime::UNIX_EPOCH + Duration::from_secs(100 + age),
6478            )
6479        };
6480        assert_eq!(at(day - 1), None, "below the threshold carries nothing");
6481        assert_eq!(at(day).as_deref(), Some(", 24h old"), "the threshold itself is marked");
6482        assert_eq!(at(2 * day - 1).as_deref(), Some(", 47h old"), "hours up to the day switch");
6483        assert_eq!(at(2 * day).as_deref(), Some(", 2d old"), "two days reads in days");
6484        assert_eq!(
6485            stale_age_clause(
6486                "100.0",
6487                Duration::ZERO,
6488                SystemTime::UNIX_EPOCH + Duration::from_secs(100),
6489            ),
6490            None,
6491            "zero turns the mark off, even for a fresh message",
6492        );
6493        assert_eq!(
6494            stale_age_clause(
6495                "100.0",
6496                Duration::from_secs(day),
6497                SystemTime::UNIX_EPOCH + Duration::from_secs(50),
6498            ),
6499            None,
6500            "a ts in the future carries nothing",
6501        );
6502        assert_eq!(
6503            stale_age_clause(
6504                "not-a-ts",
6505                Duration::from_secs(day),
6506                SystemTime::UNIX_EPOCH + Duration::from_secs(9 * day),
6507            ),
6508            None,
6509            "an unparseable ts carries nothing",
6510        );
6511    }
6512
6513    /// A redelivered correction keeps the message's ts and would read as
6514    /// a duplicate of one already delivered; the marker is what says the
6515    /// text changed.
6516    #[test]
6517    fn a_bundle_marks_an_edited_member() {
6518        let mut message = slack_msg("deploy is RED");
6519        message.edited = Some("150.0".to_owned());
6520
6521        let prose = fresh_prose(&[message]);
6522        assert!(
6523            prose.contains("[ts 100.000001 edited]"),
6524            "the correction is marked in the member's own bracket: {prose}",
6525        );
6526    }
6527
6528    /// Every member keeps its own `ts`, because that is the handle an agent
6529    /// answers one message with without touching the others.
6530    #[test]
6531    fn a_bundle_names_every_member_with_its_own_ts() {
6532        let prose =
6533            fresh_prose(&[slack_msg_at("1.1", "alice", "hi"), slack_msg_at("1.2", "bob", "there")]);
6534
6535        assert!(prose.contains("[ts 1.1]"), "the first member's ts: {prose}");
6536        assert!(prose.contains("[ts 1.2]"), "and the second's: {prose}");
6537        assert!(prose.contains("alice: hi"), "each member names its author: {prose}");
6538        assert!(prose.contains("bob: there"), "and so does the second: {prose}");
6539    }
6540
6541    /// A reply belongs under the parent it answers, whatever order the wire
6542    /// handed the two over in - history pages arrive newest first.
6543    #[test]
6544    fn a_bundle_shows_thread_replies_under_their_parent() {
6545        let mut reply = slack_msg_at("1.3", "carol", "agreed");
6546        reply.thread_ts = Some("1.1".to_owned());
6547
6548        let prose = fresh_prose(&[reply, slack_msg_at("1.1", "alice", "hi")]);
6549
6550        let parent_at = prose.find("[ts 1.1").expect("the parent is present");
6551        let reply_at = prose.find("[ts 1.3").expect("the reply is present");
6552        assert!(reply_at > parent_at, "a reply is grouped under its parent: {prose}");
6553    }
6554
6555    /// A parent ages out of the sweep window while its thread keeps moving,
6556    /// so a bundle can hold a reply whose parent it never saw. Without the
6557    /// parent's ts the agent cannot answer in that thread at all.
6558    #[test]
6559    fn a_reply_whose_parent_is_absent_still_carries_the_parent_ts() {
6560        let mut reply = slack_msg_at("1.3", "carol", "agreed");
6561        reply.thread_ts = Some("1.1".to_owned());
6562
6563        let prose = fresh_prose(&[reply]);
6564        assert!(prose.contains("in thread 1.1"), "the parent ts must survive: {prose}");
6565    }
6566
6567    /// The name the connector resolved is what the block shows. The raw id
6568    /// stays on the message and the block drops it, so the resolved name is
6569    /// the only one that reaches a reader.
6570    #[test]
6571    fn a_bundle_prefers_the_resolved_author_over_the_id() {
6572        let mut message = slack_msg_at("1.1", "U0ATEK2EAGP", "hi");
6573        message.author = Some("architect2".to_owned());
6574
6575        let prose = fresh_prose(&[message]);
6576        assert!(prose.contains("architect2: hi"), "the resolved handle is shown: {prose}");
6577        assert!(!prose.contains("U0ATEK2EAGP"), "an id is never a name to print: {prose}");
6578    }
6579
6580    /// A parent carries its own `ts` as `thread_ts`, so saying so again on
6581    /// the member line is noise - the ts is already the handle a reply into
6582    /// that thread passes.
6583    #[test]
6584    fn a_parent_does_not_restate_its_own_ts_as_a_thread() {
6585        let mut parent = slack_msg_at("1.1", "alice", "hi");
6586        parent.thread_ts = Some("1.1".to_owned());
6587
6588        let prose = fresh_prose(&[parent]);
6589        assert!(!prose.contains("in thread"), "no self-referential thread clause: {prose}");
6590    }
6591
6592    /// #1: the at-most-one-live-per-label guard lives in the shared
6593    /// `handle_spawn_worker` core, so a second dispatch for an
6594    /// already-live label - a boot re-spawn racing an MCP spawn for the
6595    /// same label - inserts no duplicate entry and never reaches the
6596    /// subprocess spawn.
6597    #[tokio::test]
6598    async fn handle_spawn_worker_dedups_already_live_label() {
6599        let (workspace, _rx) = Workspace::testing_stub();
6600        workspace.seed_test_project("forge", "/tmp/forge");
6601        let project = workspace
6602            .list_projects()
6603            .into_iter()
6604            .find(|v| v.name == "forge")
6605            .expect("seeded project present")
6606            .key;
6607        // Simulate the first (e.g. static) reviewer already live.
6608        workspace.insert_live_worker(&project, fake_worker_entry("reviewer", "reviewer-1"));
6609
6610        // A second dispatch for the same label (e.g. the dynamic reviewer)
6611        // hits the guard before any insert or spawn.
6612        let (tx, rx) = tokio::sync::oneshot::channel();
6613        handle_spawn_worker(
6614            &workspace,
6615            project.clone(),
6616            WorkerSpawnArgs {
6617                label: "reviewer".to_owned(),
6618                charter: "charter".to_owned(),
6619                kick: None,
6620                resume_kick: None,
6621                interactive: false,
6622                mcp_families: None,
6623                cwd: None,
6624            },
6625            SessionSlot::from_str_for_test("lead"),
6626            None,
6627            false,
6628            tx,
6629        );
6630
6631        assert_eq!(
6632            workspace.list_live_workers(&project).len(),
6633            1,
6634            "the duplicate spawn must not add a second live worker",
6635        );
6636        let reply = rx.await.expect("reply channel");
6637        let err = reply.expect_err("a duplicate spawn replies an error");
6638        assert!(err.contains("already live"), "error names the collision: {err}");
6639    }
6640
6641    /// A refused duplicate must leave the RUNNING worker's row exactly as
6642    /// it was. The row is written after the guard for this reason: written
6643    /// first, the refusal would have overwritten the live worker's id,
6644    /// charter and kick with the refused attempt's, and the next boot
6645    /// would resume an id no session ever ran under, orphaning the
6646    /// transcript it belongs to.
6647    ///
6648    /// The sibling test above cannot see this: it runs on a `testing_stub`
6649    /// with no DB, so the row write fails there and the ordering does not
6650    /// matter.
6651    #[tokio::test]
6652    async fn a_refused_duplicate_leaves_the_running_workers_row_alone() {
6653        let dir = tempdir().expect("tempdir");
6654        write_forge_toml(dir.path(), FIXTURE_PROJECT_PATH);
6655        let ws = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("workspace"));
6656        ws.seed_test_ready_account("Stargate");
6657        ws.seed_test_gateway_ready(true);
6658        let key = ws
6659            .list_projects()
6660            .into_iter()
6661            .find(|v| v.name == "forge")
6662            .expect("fixture project")
6663            .key;
6664        let spawn = |charter: &str| {
6665            let (tx, rx) = tokio::sync::oneshot::channel();
6666            handle_spawn_worker(
6667                &ws,
6668                key.clone(),
6669                WorkerSpawnArgs {
6670                    label: "steward".to_owned(),
6671                    charter: charter.to_owned(),
6672                    kick: None,
6673                    resume_kick: None,
6674                    interactive: false,
6675                    mcp_families: None,
6676                    cwd: None,
6677                },
6678                SessionSlot::from_str_for_test("lead"),
6679                None,
6680                false,
6681                tx,
6682            );
6683            rx
6684        };
6685
6686        let first = spawn("mind the queues")
6687            .await
6688            .expect("reply channel")
6689            .expect("the first spawn is admitted")
6690            .session_id;
6691
6692        let refused = spawn("a second, refused charter").await.expect("reply channel");
6693        let refusal = refused.expect_err("the second spawn for a live label is refused");
6694        assert!(
6695            refusal.contains("already live"),
6696            "and it is the duplicate arm that refused, not some other failure: {refusal}",
6697        );
6698
6699        let rows = ws.worker_rows_for_project(&key);
6700        let row = rows.iter().find(|r| r.label == "steward").expect("the running worker's row");
6701        assert_eq!(
6702            row.session_id.as_deref(),
6703            Some(first.as_str()),
6704            "the row still names the id the running worker was admitted under",
6705        );
6706        assert_eq!(
6707            row.charter.as_deref(),
6708            Some("mind the queues"),
6709            "and keeps the args it was admitted with, not the refused attempt's",
6710        );
6711    }
6712
6713    /// A one-commit git repo as the project, the workspace over it, and
6714    /// the label's derived worktree path. The project declares no model,
6715    /// so a spawn that gets past the worktree step dies at the account
6716    /// walk with no subprocess behind it - which is what makes the real
6717    /// spawn path callable from a test at all.
6718    fn git_spawn_fixture(
6719        label: &str,
6720    ) -> (Arc<Workspace>, ProjectKey, std::path::PathBuf, tempfile::TempDir, tempfile::TempDir)
6721    {
6722        let repo = tempdir().expect("repo tempdir");
6723        run_git(repo.path(), &["init", "-q"]);
6724        run_git(repo.path(), &["config", "user.email", "t@example.com"]);
6725        run_git(repo.path(), &["config", "user.name", "Test"]);
6726        run_git(repo.path(), &["config", "commit.gpgsign", "false"]);
6727        std::fs::write(repo.path().join("README.md"), "seed").expect("write seed");
6728        run_git(repo.path(), &["add", "."]);
6729        run_git(repo.path(), &["commit", "-q", "-m", "init"]);
6730
6731        let config = tempdir().expect("config tempdir");
6732        let repo_path_str = repo.path().to_string_lossy().replace('\\', "/");
6733        std::fs::write(
6734            forge_toml_path(config.path()),
6735            format!(
6736                "[[orgs]]\nname = \"Default\"\naccounts = [\"Stargate\"]\n\n[[orgs.projects]]\nname = \"forge\"\npath = \"{repo_path_str}\"\n\n[[accounts]]\ndisplay_name = \"Stargate\"\ntoken = \"t\"\nmodels = [\"claude-sonnet-5\"]\nprovider = \"anthropic\"\n"
6737            ),
6738        )
6739        .expect("write forge.toml");
6740
6741        let workspace =
6742            Arc::new(Workspace::new_for_test(config.path().to_owned()).expect("workspace new"));
6743        let view = workspace.list_projects().into_iter().next().expect("one project");
6744        let project_key = view.key.clone();
6745        let derived = view.path.join(".claude").join("worktrees").join(label);
6746        (workspace, project_key, derived, repo, config)
6747    }
6748
6749    /// Spawn `label` in the fixture workspace and hand back the reply.
6750    fn spawn_reply(
6751        workspace: &Arc<Workspace>,
6752        key: &ProjectKey,
6753        label: &str,
6754        from_boot_respawn: bool,
6755    ) -> tokio::sync::oneshot::Receiver<Result<WorkerSpawnReply, String>> {
6756        let (tx, rx) = tokio::sync::oneshot::channel();
6757        handle_spawn_worker(
6758            workspace,
6759            key.clone(),
6760            WorkerSpawnArgs {
6761                label: label.to_owned(),
6762                charter: "charter".to_owned(),
6763                kick: None,
6764                resume_kick: None,
6765                interactive: false,
6766                mcp_families: None,
6767                cwd: None,
6768            },
6769            SessionSlot::from_str_for_test("lead"),
6770            None,
6771            from_boot_respawn,
6772            tx,
6773        );
6774        rx
6775    }
6776
6777    /// A fresh spawn that cannot get its worktree must be refused rather
6778    /// than started. The measured case: the derived branch
6779    /// `worktree-<label>` is already checked out in another worktree, so
6780    /// nothing can be created at the path the worker is to run in - and
6781    /// unchecked, the session starts anyway and the worker reads live while
6782    /// having nowhere to work.
6783    #[tokio::test]
6784    async fn a_fresh_spawn_whose_branch_is_held_elsewhere_is_refused() {
6785        let (workspace, key, _derived, repo, _config) = git_spawn_fixture("reviewer");
6786        let elsewhere = tempdir().expect("elsewhere tempdir");
6787        run_git(
6788            repo.path(),
6789            &[
6790                "worktree",
6791                "add",
6792                "-q",
6793                "-b",
6794                "worktree-reviewer",
6795                elsewhere.path().to_str().expect("utf8 path"),
6796            ],
6797        );
6798
6799        let err = spawn_reply(&workspace, &key, "reviewer", false)
6800            .await
6801            .expect("reply channel")
6802            .expect_err("a spawn whose worktree cannot be created is refused");
6803
6804        assert!(err.contains("already checked out"), "the refusal names the obstacle: {err}");
6805        assert!(err.contains("worktree-reviewer"), "and the branch it is about: {err}");
6806        assert!(
6807            workspace.list_live_workers(&key).is_empty(),
6808            "and no worker reads live off the refused spawn"
6809        );
6810        assert!(
6811            workspace.worker_rows_for_project(&key).is_empty(),
6812            "and nothing durable survives to re-spawn it on the next boot"
6813        );
6814    }
6815
6816    /// A boot re-spawn's reply is dropped, so rolling the entry back would
6817    /// leave the row listed, nothing running, and the reason only in the
6818    /// log. The entry stays as the visible failure the async spawn
6819    /// failures already leave, carrying the reason.
6820    #[tokio::test]
6821    async fn a_refused_boot_respawn_leaves_a_failed_row_carrying_the_reason() {
6822        let (workspace, key, _derived, repo, _config) = git_spawn_fixture("reviewer");
6823        let elsewhere = tempdir().expect("elsewhere tempdir");
6824        run_git(
6825            repo.path(),
6826            &[
6827                "worktree",
6828                "add",
6829                "-q",
6830                "-b",
6831                "worktree-reviewer",
6832                elsewhere.path().to_str().expect("utf8 path"),
6833            ],
6834        );
6835
6836        let err = spawn_reply(&workspace, &key, "reviewer", true)
6837            .await
6838            .expect("reply channel")
6839            .expect_err("the boot re-spawn is refused");
6840
6841        let entry = workspace
6842            .list_live_workers(&key)
6843            .into_iter()
6844            .next()
6845            .expect("the entry stays, so the row reads as failed rather than vanishing");
6846        assert_eq!(
6847            entry.status,
6848            forge_primitives::WorkerLiveness::Failed,
6849            "the refused boot re-spawn is not live"
6850        );
6851        assert!(
6852            entry
6853                .diagnostic
6854                .as_deref()
6855                .is_some_and(|reason| reason.contains("already checked out")),
6856            "and the reason is on the row for a reader, not just some worktree word: {:?}",
6857            entry.diagnostic
6858        );
6859        assert!(err.contains("already checked out"), "the dropped reply still carries it: {err}");
6860    }
6861
6862    /// A `--new` re-spawn lands on the worktree the worker already has:
6863    /// registered at the derived path, holding its own branch, there to be
6864    /// reused. Refusing that would take out every fresh re-spawn, so the
6865    /// path comparison has to read a symlinked repo root as one directory.
6866    #[tokio::test]
6867    async fn a_fresh_spawn_onto_the_workers_own_worktree_gets_past_the_step() {
6868        let (workspace, key, derived, repo, _config) = git_spawn_fixture("reviewer");
6869        std::fs::create_dir_all(derived.parent().expect("worktrees dir")).expect("mkdir worktrees");
6870        run_git(
6871            repo.path(),
6872            &[
6873                "worktree",
6874                "add",
6875                "-q",
6876                "-b",
6877                "worktree-reviewer",
6878                derived.to_str().expect("utf8 path"),
6879            ],
6880        );
6881
6882        let err = spawn_reply(&workspace, &key, "reviewer", false)
6883            .await
6884            .expect("reply channel")
6885            .expect_err("the spawn still fails, at the account walk");
6886
6887        assert!(
6888            err.contains("declares no model"),
6889            "the spawn is admitted onto the worktree it already has: {err}"
6890        );
6891    }
6892
6893    /// Stub whose `forge.toml` caps the `forge` project at the given
6894    /// override via the entry's `max_workers`, loaded through the
6895    /// real config path because the full spawn below reads
6896    /// `config.projects`. The `notes` project stays at the default.
6897    /// Projects point at throwaway dirs: a real repo path would make
6898    /// `is_git` true and hang `--worktree <label>` spawns off the
6899    /// actual checkout.
6900    fn stub_with_project_cap(
6901        limit: usize,
6902    ) -> (Arc<Workspace>, (tempfile::TempDir, tempfile::TempDir)) {
6903        let dir = tempdir().expect("config tempdir");
6904        let projects_dir = tempdir().expect("projects tempdir");
6905        let forge_path = projects_dir.path().join("forge").display().to_string();
6906        let notes_path = projects_dir.path().join("notes").display().to_string();
6907        fs::create_dir_all(&forge_path).expect("create forge project dir");
6908        fs::create_dir_all(&notes_path).expect("create notes project dir");
6909        fs::write(
6910            forge_toml_path(dir.path()),
6911            format!(
6912                r#"
6913[[orgs]]
6914name = "Default"
6915accounts = ["Stargate"]
6916
6917[[orgs.projects]]
6918name = "forge"
6919path = "{forge_path}"
6920max_workers = {limit}
6921model = "claude-sonnet-5"
6922
6923[[orgs.projects]]
6924name = "notes"
6925path = "{notes_path}"
6926model = "claude-sonnet-5"
6927
6928[[accounts]]
6929display_name = "Stargate"
6930token = "t"
6931models = ["claude-sonnet-5"]
6932provider = "anthropic"
6933"#
6934            ),
6935        )
6936        .expect("write forge.toml");
6937        let workspace =
6938            Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("workspace new"));
6939        workspace.seed_test_ready_account("Stargate");
6940        (workspace, (dir, projects_dir))
6941    }
6942
6943    fn seeded_project(workspace: &Arc<Workspace>) -> ProjectKey {
6944        workspace
6945            .list_projects()
6946            .into_iter()
6947            .find(|v| v.name == "forge")
6948            .expect("seeded project present")
6949            .key
6950    }
6951
6952    /// #976: a lead-driven spawn at the project's worker cap
6953    /// replies a clean error naming the limit and inserts no entry.
6954    #[tokio::test]
6955    async fn spawn_worker_over_limit_refuses_and_creates_nothing() {
6956        let (workspace, _config_dir) = stub_with_project_cap(2);
6957        let project = seeded_project(&workspace);
6958        workspace.insert_live_worker(&project, fake_worker_entry("w1", "w1"));
6959        workspace.insert_live_worker(&project, fake_worker_entry("w2", "w2"));
6960
6961        let (tx, rx) = tokio::sync::oneshot::channel();
6962        handle_spawn_worker(
6963            &workspace,
6964            project.clone(),
6965            WorkerSpawnArgs {
6966                label: "w3".to_owned(),
6967                charter: "charter".to_owned(),
6968                kick: None,
6969                resume_kick: None,
6970                interactive: false,
6971                mcp_families: None,
6972                cwd: None,
6973            },
6974            SessionSlot::from_str_for_test("lead"),
6975            None,
6976            false,
6977            tx,
6978        );
6979
6980        let err = rx.await.expect("reply").expect_err("at-limit spawn must refuse");
6981        assert!(err.contains("worker limit reached"), "names the refusal: {err}");
6982        assert!(err.contains("cap is 2"), "names the cap: {err}");
6983        assert_eq!(
6984            workspace.list_live_workers(&project).len(),
6985            2,
6986            "the refused spawn creates no worker"
6987        );
6988
6989        // Label precedence: at the cap, a DUPLICATE label reports the
6990        // collision, not the cap - the lead is pointed at the remedy
6991        // that matches the actual problem.
6992        let (tx, rx) = tokio::sync::oneshot::channel();
6993        handle_spawn_worker(
6994            &workspace,
6995            project.clone(),
6996            WorkerSpawnArgs {
6997                label: "w1".to_owned(),
6998                charter: "charter".to_owned(),
6999                kick: None,
7000                resume_kick: None,
7001                interactive: false,
7002                mcp_families: None,
7003                cwd: None,
7004            },
7005            SessionSlot::from_str_for_test("lead"),
7006            None,
7007            false,
7008            tx,
7009        );
7010        let err = rx.await.expect("reply").expect_err("a live label must refuse");
7011        assert!(
7012            err.contains("already live") && !err.contains("worker limit reached"),
7013            "the collision wins over the cap: {err}"
7014        );
7015    }
7016
7017    /// The cap is PER PROJECT: the gate counts only the spawning
7018    /// project's live workers, against that project's own
7019    /// `max_workers` override, else the default of
7020    /// 2. Workers live in other projects neither consume the budget
7021    /// nor raise it.
7022    #[tokio::test]
7023    async fn the_cap_is_per_project() {
7024        let (workspace, _config_dir) = stub_with_project_cap(1);
7025        let forge_project = seeded_project(&workspace);
7026        let notes_project = workspace
7027            .list_projects()
7028            .into_iter()
7029            .find(|v| v.name == "notes")
7030            .expect("seeded notes project present")
7031            .key;
7032        // forge sits at its own cap (override 1); notes still has its
7033        // full default budget of 2.
7034        workspace.insert_live_worker(&forge_project, fake_worker_entry("w1", "w1"));
7035
7036        let (tx, rx) = tokio::sync::oneshot::channel();
7037        handle_spawn_worker(
7038            &workspace,
7039            notes_project.clone(),
7040            WorkerSpawnArgs {
7041                label: "w2".to_owned(),
7042                charter: "charter".to_owned(),
7043                kick: None,
7044                resume_kick: None,
7045                interactive: false,
7046                mcp_families: None,
7047                cwd: None,
7048            },
7049            SessionSlot::from_str_for_test("lead"),
7050            None,
7051            false,
7052            tx,
7053        );
7054        let reply = rx.await.expect("reply");
7055        assert!(
7056            reply.is_ok(),
7057            "a worker in another project must not consume notes' budget: {:?}",
7058            reply.err()
7059        );
7060        if let Ok(reply) = reply {
7061            workspace.release_session(&SessionSlot::from_str_for_test(reply.session_id));
7062        }
7063
7064        // notes fills to its own default cap of 2 and refuses past it.
7065        workspace.insert_live_worker(&notes_project, fake_worker_entry("w3", "w3"));
7066        let (tx, rx) = tokio::sync::oneshot::channel();
7067        handle_spawn_worker(
7068            &workspace,
7069            notes_project.clone(),
7070            WorkerSpawnArgs {
7071                label: "w4".to_owned(),
7072                charter: "charter".to_owned(),
7073                kick: None,
7074                resume_kick: None,
7075                interactive: false,
7076                mcp_families: None,
7077                cwd: None,
7078            },
7079            SessionSlot::from_str_for_test("lead"),
7080            None,
7081            false,
7082            tx,
7083        );
7084        let err = rx.await.expect("reply").expect_err("notes at its cap must refuse");
7085        assert!(err.contains("worker limit reached"), "names the refusal: {err}");
7086        assert!(err.contains("cap is 2"), "names notes' own cap: {err}");
7087        assert!(
7088            err.contains(&format!("project '{}' has 2 workers live", notes_project.as_str())),
7089            "notes' refusal counts only notes' workers: {err}"
7090        );
7091
7092        // forge's override still governs forge.
7093        let (tx, rx) = tokio::sync::oneshot::channel();
7094        handle_spawn_worker(
7095            &workspace,
7096            forge_project.clone(),
7097            WorkerSpawnArgs {
7098                label: "w5".to_owned(),
7099                charter: "charter".to_owned(),
7100                kick: None,
7101                resume_kick: None,
7102                interactive: false,
7103                mcp_families: None,
7104                cwd: None,
7105            },
7106            SessionSlot::from_str_for_test("lead"),
7107            None,
7108            false,
7109            tx,
7110        );
7111        let err = rx.await.expect("reply").expect_err("forge at its cap must refuse");
7112        assert!(err.contains("cap is 1"), "names forge's override: {err}");
7113        assert!(
7114            err.contains(&format!("project '{}' has 1 worker live", forge_project.as_str())),
7115            "forge's refusal counts only forge's workers, singular: {err}"
7116        );
7117    }
7118
7119    /// The override raises as well as lowers: a cap above the default
7120    /// admits past 2. Every other cap test also passes if the
7121    /// resolution clamps the override to the default, and the raise
7122    /// direction is the one the forge project's own config depends on.
7123    #[tokio::test]
7124    async fn a_cap_above_the_default_admits_past_two() {
7125        let (workspace, _config_dir) = stub_with_project_cap(3);
7126        let project = seeded_project(&workspace);
7127        workspace.insert_live_worker(&project, fake_worker_entry("w1", "w1"));
7128        workspace.insert_live_worker(&project, fake_worker_entry("w2", "w2"));
7129
7130        let (tx, rx) = tokio::sync::oneshot::channel();
7131        handle_spawn_worker(
7132            &workspace,
7133            project.clone(),
7134            WorkerSpawnArgs {
7135                label: "w3".to_owned(),
7136                charter: "charter".to_owned(),
7137                kick: None,
7138                resume_kick: None,
7139                interactive: false,
7140                mcp_families: None,
7141                cwd: None,
7142            },
7143            SessionSlot::from_str_for_test("lead"),
7144            None,
7145            false,
7146            tx,
7147        );
7148        let reply = rx.await.expect("reply");
7149        assert!(
7150            reply.is_ok(),
7151            "the override must raise the cap past the default: {:?}",
7152            reply.err()
7153        );
7154        assert_eq!(
7155            workspace.list_live_workers(&project).len(),
7156            3,
7157            "the admitted spawn creates its worker"
7158        );
7159        if let Ok(reply) = reply {
7160            workspace.release_session(&SessionSlot::from_str_for_test(reply.session_id));
7161        }
7162    }
7163
7164    /// The notice the lead reads: a worker spawned onto an account the
7165    /// walk had to take while it was bailed comes back named in the
7166    /// reply, so the lead hears it at spawn rather than when the worker
7167    /// stalls. The join from the spawned handle's account to the reply
7168    /// is the whole mechanism, and a regression there would drop the
7169    /// notice silently.
7170    #[tokio::test]
7171    async fn a_worker_spawned_onto_a_degraded_account_reports_it() {
7172        let (workspace, _config_dir) = stub_with_project_cap(2);
7173        let project = seeded_project(&workspace);
7174        workspace.account_pool().set_loading(
7175            &forge_gateway::AccountKey("Stargate".to_owned()),
7176            forge_gateway::LoadingState::Bailed,
7177        );
7178
7179        let (tx, rx) = tokio::sync::oneshot::channel();
7180        handle_spawn_worker(
7181            &workspace,
7182            project.clone(),
7183            WorkerSpawnArgs {
7184                label: "w1".to_owned(),
7185                charter: "charter".to_owned(),
7186                kick: None,
7187                resume_kick: None,
7188                interactive: false,
7189                mcp_families: None,
7190                cwd: None,
7191            },
7192            SessionSlot::from_str_for_test("lead"),
7193            None,
7194            false,
7195            tx,
7196        );
7197        let reply = rx.await.expect("reply").expect("the spawn is admitted");
7198        assert_eq!(
7199            reply.rate_limited_account.as_deref(),
7200            Some("Stargate"),
7201            "the bailed account is named in the reply, which is what raises the notice",
7202        );
7203        // This fixture's projects are throwaway dirs, not git repos, so this
7204        // worker runs in the project root: the reply must not name a worktree
7205        // that was never created.
7206        assert_eq!(reply.worktree, None, "a non-git project's worker has no worktree");
7207        workspace.release_session(&SessionSlot::from_str_for_test(reply.session_id));
7208    }
7209
7210    /// The atomicity pin, at the layer it lives: concurrent
7211    /// `insert_live_worker_if_label_absent` calls at cap 1 admit
7212    /// exactly one and refuse the rest, whatever order the lock grants.
7213    /// Direct inserts, so no spawn machinery can retire an entry
7214    /// mid-race. A barrier releases all callers at once; the count only
7215    /// stays at 1 if the cap check and the push share the lock.
7216    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
7217    async fn concurrent_cap_gate_admits_exactly_one() {
7218        let (workspace, _config_dir) = stub_with_project_cap(1);
7219        let project = seeded_project(&workspace);
7220        let barrier = Arc::new(tokio::sync::Barrier::new(32));
7221
7222        let mut handles = Vec::new();
7223        for n in 0..32 {
7224            let workspace = Arc::clone(&workspace);
7225            let project = project.clone();
7226            let barrier = Arc::clone(&barrier);
7227            handles.push(tokio::spawn(async move {
7228                barrier.wait().await;
7229                let entry = fake_worker_entry(&format!("w{n}"), &format!("w{n}-session"));
7230                workspace.insert_live_worker_if_label_absent(&project, entry, Some(1))
7231            }));
7232        }
7233
7234        let mut admitted = 0;
7235        let mut refused = 0;
7236        for handle in handles {
7237            match handle.await.expect("insert task joins") {
7238                Ok(()) => admitted += 1,
7239                Err(LiveWorkerRefusal::AtCap { live, cap }) => {
7240                    refused += 1;
7241                    assert_eq!((live, cap), (1, 1), "the refusal names the count and the cap");
7242                }
7243                Err(other) => panic!("no label collisions at distinct labels: {other:?}"),
7244            }
7245        }
7246        assert_eq!(admitted, 1, "exactly one insert wins the cap");
7247        assert_eq!(refused, 31, "every other insert is refused by the cap");
7248        assert_eq!(
7249            workspace.list_live_workers(&project).len(),
7250            1,
7251            "the pool holds exactly the one admitted entry"
7252        );
7253    }
7254
7255    /// The same gate through the full handler: eight concurrent
7256    /// lead-driven spawns at cap 1, and the live (non-`Failed`) count
7257    /// never exceeds 1.
7258    ///
7259    /// Exactly one of the eight is admitted, which the stand-in is what
7260    /// makes assertable: with a real child the reply count is tolerant,
7261    /// because a winner whose subprocess dies asynchronously is
7262    /// transitioned to `Failed` by the spawn-failure handler, freeing
7263    /// its slot for a queued spawn.
7264    ///
7265    /// That stand-in is not only for determinism - a real child booted
7266    /// against this fixture's config directory outlives the test: the
7267    /// guard that removes the tree drops as soon as the assertions hold,
7268    /// and the child then writes the CLI's `.claude.json` layout back
7269    /// into the deleted path.
7270    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
7271    async fn concurrent_spawns_keep_the_live_count_under_the_cap() {
7272        let (workspace, _config_dir) = stub_with_project_cap(1);
7273        // Held for the test's lifetime: the receiver IS the stand-in's
7274        // dispatcher, and dropping it fails the admitted spawn, which
7275        // rolls the entry back and hands the freed slot on.
7276        let (stand_in, _agent_rx) = Workspace::testing_stub_handle();
7277        workspace.install_test_spawn_handle(stand_in);
7278
7279        let mut handles = Vec::new();
7280        for n in 0..8 {
7281            let workspace = Arc::clone(&workspace);
7282            handles.push(tokio::spawn(async move {
7283                let project = seeded_project(&workspace);
7284                let (tx, rx) = tokio::sync::oneshot::channel();
7285                handle_spawn_worker(
7286                    &workspace,
7287                    project,
7288                    WorkerSpawnArgs {
7289                        label: format!("w{n}"),
7290                        charter: "charter".to_owned(),
7291                        kick: None,
7292                        resume_kick: None,
7293                        interactive: false,
7294                        mcp_families: None,
7295                        cwd: None,
7296                    },
7297                    SessionSlot::from_str_for_test("lead"),
7298                    None,
7299                    false,
7300                    tx,
7301                );
7302                rx.await.expect("reply")
7303            }));
7304        }
7305
7306        let mut winners = 0;
7307        let mut winner_sessions = Vec::new();
7308        for handle in handles {
7309            match handle.await.expect("spawn task joins") {
7310                Ok(reply) => {
7311                    winners += 1;
7312                    winner_sessions.push(SessionSlot::from_str_for_test(reply.session_id));
7313                }
7314                Err(message) => {
7315                    assert!(
7316                        message.contains("worker limit reached"),
7317                        "the only refusal is the cap, never a collision or other error: {message}"
7318                    );
7319                }
7320            }
7321        }
7322        assert_eq!(winners, 1, "the cap admits the first arrival and refuses the other seven");
7323        let project = seeded_project(&workspace);
7324        let live = workspace.list_live_workers(&project).iter().filter(|w| w.is_live()).count();
7325        assert!(live <= 1, "no overshoot past the cap; got {live} live workers");
7326        // Release closes each winner's Command channel, so the
7327        // SessionTask's disconnect() reaps the client; kill_on_drop is
7328        // the backstop.
7329        for session in winner_sessions {
7330            workspace.release_session(&session);
7331        }
7332    }
7333
7334    /// #976's open question, decided: the boot-time respawn wave is
7335    /// EXEMPT from the cap. Persisted workers are state the user
7336    /// already had, and the boot path drops the spawn reply, so an
7337    /// over-limit refusal there would strand rows silently with no
7338    /// retry. The exempt spawn proceeds past the cap.
7339    #[tokio::test]
7340    async fn boot_respawn_spawn_is_exempt_from_the_limit() {
7341        let (workspace, _config_dir) = stub_with_project_cap(2);
7342        let project = seeded_project(&workspace);
7343        workspace.insert_live_worker(&project, fake_worker_entry("w1", "w1"));
7344        workspace.insert_live_worker(&project, fake_worker_entry("w2", "w2"));
7345
7346        let (tx, rx) = tokio::sync::oneshot::channel();
7347        handle_spawn_worker(
7348            &workspace,
7349            project.clone(),
7350            WorkerSpawnArgs {
7351                label: "w3".to_owned(),
7352                charter: "charter".to_owned(),
7353                kick: None,
7354                resume_kick: None,
7355                interactive: false,
7356                mcp_families: None,
7357                cwd: None,
7358            },
7359            SessionSlot::from_str_for_test("lead"),
7360            None,
7361            true,
7362            tx,
7363        );
7364
7365        let reply = rx.await.expect("reply");
7366        assert!(reply.is_ok(), "boot respawn must not be refused by the cap: {:?}", reply.err());
7367        assert_eq!(
7368            workspace.list_live_workers(&project).len(),
7369            3,
7370            "the exempt spawn creates its worker"
7371        );
7372        if let Ok(reply) = reply {
7373            workspace.release_session(&SessionSlot::from_str_for_test(reply.session_id));
7374        }
7375    }
7376
7377    /// #976: a despawn frees its slot; the next lead-driven spawn
7378    /// succeeds.
7379    #[tokio::test]
7380    async fn despawn_frees_a_spawn_slot() {
7381        let (workspace, _config_dir) = stub_with_project_cap(1);
7382        let project = seeded_project(&workspace);
7383        workspace.insert_live_worker(&project, fake_worker_entry("w1", "w1"));
7384
7385        let (tx, rx) = tokio::sync::oneshot::channel();
7386        handle_spawn_worker(
7387            &workspace,
7388            project.clone(),
7389            WorkerSpawnArgs {
7390                label: "w2".to_owned(),
7391                charter: "charter".to_owned(),
7392                kick: None,
7393                resume_kick: None,
7394                interactive: false,
7395                mcp_families: None,
7396                cwd: None,
7397            },
7398            SessionSlot::from_str_for_test("lead"),
7399            None,
7400            false,
7401            tx,
7402        );
7403        let err = rx.await.expect("reply").expect_err("at-limit spawn must refuse");
7404        assert!(err.contains("worker limit reached"), "names the refusal: {err}");
7405
7406        let (despawn_tx, despawn_rx) = tokio::sync::oneshot::channel();
7407        handle_despawn_worker(&workspace, &project, "w1", false, despawn_tx);
7408        let outcome = despawn_rx.await.expect("despawn reply");
7409        assert!(
7410            matches!(outcome, crate::protocol::DespawnResult::Despawned { .. }),
7411            "the worker despawns: {outcome:?}"
7412        );
7413
7414        let (tx, rx) = tokio::sync::oneshot::channel();
7415        handle_spawn_worker(
7416            &workspace,
7417            project.clone(),
7418            WorkerSpawnArgs {
7419                label: "w2".to_owned(),
7420                charter: "charter".to_owned(),
7421                kick: None,
7422                resume_kick: None,
7423                interactive: false,
7424                mcp_families: None,
7425                cwd: None,
7426            },
7427            SessionSlot::from_str_for_test("lead"),
7428            None,
7429            false,
7430            tx,
7431        );
7432        let reply = rx.await.expect("reply");
7433        assert!(reply.is_ok(), "the freed slot lets the next spawn through: {:?}", reply.err());
7434        if let Ok(reply) = reply {
7435            workspace.release_session(&SessionSlot::from_str_for_test(reply.session_id));
7436        }
7437    }
7438
7439    /// A `Failed` worker holds no cap slot, matching the label-dedup
7440    /// which already treats `Failed` as not-live. One connection
7441    /// failure must not wedge every later spawn until a manual despawn.
7442    #[tokio::test]
7443    async fn a_failed_worker_does_not_consume_a_cap_slot() {
7444        let (workspace, _config_dir) = stub_with_project_cap(1);
7445        let project = seeded_project(&workspace);
7446        let mut failed = fake_worker_entry("dead", "dead-1");
7447        failed.status = forge_primitives::WorkerLiveness::Failed;
7448        workspace.insert_live_worker(&project, failed);
7449
7450        let (tx, rx) = tokio::sync::oneshot::channel();
7451        handle_spawn_worker(
7452            &workspace,
7453            project,
7454            WorkerSpawnArgs {
7455                label: "fresh".to_owned(),
7456                charter: "charter".to_owned(),
7457                kick: None,
7458                resume_kick: None,
7459                interactive: false,
7460                mcp_families: None,
7461                cwd: None,
7462            },
7463            SessionSlot::from_str_for_test("lead"),
7464            None,
7465            false,
7466            tx,
7467        );
7468        let reply = rx.await.expect("reply");
7469        assert!(reply.is_ok(), "the Failed worker must not consume the slot: {:?}", reply.err());
7470        if let Ok(reply) = reply {
7471            workspace.release_session(&SessionSlot::from_str_for_test(reply.session_id));
7472        }
7473    }
7474
7475    /// `handle_close_worker` removes the worker entry, releases the
7476    /// session, and emits `WorkerStatusChanged { Removed }`. The
7477    /// label-targeting picks the latest-spawned duplicate.
7478    #[tokio::test]
7479    async fn close_worker_removes_entry_and_emits_removed() {
7480        let (workspace, mut rx) = Workspace::testing_stub();
7481        let project = ProjectKey::new("forge");
7482        workspace.insert_live_worker(&project, fake_worker_entry("r1", "worker-1"));
7483
7484        handle_close_worker(&workspace, &project, "r1");
7485
7486        assert!(workspace.list_live_workers(&project).is_empty());
7487        let mut frames: Vec<(&str, SessionSlot)> = Vec::new();
7488        while let Ok(frame) = rx.try_recv() {
7489            match std::sync::Arc::unwrap_or_clone(frame.update) {
7490                SessionUpdate::Releasing { key } => frames.push(("releasing", key)),
7491                SessionUpdate::WorkerStatusChanged {
7492                    action: WorkerStatusAction::Removed,
7493                    status,
7494                    ..
7495                } => {
7496                    frames.push(("removed", status.slot));
7497                }
7498                _ => {}
7499            }
7500        }
7501        // The release first, so other views' rows say where the seat is going
7502        // from the gesture; the removal follows it (#1930).
7503        assert_eq!(
7504            frames.iter().map(|(kind, _)| *kind).collect::<Vec<_>>(),
7505            vec!["releasing", "removed"],
7506            "the close announces the release ahead of the removal: {frames:?}",
7507        );
7508        assert_eq!(frames[0].1, frames[1].1, "and both name the seat that closed");
7509    }
7510
7511    /// **A live worker's despawn announces the release at the gesture**, not
7512    /// at its removal: the worktree cleanup can run for minutes and the
7513    /// removal frame only follows it, so without this every other view's row
7514    /// reads that worker as working for the whole cleanup (#1930).
7515    #[tokio::test]
7516    async fn a_live_despawn_announces_the_release_before_its_removal() {
7517        let (workspace, mut rx) = Workspace::testing_stub();
7518        let project = ProjectKey::new("forge");
7519        workspace.insert_live_worker(&project, fake_worker_entry("r1", "worker-1"));
7520
7521        let (tx, resp_rx) = tokio::sync::oneshot::channel();
7522        handle_despawn_worker(&workspace, &project, "r1", false, tx);
7523        let _ = resp_rx.await.expect("despawn result");
7524
7525        let mut frames: Vec<(&str, SessionSlot)> = Vec::new();
7526        while let Ok(frame) = rx.try_recv() {
7527            match std::sync::Arc::unwrap_or_clone(frame.update) {
7528                SessionUpdate::Releasing { key } => frames.push(("releasing", key)),
7529                SessionUpdate::WorkerStatusChanged {
7530                    action: WorkerStatusAction::Removed,
7531                    status,
7532                    ..
7533                } => {
7534                    frames.push(("removed", status.slot));
7535                }
7536                _ => {}
7537            }
7538        }
7539        assert_eq!(
7540            frames.iter().map(|(kind, _)| *kind).collect::<Vec<_>>(),
7541            vec!["releasing", "removed"],
7542            "the despawn announces the release ahead of the removal: {frames:?}",
7543        );
7544        assert_eq!(frames[0].1, frames[1].1, "and both name the seat that went");
7545    }
7546
7547    /// **A spawned seat warms the browser it resolves to.** The call is off
7548    /// the spawn's own path, so the seat's first page call meets a host
7549    /// already up rather than paying the launch itself.
7550    #[tokio::test]
7551    async fn a_spawned_seat_warms_its_browser() {
7552        let (workspace, _rx) = Workspace::testing_stub();
7553        let (calls, mut heard) = tokio::sync::mpsc::unbounded_channel();
7554        workspace.install_browser_host(Arc::new(crate::browser::testing::StubHost { calls }));
7555        let seat = forge_primitives::SessionSlot::from_str_for_test("warm-seat");
7556
7557        warm_browser_seat(&workspace, &seat);
7558
7559        let (what, args) = tokio::time::timeout(std::time::Duration::from_secs(5), heard.recv())
7560            .await
7561            .expect("the fleet hears the warm")
7562            .expect("the stub's channel");
7563        assert_eq!(what, "warm", "the spawn asks the fleet to bring the profile up");
7564        assert_eq!(args, serde_json::json!(seat));
7565    }
7566
7567    /// **The warm rides the lead's spawn path, not a call of its own.** A
7568    /// lead spawned through `handle_spawn_project` asks the fleet to bring
7569    /// its profile up as it starts, so the first browser call does not pay
7570    /// the host launch.
7571    #[tokio::test]
7572    async fn a_spawned_lead_warms_its_browser_through_the_spawn_path() {
7573        let dir = tempdir().expect("tempdir");
7574        write_forge_toml(dir.path(), FIXTURE_PROJECT_PATH);
7575        let workspace = Arc::new(Workspace::new_for_test(dir.path().to_owned()).expect("new"));
7576        workspace.seed_test_ready_account("Stargate");
7577        workspace.seed_test_gateway_ready(true);
7578        let (calls, mut heard) = tokio::sync::mpsc::unbounded_channel();
7579        workspace.install_browser_host(Arc::new(crate::browser::testing::StubHost { calls }));
7580        forge_agent::set_test_sdk_binary(Some(
7581            concat!(env!("CARGO_MANIFEST_DIR"), "/../forge-sdk/tests/fixtures/mock_claude.sh")
7582                .to_owned(),
7583        ));
7584
7585        let _ = handle_spawn_project(&workspace, "forge", SessionLaunchSettings::default());
7586
7587        let (what, args) = tokio::time::timeout(std::time::Duration::from_secs(5), heard.recv())
7588            .await
7589            .expect("the spawn path hears the warm")
7590            .expect("the stub's channel");
7591        assert_eq!(what, "warm", "the spawn path is what asks the fleet");
7592        assert_eq!(
7593            args,
7594            serde_json::json!(SessionSlot::lead("Default", "forge")),
7595            "the lead's own seat is warmed",
7596        );
7597        forge_agent::set_test_sdk_binary(None);
7598    }
7599
7600    /// **A despawn sweeps the browsers to the seats that remain.** The seat
7601    /// that went is not in the set and the project's lead is, which is the
7602    /// whole of what the fleet needs to close exactly the down seat's tabs.
7603    #[tokio::test]
7604    async fn a_despawn_sweeps_the_browsers_to_the_remaining_seats() {
7605        let (workspace, _rx) = Workspace::testing_stub();
7606        let project = ProjectKey::new("forge");
7607        let dir = tempfile::tempdir().expect("a temp dir");
7608        let db = crate::store::Db::open(&dir.path().join("db.redb")).expect("the store");
7609        workspace.attach_store_for_test(db);
7610        workspace.seed_test_project("forge", "/tmp/forge-seat-sweep-test");
7611        workspace.insert_live_worker(&project, fake_worker_entry("gone", "session-gone"));
7612
7613        let (calls, mut heard) = tokio::sync::mpsc::unbounded_channel();
7614        workspace.install_browser_host(Arc::new(crate::browser::testing::StubHost { calls }));
7615
7616        let (tx, resp_rx) = tokio::sync::oneshot::channel();
7617        handle_despawn_worker(&workspace, &project, "gone", false, tx);
7618        let _ = resp_rx.await.expect("despawn result");
7619
7620        // **The close's own signals come first**: the seat is ended and its
7621        // tabs replaced before the sweep names the seats that remain.
7622        let mut ended = Vec::new();
7623        let (what, sweep) = loop {
7624            let (what, args) =
7625                tokio::time::timeout(std::time::Duration::from_secs(5), heard.recv())
7626                    .await
7627                    .expect("the sweep is asked")
7628                    .expect("the channel is fed");
7629            if what == "sweep_sessions" {
7630                break (what, args);
7631            }
7632            if what == "session_ended" {
7633                ended.push(args);
7634            }
7635        };
7636        assert_eq!(what, "sweep_sessions", "the fleet is told the seats that remain");
7637        assert_eq!(ended.len(), 1, "the despawned seat was told ended before the sweep: {ended:?}");
7638        let labels: Vec<&str> = sweep["seats"]
7639            .as_array()
7640            .expect("a seat set")
7641            .iter()
7642            .filter_map(|seat| seat.get("label").and_then(serde_json::Value::as_str))
7643            .collect();
7644        assert_eq!(labels, vec!["lead"], "the seat that went is not in the set");
7645        assert!(
7646            sweep["generation"].as_u64().is_some_and(|generation| generation >= 1),
7647            "a capture generation travels with the set, and zero is the fleet's floor: {sweep}",
7648        );
7649    }
7650
7651    /// Regression: closing ONE worker must NOT cascade to the others.
7652    /// Before the fix, `handle_close_worker` called the cascading
7653    /// `release_session` AFTER `remove_latest_worker`. The cascade-
7654    /// detection (`in_catalog && !is_worker`) saw the orphaned
7655    /// session_key as a lead (because it was already gone from
7656    /// `live_workers` by step 1) and drained every OTHER worker.
7657    /// The fix calls `release_session_inner` directly.
7658    #[tokio::test]
7659    async fn close_one_worker_leaves_other_workers_intact() {
7660        let (workspace, mut rx) = Workspace::testing_stub();
7661        let project = ProjectKey::new("forge");
7662        workspace.insert_live_worker(&project, fake_worker_entry("worker-a", "session-a"));
7663        workspace.insert_live_worker(&project, fake_worker_entry("worker-b", "session-b"));
7664        workspace.insert_live_worker(&project, fake_worker_entry("worker-c", "session-c"));
7665        assert_eq!(workspace.list_live_workers(&project).len(), 3);
7666
7667        handle_close_worker(&workspace, &project, "worker-b");
7668
7669        let remaining = workspace.list_live_workers(&project);
7670        assert_eq!(
7671            remaining.len(),
7672            2,
7673            "only the targeted worker should be removed; got {remaining:?}"
7674        );
7675        let labels: Vec<&str> = remaining.iter().map(|w| w.label.as_str()).collect();
7676        assert!(labels.contains(&"worker-a"), "worker-a must survive");
7677        assert!(labels.contains(&"worker-c"), "worker-c must survive");
7678
7679        // Exactly one Removed event for worker-b.
7680        let mut removed_labels: Vec<String> = Vec::new();
7681        while let Ok(frame) = rx.try_recv() {
7682            if let SessionUpdate::WorkerStatusChanged { action, status, .. } =
7683                std::sync::Arc::unwrap_or_clone(frame.update)
7684                && action == WorkerStatusAction::Removed
7685            {
7686                removed_labels.push(status.label);
7687            }
7688        }
7689        assert_eq!(
7690            removed_labels,
7691            vec!["worker-b".to_owned()],
7692            "exactly one Removed event for the closed worker; got {removed_labels:?}"
7693        );
7694    }
7695
7696    /// `handle_close_worker` for an unknown label logs but does not
7697    /// panic and emits no events. Defensive branch when a view's
7698    /// click races a lead-cascade or a concurrent close.
7699    #[tokio::test]
7700    async fn close_worker_unknown_label_is_noop() {
7701        let (workspace, mut rx) = Workspace::testing_stub();
7702        let project = ProjectKey::new("forge");
7703
7704        handle_close_worker(&workspace, &project, "missing");
7705
7706        assert!(rx.try_recv().is_err(), "no events emitted for unknown label");
7707    }
7708
7709    fn run_git(dir: &std::path::Path, args: &[&str]) {
7710        let status = std::process::Command::new("git")
7711            .args(args)
7712            .current_dir(dir)
7713            .status()
7714            .expect("spawn git");
7715        assert!(status.success(), "git {args:?} failed in {dir:?}");
7716    }
7717
7718    /// `git rev-parse <rev>` in `dir`, trimmed.
7719    fn git_rev(dir: &std::path::Path, rev: &str) -> String {
7720        let out = std::process::Command::new("git")
7721            .args(["rev-parse", rev])
7722            .current_dir(dir)
7723            .output()
7724            .expect("spawn git");
7725        assert!(out.status.success(), "git rev-parse {rev} failed in {dir:?}");
7726        String::from_utf8_lossy(&out.stdout).trim().to_owned()
7727    }
7728
7729    fn branch_exists(repo: &std::path::Path, branch: &str) -> bool {
7730        let out = std::process::Command::new("git")
7731            .args(["rev-parse", "--verify", "--quiet", &format!("refs/heads/{branch}")])
7732            .current_dir(repo)
7733            .output()
7734            .expect("spawn git");
7735        match out.status.code() {
7736            Some(0) => true,
7737            Some(1) => false,
7738            other => panic!("git rev-parse in {repo:?} exited {other:?}, so absence is unproven"),
7739        }
7740    }
7741
7742    fn fake_git_worker_entry(label: &str, key: &str) -> crate::mcp::workers::types::WorkerEntry {
7743        let mut entry = fake_worker_entry(label, key);
7744        entry.is_git_repo_at_spawn = true;
7745        entry
7746    }
7747
7748    /// The wave asks about the ROW's tree, not the project's: a worker
7749    /// pointed at another repository whose own worktree is gone is
7750    /// skipped, even though the project's own worktree stands - a wave
7751    /// composing the project path sees a directory and dispatches a
7752    /// re-spawn that has nowhere to start.
7753    #[test]
7754    fn the_boot_wave_reads_a_cwd_rows_own_tree() {
7755        let (workspace, _rx) = Workspace::testing_stub();
7756        workspace.enable_test_dispatch_intercept();
7757        let db_dir = tempfile::tempdir().expect("db dir");
7758        workspace.install_db_for_test(
7759            crate::store::Db::open(&db_dir.path().join("db.redb")).expect("open db"),
7760        );
7761        let repo = tempfile::tempdir().expect("git project dir");
7762        run_git(repo.path(), &["init", "-q"]);
7763        workspace.seed_test_project("proj-x", repo.path().to_str().expect("utf8 repo path"));
7764        let proj_x = workspace.project_key_for_name("proj-x").expect("seeded project");
7765        // The project's own worktree stands.
7766        std::fs::create_dir_all(repo.path().join(".claude").join("worktrees").join("moved"))
7767            .expect("project worktree");
7768        // The row's own tree has none.
7769        let sibling = tempfile::tempdir().expect("sibling repo");
7770        run_git(sibling.path(), &["init", "-q"]);
7771        let root = std::fs::canonicalize(sibling.path()).expect("canonical sibling");
7772        workspace.seed_test_worker_row_at(&proj_x, "moved", &root.to_string_lossy(), true);
7773        // A control: a project-root worker in the same wave, so a wave
7774        // that skipped everything would fail this test too.
7775        workspace.seed_test_worker_row(&proj_x, "at-home");
7776
7777        let rows = workspace.worker_rows_for_project(&proj_x);
7778        workspace.dispatch_worker_respawns(
7779            &SessionSlot::lead("TestOrg", "proj-x"),
7780            &proj_x,
7781            &rows,
7782            false,
7783        );
7784
7785        let spawned: Vec<String> = workspace
7786            .drain_test_dispatch_buffer()
7787            .into_iter()
7788            .map(|cmd| match cmd {
7789                Command::SpawnWorker { label, .. } => label,
7790                other => panic!("expected SpawnWorker, got {other:?}"),
7791            })
7792            .collect();
7793        assert!(
7794            !spawned.contains(&"moved".to_owned()),
7795            "a cwd worker whose own tree has no worktree must not be re-spawned; \
7796             spawned {spawned:?}",
7797        );
7798        assert!(
7799            spawned.contains(&"at-home".to_owned()),
7800            "and the project's own worker still re-spawns; spawned {spawned:?}",
7801        );
7802    }
7803
7804    /// A non-git worker despawns cleanly with no worktree step (the
7805    /// teardown runs, nothing touches a worktree).
7806    #[tokio::test]
7807    async fn despawn_non_git_worker_tears_down_without_worktree_step() {
7808        let (workspace, mut rx) = Workspace::testing_stub();
7809        let project = ProjectKey::new("forge");
7810        workspace.insert_live_worker(&project, fake_worker_entry("r1", "worker-1"));
7811
7812        let (tx, resp_rx) = tokio::sync::oneshot::channel();
7813        handle_despawn_worker(&workspace, &project, "r1", false, tx);
7814        let result = resp_rx.await.expect("despawn result");
7815
7816        assert!(
7817            matches!(
7818                result,
7819                crate::protocol::DespawnResult::Despawned {
7820                    worktree_cleanup_warning: None,
7821                    branch_cleanup_warning: None,
7822                }
7823            ),
7824            "non-git worker despawns cleanly: {result:?}"
7825        );
7826        assert!(workspace.list_live_workers(&project).is_empty(), "worker removed");
7827        assert_eq!(
7828            drain_removed_dispositions(&mut rx),
7829            vec![WorktreeDisposition::Absent],
7830            "a worker that never had a worktree must not claim one",
7831        );
7832    }
7833
7834    /// A stub over a configured `proj-x` with a real store, plus the key
7835    /// that project resolves under and the update receiver the stub minted.
7836    /// The despawn fall-through needs the store: it reads and writes the
7837    /// worker's durable row, so a stub with no DB never reaches the arm.
7838    struct StoreBackedStub {
7839        workspace: Arc<Workspace>,
7840        project: ProjectKey,
7841        rx: tokio::sync::mpsc::UnboundedReceiver<SeatUpdate>,
7842        /// Held so the store outlives the test.
7843        _dir: tempfile::TempDir,
7844    }
7845
7846    fn store_backed_stub() -> StoreBackedStub {
7847        let (workspace, rx) = Workspace::testing_stub();
7848        workspace.seed_test_project("proj-x", "/tmp/proj-x");
7849        let dir = tempdir().expect("tempdir");
7850        workspace.install_db_for_test(
7851            crate::store::Db::open(&dir.path().join("db.redb")).expect("open db"),
7852        );
7853        let project = ProjectKey::new(
7854            forge_agent::userdata::catalog::scan::project_key_for_directory(Some("/tmp/proj-x")),
7855        );
7856        StoreBackedStub { workspace, project, rx, _dir: dir }
7857    }
7858
7859    /// A durable row can outlive its live worker - the tag-write rollback
7860    /// removes the entry without the row - and the row is then the only
7861    /// handle on a worker the next boot re-spawns. Despawn is the tool a
7862    /// lead reaches for to remove a durable worker, so it must clear the
7863    /// row rather than report NotFound over it (#1142).
7864    #[tokio::test]
7865    async fn despawn_clears_a_stranded_row_with_no_live_worker() {
7866        let StoreBackedStub { workspace, project, mut rx, .. } = store_backed_stub();
7867        workspace
7868            .record_worker_row(
7869                &project,
7870                "stranded",
7871                "stranded-id",
7872                "charter",
7873                Some("kick"),
7874                None,
7875                false,
7876                crate::mcp::workers::types::WorkerTree { is_git_repo: false, cwd: None },
7877                None,
7878            )
7879            .expect("seed the row whose live worker is gone");
7880
7881        let (tx, resp_rx) = tokio::sync::oneshot::channel();
7882        handle_despawn_worker(&workspace, &project, "stranded", false, tx);
7883        let result = resp_rx.await.expect("despawn result");
7884
7885        assert!(
7886            matches!(result, crate::protocol::DespawnResult::Despawned { .. }),
7887            "clearing the row is a despawn, not a NotFound over the row it just removed: \
7888             {result:?}",
7889        );
7890        assert!(
7891            workspace.worker_rows_for_project(&project).is_empty(),
7892            "the stranded row must not survive the despawn that reported the worker gone",
7893        );
7894        assert!(
7895            rx.try_recv().is_err(),
7896            "nothing live was torn down, so there is no Removed event to emit",
7897        );
7898    }
7899
7900    /// A despawn takes the worker's durable records with it, and a
7901    /// stranded row's label can still own subscriptions and crons from
7902    /// before it was orphaned. Clearing the row alone would report a
7903    /// completed despawn while leaving exactly the records the live path
7904    /// exists to clear.
7905    #[tokio::test]
7906    async fn despawn_clears_a_stranded_workers_subscriptions_and_crons() {
7907        let StoreBackedStub { workspace, project, .. } = store_backed_stub();
7908        workspace
7909            .record_worker_row(
7910                &project,
7911                "stranded",
7912                "stranded-id",
7913                "c",
7914                None,
7915                None,
7916                false,
7917                crate::mcp::workers::types::WorkerTree { is_git_repo: false, cwd: None },
7918                None,
7919            )
7920            .expect("seed the stranded row");
7921        let sub = forge_primitives::GotifySubscription {
7922            id: uuid::Uuid::new_v4(),
7923            project: "proj-x".to_owned(),
7924            team_role: Some("stranded".to_owned()),
7925            applications: Vec::new(),
7926            min_priority: None,
7927            created_at: std::time::SystemTime::UNIX_EPOCH,
7928        };
7929        workspace.add_gotify_subscription(sub.clone(), true);
7930        workspace.push_cron(forge_primitives::CronEntry {
7931            id: forge_primitives::CronId::from("stranded-cron"),
7932            project_name: "proj-x".to_owned(),
7933            kind: forge_primitives::CronKind::Recurring("0 9 * * *".to_owned()),
7934            prompt: "stand-up".to_owned(),
7935            created_at: std::time::SystemTime::UNIX_EPOCH,
7936            description: None,
7937            last_fire: None,
7938            next_fire: std::time::SystemTime::UNIX_EPOCH,
7939            team_role: Some("stranded".to_owned()),
7940            wake: true,
7941        });
7942
7943        let (tx, resp_rx) = tokio::sync::oneshot::channel();
7944        handle_despawn_worker(&workspace, &project, "stranded", false, tx);
7945        assert!(
7946            matches!(
7947                resp_rx.await.expect("result"),
7948                crate::protocol::DespawnResult::Despawned { .. }
7949            ),
7950            "the row was cleared, so the despawn reports the worker gone",
7951        );
7952
7953        assert!(
7954            workspace.gotify_subscriptions_for_project("proj-x").iter().all(|s| s.id != sub.id),
7955            "the label's subscription goes with the worker it was registered for",
7956        );
7957        assert!(
7958            workspace.crons_for_project("proj-x").is_empty(),
7959            "and so do its crons, which would otherwise keep firing for a label with no worker",
7960        );
7961    }
7962
7963    /// `lead` is the project lead's own row, not a worker: it is the
7964    /// stored id an ordinary boot resumes the lead from. A despawn for it
7965    /// finds no live worker, and the fall-through must not treat that as a
7966    /// stranded worker row and clear it.
7967    #[tokio::test]
7968    async fn despawn_refuses_to_clear_the_lead_row() {
7969        let StoreBackedStub { workspace, project, mut rx, .. } = store_backed_stub();
7970        workspace
7971            .record_worker_row(
7972                &project,
7973                crate::store::sessions::LEAD_LABEL,
7974                "lead-uuid",
7975                "the lead's charter",
7976                None,
7977                None,
7978                false,
7979                crate::mcp::workers::types::WorkerTree { is_git_repo: false, cwd: None },
7980                None,
7981            )
7982            .expect("seed the lead's row");
7983
7984        let (tx, resp_rx) = tokio::sync::oneshot::channel();
7985        handle_despawn_worker(&workspace, &project, crate::store::sessions::LEAD_LABEL, false, tx);
7986
7987        assert!(
7988            matches!(resp_rx.await.expect("result"), crate::protocol::DespawnResult::NotFound),
7989            "there is no worker by that label, so the answer is NotFound",
7990        );
7991        // Read the row directly: `worker_rows_for_project` excludes the
7992        // lead's row by design, so it cannot witness this either way.
7993        let stored = {
7994            let db = workspace.db.lock();
7995            crate::store::sessions::get(
7996                db.as_ref().expect("db"),
7997                "TestOrg",
7998                "proj-x",
7999                crate::store::sessions::LEAD_LABEL,
8000            )
8001            .expect("read")
8002        };
8003        assert!(
8004            stored.is_some(),
8005            "the lead's row must survive a despawn for its label; cleared, the next boot mints a \
8006             fresh id and orphans the lead's conversation",
8007        );
8008        assert!(rx.try_recv().is_err(), "nothing live was torn down, so no Removed event");
8009    }
8010
8011    /// Despawning an unknown label reports NotFound and emits nothing. The
8012    /// store is real, so the answer comes from the fall-through's own
8013    /// "no row there" arm rather than from a stub that cannot reach it.
8014    #[tokio::test]
8015    async fn despawn_unknown_label_reports_not_found() {
8016        let StoreBackedStub { workspace, project, mut rx, .. } = store_backed_stub();
8017        // A sibling row, so "nothing was cleared" cannot pass because the
8018        // store was empty.
8019        workspace
8020            .record_worker_row(
8021                &project,
8022                "other",
8023                "other-id",
8024                "c",
8025                None,
8026                None,
8027                false,
8028                crate::mcp::workers::types::WorkerTree { is_git_repo: false, cwd: None },
8029                None,
8030            )
8031            .expect("seed a row the despawn must leave alone");
8032
8033        let (tx, resp_rx) = tokio::sync::oneshot::channel();
8034        handle_despawn_worker(&workspace, &project, "missing", false, tx);
8035
8036        assert!(matches!(resp_rx.await.expect("result"), crate::protocol::DespawnResult::NotFound));
8037        assert_eq!(
8038            workspace.worker_rows_for_project(&project).len(),
8039            1,
8040            "and the sibling row is untouched",
8041        );
8042        assert!(rx.try_recv().is_err(), "no events for unknown label");
8043    }
8044
8045    /// A despawn whose store cannot be read or written is not a despawn of
8046    /// an absent worker: reporting NotFound there sends the caller looking
8047    /// for a worker that may be right there, with the row still on disk.
8048    #[tokio::test]
8049    async fn despawn_reports_a_store_failure_apart_from_an_absent_label() {
8050        let (workspace, _rx) = Workspace::testing_stub();
8051        // A configured project with no store: the shape of a forge that
8052        // came up without its database.
8053        workspace.seed_test_project("proj-x", "/tmp/proj-x");
8054        let project = ProjectKey::new(
8055            forge_agent::userdata::catalog::scan::project_key_for_directory(Some("/tmp/proj-x")),
8056        );
8057
8058        let (tx, resp_rx) = tokio::sync::oneshot::channel();
8059        handle_despawn_worker(&workspace, &project, "stranded", false, tx);
8060
8061        let result = resp_rx.await.expect("result");
8062        let crate::protocol::DespawnResult::Failed { reason } = result else {
8063            panic!("an unreadable store must not be reported as an absent row; got {result:?}");
8064        };
8065        assert!(
8066            reason.contains("stranded"),
8067            "the failure names the label it could not clear: {reason}",
8068        );
8069    }
8070
8071    /// The same failure at the surface a caller actually reads: the facade
8072    /// must hand it back as a failed despawn, since `agents__despawn`
8073    /// renders `UnknownLabel` as "no live worker with label ...", which
8074    /// sends the lead looking for a worker that is right there.
8075    #[tokio::test]
8076    async fn a_despawn_store_failure_is_not_an_unknown_label() {
8077        let dir = tempdir().expect("tempdir");
8078        write_forge_toml(dir.path(), FIXTURE_PROJECT_PATH);
8079        let config = crate::config::load_from_dir(dir.path()).expect("load config");
8080        // No db installed: the shape of a forge that came up without its
8081        // store, where whether a row exists cannot be answered at all.
8082        let (workspace, _rx) = Workspace::testing_stub_with_config(dir.path().to_owned(), config)
8083            .expect("stub over the fixture config");
8084        let facade = crate::mcp::workers::facade::ProdWorkerFacade::from_arc(&workspace);
8085
8086        let error = facade
8087            .despawn_worker(&SessionSlot::lead("Default", "forge"), "stranded", false)
8088            .await
8089            .expect_err("a store that cannot be read is not a despawn of an absent worker");
8090
8091        assert!(
8092            matches!(error, crate::mcp::workers::facade::WorkerDespawnError::DispatchFailed { .. }),
8093            "the caller must hear a failed despawn rather than an unknown label: {error:?}",
8094        );
8095    }
8096
8097    /// Build a workspace whose single project points at a temp git
8098    /// repo, add a worktree at `<project>/.claude/worktrees/<label>`,
8099    /// and register a live git-worker under it. The worktree is built
8100    /// from the project path `list_projects` reports so the handler's
8101    /// `worker_tag_dir` resolves to the exact on-disk worktree (avoids
8102    /// the macOS /tmp symlink mismatch). Returns the pieces the test
8103    /// asserts on plus the tempdir guards (which must outlive the test).
8104    fn git_despawn_fixture(
8105        label: &str,
8106    ) -> (Arc<Workspace>, ProjectKey, std::path::PathBuf, tempfile::TempDir, tempfile::TempDir)
8107    {
8108        let repo = tempdir().expect("repo tempdir");
8109        run_git(repo.path(), &["init", "-q"]);
8110        run_git(repo.path(), &["config", "user.email", "t@example.com"]);
8111        run_git(repo.path(), &["config", "user.name", "Test"]);
8112        run_git(repo.path(), &["config", "commit.gpgsign", "false"]);
8113        std::fs::write(repo.path().join("README.md"), "seed").expect("write seed");
8114        run_git(repo.path(), &["add", "."]);
8115        run_git(repo.path(), &["commit", "-q", "-m", "init"]);
8116
8117        let config = tempdir().expect("config tempdir");
8118        let repo_path_str = repo.path().to_string_lossy().replace('\\', "/");
8119        std::fs::write(
8120            forge_toml_path(config.path()),
8121            format!(
8122                "[[orgs]]\nname = \"Default\"\naccounts = [\"Stargate\"]\n\n[[orgs.projects]]\nname = \"forge\"\npath = \"{repo_path_str}\"\n\n[[accounts]]\ndisplay_name = \"Stargate\"\ntoken = \"t\"\nmodels = [\"claude-sonnet-5\"]\nprovider = \"anthropic\"\n"
8123            ),
8124        )
8125        .expect("write forge.toml");
8126
8127        let workspace =
8128            Arc::new(Workspace::new_for_test(config.path().to_owned()).expect("workspace new"));
8129        let view = workspace.list_projects().into_iter().next().expect("one project");
8130        let project_key = view.key.clone();
8131        let project_path = view.path.clone();
8132
8133        let wt = project_path.join(".claude").join("worktrees").join(label);
8134        std::fs::create_dir_all(wt.parent().expect("wt parent")).expect("mkdir worktrees");
8135        // `-b worktree-<label>` is what claude's `--worktree <label>` does;
8136        // without it git auto-names the branch after the directory and the
8137        // reap has nothing matching to find.
8138        run_git(
8139            &project_path,
8140            &[
8141                "worktree",
8142                "add",
8143                "-q",
8144                "-b",
8145                &format!("worktree-{label}"),
8146                wt.to_str().expect("utf8 path"),
8147            ],
8148        );
8149
8150        workspace.insert_live_worker(&project_key, fake_git_worker_entry(label, "worker-1"));
8151        (workspace, project_key, wt, repo, config)
8152    }
8153
8154    /// The despawn fixture whose worker worktree holds an initialized
8155    /// submodule. The repo gains the submodule and the worktree's branch
8156    /// follows it, then `git submodule update --init` runs in the
8157    /// worktree - what a worker does - which is what leaves
8158    /// `<repo>/.git/worktrees/<label>/modules` behind.
8159    fn git_despawn_fixture_with_submodule(
8160        label: &str,
8161    ) -> (Arc<Workspace>, ProjectKey, std::path::PathBuf, tempfile::TempDir, tempfile::TempDir)
8162    {
8163        let (workspace, project_key, wt, repo, config) = git_despawn_fixture(label);
8164        let source = tempdir().expect("submodule source tempdir");
8165        run_git(source.path(), &["init", "-q"]);
8166        run_git(source.path(), &["config", "commit.gpgsign", "false"]);
8167        std::fs::write(source.path().join("sub.txt"), "submodule seed").expect("write seed");
8168        run_git(source.path(), &["add", "."]);
8169        run_git(
8170            source.path(),
8171            &[
8172                "-c",
8173                "user.email=t@example.com",
8174                "-c",
8175                "user.name=Test",
8176                "commit",
8177                "-q",
8178                "-m",
8179                "sub",
8180            ],
8181        );
8182        run_git(
8183            repo.path(),
8184            &[
8185                "-c",
8186                "protocol.file.allow=always",
8187                "submodule",
8188                "add",
8189                source.path().to_str().expect("utf8 path"),
8190                "contracts/sub",
8191            ],
8192        );
8193        run_git(repo.path(), &["commit", "-q", "-m", "add submodule"]);
8194        // The worktree is cut from the pre-submodule commit, so its branch
8195        // has to take the repo's HEAD before the submodule exists to init.
8196        run_git(&wt, &["reset", "--hard", &git_rev(repo.path(), "HEAD")]);
8197        run_git(&wt, &["-c", "protocol.file.allow=always", "submodule", "update", "--init"]);
8198        (workspace, project_key, wt, repo, config)
8199    }
8200
8201    /// The shape `git worktree remove` refuses however clean the worktree
8202    /// is: a worker that ran `git submodule update --init` in its own
8203    /// worktree, whose directory the despawn must still remove. Left
8204    /// unfixed, git's refusal reads as a disk-state failure and the
8205    /// maintainer gets a toast about a worktree that is in fact clean.
8206    #[tokio::test]
8207    async fn despawn_removes_a_clean_worktree_holding_initialized_submodules() {
8208        let (workspace, project_key, wt, repo, _config) =
8209            git_despawn_fixture_with_submodule("reviewer");
8210        // Fixture precondition, and the reason the test exists: git will
8211        // not remove this worktree unforced. Asserted rather than assumed
8212        // so a git that stops refusing fails here instead of quietly
8213        // passing without exercising the fix.
8214        assert!(
8215            repo.path().join(".git").join("worktrees").join("reviewer").join("modules").is_dir(),
8216            "fixture precondition: the worktree's gitdir holds the submodule's modules dir",
8217        );
8218        let refused = std::process::Command::new("git")
8219            .args(["worktree", "remove", wt.to_str().expect("utf8 path")])
8220            .current_dir(wt.parent().expect("wt parent"))
8221            .output()
8222            .expect("spawn git");
8223        assert!(
8224            !refused.status.success(),
8225            "fixture precondition: git refuses this worktree unforced"
8226        );
8227        assert!(wt.exists(), "and the refusal leaves it on disk");
8228        let mut rx = workspace.subscribe_answerer();
8229
8230        let (tx, resp_rx) = tokio::sync::oneshot::channel();
8231        handle_despawn_worker(&workspace, &project_key, "reviewer", false, tx);
8232        let result = resp_rx.await.expect("result");
8233
8234        assert!(
8235            matches!(
8236                result,
8237                crate::protocol::DespawnResult::Despawned {
8238                    worktree_cleanup_warning: None,
8239                    branch_cleanup_warning: None,
8240                }
8241            ),
8242            "a clean worktree git refuses by shape despawns without the caller forcing: {result:?}"
8243        );
8244        assert!(!wt.exists(), "the worktree is removed, not orphaned");
8245        assert_eq!(
8246            drain_removed_dispositions(&mut rx),
8247            vec![WorktreeDisposition::Removed],
8248            "and the toast is told the disk agrees, not that the removal failed",
8249        );
8250        assert!(
8251            !branch_exists(repo.path(), "worktree-reviewer"),
8252            "the branch behind it is reaped as usual"
8253        );
8254    }
8255
8256    /// The despawn leftover: `git worktree remove` failed on the delete
8257    /// stage - a build still writing into `target/` while the removal
8258    /// walked the tree, which git reports as "Directory not empty" - and
8259    /// because it deregisters the path before that walk, the directory
8260    /// survives with no registration. Every later `git worktree remove`
8261    /// refuses it as "not a working tree", and the despawn's clean
8262    /// re-check cannot run at all, so before the fallback this left the
8263    /// directory behind for good.
8264    ///
8265    /// The race itself is not reproducible on demand, so the test drives
8266    /// the state it leaves: the deregistered directory, with the build
8267    /// tree still in it. The despawn is forced because a deregistered
8268    /// tree cannot be read by git, which blocks an unforced despawn by
8269    /// design - in the field the check passed a moment earlier, while the
8270    /// removal was still racing the build.
8271    #[tokio::test]
8272    async fn despawn_removes_a_deregistered_leftover_directory_that_git_refuses() {
8273        let (workspace, project_key, wt, repo, _config) = git_despawn_fixture("leftover");
8274        let target = wt.join("target").join("debug");
8275        std::fs::create_dir_all(&target).expect("target dir");
8276        std::fs::write(target.join("obj.o"), "x").expect("object file");
8277        // Deregister without deleting the directory: exactly what the
8278        // failed delete above leaves.
8279        std::fs::remove_dir_all(repo.path().join(".git").join("worktrees").join("leftover"))
8280            .expect("deregister the worktree");
8281        // Fixture precondition, asserted rather than assumed so a git that
8282        // stops refusing fails here instead of passing without exercising
8283        // the fallback.
8284        let refused = std::process::Command::new("git")
8285            .args(["worktree", "remove", "--force", wt.to_str().expect("utf8 path")])
8286            .current_dir(wt.parent().expect("wt parent"))
8287            .output()
8288            .expect("spawn git");
8289        assert!(
8290            !refused.status.success(),
8291            "fixture precondition: git refuses the deregistered leftover outright"
8292        );
8293        assert!(wt.exists(), "and the refusal leaves it on disk");
8294        let mut rx = workspace.subscribe_answerer();
8295
8296        let (tx, resp_rx) = tokio::sync::oneshot::channel();
8297        handle_despawn_worker(&workspace, &project_key, "leftover", true, tx);
8298        let result = resp_rx.await.expect("result");
8299
8300        assert!(
8301            matches!(
8302                result,
8303                crate::protocol::DespawnResult::Despawned {
8304                    worktree_cleanup_warning: None,
8305                    branch_cleanup_warning: None,
8306                }
8307            ),
8308            "the leftover directory is removed, so the despawn carries no warning: {result:?}"
8309        );
8310        assert!(!wt.exists(), "the leftover directory is gone, not orphaned");
8311        assert_eq!(
8312            drain_removed_dispositions(&mut rx),
8313            vec![WorktreeDisposition::Removed],
8314            "and the toast is told the disk agrees, not that the removal failed",
8315        );
8316        assert!(
8317            !branch_exists(repo.path(), "worktree-leftover"),
8318            "the branch behind it is reaped as usual"
8319        );
8320    }
8321
8322    /// The leftover fallback anchors on the WORKER's own tree: a
8323    /// deregistered worktree under a sibling repository is removed from
8324    /// there, where a fallback composing the project's path would refuse
8325    /// the directory (it is not under the project's worktrees root) and
8326    /// leave it behind with a warning - the outcome the fallback exists
8327    /// to prevent.
8328    #[tokio::test]
8329    async fn despawn_removes_a_cwd_workers_deregistered_leftover() {
8330        let (workspace, project_key, _project_wt, _repo, _config) =
8331            git_despawn_fixture("traveller");
8332        let dir = tempfile::tempdir().expect("store dir");
8333        workspace.install_db_for_test(
8334            crate::store::Db::open(&dir.path().join("db.redb")).expect("open db"),
8335        );
8336        // The worker's own tree, with the worktree claude would fork.
8337        let sibling = tempfile::tempdir().expect("sibling repo");
8338        run_git(sibling.path(), &["init", "-q"]);
8339        run_git(sibling.path(), &["config", "commit.gpgsign", "false"]);
8340        std::fs::write(sibling.path().join("README.md"), "seed").expect("write seed");
8341        run_git(sibling.path(), &["add", "."]);
8342        run_git(
8343            sibling.path(),
8344            &[
8345                "-c",
8346                "user.email=t@example.com",
8347                "-c",
8348                "user.name=Test",
8349                "commit",
8350                "-q",
8351                "-m",
8352                "init",
8353            ],
8354        );
8355        let root = std::fs::canonicalize(sibling.path()).expect("canonical sibling");
8356        let wt = root.join(".claude").join("worktrees").join("traveller");
8357        std::fs::create_dir_all(wt.parent().expect("wt parent")).expect("mkdir worktrees");
8358        run_git(
8359            &root,
8360            &["worktree", "add", "-q", "-b", "worktree-traveller", wt.to_str().expect("utf8")],
8361        );
8362        // The state a failed delete leaves: deregistered, directory on
8363        // disk. Asserted rather than assumed, so a git that stops
8364        // refusing fails here instead of passing without the fallback.
8365        std::fs::remove_dir_all(root.join(".git").join("worktrees").join("traveller"))
8366            .expect("deregister the worktree");
8367        let refused = std::process::Command::new("git")
8368            .args(["worktree", "remove", "--force", wt.to_str().expect("utf8 path")])
8369            .current_dir(wt.parent().expect("wt parent"))
8370            .output()
8371            .expect("spawn git");
8372        assert!(
8373            !refused.status.success(),
8374            "fixture precondition: git refuses the deregistered leftover outright"
8375        );
8376        assert!(wt.exists(), "and the refusal leaves it on disk");
8377
8378        // The row and the live entry both name the sibling tree.
8379        workspace.seed_test_worker_row_at(&project_key, "traveller", &root.to_string_lossy(), true);
8380        workspace.remove_latest_worker(&project_key, "traveller");
8381        let mut entry = fake_git_worker_entry("traveller", "worker-1");
8382        entry.cwd = Some(root.clone());
8383        workspace.insert_live_worker(&project_key, entry);
8384
8385        let (tx, resp_rx) = tokio::sync::oneshot::channel();
8386        handle_despawn_worker(&workspace, &project_key, "traveller", true, tx);
8387        let result = resp_rx.await.expect("result");
8388
8389        assert!(
8390            matches!(
8391                result,
8392                crate::protocol::DespawnResult::Despawned { worktree_cleanup_warning: None, .. }
8393            ),
8394            "the leftover is removed from the worker's own tree: {result:?}"
8395        );
8396        assert!(!wt.exists(), "the leftover directory is gone, not orphaned");
8397    }
8398
8399    /// A despawn hands back with the worktree still on disk: its cleanup
8400    /// runs off the caller, and the removal happens behind it.
8401    ///
8402    /// The #1633 shape. The cleanup used to run inline in the caller's
8403    /// task, so a whole `git worktree remove` - minutes for a worker's
8404    /// build tree - sat on a runtime worker, and every timer in the
8405    /// process, the socket's frame delivery above all, stopped for the
8406    /// removal's entire duration.
8407    ///
8408    /// **The assertion is an ordering, not a clock, but it is not free of
8409    /// scheduling.** The gap it measures is the removal below - ~130ms for
8410    /// the 3,000 files here - against the caller's return, which is
8411    /// microseconds of its own work; the window it cannot close is a
8412    /// machine that deschedules THIS thread for longer than the whole
8413    /// removal, and the file count below is what keeps that window small.
8414    #[tokio::test]
8415    async fn a_despawn_hands_back_before_the_worktree_is_gone() {
8416        let (workspace, project_key, wt, _repo, _config) = git_despawn_fixture("slow");
8417        let target = wt.join("target").join("debug");
8418        std::fs::create_dir_all(&target).expect("target dir");
8419        for i in 0..3_000 {
8420            std::fs::write(target.join(format!("obj{i}.o")), b"x").expect("filler");
8421        }
8422        let (tx, rx) = tokio::sync::oneshot::channel();
8423        handle_despawn_worker(&workspace, &project_key, "slow", true, tx);
8424        assert!(
8425            wt.exists(),
8426            "the despawn ran its cleanup on the caller: the worktree is already gone by the \
8427             time it hands back",
8428        );
8429        let result = rx.await.expect("the cleanup answers");
8430        assert!(
8431            matches!(result, crate::protocol::DespawnResult::Despawned { .. }),
8432            "the cleanup still runs to completion behind the caller: {result:?}"
8433        );
8434        assert!(!wt.exists(), "and it still removes the worktree");
8435    }
8436
8437    /// The despawn's dirty probe runs off the runtime's worker. `git status`
8438    /// walks the tracked tree, so on a big repository it can hold a worker -
8439    /// and the frames it serves - for a beat.
8440    ///
8441    /// The probe is slowed deterministically through git's own fsmonitor hook,
8442    /// which waits for a RUNTIME task to release it and records whether that
8443    /// happened. The despawn and that task share the single worker below, so
8444    /// an inline probe starves the task and the hook records the timeout; a
8445    /// probe handed off to another thread lets it run.
8446    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
8447    async fn a_despawn_probes_the_worktree_off_the_runtime() {
8448        use std::os::unix::fs::PermissionsExt as _;
8449
8450        let (workspace, project_key, wt, _repo, _config) = git_despawn_fixture("probe");
8451        let dir = tempdir().expect("hook tempdir");
8452        let started = dir.path().join("probe-started");
8453        let release = dir.path().join("probe-release");
8454        let released = dir.path().join("probe-released");
8455        let hook = dir.path().join("fsmonitor.sh");
8456        // Bounded so the inline shape fails by verdict rather than by hanging
8457        // the machine's test run; the release arrives in milliseconds. The
8458        // trailing NUL-separated `/` is the hook's "all files changed" answer,
8459        // which leaves the probe's full scan - and so its verdict - untouched.
8460        fs::write(
8461            &hook,
8462            format!(
8463                "#!/bin/sh\n: > {started}\ni=0\n\
8464                 while [ ! -e {release} ] && [ $i -lt 100 ]; do sleep 0.01; i=$((i+1)); done\n\
8465                 [ -e {release} ] && : > {released}\n\
8466                 printf 'forge-probe\\0/\\0'\n",
8467                started = started.display(),
8468                release = release.display(),
8469                released = released.display(),
8470            ),
8471        )
8472        .expect("write the fsmonitor hook");
8473        std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).expect("+x");
8474        run_git(&wt, &["config", "core.fsmonitor", hook.to_str().expect("utf8 hook")]);
8475
8476        // The task the hook waits on. It shares the single worker with the
8477        // despawn, so only a probe that does not hold the worker lets it run.
8478        let (task_started, task_release) = (started.clone(), release.clone());
8479        tokio::spawn(async move {
8480            while !task_started.exists() {
8481                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
8482            }
8483            fs::write(&task_release, b"go").expect("release the probe");
8484        });
8485
8486        let (tx, rx) = tokio::sync::oneshot::channel();
8487        let despawn = tokio::spawn(async move {
8488            handle_despawn_worker(&workspace, &project_key, "probe", false, tx);
8489        });
8490        despawn.await.expect("the despawn task ran");
8491        assert!(started.exists(), "fixture precondition: git ran the fsmonitor hook");
8492        assert!(
8493            released.exists(),
8494            "the probe held the runtime: the task that releases its hook never ran",
8495        );
8496
8497        let result = rx.await.expect("despawn result");
8498        assert!(
8499            matches!(result, crate::protocol::DespawnResult::Despawned { .. }),
8500            "the verdict is still clean and the despawn still lands: {result:?}",
8501        );
8502        assert!(!wt.exists(), "and the worktree is still removed");
8503    }
8504
8505    /// A despawn's cleanup reserves its label: a spawn offered the same
8506    /// label while the worktree is mid-removal is refused by name, and the
8507    /// label frees again once the cleanup finishes.
8508    ///
8509    /// The window this closes is minutes long for a worker's build tree.
8510    /// Teardown frees the label the instant it removes the entry, while the
8511    /// removal it hands off is still deleting the directory a spawn
8512    /// admitted in that window would be handed (and the spawn path's own
8513    /// guards treat the surviving worktree as reusable).
8514    #[tokio::test]
8515    async fn a_despawn_reserves_its_label_until_the_cleanup_finishes() {
8516        let (workspace, project_key, wt, _repo, _config) = git_despawn_fixture("reserved");
8517        let target = wt.join("target").join("debug");
8518        std::fs::create_dir_all(&target).expect("target dir");
8519        for i in 0..3_000 {
8520            std::fs::write(target.join(format!("obj{i}.o")), b"x").expect("filler");
8521        }
8522        let (tx, rx) = tokio::sync::oneshot::channel();
8523        handle_despawn_worker(&workspace, &project_key, "reserved", true, tx);
8524        assert!(
8525            workspace.despawn_cleanup_pending(&project_key, "reserved"),
8526            "the label is reserved from the handoff, before the cleanup ends",
8527        );
8528
8529        let (spawn_tx, spawn_rx) = tokio::sync::oneshot::channel();
8530        handle_spawn_worker(
8531            &workspace,
8532            project_key.clone(),
8533            WorkerSpawnArgs {
8534                label: "reserved".to_owned(),
8535                charter: "c".to_owned(),
8536                kick: None,
8537                resume_kick: None,
8538                interactive: false,
8539                mcp_families: None,
8540                cwd: None,
8541            },
8542            SessionSlot::lead("Default", "forge"),
8543            None,
8544            false,
8545            spawn_tx,
8546        );
8547        let refusal = spawn_rx.await.expect("the spawn answers").expect_err("refused");
8548        assert!(
8549            refusal.contains("still cleaning up"),
8550            "the refusal names the cleanup rather than racing it: {refusal}",
8551        );
8552        assert!(
8553            wt.exists(),
8554            "and the refusal came before anything touched the worktree being removed",
8555        );
8556
8557        let result = rx.await.expect("the cleanup answers");
8558        assert!(
8559            matches!(result, crate::protocol::DespawnResult::Despawned { .. }),
8560            "the cleanup still runs to completion: {result:?}",
8561        );
8562        assert!(
8563            !workspace.despawn_cleanup_pending(&project_key, "reserved"),
8564            "and the label frees when the cleanup ends",
8565        );
8566    }
8567
8568    /// A resume for a label whose despawn cleanup is still running is
8569    /// refused before anything mints a worktree.
8570    ///
8571    /// Most of the cleanup is spent in exactly the state
8572    /// `ensure_worker_worktree` will act on: the directory gone while git
8573    /// still holds the registration. Minting there re-attaches the branch
8574    /// the cleanup is about to reap, and the reap then reports a branch it
8575    /// could not delete for a reason this spawn made. The facade refuses on
8576    /// the mark, so the ensure never runs; without that refusal this call
8577    /// comes back as a worktree-creation failure instead of naming the
8578    /// cleanup.
8579    #[tokio::test]
8580    async fn a_resume_is_refused_before_it_mints_a_worktree_for_a_pending_cleanup() {
8581        let (workspace, project_key, wt, _repo, _config) = git_despawn_fixture("resumed");
8582        std::fs::remove_dir_all(&wt).expect("clear the worktree directory");
8583        let held = workspace.mark_despawn_cleanup_pending(&project_key, "resumed");
8584        let facade = crate::mcp::workers::facade::ProdWorkerFacade::from_arc(&workspace);
8585        let refusal = facade
8586            .spawn_worker(
8587                &SessionSlot::lead("Default", "forge"),
8588                "resumed".to_owned(),
8589                "c".to_owned(),
8590                None,
8591                None,
8592                false,
8593                true,
8594                None,
8595                None,
8596            )
8597            .await
8598            .expect_err("a spawn for a label mid-cleanup is refused");
8599        let crate::mcp::workers::facade::WorkerSpawnError::DispatchFailed { message } = refusal
8600        else {
8601            panic!("the refusal names the cleanup, not a worktree failure: {refusal:?}");
8602        };
8603        assert!(message.contains("still cleaning up"), "{message}");
8604        drop(held);
8605    }
8606
8607    /// A git worker with a clean worktree despawns AND removes the
8608    /// worktree AND reaps the `worktree-<label>` branch behind it.
8609    #[tokio::test]
8610    async fn despawn_git_worker_removes_clean_worktree() {
8611        let (workspace, project_key, wt, repo, _config) = git_despawn_fixture("reviewer");
8612        assert!(wt.exists(), "worktree exists before despawn");
8613        let (tx, rx) = tokio::sync::oneshot::channel();
8614        handle_despawn_worker(&workspace, &project_key, "reviewer", false, tx);
8615        let result = rx.await.expect("result");
8616        assert!(
8617            matches!(
8618                result,
8619                crate::protocol::DespawnResult::Despawned {
8620                    worktree_cleanup_warning: None,
8621                    branch_cleanup_warning: None,
8622                }
8623            ),
8624            "clean git worktree despawns + removes: {result:?}"
8625        );
8626        assert!(workspace.list_live_workers(&project_key).is_empty(), "worker removed");
8627        assert!(!wt.exists(), "clean worktree removed");
8628        assert!(
8629            !branch_exists(repo.path(), "worktree-reviewer"),
8630            "the worktree branch is reaped, not left behind"
8631        );
8632        let record = std::fs::read_to_string(repo.path().join(".git").join("forge-worktrees"))
8633            .expect("the despawn recorded the removal");
8634        assert!(
8635            record.lines().any(|line| {
8636                line.starts_with("cleared\treviewer\t")
8637                    && line.contains(".claude/worktrees/reviewer")
8638            }),
8639            "the removal is admitted before git runs and cleared after it: {record}",
8640        );
8641    }
8642
8643    /// The git fixture with its live entry gone and a durable row left
8644    /// behind: the shape the tag-write rollback leaves, and the one a
8645    /// despawn has to reach without a live worker to ask.
8646    fn stranded_git_despawn_fixture(
8647        label: &str,
8648    ) -> (Arc<Workspace>, ProjectKey, std::path::PathBuf, tempfile::TempDir, tempfile::TempDir)
8649    {
8650        let (workspace, project_key, wt, repo, config) = git_despawn_fixture(label);
8651        workspace.remove_latest_worker(&project_key, label);
8652        workspace
8653            .record_worker_row(
8654                &project_key,
8655                label,
8656                "stranded-id",
8657                "c",
8658                Some("kick"),
8659                None,
8660                false,
8661                // The row is what says this worker runs in a worktree, so
8662                // it is what the despawn has to read the gitness from.
8663                crate::mcp::workers::types::WorkerTree { is_git_repo: true, cwd: None },
8664                None,
8665            )
8666            .expect("seed the row that outlived its worker");
8667        (workspace, project_key, wt, repo, config)
8668    }
8669
8670    /// A stranded git worker is still a worker the tool must clean up
8671    /// after: the row carries the gitness, so the despawn removes the
8672    /// worktree and reaps the branch with no live entry to ask. Skipped,
8673    /// the worktree is orphaned once the row is gone - nothing is left
8674    /// for a later despawn to resolve it from.
8675    #[tokio::test]
8676    async fn despawn_clears_a_stranded_git_workers_worktree() {
8677        let (workspace, project_key, wt, repo, _config) = stranded_git_despawn_fixture("reviewer");
8678        assert!(wt.exists(), "the worktree exists before the despawn");
8679
8680        let (tx, resp_rx) = tokio::sync::oneshot::channel();
8681        handle_despawn_worker(&workspace, &project_key, "reviewer", false, tx);
8682        let result = resp_rx.await.expect("result");
8683
8684        assert!(
8685            matches!(
8686                result,
8687                crate::protocol::DespawnResult::Despawned {
8688                    worktree_cleanup_warning: None,
8689                    branch_cleanup_warning: None,
8690                }
8691            ),
8692            "a stranded worker's clean worktree is removed like any other: {result:?}"
8693        );
8694        assert!(!wt.exists(), "the stranded worktree is removed, not orphaned");
8695        assert!(
8696            !branch_exists(repo.path(), "worktree-reviewer"),
8697            "and the branch behind it is reaped"
8698        );
8699        assert!(
8700            workspace.worker_rows_for_project(&project_key).is_empty(),
8701            "with the row it was resolved from",
8702        );
8703    }
8704
8705    /// A row the migration wrote carries no gitness until its worker's
8706    /// first spawn fills it in, and that is a worker with no known
8707    /// worktree rather than no worker. Reading "no gitness" as "no row"
8708    /// leaves exactly the class of stranded row #1142 exists to clear.
8709    #[tokio::test]
8710    async fn despawn_reaches_a_row_that_records_no_gitness() {
8711        let StoreBackedStub { workspace, project, mut rx, .. } = store_backed_stub();
8712        {
8713            let db = workspace.db.lock();
8714            crate::store::sessions::put(
8715                db.as_ref().expect("db"),
8716                &crate::store::sessions::SessionRecord {
8717                    org: "TestOrg".to_owned(),
8718                    project: "proj-x".to_owned(),
8719                    label: "migrated".to_owned(),
8720                    session_id: None,
8721                    charter: Some("c".to_owned()),
8722                    kick: None,
8723                    resume_kick: None,
8724                    interactive: None,
8725                    seat_state: None,
8726                    // The shape `migrate_from_dynamic_workers` writes.
8727                    is_git_repo: None,
8728                    mcp_families: None,
8729                    cwd: None,
8730                },
8731            )
8732            .expect("seed the row the migration writes");
8733        }
8734
8735        let (tx, resp_rx) = tokio::sync::oneshot::channel();
8736        handle_despawn_worker(&workspace, &project, "migrated", false, tx);
8737
8738        assert!(
8739            matches!(
8740                resp_rx.await.expect("result"),
8741                crate::protocol::DespawnResult::Despawned { .. }
8742            ),
8743            "a row with no recorded gitness is a worker the despawn must reach",
8744        );
8745        assert!(
8746            workspace.worker_rows_for_project(&project).is_empty(),
8747            "and it is cleared like any other",
8748        );
8749        assert!(rx.try_recv().is_err(), "nothing live was torn down");
8750    }
8751
8752    /// A stranded row whose worktree is already gone is the shape the boot
8753    /// wave skips, so the despawn has nothing to remove and must not report
8754    /// a removal failure over it. It also has nothing to protect, so it
8755    /// runs without `force`: the dirty probe cannot read a directory that
8756    /// is not there, and blocking on that would refuse a despawn over work
8757    /// no worktree can hold. The live shape reports git's failure for the
8758    /// same state, which `despawn_reports_removed_when_the_worktree_is_
8759    /// already_off_disk` pins.
8760    #[tokio::test]
8761    async fn a_stranded_row_whose_worktree_is_gone_despawns_without_force() {
8762        let (workspace, project_key, wt, repo, _config) = stranded_git_despawn_fixture("reviewer");
8763        run_git(repo.path(), &["worktree", "remove", wt.to_str().expect("utf8 path")]);
8764        assert!(!wt.exists(), "nothing on disk before the despawn");
8765
8766        let (tx, resp_rx) = tokio::sync::oneshot::channel();
8767        handle_despawn_worker(&workspace, &project_key, "reviewer", false, tx);
8768        let result = resp_rx.await.expect("result");
8769
8770        assert!(
8771            matches!(
8772                result,
8773                crate::protocol::DespawnResult::Despawned {
8774                    worktree_cleanup_warning: None,
8775                    branch_cleanup_warning: None,
8776                }
8777            ),
8778            "a worktree that was never there is not a cleanup failure: {result:?}"
8779        );
8780        assert!(
8781            workspace.worker_rows_for_project(&project_key).is_empty(),
8782            "and the row still goes, which is the whole point of the call",
8783        );
8784        assert!(
8785            !branch_exists(repo.path(), "worktree-reviewer"),
8786            "the branch outlives the worktree - `git worktree remove` never deletes one - so a \
8787             worktree that is already gone is still a branch to reap, and with the directory gone \
8788             it is not checked out",
8789        );
8790        let record = std::fs::read_to_string(repo.path().join(".git").join("forge-worktrees"))
8791            .expect("the despawn recorded the gone worktree");
8792        assert!(
8793            record.lines().any(|line| {
8794                line.starts_with("cleared\treviewer\t")
8795                    && line.contains(".claude/worktrees/reviewer")
8796            }),
8797            "the already-gone arm retracts any admission rather than leaving one to outlive it: \
8798             {record}",
8799        );
8800    }
8801
8802    /// A dirty stranded worktree blocks before anything is cleared, the
8803    /// same way a live one does: the row survives, so a later despawn (or
8804    /// `force`) still has the worker to act on.
8805    #[tokio::test]
8806    async fn a_dirty_stranded_worktree_blocks_before_the_row_goes() {
8807        let (workspace, project_key, wt, _repo, _config) = stranded_git_despawn_fixture("reviewer");
8808        std::fs::write(wt.join("uncommitted.txt"), "work in progress").expect("dirty the worktree");
8809
8810        let (tx, resp_rx) = tokio::sync::oneshot::channel();
8811        handle_despawn_worker(&workspace, &project_key, "reviewer", false, tx);
8812
8813        assert!(
8814            matches!(
8815                resp_rx.await.expect("result"),
8816                crate::protocol::DespawnResult::Blocked { .. }
8817            ),
8818            "a dirty worktree blocks the despawn",
8819        );
8820        assert!(wt.exists(), "and the worktree is left alone");
8821        assert_eq!(
8822            workspace.worker_rows_for_project(&project_key).len(),
8823            1,
8824            "a blocked despawn clears nothing, so the row it would have taken is still there",
8825        );
8826    }
8827
8828    /// Every `Removed` event's worktree disposition, in order, so a
8829    /// caller can pin the count as well as the value.
8830    fn drain_removed_dispositions(
8831        rx: &mut tokio::sync::mpsc::UnboundedReceiver<SeatUpdate>,
8832    ) -> Vec<WorktreeDisposition> {
8833        let mut seen = Vec::new();
8834        while let Ok(frame) = rx.try_recv() {
8835            if let SessionUpdate::WorkerStatusChanged { action, worktree, .. } =
8836                std::sync::Arc::unwrap_or_clone(frame.update)
8837                && action == WorkerStatusAction::Removed
8838            {
8839                seen.push(worktree);
8840            }
8841        }
8842        seen
8843    }
8844
8845    /// The `x` button leaves the worktree on disk, so its event still
8846    /// reports it intact.
8847    #[tokio::test]
8848    async fn close_worker_reports_the_worktree_intact() {
8849        let (workspace, project_key, wt, _repo, _config) = git_despawn_fixture("reviewer");
8850        let mut rx = workspace.subscribe_answerer();
8851
8852        handle_close_worker(&workspace, &project_key, "reviewer");
8853
8854        assert_eq!(drain_removed_dispositions(&mut rx), vec![WorktreeDisposition::Intact]);
8855        assert!(wt.exists(), "the x button does not touch the worktree");
8856    }
8857
8858    /// A despawn that removed the worktree says so on the event the
8859    /// close toast is built from.
8860    #[tokio::test]
8861    async fn despawn_reports_the_worktree_removed() {
8862        let (workspace, project_key, wt, _repo, _config) = git_despawn_fixture("reviewer");
8863        let mut rx = workspace.subscribe_answerer();
8864
8865        let (tx, resp_rx) = tokio::sync::oneshot::channel();
8866        handle_despawn_worker(&workspace, &project_key, "reviewer", false, tx);
8867        let _ = resp_rx.await.expect("result");
8868
8869        assert_eq!(drain_removed_dispositions(&mut rx), vec![WorktreeDisposition::Removed]);
8870        assert!(!wt.exists(), "the worktree really is gone");
8871    }
8872
8873    /// A despawn whose removal FAILED leaves something on disk, and the
8874    /// event separates that from a successful removal. The leftovers the
8875    /// fallback may delete are removed (see the deregistered-leftover test
8876    /// above), so the shape that still fails is one the fallback must
8877    /// refuse: a symlink at the worktree path. Deleting through it would
8878    /// hit whatever directory it points at, so the fallback refuses, git's
8879    /// own refusal stands, and the toast says the removal failed.
8880    #[tokio::test]
8881    async fn despawn_reports_a_failed_removal_apart_from_a_successful_one() {
8882        let (workspace, project_key, wt, repo, _config) = git_despawn_fixture("reviewer");
8883        let elsewhere = wt.parent().expect("wt parent").join("precious");
8884        std::fs::create_dir_all(&elsewhere).expect("precious dir");
8885        std::fs::write(elsewhere.join("keep.txt"), "keep").expect("keep file");
8886        // Deregister and replace the directory with a symlink to it. The
8887        // deregistration is why the despawn needs `force`: an unreadable
8888        // tree blocks the unforced path by design.
8889        std::fs::remove_dir_all(repo.path().join(".git").join("worktrees").join("reviewer"))
8890            .expect("deregister the worktree");
8891        std::fs::remove_dir_all(&wt).expect("clear the worktree dir");
8892        std::os::unix::fs::symlink(&elsewhere, &wt).expect("symlink at the worktree path");
8893        let mut rx = workspace.subscribe_answerer();
8894
8895        let (tx, resp_rx) = tokio::sync::oneshot::channel();
8896        handle_despawn_worker(&workspace, &project_key, "reviewer", true, tx);
8897        let result = resp_rx.await.expect("result");
8898
8899        assert!(
8900            matches!(
8901                result,
8902                crate::protocol::DespawnResult::Despawned { worktree_cleanup_warning: Some(_), .. }
8903            ),
8904            "the removal failed: {result:?}"
8905        );
8906        assert!(
8907            elsewhere.join("keep.txt").exists(),
8908            "the fallback refused the symlink instead of deleting through it"
8909        );
8910        assert_eq!(drain_removed_dispositions(&mut rx), vec![WorktreeDisposition::RemovalFailed]);
8911    }
8912
8913    /// A worktree already gone from disk is Removed, whatever git's exit
8914    /// code says - the toast's only claim is about the directory.
8915    #[tokio::test]
8916    async fn despawn_reports_removed_when_the_worktree_is_already_off_disk() {
8917        let (workspace, project_key, wt, repo, _config) = git_despawn_fixture("reviewer");
8918        run_git(repo.path(), &["worktree", "remove", wt.to_str().expect("utf8 path")]);
8919        assert!(!wt.exists(), "nothing on disk before the despawn");
8920        let mut rx = workspace.subscribe_answerer();
8921
8922        let (tx, resp_rx) = tokio::sync::oneshot::channel();
8923        handle_despawn_worker(&workspace, &project_key, "reviewer", true, tx);
8924        let result = resp_rx.await.expect("result");
8925
8926        assert!(
8927            matches!(
8928                result,
8929                crate::protocol::DespawnResult::Despawned { worktree_cleanup_warning: Some(_), .. }
8930            ),
8931            "git still reports its own failure to the caller: {result:?}"
8932        );
8933        assert_eq!(drain_removed_dispositions(&mut rx), vec![WorktreeDisposition::Removed]);
8934    }
8935
8936    /// The sync spawn rollback runs before claude was ever launched, so
8937    /// `--worktree <label>` never reached a process and there is no
8938    /// worktree for the toast to point at. The `ghost` project is
8939    /// advertised by the `list_projects` overlay but absent from
8940    /// `config.projects`, which is what fails the agent-handle lookup
8941    /// without spawning a subprocess.
8942    #[tokio::test]
8943    async fn spawn_rollback_reports_no_worktree_to_preserve() {
8944        let config = tempdir().expect("config tempdir");
8945        write_forge_toml(config.path(), FIXTURE_PROJECT_PATH);
8946        let workspace =
8947            Arc::new(Workspace::new_for_test(config.path().to_owned()).expect("workspace new"));
8948
8949        let repo = tempdir().expect("repo tempdir");
8950        run_git(repo.path(), &["init", "-q"]);
8951        workspace.seed_test_project("ghost", &repo.path().to_string_lossy());
8952        let project = workspace
8953            .list_projects()
8954            .into_iter()
8955            .find(|v| v.name == "ghost")
8956            .expect("seeded project present")
8957            .key;
8958        let mut rx = workspace.subscribe_answerer();
8959
8960        let (tx, resp_rx) = tokio::sync::oneshot::channel();
8961        handle_spawn_worker(
8962            &workspace,
8963            project.clone(),
8964            WorkerSpawnArgs {
8965                label: "reviewer".to_owned(),
8966                charter: "charter".to_owned(),
8967                kick: None,
8968                resume_kick: None,
8969                interactive: false,
8970                mcp_families: None,
8971                cwd: None,
8972            },
8973            SessionSlot::from_str_for_test("lead-uuid"),
8974            None,
8975            false,
8976            tx,
8977        );
8978        let err = resp_rx.await.expect("reply channel").expect_err("the agent spawn fails");
8979        // Failing earlier - at the project guard, before any insert -
8980        // would satisfy the emptiness check below without the rollback
8981        // ever running.
8982        assert!(
8983            err.contains("agent spawn failed"),
8984            "the rollback path must be the one taken: {err}"
8985        );
8986
8987        assert!(workspace.list_live_workers(&project).is_empty(), "the entry rolled back");
8988        assert_eq!(
8989            drain_removed_dispositions(&mut rx),
8990            vec![WorktreeDisposition::Absent],
8991            "nothing was created, so the toast must not name a worktree",
8992        );
8993    }
8994
8995    /// The despawn keeps a `worktree-<label>` branch the worker committed
8996    /// to, and names it in a warning rather than discarding the commits.
8997    #[tokio::test]
8998    async fn despawn_keeps_a_worktree_branch_holding_unique_commits() {
8999        let (workspace, project_key, wt, repo, _config) = git_despawn_fixture("reviewer");
9000        std::fs::write(wt.join("work.txt"), "a worker committed here").expect("write work");
9001        run_git(&wt, &["add", "."]);
9002        run_git(&wt, &["commit", "-q", "-m", "real work"]);
9003
9004        let (tx, rx) = tokio::sync::oneshot::channel();
9005        handle_despawn_worker(&workspace, &project_key, "reviewer", false, tx);
9006        let result = rx.await.expect("result");
9007        let crate::protocol::DespawnResult::Despawned {
9008            worktree_cleanup_warning: None,
9009            branch_cleanup_warning: Some(warning),
9010        } = result
9011        else {
9012            panic!("despawn succeeds with a branch warning, got {result:?}");
9013        };
9014        assert!(warning.contains("worktree-reviewer"), "warning names the branch: {warning}");
9015        assert!(
9016            warning.contains("1 commit reachable"),
9017            "warning names the commit count, not just a digit from the sha: {warning}"
9018        );
9019        assert!(warning.contains("git log"), "warning is executable: {warning}");
9020        assert!(!wt.exists(), "the worktree is still removed - the request succeeded");
9021        assert!(
9022            branch_exists(repo.path(), "worktree-reviewer"),
9023            "the branch with unique commits survives"
9024        );
9025    }
9026
9027    /// The two quiet arms: nothing to reap says nothing, an unreadable repo
9028    /// warns rather than passing for a clean despawn.
9029    #[test]
9030    fn reap_worker_branch_warns_only_when_it_cannot_verify() {
9031        let not_a_repo = tempdir().expect("tempdir");
9032        let warning =
9033            reap_worker_branch(not_a_repo.path(), "reviewer").expect("an unreadable repo warns");
9034        assert!(warning.contains("could not verify"), "names the real cause: {warning}");
9035
9036        let repo = tempdir().expect("repo tempdir");
9037        run_git(repo.path(), &["init", "-q"]);
9038        assert!(
9039            reap_worker_branch(repo.path(), "reviewer").is_none(),
9040            "no such branch is silent, not a spurious warning on every despawn",
9041        );
9042    }
9043
9044    /// The reap targets `worktree-<label>` by convention, never whatever the
9045    /// worktree has checked out - a worker that made its own branch has real
9046    /// work on that one.
9047    #[tokio::test]
9048    async fn despawn_reaps_the_conventional_branch_not_the_checked_out_one() {
9049        let (workspace, project_key, wt, repo, _config) = git_despawn_fixture("reviewer");
9050        run_git(&wt, &["checkout", "-q", "-b", "feature-x"]);
9051        std::fs::write(wt.join("work.txt"), "the worker's real work").expect("write work");
9052        run_git(&wt, &["add", "."]);
9053        run_git(&wt, &["commit", "-q", "-m", "real work"]);
9054
9055        let (tx, rx) = tokio::sync::oneshot::channel();
9056        handle_despawn_worker(&workspace, &project_key, "reviewer", false, tx);
9057        let result = rx.await.expect("result");
9058        assert!(
9059            matches!(
9060                result,
9061                crate::protocol::DespawnResult::Despawned {
9062                    worktree_cleanup_warning: None,
9063                    branch_cleanup_warning: None,
9064                }
9065            ),
9066            "the conventional branch is pristine, so nothing is kept: {result:?}"
9067        );
9068        assert!(
9069            !branch_exists(repo.path(), "worktree-reviewer"),
9070            "the conventional branch is reaped"
9071        );
9072        assert!(branch_exists(repo.path(), "feature-x"), "the worker's own branch survives");
9073    }
9074
9075    fn review_thread(id: &str) -> forge_primitives::review::ReviewThread {
9076        use forge_primitives::review::{
9077            ReviewAnchor, ReviewAuthor, ReviewComment, ReviewSide, ReviewStatus, ReviewThread,
9078        };
9079        ReviewThread {
9080            id: id.to_owned(),
9081            anchor: ReviewAnchor {
9082                path: "src/x.rs".to_owned(),
9083                side: ReviewSide::New,
9084                line: 1,
9085                content_hash: 1,
9086                context: Vec::new(),
9087                base_ref: "main".to_owned(),
9088            },
9089            comments: vec![ReviewComment {
9090                author: ReviewAuthor::User,
9091                text: "note".to_owned(),
9092                at: String::new(),
9093                review_id: None,
9094            }],
9095            status: ReviewStatus::Open,
9096            created_at: "t".to_owned(),
9097            updated_at: "t".to_owned(),
9098            commit: None,
9099        }
9100    }
9101
9102    /// A worker's own feature branch outlives its worktree and is usually
9103    /// the open PR, so the review state keyed on it must survive the
9104    /// despawn.
9105    #[tokio::test]
9106    async fn despawn_keeps_review_state_for_a_branch_that_still_exists() {
9107        let (workspace, project_key, wt, repo, _config) = git_despawn_fixture("reviewer");
9108        run_git(&wt, &["checkout", "-q", "-b", "feature-x"]);
9109        std::fs::write(wt.join("work.txt"), "the worker's real work").expect("write work");
9110        run_git(&wt, &["add", "."]);
9111        run_git(&wt, &["commit", "-q", "-m", "real work"]);
9112        workspace.upsert_review_thread("forge", "feature-x", review_thread("a"));
9113        workspace
9114            .submit_review(
9115                "forge",
9116                "feature-x",
9117                None,
9118                &[],
9119                SessionSlot::from_str_for_test("reviewer"),
9120            )
9121            .expect("seal a review on the feature branch");
9122
9123        let (tx, rx) = tokio::sync::oneshot::channel();
9124        handle_despawn_worker(&workspace, &project_key, "reviewer", false, tx);
9125        let _ = rx.await.expect("result");
9126
9127        assert!(branch_exists(repo.path(), "feature-x"), "the branch the threads key on is live");
9128        assert_eq!(
9129            workspace.load_review_threads("forge", "feature-x").expect("load").len(),
9130            1,
9131            "a live branch keeps its review threads",
9132        );
9133        assert_eq!(
9134            workspace.load_reviews("forge", "feature-x").expect("load").len(),
9135            1,
9136            "a live branch keeps its submitted reviews",
9137        );
9138    }
9139
9140    /// A pushed branch is reaped locally because the remote holds its
9141    /// commits, so the review state has to key off the remote-tracking ref
9142    /// rather than the local head that the reap just removed.
9143    #[tokio::test]
9144    async fn despawn_keeps_review_state_for_a_branch_pushed_to_a_remote() {
9145        let (workspace, project_key, wt, repo, _config) = git_despawn_fixture("reviewer");
9146        let remote = tempdir().expect("remote tempdir");
9147        run_git(remote.path(), &["init", "-q", "--bare"]);
9148        run_git(&wt, &["remote", "add", "origin", remote.path().to_str().expect("utf8 path")]);
9149        std::fs::write(wt.join("work.txt"), "pushed work").expect("write work");
9150        run_git(&wt, &["add", "."]);
9151        run_git(&wt, &["commit", "-q", "-m", "real work"]);
9152        run_git(&wt, &["push", "-q", "-u", "origin", "worktree-reviewer"]);
9153        workspace.upsert_review_thread("forge", "worktree-reviewer", review_thread("a"));
9154
9155        let (tx, rx) = tokio::sync::oneshot::channel();
9156        handle_despawn_worker(&workspace, &project_key, "reviewer", false, tx);
9157        let _ = rx.await.expect("result");
9158
9159        assert!(
9160            !branch_exists(repo.path(), "worktree-reviewer"),
9161            "the local head is reaped - without that this passes for the wrong reason",
9162        );
9163        assert_eq!(
9164            workspace.load_review_threads("forge", "worktree-reviewer").expect("load").len(),
9165            1,
9166            "a branch still on the remote keeps its review threads",
9167        );
9168    }
9169
9170    /// Despawning a git worker deletes both the review threads AND the
9171    /// submitted reviews for the torn-down worktree's branch once that
9172    /// branch is gone, leaving other branches' rows intact (else a reused
9173    /// branch inherits phantoms).
9174    #[tokio::test]
9175    async fn despawn_deletes_that_branch_reviews_and_threads() {
9176        let (workspace, project_key, wt, repo, _config) = git_despawn_fixture("reviewer");
9177        let branch = forge_agent::env::worktree::worktree_branch(&wt).expect("worktree branch");
9178        let thread = review_thread;
9179        workspace.upsert_review_thread("forge", &branch, thread("a"));
9180        workspace.upsert_review_thread("forge", "survivor", thread("b"));
9181        // Seal a review on each branch (empty thread set just mints the row).
9182        let origin = SessionSlot::from_str_for_test("reviewer");
9183        workspace
9184            .submit_review("forge", &branch, None, &[], origin.clone())
9185            .expect("seal torn-down review");
9186        workspace
9187            .submit_review("forge", "survivor", None, &[], origin)
9188            .expect("seal survivor review");
9189
9190        let (tx, rx) = tokio::sync::oneshot::channel();
9191        handle_despawn_worker(&workspace, &project_key, "reviewer", false, tx);
9192        let _ = rx.await.expect("result");
9193
9194        assert!(!branch_exists(repo.path(), &branch), "the branch the threads key on is gone");
9195        assert!(
9196            workspace.load_review_threads("forge", &branch).expect("load").is_empty(),
9197            "the torn-down branch's threads are cleaned",
9198        );
9199        assert!(
9200            workspace.load_reviews("forge", &branch).expect("load").is_empty(),
9201            "the torn-down branch's reviews are cleaned",
9202        );
9203        assert_eq!(
9204            workspace.load_review_threads("forge", "survivor").expect("load").len(),
9205            1,
9206            "another branch's threads survive",
9207        );
9208        assert_eq!(
9209            workspace.load_reviews("forge", "survivor").expect("load").len(),
9210            1,
9211            "another branch's reviews survive",
9212        );
9213    }
9214
9215    /// A detached-HEAD worktree has no resolvable branch, so despawn skips
9216    /// review-thread cleanup gracefully (warns, no panic) and leaves
9217    /// unrelated branches' threads intact.
9218    #[tokio::test]
9219    async fn despawn_with_detached_head_skips_cleanup_gracefully() {
9220        let (workspace, project_key, wt, _repo, _config) = git_despawn_fixture("reviewer");
9221        run_git(&wt, &["checkout", "--detach"]);
9222        workspace.upsert_review_thread("forge", "survivor", review_thread("a"));
9223
9224        let (tx, rx) = tokio::sync::oneshot::channel();
9225        handle_despawn_worker(&workspace, &project_key, "reviewer", false, tx);
9226        let _ = rx.await.expect("result");
9227
9228        assert_eq!(
9229            workspace.load_review_threads("forge", "survivor").expect("load").len(),
9230            1,
9231            "cleanup skipped gracefully, unrelated threads intact",
9232        );
9233    }
9234
9235    /// A dirty worktree without `force` BLOCKS the despawn: nothing is
9236    /// torn down, the worker stays live, the worktree is intact.
9237    #[tokio::test]
9238    async fn despawn_dirty_worktree_without_force_blocks() {
9239        let (workspace, project_key, wt, _repo, _config) = git_despawn_fixture("reviewer");
9240        std::fs::write(wt.join("scratch.txt"), "uncommitted").expect("write scratch");
9241        let (tx, rx) = tokio::sync::oneshot::channel();
9242        handle_despawn_worker(&workspace, &project_key, "reviewer", false, tx);
9243        let result = rx.await.expect("result");
9244        assert!(
9245            matches!(result, crate::protocol::DespawnResult::Blocked { .. }),
9246            "dirty worktree blocks without force: {result:?}"
9247        );
9248        assert_eq!(
9249            workspace.list_live_workers(&project_key).len(),
9250            1,
9251            "worker stays live when blocked"
9252        );
9253        assert!(wt.exists(), "worktree intact when blocked");
9254        assert!(
9255            !workspace.despawn_cleanup_pending(&project_key, "reviewer"),
9256            "a blocked despawn reserves nothing: no cleanup was handed off",
9257        );
9258    }
9259
9260    /// A dirty worktree WITH `force` tears down + discards the worktree.
9261    #[tokio::test]
9262    async fn despawn_dirty_worktree_force_discards() {
9263        let (workspace, project_key, wt, _repo, _config) = git_despawn_fixture("reviewer");
9264        std::fs::write(wt.join("scratch.txt"), "uncommitted").expect("write scratch");
9265        let (tx, rx) = tokio::sync::oneshot::channel();
9266        handle_despawn_worker(&workspace, &project_key, "reviewer", true, tx);
9267        let result = rx.await.expect("result");
9268        assert!(
9269            matches!(result, crate::protocol::DespawnResult::Despawned { .. }),
9270            "force despawns a dirty worktree: {result:?}"
9271        );
9272        assert!(workspace.list_live_workers(&project_key).is_empty(), "worker removed under force");
9273        assert!(!wt.exists(), "worktree discarded under force");
9274    }
9275
9276    /// A delivery that lands while the seat is mid-turn is held for the
9277    /// turn's end rather than written into the running turn, where the CLI
9278    /// would fold it in as a queued_command attachment with no standing of
9279    /// its own.
9280    #[tokio::test]
9281    async fn a_mid_turn_delivery_is_held_for_the_turn_end() {
9282        let (workspace, _rx) = Workspace::testing_stub();
9283        workspace.seed_test_project("forge", "/tmp/forge");
9284        let project = workspace
9285            .list_projects()
9286            .into_iter()
9287            .find(|v| v.name == "forge")
9288            .expect("seeded project")
9289            .key;
9290        let worker_key = SessionSlot::worker("TestOrg", "forge", "builder");
9291        workspace
9292            .insert_live_worker(&project, connected_worker_entry(&worker_key, "builder-session"));
9293        workspace.mark_session_connected_for_test(&worker_key, "builder-session");
9294        // The seat is mid-turn: a prompt is in flight at the CLI.
9295        workspace
9296            .domain_session_for(&worker_key)
9297            .expect("connected worker has a domain")
9298            .lock()
9299            .turn_pending = true;
9300        workspace.enable_test_dispatch_intercept();
9301
9302        let (tx, rx) = tokio::sync::oneshot::channel();
9303        handle_deliver_worker_prompt(
9304            &workspace,
9305            &SessionSlot::lead("TestOrg", "forge"),
9306            &project,
9307            "builder",
9308            fixture_wrapped(),
9309            Some(tx),
9310        );
9311
9312        rx.await
9313            .expect("the handler answers")
9314            .expect("a held prompt is accepted: the turn it waits on will deliver it");
9315        assert!(
9316            workspace.drain_test_dispatch_buffer().is_empty(),
9317            "nothing is written into the running turn",
9318        );
9319        assert_eq!(
9320            workspace.take_turn_held_for_slot(&worker_key).len(),
9321            1,
9322            "the prompt is held for the turn end instead",
9323        );
9324    }
9325
9326    /// The gate reads only what a `Result` clears: a session whose
9327    /// `runtime_state` alone says Running (that mirror is cleared only on
9328    /// Connected/ConnectionFailed) must not hold a delivery, or every later
9329    /// message would wait on a turn-end signal that never comes.
9330    #[tokio::test]
9331    async fn a_sticky_runtime_state_does_not_hold_a_delivery() {
9332        let (workspace, _rx) = Workspace::testing_stub();
9333        workspace.seed_test_project("forge", "/tmp/forge");
9334        let project = workspace
9335            .list_projects()
9336            .into_iter()
9337            .find(|v| v.name == "forge")
9338            .expect("seeded project")
9339            .key;
9340        let worker_key = SessionSlot::worker("TestOrg", "forge", "builder");
9341        workspace
9342            .insert_live_worker(&project, connected_worker_entry(&worker_key, "builder-session"));
9343        workspace.mark_session_connected_for_test(&worker_key, "builder-session");
9344        // No turn pending and no turn open - the Result-clearable signals
9345        // are both down - but the sticky mirror reads Running.
9346        workspace
9347            .domain_session_for(&worker_key)
9348            .expect("connected worker has a domain")
9349            .lock()
9350            .runtime_state = Some(forge_primitives::RuntimeSessionState::Running);
9351        workspace.enable_test_dispatch_intercept();
9352
9353        let (tx, rx) = tokio::sync::oneshot::channel();
9354        handle_deliver_worker_prompt(
9355            &workspace,
9356            &SessionSlot::lead("TestOrg", "forge"),
9357            &project,
9358            "builder",
9359            fixture_wrapped(),
9360            Some(tx),
9361        );
9362
9363        rx.await
9364            .expect("the handler answers")
9365            .expect("a delivery with no turn running is accepted");
9366        assert_eq!(
9367            workspace.drain_test_dispatch_buffer().len(),
9368            1,
9369            "the delivery is written now, not held behind a stale runtime_state",
9370        );
9371        assert!(
9372            workspace.take_turn_held_for_slot(&worker_key).is_empty(),
9373            "and nothing was held for a turn end that will not come",
9374        );
9375    }
9376
9377    /// `handle_deliver_worker_prompt` refuses a target label with no
9378    /// live worker, and says so down the reply channel. Mirrors the
9379    /// close_worker_unknown branch - the upstream Tool gate
9380    /// (agents__send_message facade) rejects synchronously; the spawn
9381    /// handler is defence in depth. Silence here is the defect this pins:
9382    /// the sender's ack is built from this answer, so a drop nobody
9383    /// hears is a delivery claimed that never happened.
9384    #[tokio::test]
9385    async fn deliver_worker_prompt_unknown_label_is_refused_down_the_reply_channel() {
9386        let (workspace, _rx) = Workspace::testing_stub();
9387        let project = ProjectKey::new("forge");
9388        let caller = SessionSlot::from_str_for_test("caller-1");
9389        let (tx, rx) = tokio::sync::oneshot::channel();
9390
9391        handle_deliver_worker_prompt(
9392            &workspace,
9393            &caller,
9394            &project,
9395            "missing",
9396            fixture_wrapped(),
9397            Some(tx),
9398        );
9399        let answer = rx.await.expect("the handler answers");
9400        let Err(reason) = answer else {
9401            panic!("a label with no live worker must be refused, got {answer:?}");
9402        };
9403        assert!(
9404            reason.contains("missing"),
9405            "the refusal names the label the caller addressed: {reason}"
9406        );
9407    }
9408
9409    /// A worker addressed before it Connects (session_id still None)
9410    /// must have the prompt parked for its label, not dropped - its
9411    /// Connected handler drains the bucket.
9412    #[tokio::test]
9413    async fn deliver_to_unconnected_worker_buffers_the_prompt() {
9414        let (workspace, _rx) = Workspace::testing_stub();
9415        workspace.seed_test_project("forge", "/tmp/deliver-unconnected");
9416        let project = workspace
9417            .list_projects()
9418            .into_iter()
9419            .find(|v| v.name == "forge")
9420            .expect("seeded project")
9421            .key;
9422        // The slot production derives for the seeded project's worker.
9423        let worker_key = SessionSlot::worker("TestOrg", "forge", "builder");
9424        workspace.insert_live_worker(
9425            &project,
9426            crate::mcp::workers::types::WorkerEntry {
9427                label: "builder".into(),
9428                charter: "c".into(),
9429                slot: worker_key.clone(),
9430                session_id: None,
9431                status: forge_primitives::WorkerLiveness::Spawning,
9432                spawned_at: std::time::SystemTime::UNIX_EPOCH,
9433                spawned_by: SessionSlot::lead("TestOrg", "forge"),
9434                needs_tag: false,
9435                is_git_repo_at_spawn: false,
9436                cwd: None,
9437                diagnostic: None,
9438                kick: None,
9439            },
9440        );
9441        // Pre-Connect DomainSession: session_id is None.
9442        let domain = std::sync::Arc::new(parking_lot::Mutex::new(
9443            crate::domain_session::DomainSession::new(worker_key.clone(), None),
9444        ));
9445        workspace.domain_handles.lock().insert(worker_key.clone(), domain.clone());
9446
9447        let (tx, rx) = tokio::sync::oneshot::channel();
9448        handle_deliver_worker_prompt(
9449            &workspace,
9450            &SessionSlot::lead("TestOrg", "forge"),
9451            &project,
9452            "builder",
9453            fixture_wrapped(),
9454            Some(tx),
9455        );
9456        rx.await.expect("the handler answers").expect("a parked prompt is accepted, not refused");
9457
9458        assert_eq!(
9459            workspace
9460                .parked_by_slot
9461                .lock()
9462                .get(&crate::SessionSlot::worker("TestOrg", "forge", "builder"))
9463                .map_or(0, |parked| parked.peer.len()),
9464            1,
9465            "prompt parked for the worker's Connected drain, not dropped"
9466        );
9467    }
9468
9469    /// The (source, text) of the one prompt a handler dispatched, read off
9470    /// the workspace's dispatch intercept: the pre-routing capture of the
9471    /// Command `record_queued` labels the queued row from.
9472    fn drain_the_prompt(workspace: &Workspace) -> (PromptSource, String) {
9473        workspace
9474            .drain_test_dispatch_buffer()
9475            .into_iter()
9476            .find_map(|cmd| match cmd {
9477                crate::protocol::Command::PromptUnder { source, text, .. } => Some((source, text)),
9478                _ => None,
9479            })
9480            .expect("the peer envelope is dispatched as a prompt")
9481    }
9482
9483    /// A worker entry for `label`, connected, so the delivery dispatches
9484    /// now rather than parking on the pre-Connect bucket.
9485    fn connected_worker_entry(
9486        slot: &SessionSlot,
9487        session_id: &str,
9488    ) -> crate::mcp::workers::types::WorkerEntry {
9489        crate::mcp::workers::types::WorkerEntry {
9490            label: slot.label().to_owned(),
9491            charter: "c".into(),
9492            slot: slot.clone(),
9493            session_id: Some(forge_primitives::SessionId::new(session_id)),
9494            status: forge_primitives::WorkerLiveness::Running,
9495            spawned_at: std::time::SystemTime::UNIX_EPOCH,
9496            spawned_by: SessionSlot::lead("TestOrg", "forge"),
9497            needs_tag: false,
9498            is_git_repo_at_spawn: false,
9499            cwd: None,
9500            diagnostic: None,
9501            kick: None,
9502        }
9503    }
9504
9505    /// A peer envelope delivered to a sibling worker is recorded with the
9506    /// peer source, not forge's: the prose IS a peer envelope, and every
9507    /// other peer path labels its queued row `peer`.
9508    #[tokio::test]
9509    async fn deliver_to_a_worker_records_the_peer_source() {
9510        let (workspace, _rx) = Workspace::testing_stub();
9511        workspace.seed_test_project("forge", "/tmp/deliver-worker-peer-source");
9512        let project = workspace
9513            .list_projects()
9514            .into_iter()
9515            .find(|v| v.name == "forge")
9516            .expect("seeded project")
9517            .key;
9518        let worker_key = SessionSlot::worker("TestOrg", "forge", "builder");
9519        workspace
9520            .insert_live_worker(&project, connected_worker_entry(&worker_key, "builder-session"));
9521        workspace.mark_session_connected_for_test(&worker_key, "builder-session");
9522        workspace.enable_test_dispatch_intercept();
9523
9524        handle_deliver_worker_prompt(
9525            &workspace,
9526            &SessionSlot::lead("TestOrg", "forge"),
9527            &project,
9528            "builder",
9529            fixture_wrapped(),
9530            None,
9531        );
9532
9533        let (source, text) = drain_the_prompt(&workspace);
9534        assert_eq!(
9535            source,
9536            PromptSource::Peer,
9537            "a worker-to-worker peer envelope is recorded with the peer source",
9538        );
9539        assert!(text.contains("[Message id="), "the dispatched text is the peer envelope: {text}");
9540    }
9541
9542    /// The same for the worker-to-lead path: the lead's queued row must
9543    /// read the peer source too, since the envelope is a peer message.
9544    #[tokio::test]
9545    async fn deliver_to_the_lead_records_the_peer_source() {
9546        let (workspace, _rx) = Workspace::testing_stub();
9547        let lead = SessionSlot::lead("TestOrg", "forge");
9548        // The handler drops a lead that is not pooled, and a connected
9549        // domain keeps the delivery a dispatch rather than a park.
9550        workspace.seed_test_running_session_id(&lead, "lead-session");
9551        workspace.mark_session_connected_for_test(&lead, "lead-session");
9552        workspace.enable_test_dispatch_intercept();
9553
9554        handle_deliver_worker_prompt_to_lead(
9555            &workspace,
9556            &SessionSlot::worker("TestOrg", "forge", "builder"),
9557            &lead,
9558            fixture_wrapped(),
9559            None,
9560        );
9561
9562        let (source, text) = drain_the_prompt(&workspace);
9563        assert_eq!(
9564            source,
9565            PromptSource::Peer,
9566            "a worker-to-lead peer envelope is recorded with the peer source",
9567        );
9568        assert!(text.contains("[Message id="), "the dispatched text is the peer envelope: {text}");
9569    }
9570
9571    /// `handle_deliver_worker_prompt` for a worker carrying
9572    /// `needs_tag = true` must kick off an opportunistic tag-write
9573    /// retry. The worker was spawned idle (no initial_prompt), claude
9574    /// only writes the JSONL once the first turn arrives, and this
9575    /// handler runs as that first turn is being routed.
9576    ///
9577    /// Verifies the integration end-to-end: pre-seed a Running worker
9578    /// with `needs_tag = true`, install the testing-stub agent (its
9579    /// `config_dir` resolves to `/tmp/forge-testing-stub`), seed the
9580    /// JSONL at the path `tag_session` expects, and assert that after
9581    /// `handle_deliver_worker_prompt` fires the entry's `needs_tag`
9582    /// flips to `false` (the retry succeeded). Uses a unique session_id
9583    /// to avoid collision with other parallel tests that hit the
9584    /// shared stub config_dir.
9585    #[tokio::test]
9586    async fn deliver_prompt_kicks_opportunistic_tag_retry() {
9587        // Set up a real Workspace::new with a project pointing at a
9588        // tempdir so list_projects returns the project view we need.
9589        let toml_dir = tempdir().expect("toml dir");
9590        let project_root = tempdir().expect("project root");
9591        let project_path = project_root.path().to_string_lossy().into_owned();
9592        fs::write(
9593            forge_toml_path(toml_dir.path()),
9594            format!(
9595                r#"
9596[[orgs]]
9597name = "Default"
9598accounts = ["Stargate"]
9599
9600[[orgs.projects]]
9601name = "forge"
9602path = "{project_path}"
9603
9604[[accounts]]
9605display_name = "Stargate"
9606token = "t"
9607models = ["claude-sonnet-5"]
9608provider = "anthropic"
9609"#,
9610            ),
9611        )
9612        .expect("write forge.toml");
9613        let workspace = Arc::new(Workspace::new_for_test(toml_dir.path().to_owned()).expect("new"));
9614
9615        // Resolve the project's key + path from list_projects (matches
9616        // what the spawn handler will look up).
9617        let project_view = workspace
9618            .list_projects()
9619            .into_iter()
9620            .find(|view| view.name == "forge")
9621            .expect("forge project");
9622        let project_key = project_view.key.clone();
9623
9624        // Unique session_id so concurrent test runs don't collide on
9625        // the shared /tmp/forge-testing-stub. Must be a valid UUID
9626        // since `tag_session` rejects non-UUID with `MessageParse`.
9627        let session_id = uuid::Uuid::new_v4().hyphenated().to_string();
9628        // The slot production derives for the worker under this project.
9629        let session_key = SessionSlot::worker(&project_view.org, &project_view.name, "idle");
9630
9631        // Install the testing stub so config_dir_for resolves (to
9632        // /tmp/forge-testing-stub via the bridge's default config_dir).
9633        let _agent_rx = workspace.install_testing_stub(&session_key);
9634        // A Running worker has completed its Connected handshake, so
9635        // stamp session_id - otherwise the pre-Connect buffer guard in
9636        // handle_deliver_worker_prompt would park the prompt and the
9637        // tag retry (asserted below) would never fire.
9638        if let Some(domain) = workspace.domain_session_for(&session_key) {
9639            domain.lock().session_id = Some(forge_primitives::SessionId::new(&session_id));
9640        }
9641
9642        // Pre-seed the worker entry: Running but needs_tag = true,
9643        // matching the post-deferred-NotFound state from the
9644        // `apply_worker_tag_or_rollback` deferred branch.
9645        workspace.insert_live_worker(
9646            &project_key,
9647            crate::mcp::workers::types::WorkerEntry {
9648                label: "idle".into(),
9649                charter: "c".into(),
9650                slot: session_key.clone(),
9651                // The entry carries the id the worker runs under: the
9652                // opportunistic tag retry reads it from here.
9653                session_id: Some(forge_primitives::SessionId::new(&session_id)),
9654                status: forge_primitives::WorkerLiveness::Running,
9655                spawned_at: std::time::SystemTime::UNIX_EPOCH,
9656                spawned_by: SessionSlot::lead(&project_view.org, &project_view.name),
9657                needs_tag: true,
9658                is_git_repo_at_spawn: false,
9659                cwd: None,
9660                diagnostic: None,
9661                kick: None,
9662            },
9663        );
9664
9665        // Pre-create the JSONL at the path the retry will look for.
9666        // /tmp/forge-testing-stub/projects/<sanitized>/<session_id>.jsonl
9667        let stub_config_dir = std::path::PathBuf::from("/tmp/forge-testing-stub");
9668        let sanitized =
9669            forge_agent::userdata::catalog::scan::project_key_for_directory(Some(&project_path));
9670        let projects_dir = forge_sdk::projects_dir_for(&stub_config_dir).join(&sanitized);
9671        fs::create_dir_all(&projects_dir).expect("project dir");
9672        let jsonl_path = projects_dir.join(format!("{session_id}.jsonl"));
9673        fs::write(&jsonl_path, "").expect("seed jsonl");
9674
9675        // Fire the deliver. The handler observes needs_tag = true,
9676        // looks up the project view's path, and kicks the retry task.
9677        // It also dispatches Command::Prompt which the stub agent
9678        // accepts (the side-effect we don't assert on here).
9679        let caller = SessionSlot::from_str_for_test("caller-1");
9680        handle_deliver_worker_prompt(
9681            &workspace,
9682            &caller,
9683            &project_key,
9684            "idle",
9685            fixture_wrapped(),
9686            None,
9687        );
9688
9689        // Wait briefly for the spawned task to finish the retry +
9690        // status update. The retry should succeed on first attempt
9691        // because the JSONL exists.
9692        let needs_tag = tokio::time::timeout(std::time::Duration::from_secs(2), async {
9693            loop {
9694                let entries = workspace.list_live_workers(&project_key);
9695                let entry = entries.iter().find(|e| e.slot == session_key);
9696                if let Some(entry) = entry
9697                    && !entry.needs_tag
9698                {
9699                    return false;
9700                }
9701                tokio::time::sleep(std::time::Duration::from_millis(20)).await;
9702            }
9703        })
9704        .await
9705        .expect("needs_tag must clear within budget");
9706        assert!(!needs_tag, "needs_tag cleared after opportunistic retry succeeded");
9707
9708        // Confirm the tag actually landed on disk.
9709        let body = fs::read_to_string(&jsonl_path).expect("read jsonl");
9710        assert!(
9711            body.contains("\"tag\":\"forge:worker:idle\""),
9712            "tag row appended on opportunistic retry: {body:?}"
9713        );
9714
9715        // Clean up the shared stub config_dir entries we created so
9716        // subsequent parallel runs aren't polluted.
9717        let _ = fs::remove_file(&jsonl_path);
9718    }
9719
9720    /// The opportunistic tag retry composes the tree from the worker's
9721    /// OWN tree, not the project's: a worker the spawn pointed at another
9722    /// repository keeps its JSONL under that repository's worktree, so a
9723    /// retry aimed at the project path never finds it and the worker
9724    /// stays untagged - which is what the resume-by-tag path reads.
9725    #[tokio::test]
9726    async fn deliver_prompt_kicks_the_tag_retry_at_the_workers_own_tree() {
9727        let toml_dir = tempdir().expect("toml dir");
9728        let project_root = tempdir().expect("project root");
9729        let project_path = project_root.path().to_string_lossy().into_owned();
9730        fs::write(
9731            forge_toml_path(toml_dir.path()),
9732            format!(
9733                r#"
9734[[orgs]]
9735name = "Default"
9736accounts = ["Stargate"]
9737
9738[[orgs.projects]]
9739name = "forge"
9740path = "{project_path}"
9741
9742[[accounts]]
9743display_name = "Stargate"
9744token = "t"
9745models = ["claude-sonnet-5"]
9746provider = "anthropic"
9747"#,
9748            ),
9749        )
9750        .expect("write forge.toml");
9751        let workspace = Arc::new(Workspace::new_for_test(toml_dir.path().to_owned()).expect("new"));
9752        let project_view = workspace
9753            .list_projects()
9754            .into_iter()
9755            .find(|view| view.name == "forge")
9756            .expect("forge project");
9757        let project_key = project_view.key.clone();
9758
9759        // The worker's own tree: a sibling repository the spawn pointed
9760        // it at, with the worktree claude's `--worktree` would have
9761        // forked there.
9762        let sibling = tempdir().expect("sibling repo");
9763        let sibling_root = std::fs::canonicalize(sibling.path()).expect("canonical sibling");
9764        let worker_wt = sibling_root.join(".claude").join("worktrees").join("idle");
9765
9766        let session_id = uuid::Uuid::new_v4().hyphenated().to_string();
9767        let session_key = SessionSlot::worker(&project_view.org, &project_view.name, "idle");
9768        let _agent_rx = workspace.install_testing_stub(&session_key);
9769        if let Some(domain) = workspace.domain_session_for(&session_key) {
9770            domain.lock().session_id = Some(forge_primitives::SessionId::new(&session_id));
9771        }
9772        workspace.insert_live_worker(
9773            &project_key,
9774            crate::mcp::workers::types::WorkerEntry {
9775                label: "idle".into(),
9776                charter: "c".into(),
9777                slot: session_key.clone(),
9778                session_id: Some(forge_primitives::SessionId::new(&session_id)),
9779                status: forge_primitives::WorkerLiveness::Running,
9780                spawned_at: std::time::SystemTime::UNIX_EPOCH,
9781                spawned_by: SessionSlot::lead(&project_view.org, &project_view.name),
9782                needs_tag: true,
9783                is_git_repo_at_spawn: true,
9784                cwd: Some(sibling_root.clone()),
9785                diagnostic: None,
9786                kick: None,
9787            },
9788        );
9789
9790        // The JSONL sits where the worker's own tree puts it.
9791        let stub_config_dir = std::path::PathBuf::from("/tmp/forge-testing-stub");
9792        std::fs::create_dir_all(&worker_wt).expect("worker worktree dir");
9793        let sanitized = forge_agent::userdata::catalog::scan::project_key_for_directory(Some(
9794            &worker_wt.to_string_lossy(),
9795        ));
9796        let projects_dir = forge_sdk::projects_dir_for(&stub_config_dir).join(&sanitized);
9797        fs::create_dir_all(&projects_dir).expect("project dir");
9798        let jsonl_path = projects_dir.join(format!("{session_id}.jsonl"));
9799        fs::write(&jsonl_path, "").expect("seed jsonl");
9800
9801        let caller = SessionSlot::from_str_for_test("caller-1");
9802        handle_deliver_worker_prompt(
9803            &workspace,
9804            &caller,
9805            &project_key,
9806            "idle",
9807            fixture_wrapped(),
9808            None,
9809        );
9810
9811        let cleared = tokio::time::timeout(std::time::Duration::from_secs(2), async {
9812            loop {
9813                let entries = workspace.list_live_workers(&project_key);
9814                let tagged = entries
9815                    .iter()
9816                    .find(|e| e.slot == session_key)
9817                    .is_some_and(|entry| !entry.needs_tag);
9818                if tagged {
9819                    return true;
9820                }
9821                tokio::time::sleep(std::time::Duration::from_millis(20)).await;
9822            }
9823        })
9824        .await
9825        .expect("needs_tag must clear within budget");
9826        assert!(cleared);
9827
9828        let body = fs::read_to_string(&jsonl_path).expect("read jsonl");
9829        assert!(
9830            body.contains("\"tag\":\"forge:worker:idle\""),
9831            "the tag lands in the worker's own tree: {body:?}",
9832        );
9833
9834        let _ = fs::remove_file(&jsonl_path);
9835    }
9836
9837    /// `build_worker_extra_args` returns `[("worktree", Some(label))]`
9838    /// when the project path is a git repo so the worker spawn picks
9839    /// up `--worktree=<label>` and lands inside an auto-created
9840    /// `<repo>/.claude/worktrees/<label>/`. The helper takes a
9841    /// pre-computed `is_git_repo` boolean (caller probes the path
9842    /// once and reuses the result for both the WorkerEntry flag and
9843    /// this arg list); the test seeds the bool via the same
9844    /// `forge_agent::env::worktree::is_git_repo` probe the spawn
9845    /// path uses so the two layers stay in sync.
9846    #[test]
9847    fn worker_in_git_repo_gets_worktree_flag() {
9848        let dir = tempdir().expect("tempdir");
9849        std::process::Command::new("git")
9850            .args(["init", "-q"])
9851            .current_dir(dir.path())
9852            .status()
9853            .expect("git init");
9854        let is_git = forge_agent::env::worktree::is_git_repo(dir.path());
9855        assert!(is_git, "freshly-initialised tempdir must register as git repo");
9856        let args = build_worker_extra_args(is_git, "reviewer", false);
9857        assert!(
9858            args.iter()
9859                .any(|(flag, value)| flag == "worktree" && value.as_deref() == Some("reviewer")),
9860            "expected (\"worktree\", Some(\"reviewer\")) in {args:?}"
9861        );
9862    }
9863
9864    /// `build_worker_extra_args` returns no `worktree` entry when the
9865    /// project path isn't a git repo - the worker just spawns into
9866    /// the project's plain cwd and skips the worktree path entirely.
9867    #[test]
9868    fn worker_in_non_git_repo_gets_no_worktree_flag() {
9869        let dir = tempdir().expect("tempdir"); // empty, not a repo
9870        let is_git = forge_agent::env::worktree::is_git_repo(dir.path());
9871        assert!(!is_git, "empty tempdir must not register as git repo");
9872        let args = build_worker_extra_args(is_git, "reviewer", false);
9873        assert!(
9874            !args.iter().any(|(flag, _)| flag == "worktree"),
9875            "expected no worktree entry in {args:?}"
9876        );
9877    }
9878
9879    /// The `--disallowedTools` value in `args`, failing loudly when the
9880    /// flag is absent.
9881    fn disallowed_tools_value(args: &[(String, Option<String>)]) -> String {
9882        args.iter()
9883            .find(|(flag, _)| flag == "disallowedTools")
9884            .and_then(|(_, value)| value.clone())
9885            .expect("expected a --disallowedTools entry")
9886    }
9887
9888    /// A blocked set as whole names. Comparing substrings reports a kept
9889    /// tool as blocked - `Task` is inside `TaskCreate`, and `TaskStop`
9890    /// inside nothing here but read the same way.
9891    fn blocked_names(kind: crate::mcp::SessionKind, interactive: bool) -> Vec<String> {
9892        blocked_tools_for(kind, interactive).split(',').map(str::to_owned).collect()
9893    }
9894
9895    /// The replaced tools written out rather than read off the constant
9896    /// that produces them, so dropping one from the constant fails here.
9897    const REPLACED: [&str; 11] = [
9898        "CronCreate",
9899        "CronDelete",
9900        "CronList",
9901        "TaskCreate",
9902        "TaskGet",
9903        "TaskList",
9904        "TaskUpdate",
9905        "SendMessage",
9906        "ListAgents",
9907        "Workflow",
9908        "RemoteTrigger",
9909    ];
9910
9911    /// The change's whole point: a lead loses the tools forge has
9912    /// replaced. Before this, `--disallowedTools` was the worker path's
9913    /// flag alone and a lead ran with the CLI's full set.
9914    #[test]
9915    fn a_lead_is_blocked_from_the_tools_forge_replaced() {
9916        let blocked = blocked_names(crate::mcp::SessionKind::Lead, true);
9917        for tool in [
9918            "CronCreate",
9919            "CronDelete",
9920            "CronList",
9921            "TaskCreate",
9922            "TaskGet",
9923            "TaskList",
9924            "TaskUpdate",
9925            "SendMessage",
9926            "ListAgents",
9927            "Workflow",
9928            "RemoteTrigger",
9929        ] {
9930            assert!(blocked.iter().any(|name| name == tool), "{tool} is blocked for a lead");
9931        }
9932    }
9933
9934    /// The question is a property of the spawn, not of the kind: one
9935    /// worker may ask (its row is open) and another may not.
9936    #[test]
9937    fn ask_user_question_follows_interactive_not_kind() {
9938        for (kind, interactive, expected) in [
9939            (crate::mcp::SessionKind::Worker, true, false),
9940            (crate::mcp::SessionKind::Worker, false, true),
9941            (crate::mcp::SessionKind::Lead, true, false),
9942        ] {
9943            let blocked = blocked_names(kind, interactive);
9944            let denied = blocked.iter().any(|name| name == "AskUserQuestion");
9945            assert_eq!(
9946                denied, expected,
9947                "AskUserQuestion denial is the spawn's, not the kind's; got {blocked:?}",
9948            );
9949        }
9950    }
9951
9952    #[test]
9953    fn the_kept_tools_are_never_blocked() {
9954        for blocked in [
9955            blocked_names(crate::mcp::SessionKind::Lead, true),
9956            blocked_names(crate::mcp::SessionKind::Worker, false),
9957        ] {
9958            for tool in ["ScheduleWakeup", "Task", "TaskStop", "ReportFindings"] {
9959                assert!(!blocked.iter().any(|name| name == tool), "{tool} stays enabled");
9960            }
9961        }
9962    }
9963
9964    /// The extent of the set, not just its members: every other assertion
9965    /// here says a name is present or a kept name is absent, and a twelfth
9966    /// name appended to the constant would satisfy all of them. This is the
9967    /// assertion that fails when the decided list grows by accident.
9968    #[test]
9969    fn the_blocked_set_holds_exactly_the_decided_names() {
9970        assert_eq!(
9971            blocked_names(crate::mcp::SessionKind::Lead, true).len(),
9972            11,
9973            "a lead loses the eleven replaced tools and nothing else",
9974        );
9975        assert_eq!(
9976            blocked_names(crate::mcp::SessionKind::Worker, false).len(),
9977            14,
9978            "a worker adds the two worktree pins and the question it may not ask",
9979        );
9980        assert_eq!(
9981            blocked_names(crate::mcp::SessionKind::Worker, true).len(),
9982            13,
9983            "an interactive worker keeps the question",
9984        );
9985    }
9986
9987    /// The worker's own entry carries its own denials - the worktree pins
9988    /// and, for a session that cannot ask its row's user, the question -
9989    /// alongside the replaced tools, because one builder writes every
9990    /// name. Nothing outside the decided set is swept in with them.
9991    #[test]
9992    fn a_worker_carries_its_denials_and_the_replaced_tools_in_one_entry() {
9993        let list = blocked_names(crate::mcp::SessionKind::Worker, false);
9994        for tool in [
9995            "EnterWorktree",
9996            "ExitWorktree",
9997            "AskUserQuestion",
9998            "TaskCreate",
9999            "CronCreate",
10000            "SendMessage",
10001            "RemoteTrigger",
10002        ] {
10003            assert!(
10004                list.iter().any(|name| name == tool),
10005                "{tool} must be denied to a worker; got {list:?}"
10006            );
10007        }
10008        for tool in [
10009            "ScheduleWakeup",
10010            "Task",
10011            "TaskStop",
10012            "ReportFindings",
10013            "TaskOutput",
10014            "ToolSearch",
10015            "Monitor",
10016        ] {
10017            assert!(!list.iter().any(|name| name == tool), "{tool} stays available; got {list:?}");
10018        }
10019    }
10020
10021    /// The cold spawn - the one a project's first lead and every worker go
10022    /// through - hands its child the denial. Nothing else observes this
10023    /// path: a pooled slot never reaches a launch, and a cold one otherwise
10024    /// ends in a real subprocess, so the test stands a stub handle in for
10025    /// the one the spawn would create and reads the setting it forwards.
10026    ///
10027    /// This is a lead's launch, which is the half that used to carry no
10028    /// denial at all.
10029    #[tokio::test]
10030    async fn a_cold_spawn_hands_its_child_the_replaced_tool_denial() {
10031        let dir = tempfile::tempdir().expect("tempdir");
10032        let forge_dir = crate::config::ensure_forge_data_dir(dir.path()).expect("forge dir");
10033        std::fs::write(
10034            forge_dir.join("forge.toml"),
10035            r#"
10036[[orgs]]
10037name = "TestOrg"
10038accounts = ["acct-a"]
10039
10040[[orgs.projects]]
10041name = "proj"
10042path = "/tmp/spawn-denial"
10043model = "claude-sonnet-5"
10044
10045[[accounts]]
10046display_name = "acct-a"
10047token = "t"
10048models = ["claude-sonnet-5"]
10049provider = "anthropic"
10050"#,
10051        )
10052        .expect("write forge.toml");
10053        let ws = std::sync::Arc::new(
10054            Workspace::new_for_test(dir.path().to_owned()).expect("boot from the fixture"),
10055        );
10056        ws.seed_test_ready_account("acct-a");
10057        let (handle, mut agent_rx) = Workspace::testing_stub_handle();
10058        ws.install_test_spawn_handle(handle);
10059
10060        let _ = handle_spawn_project(&ws, "proj", SessionLaunchSettings::default());
10061
10062        let Some(forge_primitives::AgentCommand::NewSession { launch_settings, .. }) =
10063            agent_rx.try_recv().ok()
10064        else {
10065            panic!("the cold spawn has to reach a launch for anything here to be observable");
10066        };
10067        let denied = launch_settings
10068            .get("extra_args")
10069            .and_then(serde_json::Value::as_array)
10070            .and_then(|pairs| {
10071                pairs.iter().find(|pair| {
10072                    pair.get(0).and_then(serde_json::Value::as_str) == Some("disallowedTools")
10073                })
10074            })
10075            .and_then(|pair| pair.get(1).and_then(serde_json::Value::as_str))
10076            .unwrap_or_default()
10077            .to_owned();
10078        for tool in REPLACED {
10079            assert!(
10080                denied.split(',').any(|name| name == tool),
10081                "{tool} must reach a spawned session's launch; got {denied:?}",
10082            );
10083        }
10084    }
10085
10086    /// A lead's settings carry no denial of their own, so the call adds
10087    /// the flag. `--disallowedTools` used to be a worker-only flag, and a
10088    /// lead kept the CLI's tools.
10089    #[test]
10090    fn the_denial_adds_the_flag_when_there_is_none() {
10091        let mut settings = SessionLaunchSettings::default();
10092        apply_blocked_tools(&mut settings, crate::mcp::SessionKind::Lead, true);
10093        let list = disallowed_tools_value(&settings.extra_args);
10094        for tool in REPLACED {
10095            assert!(
10096                list.split(',').any(|name| name == tool),
10097                "{tool} must be denied to a lead; got {list:?}",
10098            );
10099        }
10100        assert!(
10101            !list.contains("EnterWorktree") && !list.contains("ExitWorktree"),
10102            "a lead still hops worktrees; got {list:?}",
10103        );
10104        assert!(
10105            !list.contains("AskUserQuestion"),
10106            "a lead is exactly the session that must be able to ask; got {list:?}",
10107        );
10108    }
10109
10110    /// A session whose settings already carry denials - a worker's, which
10111    /// hold the worktree tools in the same value - gains the names it does
10112    /// not already have inside that value rather than a second
10113    /// `--disallowedTools` flag for the CLI to reconcile. A name both
10114    /// writers know lands once.
10115    #[test]
10116    fn the_denial_merges_into_a_list_that_is_already_there() {
10117        let mut settings = SessionLaunchSettings {
10118            extra_args: build_worker_extra_args(false, "reviewer", false),
10119            ..SessionLaunchSettings::default()
10120        };
10121        apply_blocked_tools(&mut settings, crate::mcp::SessionKind::Worker, false);
10122        let flags =
10123            settings.extra_args.iter().filter(|(flag, _)| flag == "disallowedTools").count();
10124        assert_eq!(flags, 1, "one flag, not two; got {:?}", settings.extra_args);
10125        let list = disallowed_tools_value(&settings.extra_args);
10126        for tool in REPLACED {
10127            assert_eq!(
10128                list.split(',').filter(|name| *name == tool).count(),
10129                1,
10130                "{tool} appears once; got {list:?}",
10131            );
10132        }
10133        assert!(list.contains("EnterWorktree"), "the worker's own denial survives; got {list:?}");
10134    }
10135
10136    /// A `--disallowedTools` entry with no value is still an entry: the
10137    /// names go into it rather than a second flag being pushed beside it.
10138    #[test]
10139    fn the_denial_fills_a_valueless_entry_rather_than_adding_a_second_flag() {
10140        let mut settings = SessionLaunchSettings {
10141            extra_args: vec![("disallowedTools".to_owned(), None)],
10142            ..SessionLaunchSettings::default()
10143        };
10144        apply_blocked_tools(&mut settings, crate::mcp::SessionKind::Lead, true);
10145        assert_eq!(
10146            settings.extra_args.len(),
10147            1,
10148            "the entry that is there is filled, not duplicated: {:?}",
10149            settings.extra_args,
10150        );
10151        let list = disallowed_tools_value(&settings.extra_args);
10152        for tool in REPLACED {
10153            assert!(
10154                list.split(',').any(|name| name == tool),
10155                "{tool} must be denied; got {list:?}",
10156            );
10157        }
10158        assert!(
10159            !list.starts_with(','),
10160            "a filled entry carries no leading separator; got {list:?}"
10161        );
10162    }
10163
10164    /// Workers are pinned to their worktree (when they have one) and
10165    /// must not be able to call claude's built-in `EnterWorktree` /
10166    /// `ExitWorktree` tools to hop elsewhere. `build_worker_extra_args`
10167    /// emits a `--disallowedTools` flag carrying both tool names as a
10168    /// single comma-separated value (empirically confirmed to be
10169    /// accepted by the claude CLI's variadic `<tools...>` parser).
10170    /// This test covers the git-repo case: the `worktree` flag is
10171    /// still present, and the `disallowedTools` flag is added on top.
10172    #[test]
10173    fn worker_in_git_repo_blocks_enter_and_exit_worktree() {
10174        let is_git = true;
10175        let args = build_worker_extra_args(is_git, "reviewer", false);
10176        let blocked = args.iter().find(|(flag, _)| flag == "disallowedTools");
10177        let (_, value) = blocked.expect("expected --disallowedTools entry");
10178        let value = value.as_deref().expect("expected value for --disallowedTools");
10179        assert!(value.contains("EnterWorktree"), "EnterWorktree must be blocked, got {value:?}");
10180        assert!(value.contains("ExitWorktree"), "ExitWorktree must be blocked, got {value:?}");
10181    }
10182
10183    /// The non-git-repo case still blocks the worktree-hop tools even
10184    /// though the worker isn't running inside a worktree - the tool
10185    /// surface is uniform across project shapes so workers can't be
10186    /// nudged into surprising behaviour by the project layout.
10187    #[test]
10188    fn worker_in_non_git_repo_also_blocks_worktree_tools() {
10189        let is_git = false;
10190        let args = build_worker_extra_args(is_git, "reviewer", false);
10191        let blocked = args.iter().find(|(flag, _)| flag == "disallowedTools");
10192        let (_, value) = blocked.expect("expected --disallowedTools entry even outside git-repo");
10193        let value = value.as_deref().expect("expected value for --disallowedTools");
10194        assert!(value.contains("EnterWorktree"));
10195        assert!(value.contains("ExitWorktree"));
10196    }
10197
10198    /// An answer to a worker's `AskUserQuestion` is indistinguishable
10199    /// from a decision the user made, and reaches it only if someone
10200    /// happens to be looking at that worker's row. The default is that
10201    /// the tool is never offered.
10202    #[test]
10203    fn worker_spawned_without_interactive_is_denied_ask_user_question() {
10204        let args = build_worker_extra_args(true, "reviewer", false);
10205        let blocked = args.iter().find(|(flag, _)| flag == "disallowedTools");
10206        let (_, value) = blocked.expect("expected --disallowedTools entry");
10207        let value = value.as_deref().expect("expected value for --disallowedTools");
10208        assert!(
10209            value.contains("AskUserQuestion"),
10210            "a worker spawned without interactive must not be offered AskUserQuestion, got {value:?}",
10211        );
10212    }
10213
10214    /// The opt-in keeps the tool, and changes nothing else: a worker
10215    /// spawned to be talked to directly is one whose row the user has
10216    /// open, but it is still pinned to its spawn location.
10217    #[test]
10218    fn worker_spawned_interactive_keeps_ask_user_question() {
10219        let args = build_worker_extra_args(true, "reviewer", true);
10220        let blocked = args.iter().find(|(flag, _)| flag == "disallowedTools");
10221        let (_, value) = blocked.expect("expected --disallowedTools entry");
10222        let value = value.as_deref().expect("expected value for --disallowedTools");
10223        assert!(
10224            !value.contains("AskUserQuestion"),
10225            "an interactive worker must keep AskUserQuestion, got {value:?}",
10226        );
10227        assert!(
10228            value.contains("EnterWorktree") && value.contains("ExitWorktree"),
10229            "opting into interactive must not lift the worktree-hop denial, got {value:?}",
10230        );
10231    }
10232
10233    /// A worker closed while it was still spawning drops whatever was buffered
10234    /// for it. The delivery already committed, so nothing re-delivers it, and
10235    /// the release is the last point that can record the loss.
10236    #[test]
10237    fn close_worker_records_the_delivery_parked_for_its_label() {
10238        let (workspace, _rx) = Workspace::testing_stub();
10239        workspace.seed_test_project("forge", "/tmp/close-parked");
10240        let project = workspace
10241            .list_projects()
10242            .into_iter()
10243            .find(|v| v.name == "forge")
10244            .expect("seeded project")
10245            .key;
10246        // Teardown expires the parked bucket by the entry's slot, so the
10247        // fixture carries the one production derives for this worker.
10248        let worker_key = SessionSlot::worker("TestOrg", "forge", "reviewer");
10249        let mut entry = fake_worker_entry("reviewer", "worker-1");
10250        entry.slot = worker_key.clone();
10251        workspace.insert_live_worker(&project, entry);
10252        workspace.register_domain_session(worker_key.clone(), None);
10253        workspace.park_slack(&worker_key, vec![slack_msg("buffered while spawning")]);
10254
10255        handle_close_worker(&workspace, &project, "reviewer");
10256
10257        assert_eq!(
10258            workspace.parked_by_slot.lock().get(&worker_key).map_or(0, |parked| parked.slack.len()),
10259            0,
10260            "the closed worker's parked delivery is recorded, not left for a later session",
10261        );
10262    }
10263}
10264
10265#[cfg(test)]
10266mod lead_charter_tests {
10267    use super::*;
10268
10269    /// A lead with no charter set gets the bundled one.
10270    #[test]
10271    fn lead_without_a_charter_gets_the_bundled_one() {
10272        let mut settings = SessionLaunchSettings::default();
10273        apply_lead_charter(&mut settings);
10274        assert_eq!(settings.charter.as_deref(), Some(DEFAULT_LEAD_CHARTER));
10275    }
10276
10277    /// An already-set charter (a worker spawn's inline persona) is
10278    /// never overwritten - the guard short-circuits first.
10279    #[test]
10280    fn existing_charter_is_preserved_not_overwritten() {
10281        let mut settings = SessionLaunchSettings {
10282            charter: Some("pre-existing".into()),
10283            ..SessionLaunchSettings::default()
10284        };
10285        apply_lead_charter(&mut settings);
10286        assert_eq!(settings.charter.as_deref(), Some("pre-existing"));
10287    }
10288
10289    /// A plan, spec or design the lead writes for the USER's own review
10290    /// reaches them as a document, not as a path in chat. Pinned on the
10291    /// plan step, which is the moment the file is written - a lead reads
10292    /// "write the spec/plan as a file" there and nothing else in the
10293    /// charter tells it how the USER is meant to receive it.
10294    #[test]
10295    fn lead_charter_hands_the_users_review_a_document() {
10296        assert!(
10297            DEFAULT_LEAD_CHARTER.contains(
10298                "when the plan, spec or design is for the USER's own review, deliver it through \
10299                 `documents__hand_off` with `reviewable: true`"
10300            ),
10301            "the plan step hands the USER's review a document: {DEFAULT_LEAD_CHARTER}",
10302        );
10303        assert!(
10304            DEFAULT_LEAD_CHARTER.contains("a path in chat is not a hand-off"),
10305            "the correction is pinned: {DEFAULT_LEAD_CHARTER}",
10306        );
10307    }
10308
10309    /// The bundled charter ships to every install, so it must not name
10310    /// tooling or projects that only exist in one author's environment:
10311    /// a fresh install has no user-scope skills, no plugins and no
10312    /// justfile, and `team` is not a `forge.toml` key. Most entries got
10313    /// here by being copied from an on-disk
10314    /// charter; the two path entries are pre-emptive, since prose about
10315    /// where a charter lives is the obvious place to write one.
10316    ///
10317    /// Every assertion below is a `!contains`, so all of them hold
10318    /// against an empty string - the first check is what makes the rest
10319    /// mean anything if the `include_str!` ever resolves somewhere else.
10320    #[test]
10321    fn bundled_lead_charter_assumes_no_local_environment() {
10322        assert!(
10323            DEFAULT_LEAD_CHARTER.contains("agents__spawn"),
10324            "the compiled-in charter is the real one, not an empty or wrong file",
10325        );
10326        for (token, why) in [
10327            ("pr-review-loop", "user-scope skill, absent on a fresh install"),
10328            ("superpowers", "plugin, absent on a fresh install"),
10329            ("commit-commands", "plugin, absent on a fresh install"),
10330            ("`just ", "project justfile, not every project has one"),
10331            ("hub-modules", "one user's project name"),
10332            ("team = ", "not a forge.toml key"),
10333            ("static_workers", "no longer a forge.toml key either"),
10334            ("~/.claude", "the charter must not pin a path in the user's home"),
10335            ("forge-team", "the charter must not name the deleted role filesystem"),
10336        ] {
10337            assert!(
10338                !DEFAULT_LEAD_CHARTER.contains(token),
10339                "bundled lead charter names '{token}' ({why})"
10340            );
10341        }
10342    }
10343
10344    /// A worker handed off a reviewable document and had no `review__*`
10345    /// tools to read the comments with. The spawn step is where a lead
10346    /// picks the family selection, so the requirement ships there - pinned
10347    /// so a paraphrase cannot drop it.
10348    #[test]
10349    fn the_charter_names_the_review_family_for_a_document_worker() {
10350        assert!(
10351            DEFAULT_LEAD_CHARTER.contains(
10352                "If you narrow a worker's `mcp_families`, include `review` for a worker whose \
10353                 deliverable can be handed off as a reviewable document - a fresh spawn that \
10354                 omits the field gives every family; a resume adopts the stored selection."
10355            ),
10356            "the spawn step scopes the requirement to a narrowed selection and states what a \
10357             fresh spawn and a resume each do",
10358        );
10359    }
10360
10361    /// **The project handoff is the lead's**, and a worker's note is never
10362    /// the project's: the project's memory directory holds the lead's rolling
10363    /// handoff, so a worker writing `handoff.md` there overwrites it. Both
10364    /// addresses are pinned so a reword fails here rather than in whichever
10365    /// seat writes the lead's file.
10366    #[test]
10367    fn the_charter_says_the_project_handoff_is_the_leads() {
10368        assert!(
10369            DEFAULT_LEAD_CHARTER.contains(
10370                "`handoff.md` in the project's memory directory is the project's rolling \
10371                 state, and you write it and keep it current"
10372            ),
10373            "the file, its owner and the keeping-current duty are pinned",
10374        );
10375        assert!(
10376            DEFAULT_LEAD_CHARTER.contains(
10377                "a worker's handoff note is its own file, `workers/<label>.md` in that same \
10378                 project memory directory, never the project's"
10379            ),
10380            "the worker's own address is anchored so the two cannot read alike",
10381        );
10382    }
10383
10384    /// The charter's systemone cues ship to every lead on every install, and
10385    /// the names in them are tools that can be renamed - pinned so a rename
10386    /// fails here instead of handing a lead a call it cannot make.
10387    #[test]
10388    fn the_charter_carries_its_systemone_cues() {
10389        assert!(
10390            DEFAULT_LEAD_CHARTER
10391                .contains("if your tools include `systemone__ask_noul`, make one claim-check"),
10392            "the merge-gate claim cue and its guard are pinned",
10393        );
10394        assert!(
10395            DEFAULT_LEAD_CHARTER.contains(
10396                "if your tools include `systemone__ask_choice`, weigh them with one choice over the live set"
10397            ),
10398            "the routing cue and its guard are pinned",
10399        );
10400        assert!(
10401            DEFAULT_LEAD_CHARTER.contains(
10402                "if your tools include `systemone__ask_choice`, `ask_score` or `ask_noul`, put \
10403                 a decision this session could make to the model rather than the user - the \
10404                 user's attention is the exception, not the default"
10405            ),
10406            "the escalation cue, its guard and the attention rule are pinned",
10407        );
10408        assert!(
10409            DEFAULT_LEAD_CHARTER.contains("Never park a dispatch decision on your own judgment.")
10410                && DEFAULT_LEAD_CHARTER.contains("decide whether the options genuinely differ")
10411                && DEFAULT_LEAD_CHARTER.contains(
10412                    "put the enumerated options and the context that bears on them to \
10413                     `systemone__ask_choice` when your tools include it"
10414                )
10415                && DEFAULT_LEAD_CHARTER.contains("without the tool, decide it yourself")
10416                && DEFAULT_LEAD_CHARTER
10417                    .contains("Either way, report the decision where the work is reported"),
10418            "the dispatch cue, its genuine-difference scope, its guard, its fallback and its \
10419             report duty are pinned",
10420        );
10421    }
10422
10423    /// A merge conflict is a cost, not a barrier (#2509). The charter used
10424    /// to forbid parallel branches that edit the same files, and a lead
10425    /// held a fix behind another cut for it. Only genuine shared-state
10426    /// ordering still serializes, and the old same-files ban is pinned
10427    /// absent so a reversion fails here rather than in a silent queue.
10428    #[test]
10429    fn the_charter_parallelises_independent_work() {
10430        for (clause, why) in [
10431            (
10432                "Default is NOT strictly one-at-a-time: run ad-hoc workers CONCURRENTLY on \
10433                 independent work",
10434                "the parallel-by-default headline",
10435            ),
10436            ("A merge conflict is not, by itself, a reason to hold a piece", "the conflict rule"),
10437            ("resolve the conflict and re-run the gates", "its remedy"),
10438            ("Only genuine shared-state ordering serializes:", "the narrow serialization test"),
10439            (
10440                "at most ONE in-flight migration branch at a time",
10441                "the migration sequence stays serialized",
10442            ),
10443            ("One main, one version line", "the merge queue and release line stay serialized"),
10444            (
10445                "Review is not serialization but it is a real cap",
10446                "review throughput is a cap, not a serialization claim",
10447            ),
10448        ] {
10449            assert!(
10450                DEFAULT_LEAD_CHARTER.contains(clause),
10451                "the lead charter does not carry the parallel rule ({why}): {clause}",
10452            );
10453        }
10454        assert!(
10455            !DEFAULT_LEAD_CHARTER.to_lowercase().contains("same files"),
10456            "the same-files hold is what #2509 removed; it must not come back in any casing",
10457        );
10458    }
10459
10460    /// The one-shot rule ships to every lead, and its handle is the same
10461    /// sentence the conduct block and the lead-facing tool text carry -
10462    /// pinned so a paraphrase drifting between surfaces fails here. The
10463    /// planning half rides with it: the ban starts at the plan, and a cut
10464    /// that cannot land whole goes to the user before it is cut down.
10465    #[test]
10466    fn the_charter_carries_the_one_shot_rule() {
10467        assert!(
10468            DEFAULT_LEAD_CHARTER.contains(
10469                "A work cut - one feature, one issue, one fix - lands whole: one branch, one \
10470                 review, one look, one merge. Never slice it into pieces that land one after \
10471                 another, and never show or merge an intermediate step on the way."
10472            ),
10473            "the handle and the ban, through the ban's period, are pinned together",
10474        );
10475        assert!(
10476            DEFAULT_LEAD_CHARTER.contains(
10477                "The look is the one that cut is owed: a feature waits for the user's own look \
10478                 before merge; a fix merges after the review loop as usual."
10479            ),
10480            "the look's scope is pinned",
10481        );
10482        assert!(
10483            DEFAULT_LEAD_CHARTER.contains("never sequence a cut as \"land A, then B, then C\""),
10484            "the planning ban is pinned",
10485        );
10486        assert!(
10487            DEFAULT_LEAD_CHARTER.contains(
10488                "if it genuinely cannot land whole, decide that with the user BEFORE anything is \
10489                 cut down"
10490            ),
10491            "the user-decision half is pinned",
10492        );
10493    }
10494
10495    /// Mid-turn delivery had no written duty of its own in this charter, so the
10496    /// bullet that adds one is pinned sentence by sentence.
10497    #[test]
10498    fn the_charter_answers_a_mid_turn_message() {
10499        assert!(
10500            DEFAULT_LEAD_CHARTER.contains(
10501                "- **A message from the user that arrives mid-turn is still owed an answer.**"
10502            ),
10503            "the bullet's own lead-in, so folding it into another bullet fails here",
10504        );
10505        assert!(
10506            DEFAULT_LEAD_CHARTER
10507                .contains("Answer it in that same turn, before you start the next piece of work"),
10508            "the duty falls due in the same turn, before the next piece of work",
10509        );
10510        assert!(
10511            DEFAULT_LEAD_CHARTER
10512                .contains("a question left unanswered is a dropped message, not a deferred one"),
10513            "the failure is named, so the line cannot be read as permission to defer",
10514        );
10515    }
10516
10517    /// The charter's board lines carry the landing rule (#2377): a landed
10518    /// row with no epic above it is deleted in the same pass, a landed
10519    /// child of an open epic is left terminal so the epic can read `to
10520    /// close`, and the delete is never that child's - the lead is the
10521    /// pile's author, so a charter line still saying "complete the row
10522    /// when it is merged" would reproduce the 36-completed-rows board the
10523    /// tool text was fixed to prevent.
10524    #[test]
10525    fn the_charter_carries_the_landing_rule() {
10526        for (clause, why) in [
10527            (
10528                "a landed row with no children and no epic above it is deleted in the same \
10529                 pass (`tasks__delete`), never left completed, while a landed row under an \
10530                 epic - open or already terminal - is completed and left terminal, because \
10531                 the epic reads `to close` only once every child is terminal (a verify=user \
10532                 child parks in waiting on the user and counts only once the user approves)",
10533                "the row-true bullet carries the delete and its epic carve-out together",
10534            ),
10535            (
10536                "`tasks__delete` clears a row created in error, and a landed row with no \
10537                 children and no epic above it - never a child of an open epic, whose terminal \
10538                 state its epic's close depends on",
10539                "the close-epics bullet states what the delete is for",
10540            ),
10541            (
10542                "completing the ROOT row closes it and archives the tree",
10543                "the archive that closes an epic is pinned, so the carve-out's close signal \
10544                 has an action behind it",
10545            ),
10546        ] {
10547            assert!(
10548                DEFAULT_LEAD_CHARTER.contains(clause),
10549                "the lead charter does not carry the landing rule ({why}): {clause}",
10550            );
10551        }
10552    }
10553
10554    /// Every client text box takes dictation, and a lead planning client
10555    /// work names the mechanism with the box - pinned so a charter rewrite
10556    /// keeps the rule beside the mockup step it applies from.
10557    #[test]
10558    fn the_charter_carries_the_dictation_rule() {
10559        assert!(
10560            DEFAULT_LEAD_CHARTER.contains(
10561                "the plan names dictation for every text box it adds - `DictateControl` beside \
10562                 the box"
10563            ),
10564            "the dictation rule and its control are pinned",
10565        );
10566    }
10567
10568    /// A defect the review or the look of a cut surfaces belongs to that
10569    /// cut, not to the board: the lead fixes it in line with the worker
10570    /// frozen, and only work outside the cut gets a row. Pinned because
10571    /// the charter's filing rule reads as "file everything" without the
10572    /// carve-out, which is how a finding of the cut under review was
10573    /// filed as a separate issue.
10574    #[test]
10575    fn the_charter_fixes_a_cuts_findings_in_line() {
10576        for clause in [
10577            "File what is outside the cut; fix what is inside it",
10578            "A defect that surfaces in the review or the look of a cut under review is not \
10579             filed and not handed to a worker",
10580            "fix it in line on that cut's branch, with the worker frozen",
10581            "let the same branch carry it",
10582            "except a finding of a cut under review, which you fix in line with the worker \
10583             frozen",
10584            "dispatch, except a finding of a cut under review, which is yours to fix in line",
10585            "You orchestrate; you don't write code - except a finding of a cut under review, \
10586             which you fix in line",
10587            "found anywhere else becomes a row the moment it surfaces",
10588        ] {
10589            assert!(
10590                DEFAULT_LEAD_CHARTER.contains(clause),
10591                "the lead charter does not fix a cut's findings in line: {clause}",
10592            );
10593        }
10594        assert!(
10595            !DEFAULT_LEAD_CHARTER.contains("File everything you find"),
10596            "the unconditional filing headline must not come back",
10597        );
10598    }
10599
10600    /// A user-decision wait or a `verify: user` gate is the lead's to
10601    /// mint. A worker-parked decision outlives its answer and the lead
10602    /// cannot close it afterward, so the ask travels to the lead instead
10603    /// of parking on the user from a worker's row.
10604    #[test]
10605    fn the_charter_keeps_the_user_decision_mint_with_the_lead() {
10606        for clause in [
10607            "Only you mint a `verify: user` gate or a `kind: decision` wait",
10608            "a worker hands the ask to you instead",
10609            "A decision wait a worker parks outlives its answer and you cannot close it afterward",
10610        ] {
10611            assert!(
10612                DEFAULT_LEAD_CHARTER.contains(clause),
10613                "the lead charter does not keep the user-decision mint with the lead: {clause}",
10614            );
10615        }
10616    }
10617
10618    /// A review round builds in the same checkout the worker handed
10619    /// over, so reviewers read the pushed branch and a mutation pass
10620    /// runs in a detached throwaway worktree, never the live checkout.
10621    #[test]
10622    fn the_charter_scopes_reviewers_away_from_the_live_worktree() {
10623        for clause in [
10624            "Reviewers read the pushed branch",
10625            "a mutation pass runs in a detached throwaway worktree, never the live checkout",
10626        ] {
10627            assert!(
10628                DEFAULT_LEAD_CHARTER.contains(clause),
10629                "the lead charter does not scope reviewers away from the live worktree: {clause}",
10630            );
10631        }
10632    }
10633
10634    /// A finding is a claim about the pushed branch: one written
10635    /// without reading costs the worker a round and teaches it to
10636    /// discount the rest of the list. The grep and its point-at-the-line
10637    /// bar are pinned, with the withdrawal for one that goes out wrong.
10638    #[test]
10639    fn the_charter_verifies_a_finding_against_the_branch() {
10640        for clause in [
10641            "Before a finding goes to a worker, grep the pushed branch for the exact symbol, \
10642             string or file it names and point at the line",
10643            "one you cannot point at is not sent",
10644            "withdrawn immediately and named as withdrawn",
10645            "a plausible list written without reading costs the worker a round and teaches it \
10646             to discount the rest",
10647        ] {
10648            assert!(
10649                DEFAULT_LEAD_CHARTER.contains(clause),
10650                "the lead charter does not verify a finding before sending it: {clause}",
10651            );
10652        }
10653    }
10654
10655    /// A queued message can die with the worker's session, so the
10656    /// review step re-sends its list after a restart and the health
10657    /// check proves the new session actually holds it - never by the
10658    /// earlier `sent`, which reported only the queue's receipt.
10659    #[test]
10660    fn the_charter_resends_after_a_worker_restart() {
10661        for clause in [
10662            "If the worker restarted since you last wrote to it, re-send the findings and \
10663             confirm they landed (its answer, never the earlier `sent`) before waiting on a \
10664             round",
10665            "A message still queued to a worker when its session restarted can be gone",
10666            "re-send it and confirm it landed by a distinctive phrase in the worker's transcript \
10667             or by its answer",
10668            "never by the earlier `sent`",
10669        ] {
10670            assert!(
10671                DEFAULT_LEAD_CHARTER.contains(clause),
10672                "the lead charter does not resend after a worker restart: {clause}",
10673            );
10674        }
10675    }
10676
10677    /// A commit subject, a PR body and an agent's report are claims;
10678    /// the pushed commit is the object. Pinned at the merge gate so a
10679    /// report true only of a working tree cannot read as done.
10680    #[test]
10681    fn the_charter_checks_the_object_not_the_report() {
10682        for clause in [
10683            "A commit subject, a PR body and an agent's report are claims",
10684            "before reporting a change as done off one, `git show <sha>:<path>` the thing and \
10685             quote it back",
10686            "the pushed commit is the object",
10687            "a report can be true of the working tree and false of the commit",
10688        ] {
10689            assert!(
10690                DEFAULT_LEAD_CHARTER.contains(clause),
10691                "the lead charter does not check the object before reporting done: {clause}",
10692            );
10693        }
10694    }
10695
10696    /// The merge queue is pulled from GitHub, not held only on the push
10697    /// ping: an approved-and-green PR otherwise sits silently when a
10698    /// ping is lost.
10699    #[test]
10700    fn the_charter_pulls_the_merge_queue_from_github() {
10701        for clause in [
10702            "a worker's \"PR #N ready\" ping is a prompt, not the source of truth",
10703            "on every health check and resume",
10704            "pull `gh pr list --state open` and `gh pr checks <N>` and reconcile them against \
10705             the loops you ran",
10706            "an approved-and-green PR otherwise sits silently when a ping is lost",
10707            "(on each wake)",
10708            "plus `gh pr list --state open` and `gh pr checks <N>` for the merge queue - pull \
10709             both rather than waiting on a ping",
10710        ] {
10711            assert!(
10712                DEFAULT_LEAD_CHARTER.contains(clause),
10713                "the lead charter does not pull the merge queue from GitHub: {clause}",
10714            );
10715        }
10716    }
10717
10718    /// A turn that ends on stated intent schedules nothing, so making a
10719    /// worker the trigger for a timed or external condition leaves both
10720    /// sides idle. The report-only one-shot and the ban on one that
10721    /// performs the action itself are pinned with it, because either
10722    /// alone reads as a licence to schedule the work twice.
10723    #[test]
10724    fn the_charter_forbids_making_a_worker_a_trigger() {
10725        for clause in [
10726            "Never make a worker the trigger for a timed or external condition",
10727            "A turn that ends on stated intent schedules nothing",
10728            "forge can wake a worker through a cron or a subscription it set, but not on intent",
10729            "leaves both sides idle",
10730            "schedule a report-only `cron__create` one-shot (`run_once_at`)",
10731            "Never schedule a one-shot that performs the action itself",
10732            "it races the real trigger and can double a review round or a spend",
10733        ] {
10734            assert!(
10735                DEFAULT_LEAD_CHARTER.contains(clause),
10736                "the lead charter does not forbid a worker as trigger: {clause}",
10737            );
10738        }
10739        assert!(
10740            !DEFAULT_LEAD_CHARTER.contains("A worker has no autonomous wake"),
10741            "the false absolute must not come back beside the correct wording",
10742        );
10743    }
10744
10745    /// The charter is shipped text every lead reads, so a blocked CLI tool
10746    /// named here would send it to a surface its own launch denies. The
10747    /// preamble is the other lead-facing text and carries the same test in
10748    /// its own file.
10749    #[test]
10750    fn the_lead_charter_names_no_blocked_cli_tool() {
10751        for tool in [
10752            "CronCreate",
10753            "CronDelete",
10754            "CronList",
10755            "TaskCreate",
10756            "TaskGet",
10757            "TaskList",
10758            "TaskUpdate",
10759            "SendMessage",
10760            "ListAgents",
10761            "Workflow",
10762            "RemoteTrigger",
10763        ] {
10764            assert!(
10765                !DEFAULT_LEAD_CHARTER.contains(tool),
10766                "the lead charter must not name the blocked CLI tool {tool}",
10767            );
10768        }
10769    }
10770
10771    /// The token test above checks absence only; these pin the decided
10772    /// despawn wording against silent softening in either direction.
10773    #[test]
10774    fn lead_charter_carries_the_despawn_trigger() {
10775        assert!(
10776            DEFAULT_LEAD_CHARTER.contains("despawn at the chain node"),
10777            "the chain-node despawn rule stays: {DEFAULT_LEAD_CHARTER}",
10778        );
10779        assert!(
10780            DEFAULT_LEAD_CHARTER.contains("NOT when the last downstream PR closes"),
10781            "the despawn point is the chain node, not the last PR: {DEFAULT_LEAD_CHARTER}",
10782        );
10783        assert!(
10784            DEFAULT_LEAD_CHARTER.contains("only when the next stage needs the context back"),
10785            "chain despawn closes at absorption; re-spawn waits for the next stage: {DEFAULT_LEAD_CHARTER}",
10786        );
10787    }
10788
10789    /// The wind-down used to tell leads to have the worker reset its
10790    /// worktree to main and drop its branch; measured 2026-10-17, a
10791    /// worker seat's permission engine refuses that reset, and the
10792    /// despawn reaped the branch and closed clean without it. Pinned so
10793    /// the refusable step does not come back, and the honest half - the
10794    /// tree clean, the reap the despawn's - does not drift away either.
10795    #[test]
10796    fn lead_charter_wind_down_leaves_the_reap_to_the_despawn() {
10797        assert!(
10798            DEFAULT_LEAD_CHARTER.contains("leave its worktree clean"),
10799            "the honest cleanup instruction is pinned: {DEFAULT_LEAD_CHARTER}",
10800        );
10801        assert!(
10802            DEFAULT_LEAD_CHARTER.contains("commit and push what is in flight"),
10803            "the dirty bar is commit AND push, not either: {DEFAULT_LEAD_CHARTER}",
10804        );
10805        assert!(
10806            DEFAULT_LEAD_CHARTER.contains("the despawn reaps the worktree branch"),
10807            "the reap is the despawn's, not the worker's: {DEFAULT_LEAD_CHARTER}",
10808        );
10809        assert!(
10810            DEFAULT_LEAD_CHARTER.contains("once its commits are reachable elsewhere"),
10811            "the reap's reachability bar is pinned: {DEFAULT_LEAD_CHARTER}",
10812        );
10813        assert!(
10814            DEFAULT_LEAD_CHARTER.contains("names any it must keep"),
10815            "a kept branch is named, not silently left: {DEFAULT_LEAD_CHARTER}",
10816        );
10817        assert!(
10818            !DEFAULT_LEAD_CHARTER.contains("reset to main"),
10819            "the refusable reset stays out of the wind-down: {DEFAULT_LEAD_CHARTER}",
10820        );
10821    }
10822
10823    /// A worktree is a live cut's tree, not a cache: removing one by hand to
10824    /// reclaim disk loses its branch and forces a full rebuild. Pinned with
10825    /// the reclaim's owner and the in-tree clean that survives a live worker.
10826    #[test]
10827    fn lead_charter_never_prunes_a_worktree_by_hand() {
10828        assert!(
10829            DEFAULT_LEAD_CHARTER
10830                .contains("Never prune a worker's worktree by hand to reclaim disk"),
10831            "the rule's handle is pinned: {DEFAULT_LEAD_CHARTER}",
10832        );
10833        assert!(
10834            DEFAULT_LEAD_CHARTER
10835                .contains("a tree removed mid-cut loses its branch and forces a full rebuild"),
10836            "the reason a hand prune costs work is pinned: {DEFAULT_LEAD_CHARTER}",
10837        );
10838        assert!(
10839            DEFAULT_LEAD_CHARTER.contains(
10840                "under disk pressure, have live workers push their branches and clean their \
10841                 build targets inside their own trees"
10842            ),
10843            "the reclaim that survives a live worker is pinned with what it cleans: \
10844             {DEFAULT_LEAD_CHARTER}",
10845        );
10846    }
10847
10848    /// The attention rule is the lead's posture toward the user, and both
10849    /// halves read as the opposite of themselves when softened: the shapes
10850    /// become "ask when unsure", and the spec half becomes a check-in.
10851    /// Pinned so a later edit cannot unship either silently.
10852    #[test]
10853    fn lead_charter_carries_the_attention_rule() {
10854        assert!(
10855            DEFAULT_LEAD_CHARTER.contains(
10856                "Nothing waits on the user except a hold the user \
10857                 explicitly asked for"
10858            ),
10859            "the rule's opening is pinned",
10860        );
10861        assert!(
10862            DEFAULT_LEAD_CHARTER
10863                .contains("a look they should give when they return (the board's decision wait)"),
10864            "the at-convenience look is pinned beside the hold",
10865        );
10866        assert!(
10867            DEFAULT_LEAD_CHARTER.contains(
10868                "the decision model, peer opinions and your own review already spent - something \
10869                 still genuinely ambiguous"
10870            ),
10871            "shape (c)'s channels and its bar are pinned together",
10872        );
10873        assert!(
10874            DEFAULT_LEAD_CHARTER.contains("A spec that exists is followed without check-ins"),
10875            "the spec half is pinned",
10876        );
10877        assert!(
10878            DEFAULT_LEAD_CHARTER.contains("never parked on the user"),
10879            "the new-work half is pinned",
10880        );
10881        assert!(
10882            DEFAULT_LEAD_CHARTER.contains(
10883                "A long-lived worker is a standing commitment of the user's attention rather \
10884                 than a step of work, so raise it with the user rather than deciding it yourself"
10885            ),
10886            "the long-lived-worker clause is pinned whole, its call included",
10887        );
10888    }
10889
10890    /// The charter is the only shipped text telling a lead how to maintain
10891    /// the task list, and the only task surface forge ships is `tasks__*`
10892    /// over forge's own store. Guidance written against the CLI's tools
10893    /// names fields no tool offers, and sends every lead to a tool its
10894    /// session is denied.
10895    #[test]
10896    fn lead_charter_maintains_the_task_list_through_forge_s_own_tools() {
10897        // The whole family the board is worked through: the lead cuts and
10898        // dispatches, workers claim and wait, and the list is read.
10899        for tool in [
10900            "tasks__create",
10901            "tasks__update",
10902            "tasks__delete",
10903            "tasks__claim",
10904            "tasks__wait",
10905            "tasks__list",
10906        ] {
10907            assert!(
10908                DEFAULT_LEAD_CHARTER.contains(tool),
10909                "the charter maintains the list through {tool}: {DEFAULT_LEAD_CHARTER}",
10910            );
10911        }
10912        // Case-insensitively: the CLI spells its removal status lowercase
10913        // `deleted`, and a lead reading it as a state here would go looking
10914        // for it in `tasks__update`. The landed-row rule legitimately uses
10915        // the verb - "is deleted in the same pass", `tasks__delete` named
10916        // in the same clause - so the ban is the status spelling: the
10917        // backticked form, the word named as a status, or stated as a value
10918        // ("status: deleted", "set to deleted").
10919        for spelling in
10920            ["`deleted`", "deleted status", "status deleted", "status: deleted", "to deleted"]
10921        {
10922            assert!(
10923                !DEFAULT_LEAD_CHARTER.to_lowercase().contains(spelling),
10924                "the charter names `deleted` as a status ({spelling}): {DEFAULT_LEAD_CHARTER}",
10925            );
10926        }
10927    }
10928
10929    /// #2831: the charter is where dispatch is decided, and pending is the
10930    /// board's Backlog. A ready row is a candidate, not a queued start, and
10931    /// the row is kept out of the sweep by ownership - the door the old text
10932    /// left out, and what made "backlog these rows" read as dispatched.
10933    #[test]
10934    fn lead_charter_carries_the_backlog_rule() {
10935        for clause in [
10936            "a ready row is a dispatch candidate, not work already queued to start",
10937            "Dispatch is your act: keep the row owned (the sweep dispatches only unowned rows) \
10938             to set it aside",
10939        ] {
10940            assert!(
10941                DEFAULT_LEAD_CHARTER.contains(clause),
10942                "the charter owes the backlog rule ({clause}): {DEFAULT_LEAD_CHARTER}",
10943            );
10944        }
10945    }
10946
10947    /// The board bullets above say how a row is worked; the sweep has its
10948    /// own paragraph below them. That paragraph is pinned byte for byte, so
10949    /// nothing can be added, removed or qualified inside it. The rest of the
10950    /// board section is then scanned as a net for a cron call, a cron
10951    /// parameter or a cron expression: candidate windows are parsed with
10952    /// forge's own validator, so the net accepts exactly what a session
10953    /// could schedule - names, ranges and aliases included. It is a net,
10954    /// not a closure: prose naming no tool, parameter or expression cannot
10955    /// be told apart from advice. The charter's one-shot `cron__create`
10956    /// advice sits above that section, so the scan leaves it alone.
10957    #[test]
10958    fn the_charter_carries_the_board_sweep() {
10959        const SWEEP: &str = "**Sweep the board at the start of every session, when forge injects \
10960             a sweep (it does when you have been idle, at most once every two hours, or six \
10961             hours after the last sweep), and on demand.** First `cron__list` and delete any \
10962             sweep cron an older version of this charter had you create (`cron__delete`) - it \
10963             keeps firing its sweep prompt into this session until it is removed. Never create a \
10964             standing sweep cron. End every sweep with the next three lead-owned backlog rows, \
10965             each with one line on why it comes next; they are proposals, not dispatches, and \
10966             only you start a worker on one.";
10967        const CLEANUP: &str = "First `cron__list` and delete any sweep cron an older version of \
10968             this charter had you create (`cron__delete`)";
10969        const REASON: &str =
10970            "it keeps firing its sweep prompt into this session until it is removed";
10971
10972        let section = board_section();
10973        assert!(
10974            section.contains("**Sweep the board"),
10975            "the sweep paragraph belongs in the board section, not elsewhere",
10976        );
10977        assert_eq!(
10978            DEFAULT_LEAD_CHARTER.matches("**Sweep the board").count(),
10979            1,
10980            "the charter carries exactly one sweep paragraph",
10981        );
10982        let sweep_at = section
10983            .find("**Sweep the board")
10984            .expect("the sweep paragraph belongs in the board section");
10985        let tail = &section[sweep_at..];
10986        let sweep_paragraph = &tail[..tail.find("\n\n").unwrap_or(tail.len())];
10987        assert_eq!(
10988            sweep_paragraph, SWEEP,
10989            "the sweep paragraph is pinned byte for byte: nothing added, removed or qualified",
10990        );
10991
10992        for (clause, why) in [
10993            (CLEANUP, "the old standing entry is removed, not only forbidden"),
10994            (REASON, "the reason the cleanup is owed"),
10995            (
10996                "when forge injects a sweep (it does when you have been idle, at most once every \
10997                 two hours, or six hours after the last sweep)",
10998                "the injected trigger, its throttle and its six-hour fallback",
10999            ),
11000            ("at most once every two hours", "the idle throttle"),
11001            ("six hours after the last sweep", "the fallback is measured from the last sweep"),
11002            ("End every sweep with the next three lead-owned backlog rows", "the next-pick rule"),
11003            ("each with one line on why it comes next", "each pick says why it comes next"),
11004            ("they are proposals, not dispatches", "a pick is not a dispatch"),
11005            (
11006                "settle every row waiting on the user through one door",
11007                "the sweep's scope: a dependency or resource wait has no door here",
11008            ),
11009            ("attest a look the user already gave", "the look door"),
11010            (
11011                "withdraw a moot ask (`tasks__update` canceled, its `detail` a reason the row \
11012                 does not already carry)",
11013                "the withdraw door, and the new-reason requirement",
11014            ),
11015            ("park a wait the user deferred (`parked: true`)", "the park door"),
11016            ("leave a genuine decision for the user and name it in your line", "the decision door"),
11017            (
11018                "close every epic reading `to_close`, a `verify: user` root parking on the look \
11019                 first and then being attested",
11020                "the epic close, including the gate's park-then-attest order",
11021            ),
11022            ("dispatch the next ready and unowned row", "the dispatch pick"),
11023            (
11024                "report one line: attested, withdrawn, parked, closed, dispatched, and the \
11025                 decisions still waiting on the user",
11026                "the sweep answers in one line",
11027            ),
11028        ] {
11029            assert!(
11030                DEFAULT_LEAD_CHARTER.contains(clause),
11031                "the lead charter does not carry the board sweep ({why}): {clause}",
11032            );
11033        }
11034
11035        // The two pinned sentences carry the cron tool names by necessity,
11036        // so they are cut out before the net runs over what is left.
11037        let cleanup_sentence = format!("{CLEANUP} - {REASON}.");
11038        let without_cleanup = section.replacen(&cleanup_sentence, "", 1);
11039        assert_ne!(
11040            without_cleanup, section,
11041            "the pinned cleanup sentence is one sentence: {cleanup_sentence}",
11042        );
11043        let prohibition = "Never create a standing sweep cron.";
11044        let rest = without_cleanup.replacen(prohibition, "", 1);
11045        assert_ne!(rest, without_cleanup, "the pinned prohibition is one sentence: {prohibition}");
11046        if let Some((marker, at)) = cron_marker(&rest) {
11047            let near = context_around(&rest, at);
11048            panic!("the board section carries {marker:?} near: {near}");
11049        }
11050
11051        let creations: Vec<usize> =
11052            DEFAULT_LEAD_CHARTER.match_indices("cron__create").map(|(at, _)| at).collect();
11053        let sites = if creations.is_empty() {
11054            "none".to_owned()
11055        } else {
11056            creations
11057                .iter()
11058                .map(|&at| context_around(DEFAULT_LEAD_CHARTER, at))
11059                .collect::<Vec<_>>()
11060                .join(" | ")
11061        };
11062        assert_eq!(
11063            creations.len(),
11064            1,
11065            "the charter's only cron creation is the report-only one-shot sentence; found {} at: \
11066             {sites}",
11067            creations.len(),
11068        );
11069        assert!(
11070            DEFAULT_LEAD_CHARTER
11071                .contains("schedule a report-only `cron__create` one-shot (`run_once_at`)"),
11072            "the single cron creation is the one-shot advice",
11073        );
11074    }
11075
11076    /// The net's own machinery, proved away from the charter: the charter
11077    /// carries none of these markers today, so a detector that stopped
11078    /// matching, or a scan that stopped lowercasing, would leave every
11079    /// assertion above green. The expressions are the forms forge's own
11080    /// validator accepts - wildcards, names, ranges and aliases.
11081    #[test]
11082    fn the_board_section_net_catches_cron_markers() {
11083        for (text, why) in [
11084            ("`*/30 * * * *`.", "a backtick-wrapped expression ending in a period"),
11085            ("0 6 * * 1", "five plain fields"),
11086            ("(0 9 * * 1)", "a parenthesised expression"),
11087            ("0 9 * * MON", "a day name"),
11088            ("0 9 * * MON-FRI", "a day range"),
11089            ("@daily", "an alias"),
11090        ] {
11091            assert!(cron_expression(text).is_some(), "{text} is a cron expression ({why})");
11092        }
11093        assert!(
11094            cron_marker("Create a CRON job each morning").is_some(),
11095            "the scan lowercases, so a capitalised CRON is caught",
11096        );
11097        assert_eq!(trim_cron_token("`0`."), "0", "backticks and a period are trimmed");
11098        assert_eq!(trim_cron_token("`1`,"), "1", "backticks and a comma are trimmed");
11099        assert_eq!(trim_cron_token("(2);"), "2", "parentheses and a semicolon are trimmed");
11100        assert_eq!(cron_expression("word"), None, "prose is not a cron expression");
11101    }
11102
11103    /// The board section, from its heading to the next top-level heading.
11104    fn board_section() -> &'static str {
11105        let start = DEFAULT_LEAD_CHARTER
11106            .find("## Keep the task board")
11107            .expect("the board section heading `## Keep the task board` is present");
11108        let tail = &DEFAULT_LEAD_CHARTER[start..];
11109        let end = tail.find("\n## ").unwrap_or(tail.len());
11110        &tail[..end]
11111    }
11112
11113    /// The first cron marker the board-section net recognises in `text`: a
11114    /// cron word, a `wake:` parameter or a cron expression. The haystack is
11115    /// lowercased with the ASCII-only form on purpose - it preserves byte
11116    /// offsets, so the failure context can be sliced out of the original
11117    /// text.
11118    fn cron_marker(text: &str) -> Option<(String, usize)> {
11119        let haystack = text.to_ascii_lowercase();
11120        for token in ["cron", "wake:"] {
11121            if let Some(at) = haystack.find(token) {
11122                return Some((token.to_owned(), at));
11123            }
11124        }
11125        cron_expression(&haystack).map(|at| ("cron expression".to_owned(), at))
11126    }
11127
11128    /// A whitespace-separated token with the punctuation prose wraps a
11129    /// schedule in: code-span ticks, a trailing period or comma, parentheses,
11130    /// semicolons and quotes.
11131    fn trim_cron_token(word: &str) -> &str {
11132        word.trim_matches(['`', '.', ',', '(', ')', ';', '"', '\''])
11133    }
11134
11135    /// The byte offset of the first cron expression the board-section net
11136    /// recognises in `text`: every window of five whitespace-separated
11137    /// tokens, and every `@`-prefixed token, is stripped of its wrapping
11138    /// punctuation and handed to forge's own validator, so the net accepts
11139    /// exactly what `cron__create` would - names, ranges, aliases and all -
11140    /// and nothing else by construction. A numeric run such as `2 3 4 5 6`
11141    /// parses too; that is the trade for using the real validator, it fails
11142    /// loudly, and the board section carries no such run.
11143    fn cron_expression(text: &str) -> Option<usize> {
11144        let mut tokens: Vec<(usize, &str)> = Vec::new();
11145        let mut cursor = 0;
11146        for word in text.split_whitespace() {
11147            let at = cursor + text[cursor..].find(word)?;
11148            cursor = at + word.len();
11149            let trimmed = trim_cron_token(word);
11150            if !trimmed.is_empty() {
11151                tokens.push((at, trimmed));
11152            }
11153        }
11154        for (at, token) in &tokens {
11155            if token.starts_with('@')
11156                && crate::mcp::cron::schedule::validate_cron_expr(token).is_ok()
11157            {
11158                return Some(*at);
11159            }
11160        }
11161        for window in tokens.windows(5) {
11162            let expr = window.iter().map(|(_, word)| *word).collect::<Vec<_>>().join(" ");
11163            if crate::mcp::cron::schedule::validate_cron_expr(&expr).is_ok() {
11164                return Some(window[0].0);
11165            }
11166        }
11167        None
11168    }
11169
11170    /// A window of `text` around `at`, so a failure names the lineage
11171    /// without dumping the whole charter.
11172    fn context_around(text: &str, at: usize) -> &str {
11173        let start = text[..at].char_indices().rev().nth(60).map_or(0, |(i, _)| i);
11174        let end = text[at..].char_indices().nth(120).map_or(text.len(), |(i, _)| at + i);
11175        &text[start..end]
11176    }
11177
11178    /// #1494: the parallelism rule reads the project's own `max_workers`
11179    /// rather than a figure in the prose, and a slot held below the cap
11180    /// is held by a guardrail the lead names.
11181    #[test]
11182    fn lead_charter_derives_parallelism_from_the_project_cap() {
11183        assert!(
11184            DEFAULT_LEAD_CHARTER.contains("`max_workers`"),
11185            "the fill-the-cap rule names the project's own key: {DEFAULT_LEAD_CHARTER}",
11186        );
11187        assert!(
11188            DEFAULT_LEAD_CHARTER.contains("agents__capacity"),
11189            "the rule points at the tool that reports the cap: {DEFAULT_LEAD_CHARTER}",
11190        );
11191        assert!(
11192            DEFAULT_LEAD_CHARTER
11193                .contains("Hold a slot back only when a guardrail above blocks the next piece, and say plainly which one"),
11194            "a slot held below the cap is held by a named guardrail: {DEFAULT_LEAD_CHARTER}",
11195        );
11196        assert!(
11197            !DEFAULT_LEAD_CHARTER.contains("2-3"),
11198            "no fixed worker figure in the prose - the cap is the config's: {DEFAULT_LEAD_CHARTER}",
11199        );
11200    }
11201
11202    /// The guardrails say what a lead serializes; this rule says what a
11203    /// lead must NOT - a read-only stage queued behind a write idles for
11204    /// the whole write and reads as waiting rather than blocked. The
11205    /// one-question test is pinned with its verdict, since the verdict is
11206    /// what stops a cautious lead from holding a whole chain.
11207    #[test]
11208    fn lead_charter_carries_the_serialize_the_write_rule() {
11209        assert!(
11210            DEFAULT_LEAD_CHARTER.contains("**Serialize the write, not the read.**"),
11211            "the rule's handle is pinned: {DEFAULT_LEAD_CHARTER}",
11212        );
11213        assert!(
11214            DEFAULT_LEAD_CHARTER.contains("ask which of its next stages touches shared state"),
11215            "the test before a HOLD is pinned: {DEFAULT_LEAD_CHARTER}",
11216        );
11217        assert!(
11218            DEFAULT_LEAD_CHARTER.contains(
11219                "if I start this now, what does it touch? If nothing shared, it starts now, \
11220                 whatever the merge order"
11221            ),
11222            "the test question and its verdict are pinned together: {DEFAULT_LEAD_CHARTER}",
11223        );
11224    }
11225
11226    /// The gate is independent review lenses, not the lead's own read of
11227    /// the diff - pinned with the fresh-install guard, so the sentence can
11228    /// never read as a mandate for tooling that is not there.
11229    #[test]
11230    fn lead_charter_carries_the_merge_gate_lens_rule() {
11231        assert!(
11232            DEFAULT_LEAD_CHARTER.contains("The merge gate is not met by a solo diff-read"),
11233            "the rule's handle is pinned: {DEFAULT_LEAD_CHARTER}",
11234        );
11235        assert!(
11236            DEFAULT_LEAD_CHARTER
11237                .contains("independent review lenses where they exist are what satisfies it"),
11238            "the lenses and their condition are pinned: {DEFAULT_LEAD_CHARTER}",
11239        );
11240        assert!(
11241            DEFAULT_LEAD_CHARTER
11242                .contains("Nothing here mandates tooling a fresh install does not have"),
11243            "the fresh-install guard is pinned: {DEFAULT_LEAD_CHARTER}",
11244        );
11245        assert!(
11246            DEFAULT_LEAD_CHARTER.contains("A round is not clean until every"),
11247            "a partial round is never clean: {DEFAULT_LEAD_CHARTER}",
11248        );
11249        assert!(
11250            DEFAULT_LEAD_CHARTER.contains(
11251                "not a clean round, because the one still running is the one that has not spoken"
11252            ),
11253            "a clean return does not stand for the round: {DEFAULT_LEAD_CHARTER}",
11254        );
11255        assert!(
11256            DEFAULT_LEAD_CHARTER.contains("never merge on the first one back"),
11257            "the merge waits for the last review, not the first: {DEFAULT_LEAD_CHARTER}",
11258        );
11259    }
11260
11261    /// A false claim of coverage survives author review because the diff
11262    /// cannot contradict a sentence, so one lens reads the prose. Pinned
11263    /// with what it reads, what it fails, and the phrases that hide a claim
11264    /// with nothing behind it.
11265    #[test]
11266    fn lead_charter_carries_the_prose_audit_lens() {
11267        assert!(
11268            DEFAULT_LEAD_CHARTER
11269                .contains("Every review round gets one lens that audits the prose, not the code"),
11270            "the lens, its every-round duty and its subject are pinned: {DEFAULT_LEAD_CHARTER}",
11271        );
11272        assert!(
11273            DEFAULT_LEAD_CHARTER.contains(
11274                "reads the PR body, the commit messages and the spec against what the work \
11275                 actually does"
11276            ),
11277            "what the lens reads is pinned: {DEFAULT_LEAD_CHARTER}",
11278        );
11279        assert!(
11280            DEFAULT_LEAD_CHARTER.contains(
11281                "fails any claim of coverage, a passing gate, a failure mode or a known gap \
11282                 with no evidence behind it"
11283            ),
11284            "the claim classes it fails are pinned: {DEFAULT_LEAD_CHARTER}",
11285        );
11286        assert!(
11287            DEFAULT_LEAD_CHARTER.contains(
11288                "\"covered by tests\", \"cannot happen\", \"unchanged behaviour\" and \"known \
11289                 limitation\" are the highest-risk phrases"
11290            ),
11291            "the highest-risk phrases are pinned: {DEFAULT_LEAD_CHARTER}",
11292        );
11293        assert!(
11294            DEFAULT_LEAD_CHARTER.contains("one of them reading the prose, not the code"),
11295            "the roster names the prose lens at the fan-out: {DEFAULT_LEAD_CHARTER}",
11296        );
11297    }
11298
11299    /// Two workflows fire per push and their ids sit adjacent in no stable
11300    /// order, so a verdict is only checkable when it names its workflow.
11301    /// Pinned at the merge gate and the health check, where a lead reports.
11302    #[test]
11303    fn lead_charter_quotes_the_workflow_beside_a_verdict() {
11304        assert!(
11305            DEFAULT_LEAD_CHARTER.contains("Quote the workflow name beside every verdict"),
11306            "the rule's handle is pinned: {DEFAULT_LEAD_CHARTER}",
11307        );
11308        assert!(
11309            DEFAULT_LEAD_CHARTER.contains(
11310                "read the job list off the run whose workflow is the one you are claiming"
11311            ),
11312            "the job list is read off the claimed workflow: {DEFAULT_LEAD_CHARTER}",
11313        );
11314        assert!(
11315            DEFAULT_LEAD_CHARTER.contains("one workflow's green is not another's"),
11316            "the adjacent-workflow confusion is pinned: {DEFAULT_LEAD_CHARTER}",
11317        );
11318        assert!(
11319            DEFAULT_LEAD_CHARTER.contains("quote each run's workflow name with its verdict"),
11320            "the health check repeats the rule where a lead reconciles: {DEFAULT_LEAD_CHARTER}",
11321        );
11322    }
11323
11324    /// A permission decision belongs to the conversation that raised it, so
11325    /// a lead relaying the user's go-ahead cannot clear a worker's gated
11326    /// action. Pinned at the merge step, where the lead is most likely to
11327    /// pass one on.
11328    #[test]
11329    fn lead_charter_carries_the_relayed_approval_rule() {
11330        assert!(
11331            DEFAULT_LEAD_CHARTER.contains("a relayed approval is not an approval"),
11332            "the rule's handle is pinned: {DEFAULT_LEAD_CHARTER}",
11333        );
11334        assert!(
11335            DEFAULT_LEAD_CHARTER.contains(
11336                "a worker's gated action needs the user's word in that worker's own \
11337                 conversation, not your relay of it"
11338            ),
11339            "the seat that needs the word is pinned: {DEFAULT_LEAD_CHARTER}",
11340        );
11341        assert!(
11342            DEFAULT_LEAD_CHARTER.contains("have the user paste it into that worker's chat"),
11343            "the route reconciling the rule with 'the user talks ONLY to you' is pinned: \
11344             {DEFAULT_LEAD_CHARTER}",
11345        );
11346    }
11347
11348    /// Two PRs each green on its own base merged in sequence composed red
11349    /// on main (measured 2026-08, again 2026-10-17, from a branch that
11350    /// predated the other's merge), so the merge-gate bullet carries the
11351    /// moved-main rule: origin/main merged in before the final gate, and
11352    /// main's CI read after the merge. Pinned with both halves, because
11353    /// either alone reads as a caveat with no action.
11354    #[test]
11355    fn lead_charter_carries_the_composed_main_rule() {
11356        assert!(
11357            DEFAULT_LEAD_CHARTER
11358                .contains("A branch green on its own base can compose red on a moved main"),
11359            "the composed-red fact is pinned: {DEFAULT_LEAD_CHARTER}",
11360        );
11361        assert!(
11362            DEFAULT_LEAD_CHARTER.contains("have the worker merge origin/main into its branch"),
11363            "the remedy's actor and its merge are pinned: {DEFAULT_LEAD_CHARTER}",
11364        );
11365        assert!(
11366            DEFAULT_LEAD_CHARTER.contains("after the PR merges main's CI is the only truth"),
11367            "the post-merge read is pinned: {DEFAULT_LEAD_CHARTER}",
11368        );
11369    }
11370}