1use std::sync::Arc;
16use std::time::Duration;
17
18use tracing::Instrument;
19
20use crate::protocol::SessionUpdate;
21use crate::target::ProjectKey;
22use crate::workspace::Workspace;
23
24const CRON_TICK_INTERVAL: Duration = Duration::from_secs(60);
27
28const COOLING_REWARN_INTERVAL: Duration = Duration::from_secs(30 * 60);
31
32pub(crate) const IST_OFFSET_MINUTES: i64 = 5 * 60 + 30;
35
36#[derive(Debug)]
40pub(crate) struct PendingCron {
41 pub text: String,
42 pub missed: bool,
43 pub cron_id: String,
44 pub description: Option<String>,
45}
46
47impl Workspace {
48 pub(crate) fn with_crons_mut<R>(
55 &self,
56 f: impl FnOnce(&mut Vec<forge_primitives::CronEntry>) -> R,
57 ) -> R {
58 let mut crons = self.crons.lock();
59 f(&mut crons)
60 }
61
62 fn persist_cron(&self, cron: &forge_primitives::CronEntry) {
65 let guard = self.db.lock();
66 let Some(db) = guard.as_ref() else { return };
67 if let Err(error) = crate::store::cron::upsert(db, cron) {
68 tracing::error!(
69 target: "forge_workspace::crons",
70 cron_id = %cron.id,
71 %error,
72 "persisting a cron to the store failed; it may be lost on restart",
73 );
74 }
75 }
76
77 fn delete_stored_cron(&self, id: &forge_primitives::CronId) {
79 let guard = self.db.lock();
80 let Some(db) = guard.as_ref() else { return };
81 if let Err(error) = crate::store::cron::delete(db, id) {
82 tracing::error!(
83 target: "forge_workspace::crons",
84 cron_id = %id,
85 %error,
86 "deleting a cron from the store failed; it may come back on restart",
87 );
88 }
89 }
90
91 pub(crate) fn push_cron(&self, entry: forge_primitives::CronEntry) {
93 let project_name = entry.project_name.clone();
94 self.with_crons_mut(|crons| {
95 self.persist_cron(&entry);
96 crons.push(entry);
97 });
98 self.announce_cron_schedules_changed(&project_name);
99 }
100
101 pub(crate) fn remove_cron(&self, project_name: &str, id: &forge_primitives::CronId) -> bool {
106 let removed = self.with_crons_mut(|crons| {
107 let before = crons.len();
108 crons.retain(|c| !(c.id == *id && c.project_name == project_name));
109 let removed = crons.len() != before;
110 if removed {
111 self.delete_stored_cron(id);
112 }
113 removed
114 });
115 if removed {
116 self.announce_cron_schedules_changed(project_name);
117 }
118 removed
119 }
120
121 pub(crate) fn remove_cron_owned_by(
127 &self,
128 project_name: &str,
129 id: &forge_primitives::CronId,
130 label: Option<&str>,
131 ) -> Option<forge_primitives::CronEntry> {
132 let removed = self.with_crons_mut(|crons| {
133 let pos = crons.iter().position(|c| {
134 c.id == *id && c.project_name == project_name && c.team_role.as_deref() == label
135 });
136 let removed = pos.map(|pos| crons.remove(pos));
137 if removed.is_some() {
138 self.delete_stored_cron(id);
139 }
140 removed
141 });
142 if removed.is_some() {
143 self.announce_cron_schedules_changed(project_name);
144 }
145 removed
146 }
147
148 pub(crate) fn delete_crons_for_worker(&self, project_key: &ProjectKey, label: &str) {
154 let Some(project_name) =
155 self.list_projects().into_iter().find(|v| v.key == *project_key).map(|v| v.name)
156 else {
157 tracing::warn!(
158 target: "forge_workspace::crons",
159 project = %project_key.as_str(),
160 label = %label,
161 "could not resolve a project name at worker teardown; its crons may be stranded",
162 );
163 return;
164 };
165 let removed = self.with_crons_mut(|crons| {
166 let mut removed: Vec<forge_primitives::CronId> = Vec::new();
167 crons.retain(|c| {
168 let taking =
169 c.project_name == project_name && c.team_role.as_deref() == Some(label);
170 if taking {
171 removed.push(c.id.clone());
172 }
173 !taking
174 });
175 for id in &removed {
176 self.delete_stored_cron(id);
177 }
178 !removed.is_empty()
179 });
180 if removed {
181 self.announce_cron_schedules_changed(&project_name);
182 }
183 }
184
185 fn announce_cron_schedules_changed(&self, project_name: &str) {
189 let Some(key) = self.lead_slot_for_project(project_name) else {
190 tracing::debug!(
191 target: "forge_workspace::crons",
192 event_name = "cron_schedules_changed_unroutable",
193 project = %project_name,
194 "a cron write named a project no project carries; nothing was announced",
195 );
196 return;
197 };
198 let crons = self.crons_for_project(project_name);
199 let _ = self.update_tx.send(SessionUpdate::CronSchedulesChanged { key, crons });
200 }
201
202 pub fn durable_crons_unavailable(&self) -> bool {
205 self.crons_load_failed.load(std::sync::atomic::Ordering::SeqCst)
206 }
207
208 pub fn crons_for_project(&self, project_name: &str) -> Vec<forge_primitives::CronEntry> {
214 self.crons.lock().iter().filter(|c| c.project_name == project_name).cloned().collect()
215 }
216
217 pub(crate) fn all_crons_snapshot(&self) -> Vec<forge_primitives::CronEntry> {
220 self.crons.lock().clone()
221 }
222
223 pub(crate) fn remember_cron_name(
229 &self,
230 slot: &forge_primitives::SessionSlot,
231 uuid: &str,
232 cron_id: &str,
233 description: Option<&str>,
234 ) {
235 let guard = self.db.lock();
236 let Some(db) = guard.as_ref() else {
237 return;
238 };
239 let name = forge_primitives::cron::CronName {
240 cron_id: cron_id.to_owned(),
241 description: description.map(str::to_owned),
242 };
243 if let Err(error) =
244 crate::store::cron_names::put(db, uuid, &name, std::time::SystemTime::now())
245 {
246 tracing::warn!(
247 event_name = "cron_name_store_failed",
248 org = slot.org(),
249 project = slot.project(),
250 label = slot.label(),
251 cron = %cron_id,
252 prompt = %uuid,
253 %error,
254 "the fired prompt's schedule was not recorded; a reload falls back to the prompt"
255 );
256 }
257 }
258
259 pub fn cron_names(
262 &self,
263 slot: &forge_primitives::SessionSlot,
264 uuids: &[&str],
265 ) -> std::collections::BTreeMap<String, forge_primitives::cron::CronName> {
266 let guard = self.db.lock();
267 let Some(db) = guard.as_ref() else {
268 return std::collections::BTreeMap::new();
269 };
270 crate::store::cron_names::for_uuids(db, uuids).unwrap_or_else(|error| {
271 tracing::warn!(
272 event_name = "cron_names_read_failed",
273 org = slot.org(),
274 project = slot.project(),
275 label = slot.label(),
276 prompts = uuids.len(),
277 %error,
278 "the fired prompts' schedules could not be read; their rows fall back to the prompt"
279 );
280 std::collections::BTreeMap::new()
281 })
282 }
283
284 pub(crate) fn advance_or_remove_cron(
290 &self,
291 id: &forge_primitives::CronId,
292 fired_at: std::time::SystemTime,
293 ) {
294 let mut project_name = None;
297 self.with_crons_mut(|crons| {
298 let Some(pos) = crons.iter().position(|c| &c.id == id) else { return };
299 project_name = Some(crons[pos].project_name.clone());
300 match &crons[pos].kind {
301 forge_primitives::CronKind::Once(_) => {
302 crons.remove(pos);
303 self.delete_stored_cron(id);
304 }
305 forge_primitives::CronKind::Recurring(_) => {
306 if let Some(next) =
307 crate::mcp::cron::schedule::next_fire_after(&crons[pos].kind, fired_at)
308 {
309 crons[pos].last_fire = Some(fired_at);
310 crons[pos].next_fire = next;
311 self.persist_cron(&crons[pos]);
312 } else {
313 let removed = crons.remove(pos);
317 let expr = if let forge_primitives::CronKind::Recurring(e) = &removed.kind {
318 e.as_str()
319 } else {
320 ""
321 };
322 tracing::warn!(
323 target: "forge_workspace::crons",
324 cron_id = %removed.id,
325 project = %removed.project_name,
326 expr = %expr,
327 "recurring cron has no upcoming occurrence; removed it",
328 );
329 self.delete_stored_cron(id);
330 }
331 }
332 }
333 });
334 if let Some(project_name) = project_name {
335 self.announce_cron_schedules_changed(&project_name);
336 }
337 }
338
339 pub fn fire_due_crons(self: &Arc<Self>, now: std::time::SystemTime) {
351 use crate::spawn::CronFireOutcome;
352 if self.is_shutting_down() {
356 return;
357 }
358 let snapshot = self.all_crons_snapshot();
359 let due = crate::mcp::cron::schedule::due_crons(&snapshot, now);
360 let mut still_unwakeable = std::collections::HashSet::new();
363 let mut still_cooling = std::collections::HashMap::new();
365 for id in &due {
366 let Some(cron) = snapshot.iter().find(|c| &c.id == id) else { continue };
367 let missed = now > cron.next_fire + CRON_TICK_INTERVAL * 2;
372 match crate::spawn::deliver_cron_prompt(self, cron, missed) {
373 CronFireOutcome::Delivered => self.advance_or_remove_cron(id, now),
376 CronFireOutcome::TargetGone => {
381 tracing::warn!(
382 target: "forge_workspace::crons",
383 event_name = "cron_owner_gone",
384 project = %cron.project_name,
385 cron_id = %id,
386 label = cron.team_role.as_deref().unwrap_or("lead"),
387 "cron slot has no session; removing the cron",
388 );
389 self.remove_cron(&cron.project_name, id);
390 }
391 CronFireOutcome::TargetCannotBeWoken { directory } => {
399 let recurring = matches!(cron.kind, forge_primitives::CronKind::Recurring(_));
400 if self.cron_unwakeable_is_new(id) {
405 tracing::warn!(
406 target: "forge_workspace::crons",
407 event_name = "cron_owner_cannot_be_woken",
408 project = %cron.project_name,
409 cron_id = %id,
410 label = cron.team_role.as_deref().unwrap_or("lead"),
411 directory = %directory.display(),
412 recurring,
413 "cron owner cannot be woken, so this fire did not land; the owner's \
414 row is kept",
415 );
416 } else {
417 tracing::debug!(
418 target: "forge_workspace::crons",
419 event_name = "cron_owner_cannot_be_woken",
420 project = %cron.project_name,
421 cron_id = %id,
422 "cron owner still cannot be woken; the first warning for this \
423 cron already names it",
424 );
425 }
426 still_unwakeable.insert(id.clone());
427 if recurring {
428 self.advance_or_remove_cron(id, now);
429 }
430 }
431 CronFireOutcome::SkippedAsleep => {
436 tracing::debug!(
437 target: "forge_workspace::crons",
438 event_name = "cron_skipped_asleep",
439 project = %cron.project_name,
440 cron_id = %id,
441 "the cron's owner is asleep and this cron does not wake it; the fire \
442 stays due",
443 );
444 }
445 CronFireOutcome::DeferredCooling { cooling } => {
449 let reset = cooling_reset_text(cooling.reset_in, now);
450 let warned_before = self.cron_cooling_warned_at(id);
451 let warn_now = warned_before.is_none_or(|at| {
452 now.duration_since(at).unwrap_or(Duration::ZERO) >= COOLING_REWARN_INTERVAL
453 });
454 if warn_now {
455 tracing::warn!(
456 target: "forge_workspace::crons",
457 event_name = "cron_deferred_cooling",
458 project = %cron.project_name,
459 cron_id = %id,
460 label = cron.team_role.as_deref().unwrap_or("lead"),
461 org = %cooling.org,
462 model = %cooling.model,
463 cooling_accounts = cooling.tried,
464 reset_in_secs = cooling.reset_in.as_secs(),
465 resets_at = %reset,
466 "cron fire deferred: every account serving this project's model is \
467 cooling, so the wake would be refused; it stays due and retries",
468 );
469 let _ = self.update_tx.send(SessionUpdate::ServiceStatus {
470 severity:
471 forge_primitives::cloud::service_status::ServiceSeverity::Warning,
472 message: format!(
473 "Cron in '{}' deferred: no account serving {} in {} is free \
474 ({} account(s) cooling in the org); resets at {reset}",
475 cron.project_name, cooling.model, cooling.org, cooling.tried,
476 ),
477 });
478 } else {
479 tracing::debug!(
480 target: "forge_workspace::crons",
481 event_name = "cron_deferred_cooling",
482 project = %cron.project_name,
483 cron_id = %id,
484 label = cron.team_role.as_deref().unwrap_or("lead"),
485 org = %cooling.org,
486 model = %cooling.model,
487 cooling_accounts = cooling.tried,
488 reset_in_secs = cooling.reset_in.as_secs(),
489 resets_at = %reset,
490 "cron fire still deferred by the same cooling; the warning for this \
491 episode already names it",
492 );
493 }
494 let warned_at = warned_before.filter(|_| !warn_now).unwrap_or(now);
497 still_cooling.insert(id.clone(), warned_at);
498 }
499 CronFireOutcome::DispatchFailed => {
503 tracing::warn!(
504 target: "forge_workspace::crons",
505 project = %cron.project_name,
506 cron_id = %id,
507 "cron fire deferred; leaving it due to retry",
508 );
509 let _ = self.update_tx.send(SessionUpdate::ServiceStatus {
510 severity: forge_primitives::cloud::service_status::ServiceSeverity::Warning,
511 message: format!(
512 "Cron in '{}' could not fire yet (its session is shutting down, or a \
513 spawn would be refused right now); it stays due and retries",
514 cron.project_name
515 ),
516 });
517 }
518 }
519 }
520 self.cron_unwakeable_commit(still_unwakeable);
525 self.cron_cooling_commit(still_cooling);
526 }
527
528 pub fn start_cron_scheduler(self: &Arc<Self>) {
533 let weak = Arc::downgrade(self);
534 let span = tracing::info_span!("cron_scheduler");
535 self.supervisor.spawn("cron_scheduler", forge_supervise::Policy::backoff(), move || {
536 let weak = weak.clone();
537 let span = span.clone();
538 async move {
539 let mut interval = tokio::time::interval_at(
540 tokio::time::Instant::now() + CRON_TICK_INTERVAL,
541 CRON_TICK_INTERVAL,
542 );
543 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
544 loop {
545 interval.tick().await;
546 let Some(workspace) = weak.upgrade() else {
547 return;
548 };
549 workspace.fire_due_crons(std::time::SystemTime::now());
550 }
551 }
552 .instrument(span)
553 });
554 }
555}
556
557fn cooling_reset_text(reset_in: Duration, now: std::time::SystemTime) -> String {
562 let ist = now
563 .checked_add(reset_in)
564 .and_then(|at| at.duration_since(std::time::SystemTime::UNIX_EPOCH).ok())
565 .and_then(|since_epoch| {
566 chrono::DateTime::from_timestamp(
567 i64::try_from(since_epoch.as_secs()).ok()?,
568 since_epoch.subsec_nanos(),
569 )
570 })
571 .and_then(|at| at.checked_add_signed(chrono::TimeDelta::minutes(IST_OFFSET_MINUTES)));
572 match ist {
573 Some(at) => format!("{} IST (in {})", at.format("%H:%M"), wait_text(reset_in)),
574 None => format!("in {}", wait_text(reset_in)),
575 }
576}
577
578fn wait_text(wait: Duration) -> String {
580 let (hours, minutes, seconds) =
581 (wait.as_secs() / 3600, wait.as_secs() % 3600 / 60, wait.as_secs() % 60);
582 if hours > 0 {
583 format!("{hours}h {minutes}m")
584 } else if minutes > 0 {
585 format!("{minutes}m {seconds}s")
586 } else {
587 format!("{seconds}s")
588 }
589}
590
591#[cfg(any(test, feature = "testing"))]
592impl Workspace {
593 pub fn seed_test_cron(&self, entry: forge_primitives::CronEntry) {
601 self.with_crons_mut(|crons| {
602 self.persist_cron(&entry);
603 crons.push(entry);
604 });
605 }
606}
607
608#[cfg(test)]
609mod tests {
610 use super::*;
611
612 use tempfile::tempdir;
613
614 use crate::SessionSlot;
615 use crate::protocol::{Command, SessionUpdate};
616 use crate::workspace::PooledAgent;
617 use forge_gateway::AccountKey;
618
619 fn parked_crons(ws: &crate::Workspace, project: &str, label: Option<&str>) -> Vec<String> {
622 let org =
623 ws.list_projects().into_iter().find(|v| v.name == project).expect("seeded project").org;
624 ws.parked_by_slot
625 .lock()
626 .get(&crate::SessionSlot::for_label(&org, project, label))
627 .map(|parked| parked.cron.iter().map(|p| p.text.clone()).collect())
628 .unwrap_or_default()
629 }
630
631 fn seed_worker_row(ws: &crate::Workspace, project_key: &ProjectKey, label: &str) {
636 ws.seed_test_worker_row(project_key, label);
637 assert!(
638 ws.stored_worker_row(project_key, label).expect("read the seeded row").is_some(),
639 "the worker row for {label} did not land; does {project_key:?} resolve to a project?",
640 );
641 }
642
643 fn stranded_owner_fixture(
652 label: &str,
653 ) -> (Arc<Workspace>, ProjectKey, tempfile::TempDir, tempfile::TempDir) {
654 let (ws, _rx) = Workspace::testing_stub();
655 let dir = tempdir().expect("tempdir");
656 ws.install_db_for_test(
657 crate::store::Db::open(&dir.path().join("db.redb")).expect("open db"),
658 );
659 let project_dir = tempdir().expect("project dir");
660 let key = seed_project_with_a_real_root(&ws, &project_dir);
661 ws.record_worker_row(
662 &key,
663 label,
664 &format!("{label}-uuid"),
665 "c",
666 None,
667 None,
668 false,
669 crate::mcp::workers::types::WorkerTree { is_git_repo: true, cwd: None },
670 None,
671 )
672 .expect("seed the stranded worker's row");
673 (ws, key, dir, project_dir)
674 }
675
676 fn seed_project_with_a_real_root(ws: &crate::Workspace, dir: &tempfile::TempDir) -> ProjectKey {
677 let root = dir.path().join("proj-root");
678 std::fs::create_dir_all(&root).expect("create the project dir");
679 ws.seed_test_project("proj", &root.to_string_lossy());
680 ws.project_key_for_name("proj").expect("seeded project")
681 }
682
683 fn live_worker_entry(project: &str, label: &str) -> crate::mcp::workers::types::WorkerEntry {
684 crate::mcp::workers::types::WorkerEntry {
685 label: label.to_owned(),
686 charter: "c".to_owned(),
687 slot: SessionSlot::worker("TestOrg", project, label),
688 session_id: None,
689 status: forge_primitives::WorkerLiveness::Running,
690 spawned_at: std::time::SystemTime::UNIX_EPOCH,
691 spawned_by: SessionSlot::lead("TestOrg", project),
692 needs_tag: false,
693 is_git_repo_at_spawn: false,
694 cwd: None,
695 diagnostic: None,
696 kick: None,
697 }
698 }
699
700 fn drain_updates(
702 rx: &mut tokio::sync::mpsc::UnboundedReceiver<crate::SeatUpdate>,
703 ) -> Vec<crate::protocol::SessionUpdate> {
704 let mut out = Vec::new();
705 while let Ok(frame) = rx.try_recv() {
706 out.push(std::sync::Arc::unwrap_or_clone(frame.update));
707 }
708 out
709 }
710
711 fn project_expanded_path(workspace: &Workspace, name: &str) -> String {
712 workspace.list_projects().into_iter().find(|p| p.name == name).map_or_else(
713 || panic!("project '{name}' missing from workspace"),
714 |p| p.path.to_string_lossy().into_owned(),
715 )
716 }
717
718 fn schedule(
720 id: &str,
721 project: &str,
722 prompt: &str,
723 owner: Option<&str>,
724 ) -> forge_primitives::cron::CronEntry {
725 use forge_primitives::cron::{CronEntry, CronId, CronKind};
726 CronEntry {
727 id: CronId::from(id),
728 project_name: project.to_owned(),
729 kind: CronKind::Recurring("0 9 * * *".to_owned()),
730 prompt: prompt.to_owned(),
731 created_at: std::time::SystemTime::UNIX_EPOCH,
732 description: None,
733 last_fire: None,
734 next_fire: std::time::SystemTime::UNIX_EPOCH,
735 team_role: owner.map(str::to_owned),
736 wake: true,
737 }
738 }
739
740 fn deliver_cron(
744 ws: &Arc<Workspace>,
745 project: &str,
746 owner: Option<&str>,
747 prompt: &str,
748 missed: bool,
749 ) -> crate::spawn::CronFireOutcome {
750 crate::spawn::deliver_cron_prompt(
751 ws,
752 &schedule("delivered-cron", project, prompt, owner),
753 missed,
754 )
755 }
756
757 fn next_crons_changed(
760 rx: &mut tokio::sync::mpsc::UnboundedReceiver<crate::SeatUpdate>,
761 ) -> (SessionSlot, Vec<forge_primitives::cron::CronEntry>) {
762 match rx.try_recv().map(|frame| std::sync::Arc::unwrap_or_clone(frame.update)) {
763 Ok(SessionUpdate::CronSchedulesChanged { key, crons }) => (key, crons),
764 other => panic!("expected a CronSchedulesChanged on the stream, got {other:?}"),
765 }
766 }
767
768 #[test]
774 fn a_cron_write_announces_the_projects_schedules() {
775 use forge_primitives::cron::CronId;
776 let dir = tempdir().expect("tempdir");
777 let (ws, mut rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
778 ws.seed_test_project("proj", "/tmp/tp-crons");
779 let lead = SessionSlot::lead("TestOrg", "proj");
780
781 ws.push_cron(schedule("c1", "proj", "stand-up", None));
782 let (key, crons) = next_crons_changed(&mut rx);
783 assert_eq!(key, lead, "a create routes on the project's lead seat");
784 assert_eq!(
785 crons.iter().map(|cron| cron.prompt.clone()).collect::<Vec<_>>(),
786 ["stand-up"],
787 "and carries the set the core now holds",
788 );
789
790 assert!(
791 ws.remove_cron_owned_by("proj", &CronId::from("c1"), None).is_some(),
792 "precondition: the lead deletes its own cron",
793 );
794 let (_, crons) = next_crons_changed(&mut rx);
795 assert!(crons.is_empty(), "a delete announces the set it left behind");
796 }
797
798 #[test]
805 fn a_cron_write_that_changed_nothing_announces_nothing() {
806 use forge_primitives::cron::CronId;
807 let dir = tempdir().expect("tempdir");
808 let (ws, mut rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
809 ws.seed_test_project("proj", "/tmp/tp-crons-noop");
810 let key = ws.project_key_for_name("proj").expect("seeded project");
811
812 assert!(
813 !ws.remove_cron("proj", &CronId::from("ghost")),
814 "precondition: no entry carries the id",
815 );
816 assert!(
817 ws.remove_cron_owned_by("proj", &CronId::from("ghost"), None).is_none(),
818 "precondition: no entry carries the id for that owner either",
819 );
820 ws.seed_test_cron(schedule("c1", "proj", "stand-up", Some("reviewer")));
821 assert!(
822 ws.remove_cron_owned_by("proj", &CronId::from("c1"), None).is_none(),
823 "precondition: the entry belongs to the worker, so the lead's delete is refused",
824 );
825 ws.delete_crons_for_worker(&key, "nobody");
826 ws.advance_or_remove_cron(&CronId::from("ghost"), std::time::SystemTime::now());
827
828 let announced = rx.try_recv();
829 assert!(
830 announced.is_err(),
831 "no write here changed a schedule, so nothing is announced, and this was: {announced:?}",
832 );
833 }
834
835 #[test]
845 fn a_fired_schedule_and_a_teardown_announce_what_they_left() {
846 use forge_primitives::cron::{CronId, CronKind};
847 let dir = tempdir().expect("tempdir");
848 let (ws, mut rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
849 ws.seed_test_project("proj", "/tmp/tp-crons-fire");
850 let key = ws.project_key_for_name("proj").expect("seeded project");
851 let lead = SessionSlot::lead("TestOrg", "proj");
852 let now = std::time::SystemTime::now();
853
854 ws.seed_test_cron(schedule("r", "proj", "stand-up", None));
855 ws.advance_or_remove_cron(&CronId::from("r"), now);
856 let (announced_key, crons) = next_crons_changed(&mut rx);
857 assert_eq!(announced_key, lead, "a fire routes on the project's lead seat");
858 assert_eq!(crons.len(), 1, "a fired recurring keeps its entry");
859 assert!(
860 crons[0].next_fire > now,
861 "and the announcement carries the schedule it advanced to"
862 );
863 assert_eq!(crons[0].last_fire, Some(now), "with the fire it recorded");
864
865 ws.seed_test_cron(forge_primitives::cron::CronEntry {
868 kind: CronKind::Once(now),
869 ..schedule("o", "proj", "deploy", None)
870 });
871 ws.advance_or_remove_cron(&CronId::from("o"), now);
872 let (_, crons) = next_crons_changed(&mut rx);
873 assert!(
874 crons.iter().all(|cron| cron.id != CronId::from("o")),
875 "the fired run-once is announced as gone",
876 );
877
878 ws.seed_test_cron(schedule("orphan", "proj", "no owner", None));
882 assert!(
883 ws.remove_cron("proj", &CronId::from("orphan")),
884 "precondition: the router's removal finds the entry",
885 );
886 let (announced_key, crons) = next_crons_changed(&mut rx);
887 assert_eq!(announced_key, lead, "a dead-slot removal routes the same way");
888 assert!(
889 crons.iter().all(|cron| cron.id != CronId::from("orphan")),
890 "and announces the set it left: {crons:?}",
891 );
892
893 ws.seed_test_cron(schedule("w", "proj", "review", Some("reviewer")));
895 ws.delete_crons_for_worker(&key, "reviewer");
896 let (announced_key, crons) = next_crons_changed(&mut rx);
897 assert_eq!(announced_key, lead, "a teardown routes on the project's lead seat");
898 assert!(
899 crons.iter().all(|cron| cron.team_role.as_deref() != Some("reviewer")),
900 "and announces what the teardown left: {crons:?}",
901 );
902 }
903
904 #[test]
905 fn cron_methods_push_list_remove_and_persist() {
906 use forge_primitives::cron::{CronEntry, CronId, CronKind};
907 let dir = tempdir().expect("tempdir");
908 let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
909 let db = crate::store::Db::open(&dir.path().join("db.redb")).expect("open db");
910 ws.install_db_for_test(db);
911 let persisted = || {
912 crate::store::cron::list(ws.db.lock().as_ref().expect("db installed")).expect("list")
913 };
914
915 let entry = CronEntry {
916 id: CronId::from("c1"),
917 project_name: "forge".to_owned(),
918 kind: CronKind::Recurring("0 9 * * *".to_owned()),
919 prompt: "stand-up".to_owned(),
920 created_at: std::time::SystemTime::UNIX_EPOCH,
921 description: None,
922 last_fire: None,
923 next_fire: std::time::SystemTime::UNIX_EPOCH,
924 team_role: None,
925 wake: true,
926 };
927
928 ws.push_cron(entry.clone());
929 assert_eq!(ws.crons_for_project("forge"), vec![entry.clone()], "listed for its project");
930 assert!(ws.crons_for_project("other").is_empty(), "scoped by project name");
931 assert_eq!(persisted(), vec![entry.clone()], "push persisted to the store");
932
933 assert!(!ws.remove_cron("other", &entry.id), "delete is scoped to the owning project");
935 assert_eq!(ws.crons_for_project("forge").len(), 1);
936
937 assert!(ws.remove_cron("forge", &entry.id), "the owning project removes it");
938 assert!(ws.crons_for_project("forge").is_empty());
939 assert!(persisted().is_empty(), "removal persisted");
940
941 assert!(!ws.remove_cron("forge", &CronId::from("c1")), "removing a gone id reports false");
942 }
943
944 #[test]
959 fn cron_mutations_over_a_failed_boot_read_do_not_erase_the_store() {
960 use forge_primitives::cron::{CronEntry, CronId, CronKind};
961 let dir = tempdir().expect("tempdir");
962 let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
963 ws.seed_test_project("proj", "/tmp/tp-crons-failed-load");
964 let db = crate::store::Db::open(&dir.path().join("db.redb")).expect("open db");
965 crate::store::cron::upsert(&db, &schedule("seed-1", "proj", "one", None)).expect("seed 1");
967 crate::store::cron::upsert(&db, &schedule("seed-2", "proj", "two", None)).expect("seed 2");
968 ws.install_db_for_test(db);
969 let persisted = || {
970 crate::store::cron::list(ws.db.lock().as_ref().expect("db installed")).expect("list")
971 };
972 let seeded = |rows: &[forge_primitives::cron::CronEntry]| {
973 rows.iter().filter(|c| c.id.as_str().starts_with("seed-")).count()
974 };
975 let still_seeded = |door: &str| {
978 assert_eq!(
979 seeded(&persisted()),
980 2,
981 "after {door} the rows the failed read never loaded must still be there",
982 );
983 };
984 let key = ws.project_key_for_name("proj").expect("seeded project");
985 let now = std::time::SystemTime::now();
986
987 ws.push_cron(schedule("created", "proj", "new", None));
988 still_seeded("a create");
989
990 assert!(
993 !ws.remove_cron("proj", &CronId::from("seed-1")),
994 "precondition: the empty in-memory set holds no such row",
995 );
996 still_seeded("a delete that removed nothing");
997
998 assert!(
999 ws.remove_cron_owned_by("proj", &CronId::from("seed-1"), None).is_none(),
1000 "precondition: the empty in-memory set holds no such row for that owner either",
1001 );
1002 still_seeded("an owner-scoped delete that matched nothing");
1003
1004 ws.delete_crons_for_worker(&key, "worker");
1005 still_seeded("a worker teardown with no matching crons");
1006
1007 ws.advance_or_remove_cron(&CronId::from("seed-2"), now);
1008 still_seeded("a fire-advance over an unloaded row");
1009
1010 assert!(
1013 ws.remove_cron("proj", &CronId::from("created")),
1014 "precondition: the loaded row is removed",
1015 );
1016 still_seeded("a delete of a loaded row");
1017
1018 ws.push_cron(schedule("owned", "proj", "own", None));
1019 assert!(
1020 ws.remove_cron_owned_by("proj", &CronId::from("owned"), None).is_some(),
1021 "precondition: the loaded row is removed through the owner's door",
1022 );
1023 still_seeded("an owner-scoped delete of a loaded row");
1024
1025 ws.push_cron(schedule("worker-cron", "proj", "w", Some("worker")));
1026 ws.delete_crons_for_worker(&key, "worker");
1027 still_seeded("a worker teardown that took a cron");
1028
1029 ws.push_cron(CronEntry {
1030 kind: CronKind::Once(now),
1031 ..schedule("once", "proj", "fire", None)
1032 });
1033 ws.advance_or_remove_cron(&CronId::from("once"), now);
1034 still_seeded("a fire that removed a run-once");
1035
1036 ws.push_cron(schedule("recurring", "proj", "again", None));
1037 ws.advance_or_remove_cron(&CronId::from("recurring"), now);
1038 still_seeded("a fire that advanced a recurring");
1039
1040 ws.push_cron(CronEntry {
1041 kind: CronKind::Recurring("0 0 30 2 *".to_owned()),
1042 ..schedule("never", "proj", "never", None)
1043 });
1044 ws.advance_or_remove_cron(&CronId::from("never"), now);
1045 still_seeded("a fire that removed a never-occurring recurring");
1046 }
1047
1048 #[test]
1049 fn advance_or_remove_cron_recurring_advances_run_once_removes() {
1050 use forge_primitives::cron::{CronEntry, CronId, CronKind};
1051 let dir = tempdir().expect("tempdir");
1052 let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
1053 let fired = std::time::SystemTime::now();
1054
1055 ws.push_cron(CronEntry {
1056 id: CronId::from("r"),
1057 project_name: "forge".to_owned(),
1058 kind: CronKind::Recurring("*/5 * * * *".to_owned()),
1059 prompt: "p".to_owned(),
1060 created_at: fired,
1061 description: None,
1062 last_fire: None,
1063 next_fire: std::time::SystemTime::UNIX_EPOCH,
1064 team_role: None,
1065 wake: true,
1066 });
1067 ws.advance_or_remove_cron(&CronId::from("r"), fired);
1068 let after = ws.crons_for_project("forge");
1069 assert_eq!(after.len(), 1, "a recurring cron stays after firing");
1070 assert!(after[0].next_fire > fired, "next_fire advanced to a future slot");
1071 assert_eq!(after[0].last_fire, Some(fired), "last_fire recorded");
1072
1073 ws.push_cron(CronEntry {
1074 id: CronId::from("o"),
1075 project_name: "forge".to_owned(),
1076 kind: CronKind::Once(std::time::SystemTime::UNIX_EPOCH),
1077 prompt: "p".to_owned(),
1078 created_at: fired,
1079 description: None,
1080 last_fire: None,
1081 next_fire: std::time::SystemTime::UNIX_EPOCH,
1082 team_role: None,
1083 wake: true,
1084 });
1085 ws.advance_or_remove_cron(&CronId::from("o"), fired);
1086 assert!(
1087 ws.crons_for_project("forge").iter().all(|c| c.id != CronId::from("o")),
1088 "a run-once is removed after firing",
1089 );
1090
1091 ws.advance_or_remove_cron(&CronId::from("ghost"), fired);
1093 }
1094
1095 #[test]
1096 fn fire_due_crons_spawns_asleep_project_buffers_prompt_and_skips_future() {
1097 use forge_primitives::cron::{CronEntry, CronId, CronKind};
1098 let dir = tempdir().expect("tempdir");
1099 let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
1100 ws.seed_test_project("forge", "/tmp/forge");
1101
1102 let now = std::time::SystemTime::now();
1103 let past = std::time::SystemTime::UNIX_EPOCH;
1104 let far_future = now + std::time::Duration::from_secs(86_400);
1105
1106 ws.push_cron(CronEntry {
1107 id: CronId::from("due"),
1108 project_name: "forge".to_owned(),
1109 kind: CronKind::Recurring("*/5 * * * *".to_owned()),
1110 prompt: "morning".to_owned(),
1111 created_at: past,
1112 description: None,
1113 last_fire: None,
1114 next_fire: past,
1115 team_role: None,
1116 wake: true,
1117 });
1118 ws.push_cron(CronEntry {
1119 id: CronId::from("later"),
1120 project_name: "forge".to_owned(),
1121 kind: CronKind::Recurring("*/5 * * * *".to_owned()),
1122 prompt: "later".to_owned(),
1123 created_at: past,
1124 description: None,
1125 last_fire: None,
1126 next_fire: far_future,
1127 team_role: None,
1128 wake: true,
1129 });
1130
1131 ws.enable_test_dispatch_intercept();
1132 ws.fire_due_crons(now);
1133 let dispatched = ws.drain_test_dispatch_buffer();
1134
1135 let spawns = dispatched
1138 .iter()
1139 .filter(|c| {
1140 matches!(c, crate::protocol::Command::SpawnProject { project_name, .. }
1141 if project_name == "forge")
1142 })
1143 .count();
1144 assert_eq!(spawns, 1, "one spawn for the single due cron");
1145
1146 let buffered = parked_crons(&ws, "forge", None);
1149 assert_eq!(buffered, vec!["morning".to_owned()], "the due cron's prompt was buffered");
1150
1151 let crons = ws.crons_for_project("forge");
1153 let due = crons.iter().find(|c| c.id == CronId::from("due")).expect("due present");
1154 assert!(due.next_fire > now, "the fired cron advanced past now");
1155 let later = crons.iter().find(|c| c.id == CronId::from("later")).expect("later present");
1156 assert_eq!(later.next_fire, far_future, "the not-yet-due cron is untouched");
1157 }
1158
1159 #[test]
1160 fn boot_catch_up_fires_overdue_once_advances_persists_and_does_not_refire() {
1161 use forge_primitives::cron::{CronEntry, CronId, CronKind};
1162 let dir = tempdir().expect("tempdir");
1163 let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
1164 let db = crate::store::Db::open(&dir.path().join("db.redb")).expect("open db");
1165 ws.install_db_for_test(db);
1166 ws.seed_test_project("forge", "/tmp/forge-catchup");
1167
1168 let now = std::time::SystemTime::now();
1169 let past = std::time::SystemTime::UNIX_EPOCH;
1170 let far_future = now + std::time::Duration::from_secs(86_400);
1171
1172 for entry in [
1176 CronEntry {
1177 id: CronId::from("rec"),
1178 project_name: "forge".to_owned(),
1179 kind: CronKind::Recurring("*/5 * * * *".to_owned()),
1180 prompt: "morning".to_owned(),
1181 created_at: past,
1182 description: None,
1183 last_fire: None,
1184 next_fire: past,
1185 team_role: None,
1186 wake: true,
1187 },
1188 CronEntry {
1189 id: CronId::from("once"),
1190 project_name: "forge".to_owned(),
1191 kind: CronKind::Once(past),
1192 prompt: "deploy".to_owned(),
1193 created_at: past,
1194 description: None,
1195 last_fire: None,
1196 next_fire: past,
1197 team_role: None,
1198 wake: true,
1199 },
1200 CronEntry {
1201 id: CronId::from("future"),
1202 project_name: "forge".to_owned(),
1203 kind: CronKind::Recurring("*/5 * * * *".to_owned()),
1204 prompt: "later".to_owned(),
1205 created_at: past,
1206 description: None,
1207 last_fire: None,
1208 next_fire: far_future,
1209 team_role: None,
1210 wake: true,
1211 },
1212 ] {
1213 ws.push_cron(entry);
1214 }
1215 assert_eq!(ws.crons_for_project("forge").len(), 3, "all three crons loaded");
1216
1217 ws.enable_test_dispatch_intercept();
1219 ws.fire_due_crons(now);
1220 let first = ws.drain_test_dispatch_buffer();
1221 let spawns = first
1222 .iter()
1223 .filter(|c| {
1224 matches!(c, crate::protocol::Command::SpawnProject { project_name, .. }
1225 if project_name == "forge")
1226 })
1227 .count();
1228 assert_eq!(spawns, 2, "boot fires the two overdue crons, not the future one");
1229
1230 let crons = ws.crons_for_project("forge");
1233 assert!(crons.iter().all(|c| c.id != CronId::from("once")), "overdue run-once removed");
1234 let rec = crons.iter().find(|c| c.id == CronId::from("rec")).expect("recurring present");
1235 assert!(rec.next_fire > now, "overdue recurring advanced past now");
1236 assert_eq!(rec.last_fire, Some(now), "the fired recurring recorded last_fire");
1237 let fut = crons.iter().find(|c| c.id == CronId::from("future")).expect("future present");
1238 assert_eq!(fut.next_fire, far_future, "the future cron is untouched");
1239
1240 let persisted =
1242 crate::store::cron::list(ws.db.lock().as_ref().expect("db installed")).expect("list");
1243 assert!(persisted.iter().all(|c| c.id != CronId::from("once")), "removal persisted");
1244 assert!(
1245 persisted
1246 .iter()
1247 .find(|c| c.id == CronId::from("rec"))
1248 .expect("rec persisted")
1249 .next_fire
1250 > now,
1251 "advance persisted",
1252 );
1253
1254 ws.fire_due_crons(now);
1256 let refires = ws
1257 .drain_test_dispatch_buffer()
1258 .iter()
1259 .filter(|c| matches!(c, crate::protocol::Command::SpawnProject { .. }))
1260 .count();
1261 assert_eq!(refires, 0, "advanced crons are not due again; no double-fire");
1262 }
1263
1264 #[test]
1270 fn deliver_cron_prompt_to_running_lead_emits_cron_prompt_appended() {
1271 let dir = tempdir().expect("tempdir");
1272 let (ws, mut rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
1273 ws.install_db_for_test(
1276 crate::store::Db::open(&dir.path().join("db.redb")).expect("open db"),
1277 );
1278 ws.seed_test_project("cronlead", "/tmp/cron-lead");
1279 let cwd = project_expanded_path(&ws, "cronlead");
1282 ws.record_connected_session(&cwd, "lead-uuid", None);
1283 let lead_key = SessionSlot::lead("TestOrg", "cronlead");
1284 let (handle, _agent_rx) = Workspace::testing_stub_handle();
1285 ws.pool.lock().insert(
1286 lead_key.clone(),
1287 PooledAgent {
1288 handle: Arc::new(handle),
1289 account: AccountKey("test".to_owned()),
1290 permission_mode: None,
1291 registration: None,
1292 session_id: "pooled-session".to_owned(),
1293 },
1294 );
1295
1296 ws.mark_session_connected_for_test(&lead_key, "lead-uuid");
1297 ws.enable_test_dispatch_intercept();
1298 let mut fired = schedule("c1", "cronlead", "morning", None);
1299 fired.description = Some("Morning summary".to_owned());
1300 let outcome = crate::spawn::deliver_cron_prompt(&ws, &fired, false);
1301 assert!(matches!(outcome, crate::spawn::CronFireOutcome::Delivered));
1302
1303 let dispatched = ws.drain_test_dispatch_buffer();
1305 let wired_id = dispatched
1306 .iter()
1307 .find_map(|c| match c {
1308 Command::PromptUnder { key, text, uuid, .. }
1309 if key == &lead_key && text == "morning" =>
1310 {
1311 Some(uuid.clone())
1312 }
1313 _ => None,
1314 })
1315 .expect("the running lead receives the fired cron prompt verbatim, wired under an id");
1316
1317 let echoed = drain_updates(&mut rx).into_iter().find_map(|u| match u {
1324 SessionUpdate::CronPromptAppended { key, text, uuid, cron_id, description }
1325 if key == lead_key && text == "morning" =>
1326 {
1327 Some((uuid, cron_id, description))
1328 }
1329 _ => None,
1330 });
1331 let (echoed_id, cron_id, description) =
1332 echoed.expect("the delivery echoes a cron block for the fired prompt");
1333 assert_eq!(
1334 echoed_id.as_str(),
1335 wired_id.as_str(),
1336 "the block and the fired prompt are one thing by id",
1337 );
1338 assert!(!wired_id.is_empty(), "and the id is real, not a blank");
1339 assert_eq!(cron_id, "c1", "the echo names the entry that fired");
1340 assert_eq!(description.as_deref(), Some("Morning summary"), "and its description");
1341 let stored = crate::store::cron_names::for_uuids(
1346 ws.db.lock().as_ref().expect("db installed"),
1347 &[wired_id.as_str()],
1348 )
1349 .expect("read the stored pairing");
1350 assert_eq!(
1351 stored.get(wired_id.as_str()),
1352 Some(&forge_primitives::cron::CronName {
1353 cron_id: "c1".to_owned(),
1354 description: Some("Morning summary".to_owned()),
1355 }),
1356 "the fired prompt's schedule is readable after the frame is gone"
1357 );
1358 }
1359
1360 #[test]
1361 fn deliver_worker_cron_to_a_live_worker_prompts_the_worker() {
1362 let (ws, _rx) = Workspace::testing_stub();
1363 ws.seed_test_project("proj", "/tmp/wc-live");
1364 let key = ws.list_projects().into_iter().find(|v| v.name == "proj").expect("view").key;
1365 ws.insert_live_worker(&key, live_worker_entry("proj", "reviewer"));
1366 let worker_key = SessionSlot::worker("TestOrg", "proj", "reviewer");
1367 ws.mark_session_connected_for_test(&worker_key, "worker-uuid");
1368
1369 ws.enable_test_dispatch_intercept();
1370 let outcome = deliver_cron(&ws, "proj", Some("reviewer"), "review the diff", false);
1371 assert!(matches!(outcome, crate::spawn::CronFireOutcome::Delivered));
1372 let dispatched = ws.drain_test_dispatch_buffer();
1373 assert!(
1374 dispatched.iter().any(|c| matches!(
1375 c, Command::Prompt { key, text, .. } | Command::PromptUnder { key, text, .. }
1376 if key == &worker_key && text == "review the diff"
1377 )),
1378 "a live worker's cron fires straight into the worker",
1379 );
1380 }
1381
1382 #[derive(Clone, Default)]
1384 struct Caught(std::sync::Arc<std::sync::Mutex<Vec<(tracing::Level, String, String)>>>);
1385
1386 #[derive(Default)]
1387 struct CaughtFields(String);
1388
1389 impl tracing::field::Visit for CaughtFields {
1390 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
1391 if !self.0.is_empty() {
1392 self.0.push(' ');
1393 }
1394 let _ = std::fmt::write(&mut self.0, format_args!("{}={value:?}", field.name()));
1395 }
1396 }
1397
1398 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for Caught {
1399 fn on_event(
1400 &self,
1401 event: &tracing::Event<'_>,
1402 _ctx: tracing_subscriber::layer::Context<'_, S>,
1403 ) {
1404 let mut fields = CaughtFields::default();
1405 event.record(&mut fields);
1406 self.0.lock().expect("capture").push((
1407 *event.metadata().level(),
1408 event.metadata().target().to_owned(),
1409 fields.0,
1410 ));
1411 }
1412 }
1413
1414 fn cooling_records(caught: &Caught, level: tracing::Level) -> Vec<String> {
1417 caught
1418 .0
1419 .lock()
1420 .expect("capture")
1421 .iter()
1422 .filter(|(recorded, target, fields)| {
1423 *recorded == level
1424 && target == "forge_workspace::crons"
1425 && fields.contains("cron_deferred_cooling")
1426 })
1427 .map(|(_, _, fields)| fields.clone())
1428 .collect()
1429 }
1430
1431 fn service_notices(
1433 rx: &mut tokio::sync::mpsc::UnboundedReceiver<crate::SeatUpdate>,
1434 ) -> Vec<String> {
1435 drain_updates(rx)
1436 .into_iter()
1437 .filter_map(|update| match update {
1438 SessionUpdate::ServiceStatus { message, .. } => Some(message),
1439 _ => None,
1440 })
1441 .collect()
1442 }
1443
1444 fn unix_now() -> u64 {
1445 std::time::SystemTime::now()
1446 .duration_since(std::time::SystemTime::UNIX_EPOCH)
1447 .expect("clock after the epoch")
1448 .as_secs()
1449 }
1450
1451 fn cool_for(ws: &Workspace, wait: std::time::Duration) {
1454 cool_until(ws, unix_now() + wait.as_secs());
1455 }
1456
1457 fn cool_until(ws: &Workspace, reset_epoch: u64) {
1459 ws.gateway.report_probe_limit(&AccountKey("acct-a".to_owned()), Some(reset_epoch));
1460 }
1461
1462 fn reset_mid_minute(now: std::time::SystemTime, wait: std::time::Duration) -> u64 {
1466 let target = now
1467 .duration_since(std::time::SystemTime::UNIX_EPOCH)
1468 .expect("clock after the epoch")
1469 .as_secs()
1470 + wait.as_secs();
1471 target - target % 60 + 30
1472 }
1473
1474 fn ist_clock(unix_seconds: u64) -> String {
1476 let at = chrono::DateTime::<chrono::Utc>::from(
1477 std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(unix_seconds),
1478 );
1479 (at + chrono::TimeDelta::minutes(5 * 60 + 30)).format("%H:%M").to_string()
1480 }
1481
1482 fn clear_cooling(ws: &Workspace) {
1484 ws.gateway.report_probe_limit(&AccountKey("acct-a".to_owned()), Some(unix_now() - 60));
1485 }
1486
1487 #[tokio::test]
1498 async fn a_cooling_walk_defers_the_cron_and_warns_once_per_episode() {
1499 use tracing_subscriber::layer::SubscriberExt as _;
1500
1501 let (ws, _dir) = workspace_with_one_unsettled_account();
1502 ws.seed_test_ready_account("acct-a");
1503 let mut rx = ws.subscribe_answerer();
1504 ws.seed_test_cron(schedule("cool", "proj", "stand-up", None));
1505 let now = std::time::SystemTime::now();
1506 let reset_epoch = reset_mid_minute(now, std::time::Duration::from_secs(2 * 60 * 60));
1507 cool_until(&ws, reset_epoch);
1508
1509 let caught = Caught::default();
1510 let subscriber = tracing_subscriber::registry().with(caught.clone());
1511 let _guard = tracing::subscriber::set_default(subscriber);
1512
1513 ws.fire_due_crons(now);
1514 let warned = cooling_records(&caught, tracing::Level::WARN);
1515 assert_eq!(warned.len(), 1, "the first cooling tick warns once");
1516 let fields = &warned[0];
1517 let clock = ist_clock(reset_epoch);
1518 assert!(fields.contains("org=TestOrg"), "the warning names the org: {fields}");
1519 assert!(fields.contains("model=claude-sonnet-5"), "the warning names the model: {fields}");
1520 assert!(
1521 fields.contains(&format!("resets_at={clock} IST")),
1522 "the warning names when the reset lands: {fields}",
1523 );
1524 assert!(
1525 fields.contains("cooling_accounts=1"),
1526 "the warning counts the org's cooling accounts: {fields}",
1527 );
1528 let notices = service_notices(&mut rx);
1529 assert_eq!(notices.len(), 1, "and sends the client one notice: {notices:?}");
1530 let notice = ¬ices[0];
1531 assert!(notice.contains("claude-sonnet-5"), "the notice names the model: {notice}");
1532 assert!(notice.contains("TestOrg"), "the notice names the org: {notice}");
1533 assert!(
1534 notice.contains(&format!("{clock} IST")),
1535 "the notice names when the reset lands: {notice}",
1536 );
1537 assert!(
1538 notice.contains("1 account(s) cooling in the org"),
1539 "and counts the accounts the walk skipped: {notice}",
1540 );
1541
1542 ws.fire_due_crons(now + std::time::Duration::from_secs(61));
1543 assert_eq!(
1544 cooling_records(&caught, tracing::Level::WARN).len(),
1545 1,
1546 "the repeat tick is debug, not another warning",
1547 );
1548 assert_eq!(
1549 cooling_records(&caught, tracing::Level::DEBUG).len(),
1550 1,
1551 "and says so at debug rather than staying silent",
1552 );
1553 assert!(service_notices(&mut rx).is_empty(), "and sends the client nothing");
1554
1555 ws.fire_due_crons(now + std::time::Duration::from_secs(30 * 60 + 60));
1558 assert_eq!(
1559 cooling_records(&caught, tracing::Level::WARN).len(),
1560 2,
1561 "the 30-minute floor re-warns while it lasts",
1562 );
1563
1564 let entry = ws
1565 .crons_for_project("proj")
1566 .into_iter()
1567 .find(|cron| cron.id == forge_primitives::CronId::from("cool"))
1568 .expect("a deferral keeps the entry");
1569 assert_eq!(
1570 entry.next_fire,
1571 std::time::SystemTime::UNIX_EPOCH,
1572 "and leaves it due rather than advancing",
1573 );
1574 }
1575
1576 #[tokio::test]
1582 async fn a_cooling_episode_that_clears_warns_again_next_time() {
1583 let (ws, _dir) = workspace_with_one_unsettled_account();
1584 ws.seed_test_ready_account("acct-a");
1585 let mut rx = ws.subscribe_answerer();
1586 ws.seed_test_cron(schedule("cool", "proj", "stand-up", None));
1587 let now = std::time::SystemTime::now();
1588
1589 cool_for(&ws, std::time::Duration::from_secs(2 * 60 * 60));
1590 ws.fire_due_crons(now);
1591 assert_eq!(service_notices(&mut rx).len(), 1, "the first episode warns once");
1592
1593 ws.enable_test_dispatch_intercept();
1596 clear_cooling(&ws);
1597 ws.fire_due_crons(now + std::time::Duration::from_secs(61));
1598 assert!(service_notices(&mut rx).is_empty(), "a recovered walk has nothing to warn about");
1599 assert!(
1600 ws.drain_test_dispatch_buffer().iter().any(|command| matches!(
1601 command,
1602 Command::SpawnProject { project_name, .. } if project_name == "proj"
1603 )),
1604 "precondition: the recovered fire landed rather than deferring again",
1605 );
1606
1607 cool_for(&ws, std::time::Duration::from_secs(2 * 60 * 60));
1609 ws.with_crons_mut(|crons| {
1610 let entry = crons
1611 .iter_mut()
1612 .find(|cron| cron.id == forge_primitives::CronId::from("cool"))
1613 .expect("the recovered fire kept the recurring entry");
1614 entry.next_fire = now + std::time::Duration::from_secs(120);
1615 });
1616 ws.fire_due_crons(now + std::time::Duration::from_secs(120));
1617 assert_eq!(service_notices(&mut rx).len(), 1, "a new episode warns again");
1618 }
1619
1620 #[test]
1625 fn an_unformattable_reset_falls_back_to_the_wait() {
1626 let last_second = chrono::DateTime::<chrono::Utc>::MAX_UTC.timestamp();
1627 let near_the_end = std::time::SystemTime::UNIX_EPOCH
1628 + std::time::Duration::from_secs(
1629 u64::try_from(last_second - 60).expect("a positive second count"),
1630 );
1631 for (reset_in, now) in
1632 [(Duration::MAX, std::time::SystemTime::now()), (Duration::from_secs(60), near_the_end)]
1633 {
1634 let text = cooling_reset_text(reset_in, now);
1635 assert!(text.starts_with("in "), "the wait survives without a clock: {text}");
1636 assert!(!text.contains("IST"), "and no wall clock is claimed: {text}");
1637 }
1638 }
1639
1640 #[test]
1641 fn deliver_asleep_worker_cron_buffers_and_wakes_the_project() {
1642 let (ws, _rx) = Workspace::testing_stub();
1643 let dir = tempdir().expect("tempdir");
1644 ws.install_db_for_test(
1645 crate::store::Db::open(&dir.path().join("db.redb")).expect("open db"),
1646 );
1647 let key = seed_project_with_a_real_root(&ws, &dir);
1650 seed_worker_row(&ws, &key, "reviewer");
1651
1652 ws.enable_test_dispatch_intercept();
1653 let outcome = deliver_cron(&ws, "proj", Some("reviewer"), "nightly", false);
1654 assert!(matches!(outcome, crate::spawn::CronFireOutcome::Delivered));
1655 let dispatched = ws.drain_test_dispatch_buffer();
1656 assert!(
1657 dispatched.iter().any(|c| matches!(
1658 c, Command::SpawnProject { project_name, .. } if project_name == "proj"
1659 )),
1660 "an asleep worker cron wakes the whole project via SpawnProject",
1661 );
1662 let buffered = parked_crons(&ws, "proj", Some("reviewer"));
1663 assert_eq!(buffered, vec!["nightly".to_owned()], "buffered for the worker owner");
1664 }
1665
1666 #[test]
1671 fn deliver_cron_to_spawning_worker_buffers_via_owner() {
1672 let (ws, _rx) = Workspace::testing_stub();
1673 let dir = tempdir().expect("tempdir");
1674 ws.install_db_for_test(
1675 crate::store::Db::open(&dir.path().join("db.redb")).expect("open db"),
1676 );
1677 let key = seed_project_with_a_real_root(&ws, &dir);
1678 seed_worker_row(&ws, &key, "reviewer");
1679 let worker_key = SessionSlot::worker("TestOrg", "proj", "reviewer");
1680 ws.insert_live_worker(&key, live_worker_entry("proj", "reviewer"));
1681 ws.register_domain_session(worker_key.clone(), None);
1683
1684 ws.enable_test_dispatch_intercept();
1685 let outcome = deliver_cron(&ws, "proj", Some("reviewer"), "nightly", false);
1686 assert!(matches!(outcome, crate::spawn::CronFireOutcome::Delivered));
1687 let dispatched = ws.drain_test_dispatch_buffer();
1688 assert!(
1689 !dispatched.iter().any(|c| matches!(
1690 c, Command::Prompt { key, .. } | Command::PromptUnder { key, .. }
1691 if key == &worker_key
1692 )),
1693 "no bare Prompt to the still-spawning worker (would be dropped)",
1694 );
1695 let buffered = parked_crons(&ws, "proj", Some("reviewer"));
1696 assert_eq!(
1697 buffered,
1698 vec!["nightly".to_owned()],
1699 "buffered for the worker's Connected drain"
1700 );
1701 }
1702
1703 #[test]
1708 fn deliver_worker_cron_parks_for_an_owner_that_is_still_spawning() {
1709 let (ws, key, _db, _project) = stranded_owner_fixture("reviewer");
1712 ws.insert_live_worker(&key, live_worker_entry("proj", "reviewer"));
1713
1714 ws.enable_test_dispatch_intercept();
1715 let outcome = deliver_cron(&ws, "proj", Some("reviewer"), "nightly", false);
1716 assert!(
1717 matches!(outcome, crate::spawn::CronFireOutcome::Delivered),
1718 "a worker still spawning is not unwakeable - its worktree is being created \
1719 along with it; got {}",
1720 outcome_name(&outcome),
1721 );
1722 assert_eq!(
1723 parked_crons(&ws, "proj", Some("reviewer")),
1724 vec!["nightly".to_owned()],
1725 "the prompt parks for the spawning worker's Connected drain",
1726 );
1727 }
1728
1729 #[test]
1734 fn deliver_worker_cron_does_not_park_for_a_failed_entry() {
1735 let (ws, key, _db, _project) = stranded_owner_fixture("reviewer");
1736 let mut failed = live_worker_entry("proj", "reviewer");
1737 failed.status = forge_primitives::WorkerLiveness::Failed;
1738 ws.insert_live_worker(&key, failed);
1739
1740 ws.enable_test_dispatch_intercept();
1741 let outcome = deliver_cron(&ws, "proj", Some("reviewer"), "nightly", false);
1742 assert!(
1743 matches!(outcome, crate::spawn::CronFireOutcome::TargetCannotBeWoken { .. }),
1744 "a Failed entry is not a worker the wave will start, so its fire must not \
1745 park; got {}",
1746 outcome_name(&outcome),
1747 );
1748 assert_eq!(
1749 parked_crons(&ws, "proj", Some("reviewer")).len(),
1750 0,
1751 "nothing parks for a worker that will never connect",
1752 );
1753 }
1754
1755 #[test]
1758 fn deliver_worker_cron_does_not_dispatch_to_a_connected_failed_entry() {
1759 let (ws, key, _db, _project) = stranded_owner_fixture("reviewer");
1760 let mut failed = live_worker_entry("proj", "reviewer");
1761 failed.status = forge_primitives::WorkerLiveness::Failed;
1762 ws.insert_live_worker(&key, failed.clone());
1763 ws.mark_session_connected_for_test(&failed.slot, "reviewer-uuid");
1764
1765 ws.enable_test_dispatch_intercept();
1766 let outcome = deliver_cron(&ws, "proj", Some("reviewer"), "nightly", false);
1767 assert!(
1768 matches!(outcome, crate::spawn::CronFireOutcome::TargetCannotBeWoken { .. }),
1769 "a failed worker's slot is not a dispatch target, however connected it still \
1770 reads; got {}",
1771 outcome_name(&outcome),
1772 );
1773 let dispatched = ws.drain_test_dispatch_buffer();
1774 assert!(
1775 !dispatched.iter().any(|c| matches!(
1776 c,
1777 Command::Prompt { key, .. } | Command::PromptUnder { key, .. }
1778 if key == &failed.slot
1779 )),
1780 "and no prompt was dispatched into the failed worker: {dispatched:?}",
1781 );
1782 }
1783
1784 #[tokio::test]
1785 async fn deliver_asleep_dynamic_worker_cron_buffers_and_wakes_the_project() {
1786 let (ws, _rx) = Workspace::testing_stub();
1787 let dir = tempdir().expect("tempdir");
1788 ws.install_db_for_test(
1789 crate::store::Db::open(&dir.path().join("db.redb")).expect("open db"),
1790 );
1791 let key = seed_project_with_a_real_root(&ws, &dir);
1792 seed_worker_row(&ws, &key, "scratch");
1794
1795 ws.enable_test_dispatch_intercept();
1796 let outcome = deliver_cron(&ws, "proj", Some("scratch"), "hourly", false);
1797 assert!(matches!(outcome, crate::spawn::CronFireOutcome::Delivered));
1798 let dispatched = ws.drain_test_dispatch_buffer();
1799 assert!(
1800 dispatched.iter().any(|c| matches!(
1801 c, Command::SpawnProject { project_name, .. } if project_name == "proj"
1802 )),
1803 "an asleep dynamic worker cron wakes the project too",
1804 );
1805 assert_eq!(
1806 parked_crons(&ws, "proj", Some("scratch")).len(),
1807 1,
1808 "buffered for the dynamic worker owner"
1809 );
1810 }
1811
1812 #[test]
1817 fn a_no_wake_cron_skips_a_sleeping_owner_and_stays_due() {
1818 let (ws, _rx) = Workspace::testing_stub();
1819 ws.seed_test_project("proj", "/tmp/wc-nowake");
1820 ws.enable_test_dispatch_intercept();
1821 let now = std::time::SystemTime::now();
1822
1823 let mut sweep = schedule("sweep", "proj", "sweep", None);
1824 sweep.next_fire = std::time::SystemTime::UNIX_EPOCH;
1825 sweep.wake = false;
1826 ws.push_cron(sweep);
1827 ws.fire_due_crons(now);
1828
1829 let stored = ws.crons_for_project("proj");
1830 assert_eq!(stored.len(), 1, "the skipped fire left the cron in place");
1831 assert_eq!(stored[0].last_fire, None, "and did not count as a fire");
1832 assert_eq!(
1833 stored[0].next_fire,
1834 std::time::SystemTime::UNIX_EPOCH,
1835 "the entry stays due, so a session start runs it",
1836 );
1837 assert!(ws.drain_test_dispatch_buffer().is_empty(), "no session was woken");
1838 assert!(parked_crons(&ws, "proj", None).is_empty(), "and nothing was parked");
1839
1840 let mut wakeful = schedule("wakeful", "proj", "stand-up", None);
1842 wakeful.next_fire = std::time::SystemTime::UNIX_EPOCH;
1843 ws.push_cron(wakeful);
1844 ws.fire_due_crons(now);
1845 assert!(
1846 !ws.drain_test_dispatch_buffer().is_empty(),
1847 "a wakeful cron still wakes its owner",
1848 );
1849 assert_eq!(
1850 ws.crons_for_project("proj")
1851 .iter()
1852 .find(|cron| cron.id == forge_primitives::cron::CronId::from("wakeful"))
1853 .and_then(|cron| cron.last_fire),
1854 Some(now),
1855 "and its fire advances the schedule",
1856 );
1857 }
1858
1859 #[test]
1863 fn a_no_wake_cron_fires_normally_while_the_owner_is_awake() {
1864 let dir = tempdir().expect("tempdir");
1865 let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
1866 ws.seed_test_project("proj", "/tmp/wc-nowake-awake");
1867 let cwd = project_expanded_path(&ws, "proj");
1868 ws.record_connected_session(&cwd, "lead-uuid", None);
1869 let lead = SessionSlot::lead("TestOrg", "proj");
1870 let (handle, _agent_rx) = Workspace::testing_stub_handle();
1871 ws.pool.lock().insert(
1872 lead.clone(),
1873 PooledAgent {
1874 handle: Arc::new(handle),
1875 account: AccountKey("test".to_owned()),
1876 permission_mode: None,
1877 registration: None,
1878 session_id: "pooled-session".to_owned(),
1879 },
1880 );
1881 ws.mark_session_connected_for_test(&lead, "lead-uuid");
1882
1883 let now = std::time::SystemTime::now();
1884 let mut sweep = schedule("sweep", "proj", "sweep", None);
1885 sweep.next_fire = std::time::SystemTime::UNIX_EPOCH;
1886 sweep.wake = false;
1887 ws.push_cron(sweep);
1888 ws.enable_test_dispatch_intercept();
1889
1890 ws.fire_due_crons(now);
1891 let texts: Vec<String> = ws
1892 .drain_test_dispatch_buffer()
1893 .iter()
1894 .filter_map(|command| match command {
1895 Command::Prompt { text, .. } | Command::PromptUnder { text, .. } => {
1896 Some(text.clone())
1897 }
1898 _ => None,
1899 })
1900 .collect();
1901 assert!(
1902 texts.iter().any(|text| text.contains("sweep")),
1903 "the awake owner is given its sweep: {texts:?}",
1904 );
1905 assert_eq!(
1906 ws.crons_for_project("proj")
1907 .iter()
1908 .find(|cron| cron.id == forge_primitives::cron::CronId::from("sweep"))
1909 .and_then(|cron| cron.last_fire),
1910 Some(now),
1911 "and the fire advances the schedule",
1912 );
1913 }
1914
1915 #[test]
1916 fn deliver_worker_cron_with_owner_conclusively_gone_is_target_gone() {
1917 let (ws, _rx) = Workspace::testing_stub();
1918 let dir = tempdir().expect("tempdir");
1919 ws.install_db_for_test(
1920 crate::store::Db::open(&dir.path().join("db.redb")).expect("open db"),
1921 );
1922 ws.seed_test_project("proj", "/tmp/wc-gone");
1927 let outcome = deliver_cron(&ws, "proj", Some("ghost"), "x", false);
1928 assert!(
1929 matches!(outcome, crate::spawn::CronFireOutcome::TargetGone),
1930 "a label with no row in the session store is conclusively gone",
1931 );
1932 }
1933
1934 #[test]
1945 fn deliver_worker_cron_with_an_unwakeable_owner_is_neither_parked_nor_delivered() {
1946 let (ws, _rx) = Workspace::testing_stub();
1947 let dir = tempdir().expect("tempdir");
1948 ws.install_db_for_test(
1949 crate::store::Db::open(&dir.path().join("db.redb")).expect("open db"),
1950 );
1951 let project_dir = tempdir().expect("project dir");
1952 ws.seed_test_project("proj", &project_dir.path().to_string_lossy());
1953 let key = ws.project_key_for_name("proj").expect("seeded project");
1954 ws.record_worker_row(
1955 &key,
1956 "steward",
1957 "steward-uuid",
1958 "c",
1959 None,
1960 None,
1961 false,
1962 crate::mcp::workers::types::WorkerTree { is_git_repo: true, cwd: None },
1963 None,
1964 )
1965 .expect("seed the worker's row");
1966
1967 let outcome = deliver_cron(&ws, "proj", Some("steward"), "x", false);
1968 let crate::spawn::CronFireOutcome::TargetCannotBeWoken { directory } = outcome else {
1969 panic!("an owner whose worktree is gone cannot be woken, so its fire is not delivered");
1970 };
1971 assert_eq!(
1972 directory,
1973 project_dir.path().join(".claude").join("worktrees").join("steward"),
1974 "the outcome has to carry the directory the fire was waiting for: it is the one \
1975 thing an operator can restore",
1976 );
1977 assert_eq!(
1978 parked_crons(&ws, "proj", Some("steward")).len(),
1979 0,
1980 "nothing parks for an owner that can never drain the bucket",
1981 );
1982
1983 std::fs::create_dir_all(
1984 project_dir.path().join(".claude").join("worktrees").join("steward"),
1985 )
1986 .expect("restore the worktree");
1987 let delivered = deliver_cron(&ws, "proj", Some("steward"), "x", false);
1988 assert!(
1989 matches!(delivered, crate::spawn::CronFireOutcome::Delivered),
1990 "the same fire is delivered once the worktree is back, so the refusal above \
1991 was the missing directory and not the fixture",
1992 );
1993 assert_eq!(
1994 parked_crons(&ws, "proj", Some("steward")).len(),
1995 1,
1996 "and its prompt is parked for the owner to drain on connect",
1997 );
1998 }
1999
2000 #[test]
2007 fn a_cron_wake_reads_the_rows_own_tree() {
2008 let (ws, _rx) = Workspace::testing_stub();
2009 let dir = tempdir().expect("tempdir");
2010 ws.install_db_for_test(
2011 crate::store::Db::open(&dir.path().join("db.redb")).expect("open db"),
2012 );
2013 let project_dir = tempdir().expect("project dir");
2014 ws.seed_test_project("proj", &project_dir.path().to_string_lossy());
2015 let key = ws.project_key_for_name("proj").expect("seeded project");
2016 std::fs::create_dir_all(
2019 project_dir.path().join(".claude").join("worktrees").join("steward"),
2020 )
2021 .expect("create the project worktree");
2022 let worker_tree = tempdir().expect("worker tree");
2024 let root = std::fs::canonicalize(worker_tree.path()).expect("canonical tree");
2025 ws.seed_test_worker_row_at(&key, "steward", &root.to_string_lossy(), true);
2026
2027 let outcome = deliver_cron(&ws, "proj", Some("steward"), "x", false);
2028 let crate::spawn::CronFireOutcome::TargetCannotBeWoken { directory } = outcome else {
2029 panic!("an owner whose own tree has no worktree cannot be woken");
2030 };
2031 assert_eq!(
2032 directory,
2033 root.join(".claude").join("worktrees").join("steward"),
2034 "the outcome names the tree an operator has to restore - the worker's own",
2035 );
2036 }
2037
2038 #[test]
2039 fn deliver_worker_cron_leaves_it_when_the_owner_check_cannot_read() {
2040 let (ws, _rx) = Workspace::testing_stub();
2044 ws.seed_test_project("proj", "/tmp/wc-unknown");
2045 let outcome = deliver_cron(&ws, "proj", Some("scratch"), "x", false);
2046 assert!(
2047 matches!(outcome, crate::spawn::CronFireOutcome::DispatchFailed),
2048 "a failed owner check leaves the cron for the next tick, not TargetGone",
2049 );
2050 }
2051
2052 #[test]
2057 fn deliver_cron_with_the_listener_unbound_leaves_the_fire_for_the_next_tick() {
2058 let (ws, _rx) = Workspace::testing_stub();
2059 ws.seed_test_project("proj", "/tmp/wc-gate");
2060 ws.enable_test_dispatch_intercept();
2061
2062 let bound = deliver_cron(&ws, "proj", None, "x", false);
2065 assert!(
2066 matches!(bound, crate::spawn::CronFireOutcome::Delivered),
2067 "the same fire with the listener bound is delivered",
2068 );
2069 ws.drain_test_dispatch_buffer();
2070
2071 ws.seed_test_gateway_ready(false);
2072 let unbound = deliver_cron(&ws, "proj", None, "x", false);
2073 assert!(
2074 matches!(unbound, crate::spawn::CronFireOutcome::DispatchFailed),
2075 "an unbound listener is a transient refusal, not a delivered fire",
2076 );
2077 assert_eq!(
2078 parked_crons(&ws, "proj", None).len(),
2079 1,
2080 "the deferred fire parked nothing, so the retry parks it once rather than twice",
2081 );
2082 }
2083
2084 #[test]
2090 fn deliver_cron_with_an_unsettled_account_map_leaves_the_fire_for_the_next_tick() {
2091 let (ws, _dir) = workspace_with_one_unsettled_account();
2092 ws.enable_test_dispatch_intercept();
2093
2094 let unsettled = deliver_cron(&ws, "proj", None, "x", false);
2097 assert_eq!(
2098 outcome_name(&unsettled),
2099 "DispatchFailed",
2100 "an account still loading is a transient refusal, not a delivered fire",
2101 );
2102 assert!(
2103 parked_crons(&ws, "proj", None).is_empty(),
2104 "and nothing is parked, so the retry parks it once rather than twice",
2105 );
2106
2107 ws.seed_test_ready_account("acct-a");
2110 let ready = deliver_cron(&ws, "proj", None, "x", false);
2111 assert_eq!(
2112 outcome_name(&ready),
2113 "Delivered",
2114 "the same fire with the account settled is delivered",
2115 );
2116
2117 ws.seed_test_account_state("acct-a", forge_gateway::LoadingState::Bailed);
2120 let bailed = deliver_cron(&ws, "proj", None, "x", false);
2121 assert_eq!(
2122 outcome_name(&bailed),
2123 "Delivered",
2124 "a Bailed account settles the map and the walk falls back to it",
2125 );
2126 assert_eq!(
2127 parked_crons(&ws, "proj", None).len(),
2128 2,
2129 "both settled fires parked their prompt",
2130 );
2131 }
2132
2133 #[test]
2137 fn deliver_cron_during_a_cooldown_leaves_the_fire_for_the_next_tick() {
2138 let (ws, _dir) = workspace_with_one_unsettled_account();
2139 ws.seed_test_ready_account("acct-a");
2140 ws.enable_test_dispatch_intercept();
2141
2142 let served = deliver_cron(&ws, "proj", None, "x", false);
2145 assert_eq!(
2146 outcome_name(&served),
2147 "Delivered",
2148 "the same fire with the account serving is delivered",
2149 );
2150
2151 let reset_at = std::time::SystemTime::now()
2154 .duration_since(std::time::UNIX_EPOCH)
2155 .expect("epoch")
2156 .as_secs()
2157 + 60;
2158 ws.gateway
2159 .report_probe_limit(&forge_gateway::AccountKey("acct-a".to_owned()), Some(reset_at));
2160 let cooling = deliver_cron(&ws, "proj", None, "x", false);
2161 assert_eq!(
2162 outcome_name(&cooling),
2163 "DeferredCooling",
2164 "a cooling walk is a transient refusal, not a delivered fire",
2165 );
2166 assert_eq!(
2167 parked_crons(&ws, "proj", None).len(),
2168 1,
2169 "the deferred fire parked nothing, so the retry parks it once rather than twice",
2170 );
2171 }
2172
2173 fn outcome_name(outcome: &crate::spawn::CronFireOutcome) -> &'static str {
2176 match outcome {
2177 crate::spawn::CronFireOutcome::Delivered => "Delivered",
2178 crate::spawn::CronFireOutcome::TargetGone => "TargetGone",
2179 crate::spawn::CronFireOutcome::TargetCannotBeWoken { .. } => "TargetCannotBeWoken",
2180 crate::spawn::CronFireOutcome::DeferredCooling { .. } => "DeferredCooling",
2181 crate::spawn::CronFireOutcome::DispatchFailed => "DispatchFailed",
2182 crate::spawn::CronFireOutcome::SkippedAsleep => "SkippedAsleep",
2183 }
2184 }
2185
2186 fn workspace_with_one_unsettled_account()
2197 -> (std::sync::Arc<crate::Workspace>, tempfile::TempDir) {
2198 let dir = tempdir().expect("tempdir");
2199 let forge_dir = crate::config::ensure_forge_data_dir(dir.path()).expect("forge dir");
2200 std::fs::write(
2201 forge_dir.join("forge.toml"),
2202 r#"
2203[[orgs]]
2204name = "TestOrg"
2205accounts = ["acct-a"]
2206
2207[[orgs.projects]]
2208name = "proj"
2209path = "/tmp/wc-unsettled"
2210model = "claude-sonnet-5"
2211
2212[[accounts]]
2213display_name = "acct-a"
2214token = "t"
2215models = ["claude-sonnet-5"]
2216provider = "anthropic"
2217"#,
2218 )
2219 .expect("write forge.toml");
2220 let ws = std::sync::Arc::new(
2221 crate::Workspace::new_for_test(dir.path().to_owned()).expect("boot from the fixture"),
2222 );
2223 (ws, dir)
2224 }
2225
2226 #[test]
2227 fn deliver_cron_marks_an_overdue_fire_as_missed() {
2228 let dir = tempdir().expect("tempdir");
2229 let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
2230 ws.seed_test_project("proj", "/tmp/wc-missed");
2231 let cwd = project_expanded_path(&ws, "proj");
2232 ws.record_connected_session(&cwd, "lead-uuid", None);
2233 let lead_key = SessionSlot::lead("TestOrg", "proj");
2234 let (handle, _agent_rx) = Workspace::testing_stub_handle();
2235 ws.pool.lock().insert(
2236 lead_key.clone(),
2237 PooledAgent {
2238 handle: Arc::new(handle),
2239 account: AccountKey("test".to_owned()),
2240 permission_mode: None,
2241 registration: None,
2242 session_id: "pooled-session".to_owned(),
2243 },
2244 );
2245
2246 ws.mark_session_connected_for_test(&lead_key, "lead-uuid");
2247 ws.enable_test_dispatch_intercept();
2248 deliver_cron(&ws, "proj", None, "standup", true);
2249 deliver_cron(&ws, "proj", None, "standup", false);
2250 let dispatched = ws.drain_test_dispatch_buffer();
2251 let texts: Vec<String> = dispatched
2252 .iter()
2253 .filter_map(|c| match c {
2254 Command::Prompt { text, .. } | Command::PromptUnder { text, .. } => {
2255 Some(text.clone())
2256 }
2257 _ => None,
2258 })
2259 .collect();
2260 assert!(
2261 texts.contains(&"[missed cron] standup".to_owned()),
2262 "an overdue fire is delivered with the missed marker",
2263 );
2264 assert!(texts.contains(&"standup".to_owned()), "an on-time fire has no marker");
2265 }
2266
2267 #[test]
2268 fn fire_due_crons_missed_threshold_is_two_ticks() {
2269 use forge_primitives::cron::{CronEntry, CronId, CronKind};
2270 let dir = tempdir().expect("tempdir");
2271 let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
2272 ws.seed_test_project("proj", "/tmp/wc-thresh");
2273 let cwd = project_expanded_path(&ws, "proj");
2274 ws.record_connected_session(&cwd, "lead-uuid", None);
2275 let lead_key = SessionSlot::lead("TestOrg", "proj");
2276 let (handle, _agent_rx) = Workspace::testing_stub_handle();
2277 ws.pool.lock().insert(
2278 lead_key.clone(),
2279 PooledAgent {
2280 handle: Arc::new(handle),
2281 account: AccountKey("test".to_owned()),
2282 permission_mode: None,
2283 registration: None,
2284 session_id: "pooled-session".to_owned(),
2285 },
2286 );
2287 ws.mark_session_connected_for_test(&lead_key, "lead-uuid");
2288
2289 let now = std::time::SystemTime::now();
2290 let cron = |id: &str, prompt: &str, next_fire| CronEntry {
2291 id: CronId::from(id),
2292 project_name: "proj".to_owned(),
2293 kind: CronKind::Recurring("*/5 * * * *".to_owned()),
2294 prompt: prompt.to_owned(),
2295 created_at: std::time::SystemTime::UNIX_EPOCH,
2296 description: None,
2297 last_fire: None,
2298 next_fire,
2299 team_role: None,
2300 wake: true,
2301 };
2302 ws.push_cron(cron("recent", "recent", now - std::time::Duration::from_secs(60)));
2305 ws.push_cron(cron("stale", "stale", now - std::time::Duration::from_secs(180)));
2306
2307 ws.enable_test_dispatch_intercept();
2308 ws.fire_due_crons(now);
2309 let texts: Vec<String> = ws
2310 .drain_test_dispatch_buffer()
2311 .iter()
2312 .filter_map(|c| match c {
2313 Command::Prompt { text, .. } | Command::PromptUnder { text, .. } => {
2314 Some(text.clone())
2315 }
2316 _ => None,
2317 })
2318 .collect();
2319 assert!(texts.contains(&"recent".to_owned()), "a one-tick-late fire is on-time");
2320 assert!(
2321 texts.contains(&"[missed cron] stale".to_owned()),
2322 "a three-tick-late fire is marked missed",
2323 );
2324 }
2325
2326 #[test]
2327 fn fire_due_crons_removes_a_cron_whose_project_is_gone() {
2328 use forge_primitives::cron::{CronEntry, CronId, CronKind};
2329 let (ws, _rx) = Workspace::testing_stub();
2331 let now = std::time::SystemTime::now();
2332
2333 ws.push_cron(CronEntry {
2334 id: CronId::from("orphan"),
2335 project_name: "deleted-project".to_owned(),
2336 kind: CronKind::Recurring("*/5 * * * *".to_owned()),
2337 prompt: "p".to_owned(),
2338 created_at: std::time::SystemTime::UNIX_EPOCH,
2339 description: None,
2340 last_fire: None,
2341 next_fire: std::time::SystemTime::UNIX_EPOCH, team_role: None,
2343 wake: true,
2344 });
2345
2346 ws.enable_test_dispatch_intercept();
2347 ws.fire_due_crons(now);
2348 let dispatched = ws.drain_test_dispatch_buffer();
2349
2350 assert!(
2351 dispatched.iter().all(|c| !matches!(c, crate::protocol::Command::SpawnProject { .. })),
2352 "a cron whose project is gone gets no spawn",
2353 );
2354 assert!(
2355 ws.crons_for_project("deleted-project").is_empty(),
2356 "an overdue cron whose project left forge.toml is removed, not advanced forever",
2357 );
2358 }
2359
2360 #[test]
2368 fn fire_due_crons_advances_recurring_and_keeps_a_one_shot_when_the_owner_cannot_be_woken() {
2369 use forge_primitives::cron::{CronEntry, CronId, CronKind};
2370 let (ws, _key, _db, _project) = stranded_owner_fixture("steward");
2371
2372 let now = std::time::SystemTime::now();
2373 let past = std::time::SystemTime::UNIX_EPOCH;
2374 for (id, kind) in [
2375 ("recurring", CronKind::Recurring("*/5 * * * *".to_owned())),
2376 ("one-shot", CronKind::Once(past)),
2377 ] {
2378 ws.push_cron(CronEntry {
2379 id: CronId::from(id),
2380 project_name: "proj".to_owned(),
2381 kind,
2382 prompt: "p".to_owned(),
2383 created_at: past,
2384 description: None,
2385 last_fire: None,
2386 next_fire: past,
2387 team_role: Some("steward".to_owned()),
2388 wake: true,
2389 });
2390 }
2391
2392 ws.enable_test_dispatch_intercept();
2393 ws.fire_due_crons(now);
2394
2395 let after = ws.crons_for_project("proj");
2396 let ids: Vec<String> = after.iter().map(|c| c.id.0.clone()).collect();
2397 let recurring = after.iter().find(|c| c.id == CronId::from("recurring"));
2398 assert!(
2399 recurring.is_some_and(|c| c.last_fire.is_some()),
2400 "a recurring cron's fire is dropped and its schedule advances, rather than \
2401 the entry being removed; entries left {ids:?}",
2402 );
2403 assert!(
2404 after.iter().any(|c| c.id == CronId::from("one-shot")),
2405 "a one-shot has no later slot, so it stays due rather than being removed \
2406 along with its prompt; entries left {ids:?}",
2407 );
2408 assert_eq!(
2409 parked_crons(&ws, "proj", Some("steward")).len(),
2410 0,
2411 "nothing parks for an owner that can never drain the bucket",
2412 );
2413 }
2414
2415 #[test]
2416 fn advance_or_remove_cron_removes_never_occurring_recurring() {
2417 use forge_primitives::cron::{CronEntry, CronId, CronKind};
2418 let dir = tempdir().expect("tempdir");
2419 let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
2420 ws.push_cron(CronEntry {
2424 id: CronId::from("impossible"),
2425 project_name: "forge".to_owned(),
2426 kind: CronKind::Recurring("0 0 30 2 *".to_owned()),
2427 prompt: "p".to_owned(),
2428 created_at: std::time::SystemTime::UNIX_EPOCH,
2429 description: None,
2430 last_fire: None,
2431 next_fire: std::time::SystemTime::UNIX_EPOCH,
2432 team_role: None,
2433 wake: true,
2434 });
2435 ws.advance_or_remove_cron(&CronId::from("impossible"), std::time::SystemTime::now());
2436 assert!(
2437 ws.crons_for_project("forge").is_empty(),
2438 "a recurring cron with no upcoming occurrence is removed, not left stuck",
2439 );
2440 }
2441
2442 #[tokio::test(start_paused = true)]
2447 async fn start_cron_scheduler_fires_a_due_cron_once_despite_a_second_start() {
2448 use forge_primitives::cron::{CronEntry, CronId, CronKind};
2449 let dir = tempdir().expect("tempdir");
2450 let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
2451 ws.seed_test_project("proj", "/tmp/wc-scheduler-once");
2452
2453 ws.push_cron(CronEntry {
2454 id: CronId::from("due"),
2455 project_name: "proj".to_owned(),
2456 kind: CronKind::Recurring("*/5 * * * *".to_owned()),
2457 prompt: "tick".to_owned(),
2458 created_at: std::time::SystemTime::UNIX_EPOCH,
2459 description: None,
2460 last_fire: None,
2461 next_fire: std::time::SystemTime::UNIX_EPOCH, team_role: None,
2463 wake: true,
2464 });
2465
2466 ws.enable_test_dispatch_intercept();
2467 ws.start_cron_scheduler();
2468 assert!(
2469 ws.supervisor().snapshot().iter().any(|member| member.name == "cron_scheduler"),
2470 "the first start registers the scheduler",
2471 );
2472
2473 tokio::task::yield_now().await;
2476 tokio::task::yield_now().await;
2477 tokio::time::advance(CRON_TICK_INTERVAL).await;
2478 tokio::task::yield_now().await;
2479 tokio::task::yield_now().await;
2480 let first_tick = count_proj_spawns(&ws);
2481 assert_eq!(
2482 first_tick, 1,
2483 "the first start runs the scheduler; a guard that never opens fires nothing",
2484 );
2485
2486 let weak_before = Arc::downgrade(&ws).weak_count();
2493 ws.start_cron_scheduler();
2494 let weak_after = Arc::downgrade(&ws).weak_count();
2495 assert_eq!(weak_after, weak_before, "a second start must not spawn a second tick loop");
2496 }
2497
2498 fn count_proj_spawns(ws: &Workspace) -> usize {
2499 ws.drain_test_dispatch_buffer()
2500 .iter()
2501 .filter(|c| {
2502 matches!(c, crate::protocol::Command::SpawnProject { project_name, .. }
2503 if project_name == "proj")
2504 })
2505 .count()
2506 }
2507
2508 #[tokio::test(start_paused = true)]
2514 async fn the_scheduler_task_does_not_hold_the_workspace_alive() {
2515 let dir = tempdir().expect("tempdir");
2516 let weak = {
2517 let (ws, _rx) = Workspace::testing_stub_with_config_dir(dir.path().to_owned());
2518 ws.start_cron_scheduler();
2519 tokio::task::yield_now().await;
2522 tokio::task::yield_now().await;
2523 Arc::downgrade(&ws)
2524 };
2525 tokio::time::advance(CRON_TICK_INTERVAL * 2).await;
2526 tokio::task::yield_now().await;
2527 tokio::task::yield_now().await;
2528 assert!(weak.upgrade().is_none(), "the scheduler task must hold only a Weak");
2529 }
2530}