Skip to main content

forge_gateway/
forward.rs

1//! The route handler: hello probe, path parsing, splice, re-header,
2//! and the streaming forward.
3//!
4//! Routing rules, in the order they are checked:
5//! - `<segments>/api/hello` answers 404 and never constructs the
6//!   upstream leg - the CLI's connectivity probe must not become an
7//!   authenticated request to a third party on a path that needs no
8//!   auth.
9//! - `<segments>/v1/messages` (query ignored) resolves the binding and
10//!   forwards to the bound account's upstream with the real credential
11//!   attached and the CLI's dummy stripped. A bound session keeps its
12//!   account for its whole life: an account the config removes is retired
13//!   rather than dropped, and a model the account does not declare is still
14//!   forwarded, because the account was the spawn's choice and a running
15//!   session is not reconfigured. An unbound session selects an account
16//!   now, from the model in its own body.
17//! - The failing response streams back to the CLI untouched, but the
18//!   triggers it carries mark the account exhausted and rotate the
19//!   binding, so the CLI's retry lands on the next account with
20//!   nothing replayed.
21
22use std::collections::HashMap;
23use std::sync::Arc;
24use std::time::{Duration, SystemTime};
25
26use bytes::Bytes;
27use futures_util::StreamExt;
28use hyper::body::Frame;
29use hyper::header::{CONTENT_LENGTH, TRANSFER_ENCODING};
30use hyper::{HeaderMap, Method, Request, StatusCode};
31
32use crate::account::AccountKey;
33use crate::binding::{Bindings, DUMMY_CREDENTIAL, credential_variable_for};
34use crate::listener::{RouteHandler, StreamBody, text_response};
35use crate::rotation::RotationState;
36use crate::splice::splice_model;
37
38/// The upstream an account with no base URL of its own talks to: the
39/// four Anthropic accounts declare none today, and the CLI's default
40/// host is what they have always used.
41const ANTHROPIC_UPSTREAM: &str = "https://api.anthropic.com";
42
43/// How long the upstream may produce nothing at all before the
44/// forwarded turn is abandoned. An idle bound, not a deadline for the
45/// whole response: a turn streams for as long as the model takes, and
46/// only silence means the leg is wedged.
47pub const FORWARD_IDLE_TIMEOUT: Duration = Duration::from_secs(300);
48
49/// How long the connect and the TLS handshake may take before the
50/// request is abandoned. Well under the idle bound: a host that
51/// blackholes the SYN never reaches the first byte, so the idle bound
52/// alone would hold the turn open for it, longer than the kernel's own
53/// connect timeout did before the leg carried any bound at all.
54pub const FORWARD_CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
55
56/// The error type a streamed body yields.
57type BoxError = Box<dyn std::error::Error + Send + Sync>;
58
59/// Why a request could not be bound to an account.
60#[derive(Debug)]
61pub enum SelectFailure {
62    /// The org in the path is not in the config.
63    UnknownOrg { org: String },
64    /// No account in the walk declares the model.
65    NoEligibleAccount { model: String, org: String },
66    /// The whole walk is inside a rotation cooldown: they serve the
67    /// model, they are cooling until `reset_in`.
68    BudgetExhausted { tried: usize, org: String, model: String, reset_in: Duration },
69}
70
71/// The gateway's route handler: bindings + the account pool +
72/// rotation state + the upstream client.
73pub struct Gateway {
74    pub bindings: Bindings,
75    pool: Arc<crate::AccountPool>,
76    client: reqwest::Client,
77    /// Each org's walk order, from the config the caller loaded. Read
78    /// when a session has no binding and selection must run.
79    org_pins: parking_lot::Mutex<HashMap<String, crate::selection::OrgPin>>,
80    /// Exhaustion marks and cooldowns, shared with the workspace's
81    /// `rate_limit_event` reports. The numbers inside arrive from
82    /// `[gateway]` at boot and again on a config reload; the cooldown and
83    /// streak maps are the running state and are not replaced by a reload
84    /// that leaves the numbers alone.
85    rotation: parking_lot::Mutex<RotationState>,
86    /// Bumped by every cooldown write that can change what [`Self::select_for`]
87    /// answers, so a reader outside this crate can announce the change
88    /// instead of holding a roster snapshot the clock has moved past.
89    rotation_generation: tokio::sync::watch::Sender<u64>,
90}
91
92impl Gateway {
93    /// The caller builds `client` through `ProviderHost`, so the
94    /// forward leg carries the TLS trust and the bounds every other
95    /// outbound path does:
96    /// `host.streaming_http_client(FORWARD_CONNECT_TIMEOUT, FORWARD_IDLE_TIMEOUT)`.
97    pub fn new(pool: Arc<crate::AccountPool>, client: reqwest::Client) -> Self {
98        Self {
99            bindings: Bindings::default(),
100            pool,
101            client,
102            org_pins: parking_lot::Mutex::new(HashMap::new()),
103            rotation: parking_lot::Mutex::new(RotationState::new()),
104            rotation_generation: tokio::sync::watch::channel(0).0,
105        }
106    }
107
108    /// Subscribe to the rotation's write notifications: every cooldown that
109    /// starts (or is replaced) bumps the generation, so a reader wakes and
110    /// re-reads what selection would answer.
111    pub fn rotation_watch(&self) -> tokio::sync::watch::Receiver<u64> {
112        self.rotation_generation.subscribe()
113    }
114
115    /// The soonest live cooldown's end, or `None` when nothing is cooling.
116    /// A reader sleeps until it to announce the thaw, which the gateway
117    /// cannot publish - the clock drives it, not a write.
118    pub fn soonest_cooldown_end(&self, now: SystemTime) -> Option<SystemTime> {
119        self.rotation.lock().soonest_cooldown_end(now)
120    }
121
122    fn note_rotation_changed(&self) {
123        self.rotation_generation.send_modify(|generation| *generation += 1);
124    }
125
126    /// The rotation numbers from `[gateway]`: the 429 streak count and
127    /// window, and the cooldown when no reset time is known. Called at
128    /// boot and again by a config reload that moved them.
129    pub fn set_rotation_numbers(&self, numbers: crate::rotation::RotationNumbers) {
130        *self.rotation.lock() = RotationState::with_numbers(numbers);
131        self.note_rotation_changed();
132    }
133
134    /// Publish each org's walk order (primary pin then fallbacks).
135    /// Called at construction and again by a config reload.
136    pub fn set_org_pins(&self, pins: impl IntoIterator<Item = (String, crate::selection::OrgPin)>) {
137        *self.org_pins.lock() = pins.into_iter().collect();
138    }
139
140    /// The published walk order per org, sorted by org name so a view
141    /// renders in a stable order (the store underneath is a HashMap).
142    pub fn org_pins(&self) -> Vec<(String, crate::selection::OrgPin)> {
143        let mut rows: Vec<(String, crate::selection::OrgPin)> =
144            self.org_pins.lock().iter().map(|(org, pin)| (org.clone(), pin.clone())).collect();
145        rows.sort_by(|left, right| left.0.cmp(&right.0));
146        rows
147    }
148
149    /// Which account serves `model` for `org`: the org pin lookup, the
150    /// cooling filter and the declared-model walk, WITHOUT binding.
151    /// This is the one selection decision - the forward binds what it
152    /// returns, and a spawn or the launchpad reads it as the answer.
153    pub fn select_for(&self, org: &str, model: &str) -> Result<AccountKey, SelectFailure> {
154        self.walk(org, model)
155    }
156
157    /// The account a spawn lands on first once the walk thaws: among the
158    /// cooling accounts declaring `model`, the one whose cooldown ends
159    /// soonest. That is the walk's answer at the first moment any of them
160    /// is free, which pin order alone does not give - a later-pinned
161    /// account can thaw first.
162    ///
163    /// A prediction made while the walk refuses, not a selection: an
164    /// account still `Loading` when the cooldown lifts is skipped by the
165    /// walk, so the read can name one that has not settled. A caller uses
166    /// it only where `select_for` fails; the walk's own answer replaces it
167    /// as soon as the walk succeeds. Selection never calls it.
168    pub fn cooling_account_serving(&self, org: &str, model: &str) -> Option<AccountKey> {
169        let pin = self.org_pins.lock().get(org).cloned()?;
170        let now = SystemTime::now();
171        let rotation = self.rotation.lock();
172        pin.accounts
173            .iter()
174            .chain(pin.fallback_accounts.iter())
175            .filter_map(|name| {
176                let key = AccountKey(name.clone());
177                if !rotation.is_cooling_down(&key, now) || !self.pool.declares(&key, model) {
178                    return None;
179                }
180                Some((key.clone(), rotation.cooldown_end(&key)?))
181            })
182            // Ties keep pin order: `min_by_key` returns the first minimum.
183            .min_by_key(|(_, until)| *until)
184            .map(|(key, _)| key)
185    }
186
187    fn walk(&self, org: &str, model: &str) -> Result<AccountKey, SelectFailure> {
188        let Some(pin) = self.org_pins.lock().get(org).cloned() else {
189            return Err(SelectFailure::UnknownOrg { org: org.to_owned() });
190        };
191        let now = SystemTime::now();
192        let rotation = self.rotation.lock();
193        let cooling = |name: &String| rotation.is_cooling_down(&AccountKey(name.clone()), now);
194        let cooled: Vec<String> = pin
195            .accounts
196            .iter()
197            .chain(pin.fallback_accounts.iter())
198            .filter(|n| cooling(n))
199            .cloned()
200            .collect();
201        let pin = crate::selection::OrgPin {
202            accounts: pin.accounts.into_iter().filter(|name| !cooling(name)).collect(),
203            fallback_accounts: pin
204                .fallback_accounts
205                .into_iter()
206                .filter(|name| !cooling(name))
207                .collect(),
208        };
209        match self.pool.select_account(&pin, org, model) {
210            Ok(account) => Ok(account),
211            // The walk emptied. When cooling accounts were filtered out,
212            // the real cause is exhaustion: they DO serve the model,
213            // they are cooling until a knowable time, and the budget
214            // failure carries the soonest of THEIR resets. Another org's
215            // account clearing sooner is not a time this walk recovers
216            // at. A walk empty without cooling is a declared-model miss.
217            Err(crate::selection::SelectionError::NoEligibleAccount { model, org }) => {
218                let cooled: Vec<AccountKey> = cooled.into_iter().map(AccountKey).collect();
219                if let Some(reset) = rotation.soonest_reset_among(&cooled, now) {
220                    return Err(SelectFailure::BudgetExhausted {
221                        tried: cooled.len(),
222                        org,
223                        model,
224                        reset_in: reset.duration_since(now).unwrap_or(Duration::ZERO),
225                    });
226                }
227                Err(SelectFailure::NoEligibleAccount { model, org })
228            }
229        }
230    }
231
232    /// Select the account for an unbound session and bind it, so the
233    /// session keeps it.
234    fn select_and_bind(
235        &self,
236        org: &str,
237        project: &str,
238        session: &str,
239        model: &str,
240    ) -> Result<AccountKey, SelectFailure> {
241        let account = self.select_for(org, model)?;
242        self.bindings.bind(org, project, session, account.clone());
243        Ok(account)
244    }
245
246    /// Read the real credential and upstream base out of the bound
247    /// account's env. `None` when the account carries no credential in
248    /// the variable its provider authenticates with.
249    fn credential_for(&self, account: &AccountKey) -> Option<(String, String)> {
250        let (provider, env) = self.pool.provider_and_env(account)?;
251        let variable = credential_variable_for(provider);
252        let credential = env.get(variable)?.trim().to_owned();
253        if credential.is_empty() || credential == DUMMY_CREDENTIAL {
254            return None;
255        }
256        let upstream = env
257            .get("ANTHROPIC_BASE_URL")
258            .map(|v| v.trim().trim_end_matches('/').to_owned())
259            .filter(|v| !v.is_empty())
260            .unwrap_or_else(|| ANTHROPIC_UPSTREAM.to_owned());
261        Some((upstream, credential))
262    }
263
264    /// The reset the account's own usage probe reports for its
265    /// exhausted window, when still ahead. Absent for every
266    /// non-window-billed account.
267    fn probe_reset_for(&self, account: &AccountKey) -> Option<Duration> {
268        let now = SystemTime::now();
269        let reset = self.pool.usage(&account.0)?.binding_reset_at()?;
270        reset.duration_since(now).ok()
271    }
272
273    /// The cooldown for a failing response: the response's Retry-After,
274    /// else the account's own probe reset, else the configured default.
275    fn cooldown_for(&self, account: &AccountKey, headers: &HeaderMap) -> Duration {
276        retry_after(headers)
277            .or_else(|| self.probe_reset_for(account))
278            .unwrap_or_else(|| self.rotation.lock().no_reset_cooldown())
279    }
280
281    /// Selection, or the loud response its failure produces.
282    fn select_or_fail(
283        &self,
284        org: &str,
285        project: &str,
286        session: &str,
287        model: &str,
288    ) -> Result<AccountKey, Box<hyper::Response<StreamBody>>> {
289        match self.select_and_bind(org, project, session, model) {
290            Ok(account) => Ok(account),
291            Err(SelectFailure::UnknownOrg { org }) => Err(Box::new(text_response(
292                StatusCode::NOT_FOUND,
293                format!("org '{org}' is not known to the gateway"),
294            ))),
295            Err(SelectFailure::NoEligibleAccount { model, org }) => Err(Box::new(text_response(
296                StatusCode::SERVICE_UNAVAILABLE,
297                format!(
298                    "no account in org '{org}' serves model '{model}'; the session is \
299                     unbound and nothing was forwarded"
300                ),
301            ))),
302            Err(SelectFailure::BudgetExhausted { tried, org, model, reset_in }) => {
303                Err(Box::new(text_response(
304                    StatusCode::SERVICE_UNAVAILABLE,
305                    format!(
306                        "every account serving model '{model}' in org '{org}' is cooling; \
307                         tried {tried} account(s), soonest reset in {}s",
308                        reset_in.as_secs()
309                    ),
310                )))
311            }
312        }
313    }
314
315    /// Apply the rotation triggers the held response proves, then
316    /// stream it back untouched - the CLI's retry carries the same
317    /// session id, so rotating the binding here is what routes the
318    /// retry to the next account.
319    fn note_response(
320        &self,
321        account: &AccountKey,
322        org: &str,
323        project: &str,
324        session: &str,
325        status: StatusCode,
326        headers: &HeaderMap,
327    ) {
328        let rejected = header_value_is(headers, "anthropic-ratelimit-unified-status", "rejected")
329            || header_value_is(headers, "anthropic-ratelimit-unified-overage-status", "rejected");
330        let now = SystemTime::now();
331        if status == StatusCode::TOO_MANY_REQUESTS && self.rotation.lock().record_429(account, now)
332        {
333            let cooldown = self.cooldown_for(account, headers);
334            self.rotate_off(account, org, project, session, now + cooldown);
335        }
336        // The header family marks exhaustion on "the failing response"
337        // (spec, Rotation): an account not paying for overage reports
338        // `-overage-status: rejected` on every response, successes included.
339        if rejected && !status.is_success() {
340            let cooldown = self.cooldown_for(account, headers);
341            self.rotate_off(account, org, project, session, now + cooldown);
342        }
343        if status.is_success() {
344            self.rotation.lock().reset_streak(account);
345        }
346    }
347
348    /// Mark `account` exhausted until `until` and drop the session's
349    /// binding, so the next request for it re-selects with the
350    /// exhausted account skipped.
351    fn rotate_off(
352        &self,
353        account: &AccountKey,
354        org: &str,
355        project: &str,
356        session: &str,
357        until: SystemTime,
358    ) {
359        self.rotation.lock().cool_down(account, until);
360        self.bindings.unbind(org, project, session);
361        self.note_rotation_changed();
362    }
363
364    /// The workspace's `rate_limit_event` report: the CLI told us the
365    /// bound account's window is not `allowed`, and the frame's reset
366    /// time is the cooldown. Both the exhausted account and the probe
367    /// agree on a reset time or its absence; nothing else is read.
368    /// A session with no binding (gateway not in use) has nothing to
369    /// rotate.
370    pub fn report_rate_limit(
371        &self,
372        org: &str,
373        project: &str,
374        session: &str,
375        reset_at: Option<u64>,
376    ) {
377        let Some(account) = self.bindings.take_binding(org, project, session) else {
378            return;
379        };
380        let now = SystemTime::now();
381        // A zero reset means no usable reset: it would end the cooldown
382        // the instant it starts, so the configured default applies.
383        let until = reset_at
384            .filter(|secs| *secs > 0)
385            .map_or(now + self.rotation.lock().no_reset_cooldown(), crate::rotation::reset_instant);
386        self.rotation.lock().cool_down(&account, until);
387        self.note_rotation_changed();
388    }
389
390    /// Whether the rotation is holding `account` out of selection right now -
391    /// the cooldown a probe at a window's cap writes.
392    ///
393    /// A view reads it to state the selection signal without stating the
394    /// usage: a probe landing at the cap moves this and nothing on the
395    /// account's loading row.
396    pub fn is_cooling(&self, account: &AccountKey) -> bool {
397        self.rotation.lock().is_cooling_down(account, SystemTime::now())
398    }
399
400    /// The usage probe's verdict: a snapshot with any window at the
401    /// cap and a reset ahead proves exhaustion until that reset. Every
402    /// session bound to the account rotates.
403    pub fn report_probe_limit(&self, account: &AccountKey, reset_at: Option<u64>) {
404        let now = SystemTime::now();
405        let until = reset_at
406            .filter(|secs| *secs > 0)
407            .map_or(now + self.rotation.lock().no_reset_cooldown(), crate::rotation::reset_instant);
408        self.rotation.lock().cool_down(account, until);
409        self.bindings.unbind_for_account(account);
410        self.note_rotation_changed();
411    }
412}
413
414/// The `Retry-After` header, as seconds. Zero reads as absent: a
415/// cooldown ending the instant it starts is the same as none, and the
416/// next arm of the cooldown rule should decide instead.
417fn retry_after(headers: &HeaderMap) -> Option<Duration> {
418    headers
419        .get(hyper::header::RETRY_AFTER)
420        .and_then(|v| v.to_str().ok())
421        .and_then(|v| v.parse::<u64>().ok())
422        .filter(|secs| *secs > 0)
423        .map(Duration::from_secs)
424}
425
426/// Case-insensitive header comparison against a wire literal.
427fn header_value_is(headers: &HeaderMap, name: &str, value: &str) -> bool {
428    headers.get(name).and_then(|v| v.to_str().ok()).is_some_and(|v| v.eq_ignore_ascii_case(value))
429}
430
431#[async_trait::async_trait]
432impl RouteHandler for Gateway {
433    async fn route(&self, request: Request<Bytes>) -> hyper::Response<StreamBody> {
434        let (parts, body) = request.into_parts();
435        let segments: Vec<&str> = parts.uri.path().split('/').filter(|s| !s.is_empty()).collect();
436        if segments.len() < 4 {
437            return text_response(StatusCode::NOT_FOUND, "not found");
438        }
439        let (org, project, session) = (segments[0], segments[1], segments[2]);
440        let api_path = segments[3..].join("/");
441
442        // The CLI's connectivity probe. 404 is the answer it proceeds
443        // on; no upstream leg, no credentials.
444        if api_path == "api/hello" {
445            return text_response(StatusCode::NOT_FOUND, "not found");
446        }
447        if parts.method != Method::POST || api_path != "v1/messages" {
448            return text_response(StatusCode::NOT_FOUND, "not found");
449        }
450
451        let (spliced, model) = match splice_model(&body, None) {
452            Ok(spliced) => spliced,
453            Err(error) => return text_response(StatusCode::BAD_REQUEST, error.to_string()),
454        };
455
456        // A bound session keeps its account: a model change is forwarded
457        // on the account the session spawned under, because that choice
458        // was the spawn's and a running session is not reconfigured. An
459        // unbound session selects now, from the model in its own body and
460        // the org named in its path; the selection failure names both.
461        let account = match self.bindings.binding_for(org, project, session) {
462            Some(bound) => bound,
463            None => match self.select_or_fail(org, project, session, &model) {
464                Ok(account) => account,
465                Err(response) => return *response,
466            },
467        };
468        let Some((upstream, credential)) = self.credential_for(&account) else {
469            return text_response(
470                StatusCode::SERVICE_UNAVAILABLE,
471                format!(
472                    "account '{}' has no usable credential for session '{session}'; \
473                     nothing was forwarded",
474                    account.0
475                ),
476            );
477        };
478
479        let query = match parts.uri.query() {
480            Some(q) => format!("?{q}"),
481            None => "?beta=true".to_owned(),
482        };
483
484        let url = format!("{upstream}/v1/messages{query}");
485
486        // The body carries the canonical model name; the upstream may
487        // spell it differently. When the account declares a slug for
488        // the model, splice the slug in - never deserializing.
489        let spliced = match self.pool.model_slug_for(&account, &model) {
490            Some(slug) => {
491                splice_model(&spliced, Some(&slug)).map(|(body, _)| body).unwrap_or(spliced)
492            }
493            None => spliced,
494        };
495
496        // The CLI's headers forward verbatim - anthropic-beta,
497        // anthropic-version, user-agent, x-app: the betas and the API
498        // version are entitlements the upstream validates, and dropping
499        // them changes what the request is allowed to do. The
500        // credentials are the exception: both header forms are stripped
501        // (HeaderMap keys are case-insensitive, and remove() clears
502        // every value under the name), then the real credential is
503        // attached. Framing headers go: reqwest derives them from the
504        // spliced bytes, which is what keeps a rewrite from leaving a
505        // stale length behind.
506        let mut outbound_headers = parts.headers;
507        while outbound_headers.remove(hyper::header::AUTHORIZATION).is_some() {}
508        while outbound_headers.remove("x-api-key").is_some() {}
509        outbound_headers.remove(hyper::header::HOST);
510        outbound_headers.remove(CONTENT_LENGTH);
511        outbound_headers.remove(hyper::header::TRANSFER_ENCODING);
512
513        let upstream_response = match self
514            .client
515            .post(url)
516            .headers(outbound_headers)
517            .header(hyper::header::AUTHORIZATION, format!("Bearer {credential}"))
518            .body(spliced)
519            .send()
520            .await
521        {
522            Ok(response) => response,
523            Err(error) => {
524                return text_response(
525                    StatusCode::BAD_GATEWAY,
526                    format!("upstream request failed: {error}"),
527                );
528            }
529        };
530
531        // The upstream's headers stream back with the status -
532        // retry-after, request-id and the rate-limit set are the CLI's
533        // backoff inputs, and dropping them degrades it to blind
534        // pacing. Framing headers are omitted: hyper re-frames the
535        // streamed body itself.
536        let status = upstream_response.status();
537        let response_headers = upstream_response.headers().clone();
538        self.note_response(&account, org, project, session, status, &response_headers);
539        let mut response = hyper::Response::builder()
540            .status(StatusCode::from_u16(status.as_u16()).unwrap_or(StatusCode::BAD_GATEWAY));
541        for (name, value) in upstream_response.headers() {
542            if name == CONTENT_LENGTH || name == TRANSFER_ENCODING {
543                continue;
544            }
545            response = response.header(name, value);
546        }
547        let stream: std::pin::Pin<
548            Box<dyn futures_util::Stream<Item = Result<Frame<Bytes>, BoxError>> + Send + Sync>,
549        > = Box::pin(
550            upstream_response
551                .bytes_stream()
552                .map(|chunk| chunk.map(Frame::data).map_err(|error| Box::new(error) as BoxError)),
553        );
554        let streamed: StreamBody =
555            http_body_util::combinators::BoxBody::new(http_body_util::StreamBody::new(stream));
556        match response.body(streamed) {
557            Ok(response) => response,
558            Err(error) => {
559                text_response(StatusCode::BAD_GATEWAY, format!("response build failed: {error}"))
560            }
561        }
562    }
563}
564
565#[cfg(test)]
566mod tests {
567    use super::*;
568    use crate::binding::Registration;
569    use crate::listener::GatewayListener;
570    use hyper::header::CONTENT_TYPE;
571    use std::collections::HashMap;
572    use std::net::Ipv4Addr;
573    use std::time::{Duration, Instant};
574
575    /// One request the stub upstream received.
576    #[derive(Debug, Clone)]
577    struct Recorded {
578        path_with_query: String,
579        headers: HashMap<String, Vec<String>>,
580        body: String,
581    }
582
583    impl Recorded {
584        fn all(&self, name: &str) -> Vec<&str> {
585            self.headers
586                .iter()
587                .filter(|(k, _)| k.eq_ignore_ascii_case(name))
588                .flat_map(|(_, v)| v.iter().map(String::as_str))
589                .collect()
590        }
591
592        fn first(&self, name: &str) -> Option<&str> {
593            self.all(name).into_iter().next()
594        }
595    }
596
597    /// A stub upstream that records what arrived and streams its
598    /// response in two chunks separated by `chunk_delay`, so a
599    /// buffering gateway is measurable from the client side.
600    struct RecordingUpstream {
601        requests: Arc<parking_lot::Mutex<Vec<Recorded>>>,
602        chunk_delay: Duration,
603        /// One scripted (status, extra headers) pair per request.
604        script: ScriptHandle,
605    }
606
607    type ScriptHandle =
608        Arc<parking_lot::Mutex<std::collections::VecDeque<(u16, Vec<(String, String)>)>>>;
609
610    #[async_trait::async_trait]
611    impl RouteHandler for RecordingUpstream {
612        async fn route(&self, request: Request<Bytes>) -> hyper::Response<StreamBody> {
613            let mut headers: HashMap<String, Vec<String>> = HashMap::new();
614            for (name, value) in request.headers() {
615                let entry = headers.entry(name.as_str().to_ascii_lowercase()).or_default();
616                entry.push(String::from_utf8_lossy(value.as_bytes()).into_owned());
617            }
618            self.requests.lock().push(Recorded {
619                path_with_query: request
620                    .uri()
621                    .path_and_query()
622                    .map(|p| p.as_str().to_owned())
623                    .unwrap_or_default(),
624                headers,
625                body: String::from_utf8_lossy(request.body()).into_owned(),
626            });
627            // Each scripted (status, headers) pair answers one request;
628            // unscripted requests get the healthy default. A scripted
629            // header overrides its default rather than stacking a
630            // second value under the same name.
631            let (status, extra) =
632                self.script.lock().pop_front().unwrap_or((StatusCode::OK.as_u16(), Vec::new()));
633            let scripted = |name: &str| extra.iter().any(|(n, _)| n.eq_ignore_ascii_case(name));
634            let delay = self.chunk_delay;
635            let stream = futures_util::stream::unfold(
636                (0u8, delay),
637                move |state: (u8, Duration)| async move {
638                    type BoxError = Box<dyn std::error::Error + Send + Sync>;
639                    match state.0 {
640                        0 => Some((
641                            Ok::<Frame<Bytes>, BoxError>(Frame::data(Bytes::from_static(
642                                b"event: first\n",
643                            ))),
644                            (1u8, delay),
645                        )),
646                        1 => {
647                            tokio::time::sleep(delay).await;
648                            Some((
649                                Ok(Frame::data(Bytes::from_static(b"event: second\n"))),
650                                (2u8, delay),
651                            ))
652                        }
653                        _ => None,
654                    }
655                },
656            );
657            let mut builder = hyper::Response::builder()
658                .status(StatusCode::from_u16(status).expect("test status"))
659                .header(CONTENT_TYPE, "text/event-stream")
660                .header("request-id", "req-test");
661            if !scripted("retry-after") {
662                builder = builder.header("retry-after", "30");
663            }
664            if !scripted("anthropic-ratelimit-unified-status") {
665                builder = builder.header("anthropic-ratelimit-unified-status", "allowed");
666            }
667            for (name, value) in extra {
668                builder = builder.header(name, value);
669            }
670            builder
671                .body(http_body_util::combinators::BoxBody::new(http_body_util::StreamBody::new(
672                    stream,
673                )))
674                .expect("static response parts always build")
675        }
676    }
677
678    /// A free port, for the stub and the listener under test.
679    async fn free_port() -> u16 {
680        let probe = tokio::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.expect("probe");
681        probe.local_addr().expect("addr").port()
682    }
683
684    struct Harness {
685        /// The URL a CLI was told to talk to.
686        client_url: String,
687        /// The stub upstream, so a test that rewrites the pool's env can
688        /// keep the request off the real network.
689        upstream_url: String,
690        gateway: Arc<Gateway>,
691        requests: Arc<parking_lot::Mutex<Vec<Recorded>>>,
692        script: ScriptHandle,
693    }
694
695    /// The host the harness builds its forward client through, so the
696    /// leg runs against the port's own construction rather than a
697    /// hand-built client.
698    struct TestHost;
699
700    #[async_trait::async_trait]
701    impl crate::ProviderHost for TestHost {
702        fn http_client(&self, _timeout: Duration) -> Result<reqwest::Client, String> {
703            unreachable!("the forward harness runs no probe")
704        }
705
706        fn streaming_http_client(
707            &self,
708            connect_timeout: Duration,
709            idle_timeout: Duration,
710        ) -> Result<reqwest::Client, String> {
711            reqwest::Client::builder()
712                .connect_timeout(connect_timeout)
713                .read_timeout(idle_timeout)
714                .build()
715                .map_err(|e| e.to_string())
716        }
717
718        async fn user_agent(&self) -> Result<String, String> {
719            unreachable!("the forward harness runs no probe")
720        }
721    }
722
723    /// Stub upstream + gateway listener + one registered session whose
724    /// account's upstream is the stub. The pool holds one account per
725    /// provider, both with the stub as their upstream, so a test that
726    /// asserts "no upstream request" is airtight whichever account a
727    /// broken selection could have picked.
728    async fn harness(chunk_delay: Duration) -> Harness {
729        harness_with_idle_timeout(chunk_delay, FORWARD_IDLE_TIMEOUT).await
730    }
731
732    /// [`harness`] with the forward leg's idle bound under the test's
733    /// control, so a stalled upstream is reachable inside a test.
734    async fn harness_with_idle_timeout(chunk_delay: Duration, idle_timeout: Duration) -> Harness {
735        let script: ScriptHandle =
736            Arc::new(parking_lot::Mutex::new(std::collections::VecDeque::new()));
737        let requests = Arc::new(parking_lot::Mutex::new(Vec::new()));
738        let upstream = Arc::new(RecordingUpstream {
739            requests: Arc::clone(&requests),
740            chunk_delay,
741            script: Arc::clone(&script),
742        });
743        let stub_port = free_port().await;
744        let stub_listener = GatewayListener::bind(stub_port).await.expect("stub bind");
745        let stub_url = format!("http://127.0.0.1:{stub_port}");
746        let upstream_url = stub_url.clone();
747        let stub_handler: Arc<dyn RouteHandler> = upstream.clone();
748        tokio::spawn(async move { stub_listener.run(stub_handler).await });
749
750        let account_env = HashMap::from([
751            ("ANTHROPIC_BASE_URL".to_owned(), stub_url.clone()),
752            ("ANTHROPIC_AUTH_TOKEN".to_owned(), "real-openrouter-key".to_owned()),
753            ("ANTHROPIC_API_KEY".to_owned(), String::new()),
754        ]);
755        let anthropic_env = HashMap::from([
756            ("ANTHROPIC_BASE_URL".to_owned(), stub_url),
757            ("CLAUDE_CODE_OAUTH_TOKEN".to_owned(), "real-oauth-token".to_owned()),
758            ("ANTHROPIC_API_KEY".to_owned(), String::new()),
759        ]);
760        let pool = Arc::new(crate::AccountPool::new(&[
761            forge_primitives::account::LoadedAccount {
762                display_name: "OpenRouter".to_owned(),
763                provider: forge_primitives::account::Provider::Openrouter,
764                base_url: None,
765                models: vec!["glm-5.3-flash".to_owned()],
766                model_aliases: std::collections::HashMap::new(),
767                model_slugs: std::collections::HashMap::from([(
768                    "glm-5.3-flash".to_owned(),
769                    "zai-org/glm-5.3-flash".to_owned(),
770                )]),
771                env: account_env.clone(),
772            },
773            forge_primitives::account::LoadedAccount {
774                display_name: "Anthropic".to_owned(),
775                provider: forge_primitives::account::Provider::Anthropic,
776                base_url: None,
777                models: vec!["claude-sonnet-5".to_owned(), "claude-opus-5".to_owned()],
778                model_aliases: std::collections::HashMap::new(),
779                model_slugs: std::collections::HashMap::new(),
780                env: anthropic_env,
781            },
782        ]));
783        let client = crate::ProviderHost::streaming_http_client(
784            &TestHost,
785            FORWARD_CONNECT_TIMEOUT,
786            idle_timeout,
787        )
788        .expect("the test host builds a client");
789        let gateway = Arc::new(Gateway::new(Arc::clone(&pool), client));
790
791        let listener_port = free_port().await;
792        let listener = GatewayListener::bind(listener_port).await.expect("gateway bind");
793        let listener_base = format!("http://127.0.0.1:{listener_port}");
794        let client_url = format!("{listener_base}/Busytools/forge/session-1");
795        let gateway_handler: Arc<dyn RouteHandler> = gateway.clone();
796        tokio::spawn(async move { listener.run(gateway_handler).await });
797
798        gateway.bindings.register(
799            &Registration {
800                org: "Busytools".to_owned(),
801                project: "forge".to_owned(),
802                session: "session-1".to_owned(),
803                account: AccountKey("OpenRouter".to_owned()),
804                provider: forge_primitives::account::Provider::Openrouter,
805            },
806            &listener_base,
807            &account_env,
808        );
809
810        Harness { client_url, upstream_url, gateway, requests, script }
811    }
812
813    /// A CLI-shaped POST: the dummy credential in the auth variable's
814    /// header form, a junk `x-api-key`, and the entitlement headers the
815    /// real CLI emits - exactly what a launching env with
816    /// `ANTHROPIC_API_KEY` set would produce.
817    async fn post_as_cli(url: &str) -> reqwest::Response {
818        // The harness session is bound to the OpenRouter account, so
819        // the body carries a model that account declares: a claude
820        // model here would rotate the binding instead of forwarding.
821        reqwest::Client::new()
822            .post(url)
823            .header(hyper::header::AUTHORIZATION, "Bearer forge-gateway-unused")
824            .header("x-api-key", "junk-key")
825            .header(CONTENT_TYPE, "application/json")
826            .header("anthropic-beta", "oauth-2025-04-20,extended-cache-ttl-2025-04-11")
827            .header("anthropic-version", "2023-06-01")
828            .header("user-agent", "claude-code/2.1.263")
829            .body(r#"{"model":"glm-5.3-flash","messages":[]}"#)
830            .send()
831            .await
832            .expect("gateway responds")
833    }
834
835    #[tokio::test]
836    async fn the_hello_probe_answers_404_and_never_reaches_the_upstream() {
837        let harness = harness(Duration::ZERO).await;
838        let response = reqwest::Client::new()
839            .head(format!("{}/api/hello", harness.client_url))
840            .send()
841            .await
842            .expect("gateway responds");
843        assert_eq!(response.status(), StatusCode::NOT_FOUND);
844        assert!(
845            harness.requests.lock().is_empty(),
846            "the connectivity probe must not construct an upstream request",
847        );
848    }
849
850    #[tokio::test]
851    async fn a_forwarded_message_carries_the_cli_headers_and_the_real_credential_only() {
852        let harness = harness(Duration::ZERO).await;
853        let response = post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
854        assert_eq!(response.status(), StatusCode::OK);
855
856        let requests = harness.requests.lock();
857        assert_eq!(requests.len(), 1, "exactly one upstream request");
858        let recorded = &requests[0];
859        assert_eq!(
860            recorded.path_with_query, "/v1/messages?beta=true",
861            "the API path forwards with its query"
862        );
863        assert_eq!(
864            recorded.first("authorization"),
865            Some("Bearer real-openrouter-key"),
866            "the real credential rides the auth header",
867        );
868        assert_eq!(
869            recorded.all("x-api-key"),
870            Vec::<&str>::new(),
871            "x-api-key is stripped, never forwarded",
872        );
873        assert!(!recorded.body.contains(DUMMY_CREDENTIAL), "the dummy never reaches the upstream");
874        // The entitlement headers the spec's wire section rests on:
875        // they forward verbatim or an OAuth forward loses its beta and
876        // a versioned API loses its version.
877        assert_eq!(
878            recorded.first("anthropic-beta"),
879            Some("oauth-2025-04-20,extended-cache-ttl-2025-04-11"),
880            "anthropic-beta forwards verbatim",
881        );
882        assert_eq!(
883            recorded.first("anthropic-version"),
884            Some("2023-06-01"),
885            "anthropic-version forwards verbatim",
886        );
887        assert_eq!(
888            recorded.first("user-agent"),
889            Some("claude-code/2.1.263"),
890            "user-agent forwards verbatim",
891        );
892        assert_eq!(
893            recorded.first("content-type"),
894            Some("application/json"),
895            "content-type forwards verbatim",
896        );
897    }
898
899    #[tokio::test]
900    async fn upstream_response_headers_reach_the_cli() {
901        let harness = harness(Duration::ZERO).await;
902        let response = post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
903        // retry-after is the CLI's backoff input; the rate-limit set is
904        // its window display. Dropping them degrades the client to
905        // blind pacing.
906        for (name, expected) in
907            [("retry-after", "30"), ("anthropic-ratelimit-unified-status", "allowed")]
908        {
909            assert_eq!(
910                response.headers().get(name).and_then(|v| v.to_str().ok()),
911                Some(expected),
912                "{name} must reach the CLI",
913            );
914        }
915    }
916
917    #[tokio::test]
918    async fn an_unbound_session_selects_from_the_org_pin_and_binds() {
919        let harness = harness(Duration::ZERO).await;
920        // The harness registered session-1 → Openrouter. A ghost
921        // session with no binding selects now: the model is
922        // non-claude, so the first non-Anthropic account in walk order
923        // wins, and the result is bound for the session's lifetime.
924        harness.gateway.set_org_pins([(
925            "Busytools".to_owned(),
926            crate::selection::OrgPin {
927                accounts: vec!["OpenRouter".to_owned(), "Stargate".to_owned()],
928                fallback_accounts: Vec::new(),
929            },
930        )]);
931        // The harness pool's OpenRouter account defaults to Loading;
932        // selection skips non-terminal accounts.
933        harness
934            .gateway
935            .pool
936            .set_loading(&AccountKey("OpenRouter".to_owned()), crate::LoadingState::Ready);
937
938        let ghost_url = harness.client_url.replacen("session-1", "ghost", 1);
939        let response = reqwest::Client::new()
940            .post(format!("{ghost_url}/v1/messages?beta=true"))
941            .header(CONTENT_TYPE, "application/json")
942            .body(r#"{"model":"glm-5.3-flash","messages":[]}"#)
943            .send()
944            .await
945            .expect("gateway responds");
946        let status = response.status();
947        let failure_body = response.text().await.expect("body");
948        assert_eq!(status, StatusCode::OK, "selection serves the request: {failure_body}");
949        assert_eq!(
950            harness.gateway.bindings.binding_for("Busytools", "forge", "ghost"),
951            Some(AccountKey("OpenRouter".to_owned())),
952            "the selected account is bound for the session's lifetime",
953        );
954    }
955
956    #[tokio::test]
957    async fn an_unknown_org_is_a_loud_404_and_never_reaches_the_upstream() {
958        let harness = harness(Duration::ZERO).await;
959        let ghost_url = harness.client_url.replacen("/Busytools/", "/GhostOrg/", 1);
960        let response = post_as_cli(&format!("{ghost_url}/v1/messages?beta=true")).await;
961        assert_eq!(response.status(), StatusCode::NOT_FOUND);
962        let body = response.text().await.expect("body");
963        assert!(body.contains("GhostOrg"), "the failure names the unknown org, got: {body}");
964        assert!(
965            harness.requests.lock().is_empty(),
966            "an unknown org must not construct an upstream request",
967        );
968    }
969
970    #[tokio::test]
971    async fn a_model_no_account_serves_is_a_loud_503_and_never_reaches_the_upstream() {
972        let harness = harness(Duration::ZERO).await;
973        // The pin's only account is Anthropic and that account is
974        // Ready; a non-claude model has no eligible account in this
975        // org. The refusal is the declared-model gate's, not a loading
976        // state or an unknown name.
977        harness.gateway.set_org_pins([(
978            "Busytools".to_owned(),
979            crate::selection::OrgPin {
980                accounts: vec!["Anthropic".to_owned()],
981                fallback_accounts: Vec::new(),
982            },
983        )]);
984        harness
985            .gateway
986            .pool
987            .set_loading(&AccountKey("Anthropic".to_owned()), crate::LoadingState::Ready);
988
989        let ghost_url = harness.client_url.replacen("session-1", "ghost", 1);
990        let response = reqwest::Client::new()
991            .post(format!("{ghost_url}/v1/messages?beta=true"))
992            .header(CONTENT_TYPE, "application/json")
993            .body(r#"{"model":"glm-5.3-flash","messages":[]}"#)
994            .send()
995            .await
996            .expect("gateway responds");
997        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
998        let body = response.text().await.expect("body");
999        assert!(
1000            body.contains("glm-5.3-flash") && body.contains("Busytools"),
1001            "the failure names the model and the org, got: {body}",
1002        );
1003        assert!(
1004            harness.requests.lock().is_empty(),
1005            "an unselectable request must not construct an upstream request",
1006        );
1007    }
1008
1009    #[tokio::test]
1010    async fn a_non_claude_model_in_a_non_anthropic_org_binds_and_forwards() {
1011        let harness = harness(Duration::ZERO).await;
1012        // Mirror of the refusal above: the org's one account is the
1013        // OpenRouter one, glm is in its declared models, so the same
1014        // route selects, binds, and forwards.
1015        harness.gateway.set_org_pins([(
1016            "Busytools".to_owned(),
1017            crate::selection::OrgPin {
1018                accounts: vec!["OpenRouter".to_owned()],
1019                fallback_accounts: Vec::new(),
1020            },
1021        )]);
1022        harness
1023            .gateway
1024            .pool
1025            .set_loading(&AccountKey("OpenRouter".to_owned()), crate::LoadingState::Ready);
1026
1027        let ghost_url = harness.client_url.replacen("session-1", "ghost", 1);
1028        let response = reqwest::Client::new()
1029            .post(format!("{ghost_url}/v1/messages?beta=true"))
1030            .header(CONTENT_TYPE, "application/json")
1031            .body(r#"{"model":"glm-5.3-flash","messages":[]}"#)
1032            .send()
1033            .await
1034            .expect("gateway responds");
1035        assert_eq!(
1036            response.status(),
1037            StatusCode::OK,
1038            "the model gate admits the model to a declaring account",
1039        );
1040        assert_eq!(
1041            harness.gateway.bindings.binding_for("Busytools", "forge", "ghost"),
1042            Some(AccountKey("OpenRouter".to_owned())),
1043            "the route binds what the declared-model gate admitted",
1044        );
1045        assert_eq!(
1046            harness.requests.lock().len(),
1047            1,
1048            "the forwarded request reached the bound account's upstream",
1049        );
1050    }
1051
1052    #[tokio::test]
1053    async fn the_response_streams_while_the_upstream_is_still_sending() {
1054        let harness = harness(Duration::from_millis(1000)).await;
1055        let started = Instant::now();
1056        let mut response =
1057            post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
1058
1059        let first = response.chunk().await.expect("a body chunk").expect("chunk bytes");
1060        assert!(
1061            started.elapsed() < Duration::from_millis(800),
1062            "the first chunk arrives while the upstream is still streaming, not after it \
1063             finished; took {:?}",
1064            started.elapsed(),
1065        );
1066        assert!(!first.is_empty());
1067
1068        // The tail arrives after the stub's gap, so the whole body
1069        // really was split across the stream.
1070        let second = response.chunk().await.expect("a second chunk").expect("chunk bytes");
1071        assert!(!second.is_empty());
1072        assert!(started.elapsed() >= Duration::from_millis(1000), "the tail respects the gap");
1073    }
1074
1075    /// The connect bound is the tighter inner one, and that ordering is
1076    /// its whole reason for existing: it is what a host that never
1077    /// produces a first byte runs into, where the idle bound would hold
1078    /// the turn open far longer than a connect ever should.
1079    #[test]
1080    fn the_connect_bound_stays_well_under_the_idle_bound() {
1081        assert!(
1082            FORWARD_CONNECT_TIMEOUT <= FORWARD_IDLE_TIMEOUT / 10,
1083            "the connect bound is the tighter one; {FORWARD_CONNECT_TIMEOUT:?} against an idle \
1084             bound of {FORWARD_IDLE_TIMEOUT:?} lets a blackholed connect outlast the wedge the \
1085             leg is bounded to stop",
1086        );
1087    }
1088
1089    /// The forward leg's client is the host's, and a host that bounds
1090    /// idle reads ends a wedged upstream: the CLI's turn is abandoned
1091    /// instead of held open until the upstream produces anything.
1092    #[tokio::test]
1093    async fn a_stalled_upstream_ends_the_forwarded_stream_at_the_hosts_idle_bound() {
1094        let harness =
1095            harness_with_idle_timeout(Duration::from_secs(5), Duration::from_millis(200)).await;
1096        let started = Instant::now();
1097        let mut response =
1098            post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
1099
1100        let first = response.chunk().await.expect("a body chunk").expect("chunk bytes");
1101        assert!(!first.is_empty(), "the first chunk arrives while the upstream is healthy");
1102
1103        let stalled = tokio::time::timeout(Duration::from_secs(8), response.chunk())
1104            .await
1105            .expect("the idle bound ends the stream; an unbounded client never returns here");
1106        assert!(
1107            stalled.is_err(),
1108            "the stream is cut mid-body when the upstream goes quiet; got {stalled:?}",
1109        );
1110        assert!(
1111            started.elapsed() < Duration::from_secs(5),
1112            "the host's bound ended the stream, not the upstream's next chunk; took {:?}",
1113            started.elapsed(),
1114        );
1115    }
1116
1117    #[tokio::test]
1118    async fn the_fifth_429_rotates_the_binding_and_the_cooled_account_is_skipped() {
1119        let harness = harness(Duration::ZERO).await;
1120        harness.gateway.set_org_pins([(
1121            "Busytools".to_owned(),
1122            crate::selection::OrgPin {
1123                accounts: vec!["OpenRouter".to_owned()],
1124                fallback_accounts: Vec::new(),
1125            },
1126        )]);
1127        for _ in 0..4 {
1128            harness
1129                .script
1130                .lock()
1131                .push_back((429, vec![("retry-after".to_owned(), "30".to_owned())]));
1132            let response =
1133                post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
1134            assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS);
1135        }
1136        // Four transient 429s: retried in place, the binding holds.
1137        assert_eq!(
1138            harness.gateway.bindings.binding_for("Busytools", "forge", "session-1"),
1139            Some(AccountKey("OpenRouter".to_owned())),
1140            "transient 429s do not rotate",
1141        );
1142        harness.script.lock().push_back((429, vec![("retry-after".to_owned(), "30".to_owned())]));
1143        let response = post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
1144        assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS);
1145        // The streak fired: the binding is gone and the account cools,
1146        // so the CLI's retry cannot come back to it.
1147        assert_eq!(
1148            harness.gateway.bindings.binding_for("Busytools", "forge", "session-1"),
1149            None,
1150            "the fifth 429 drops the binding",
1151        );
1152        let retry = post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
1153        assert_eq!(
1154            retry.status(),
1155            StatusCode::SERVICE_UNAVAILABLE,
1156            "the cooled account is skipped and nothing else can serve the session",
1157        );
1158    }
1159
1160    #[tokio::test]
1161    async fn a_rejected_unified_status_header_rotates_on_the_first_response() {
1162        let harness = harness(Duration::ZERO).await;
1163        harness.gateway.set_org_pins([(
1164            "Busytools".to_owned(),
1165            crate::selection::OrgPin {
1166                accounts: vec!["OpenRouter".to_owned()],
1167                fallback_accounts: Vec::new(),
1168            },
1169        )]);
1170        harness.script.lock().push_back((
1171            StatusCode::PAYMENT_REQUIRED.as_u16(),
1172            vec![
1173                ("retry-after".to_owned(), "45".to_owned()),
1174                ("anthropic-ratelimit-unified-status".to_owned(), "rejected".to_owned()),
1175            ],
1176        ));
1177        let response = post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
1178        assert_eq!(response.status(), StatusCode::PAYMENT_REQUIRED);
1179        assert_eq!(
1180            harness.gateway.bindings.binding_for("Busytools", "forge", "session-1"),
1181            None,
1182            "proven exhaustion drops the binding without waiting for a streak",
1183        );
1184        let retry = post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
1185        assert_eq!(retry.status(), StatusCode::SERVICE_UNAVAILABLE);
1186    }
1187
1188    /// The spec scopes the header trigger to `rejected` "on the failing
1189    /// response" (spec, Rotation), so a 200 carrying it cools nothing.
1190    /// The overage arm is the one that bites: an account not paying for
1191    /// overage reports `overage-status: rejected` on every response it
1192    /// sends, successful ones included.
1193    #[tokio::test]
1194    async fn a_success_carrying_rejected_cools_nothing() {
1195        for header in
1196            ["anthropic-ratelimit-unified-status", "anthropic-ratelimit-unified-overage-status"]
1197        {
1198            let harness = harness(Duration::ZERO).await;
1199            harness.script.lock().push_back((
1200                StatusCode::OK.as_u16(),
1201                vec![(header.to_owned(), "rejected".to_owned())],
1202            ));
1203            let response =
1204                post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
1205            assert_eq!(response.status(), StatusCode::OK, "the status streams back untouched");
1206            assert_eq!(
1207                harness.gateway.bindings.binding_for("Busytools", "forge", "session-1"),
1208                Some(AccountKey("OpenRouter".to_owned())),
1209                "{header}: a success is not a failing response, so the binding survives",
1210            );
1211            assert!(
1212                !harness
1213                    .gateway
1214                    .rotation
1215                    .lock()
1216                    .is_cooling_down(&AccountKey("OpenRouter".to_owned()), SystemTime::now()),
1217                "{header}: a success must not start a cooldown",
1218            );
1219        }
1220    }
1221
1222    /// A rejected FAILING response is not a success: it rotates without
1223    /// clearing the streak on its way past. `retry-after: 0` reads as no
1224    /// reset time and a zero no-reset cooldown cools the account to the
1225    /// instant, so the rotated account is selectable again at once and
1226    /// the streak alone decides whether the next 429 is the fifth.
1227    #[tokio::test]
1228    async fn a_rejected_failing_response_does_not_clear_the_429_streak() {
1229        let harness = harness(Duration::ZERO).await;
1230        harness.gateway.set_org_pins([(
1231            "Busytools".to_owned(),
1232            crate::selection::OrgPin {
1233                accounts: vec!["OpenRouter".to_owned()],
1234                fallback_accounts: Vec::new(),
1235            },
1236        )]);
1237        harness
1238            .gateway
1239            .pool
1240            .set_loading(&AccountKey("OpenRouter".to_owned()), crate::LoadingState::Ready);
1241        harness.gateway.set_rotation_numbers(crate::rotation::RotationNumbers {
1242            no_reset_cooldown: Duration::ZERO,
1243            ..Default::default()
1244        });
1245        for _ in 0..4 {
1246            harness
1247                .script
1248                .lock()
1249                .push_back((429, vec![("retry-after".to_owned(), "30".to_owned())]));
1250            let response =
1251                post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
1252            assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS, "the streak is counting");
1253        }
1254        harness.script.lock().push_back((
1255            StatusCode::PAYMENT_REQUIRED.as_u16(),
1256            vec![
1257                ("anthropic-ratelimit-unified-status".to_owned(), "rejected".to_owned()),
1258                ("retry-after".to_owned(), "0".to_owned()),
1259            ],
1260        ));
1261        post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
1262        assert_eq!(
1263            harness.gateway.bindings.binding_for("Busytools", "forge", "session-1"),
1264            None,
1265            "the 402's rejected header rotated the binding, not a streak",
1266        );
1267        harness.script.lock().push_back((429, vec![("retry-after".to_owned(), "30".to_owned())]));
1268        let fifth = post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
1269        assert_eq!(
1270            fifth.status(),
1271            StatusCode::TOO_MANY_REQUESTS,
1272            "the fifth 429 reached the upstream, so the binding below is the streak's verdict",
1273        );
1274        assert_eq!(
1275            harness.gateway.bindings.binding_for("Busytools", "forge", "session-1"),
1276            None,
1277            "the rejection left the four 429s standing, so this one is the fifth",
1278        );
1279    }
1280
1281    /// A 429 carrying the standing `-overage-status: rejected` rotates
1282    /// below the streak threshold: the header arm reads the status and a
1283    /// 429 is not a success. That is newer than pre-regression, where the
1284    /// 429 arm was exclusive, so it is pinned rather than fixed. The
1285    /// standing-state distinction (`-overage-disabled-reason: out_of_credits`)
1286    /// is the architectural option if it ever needs one.
1287    #[tokio::test]
1288    async fn a_429_carrying_the_standing_rejection_rotates_below_the_streak() {
1289        let harness = harness(Duration::ZERO).await;
1290        pin_only(&harness, "OpenRouter");
1291        harness.script.lock().push_back((
1292            429,
1293            vec![("anthropic-ratelimit-unified-overage-status".to_owned(), "rejected".to_owned())],
1294        ));
1295        let response = post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
1296        assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS);
1297        assert_eq!(
1298            harness.gateway.bindings.binding_for("Busytools", "forge", "session-1"),
1299            None,
1300            "one 429 is not a streak, so the header arm is what rotated it",
1301        );
1302    }
1303
1304    /// The spec: "any success resets it". The success here carries the
1305    /// standing overage rejection, which is what an account not paying
1306    /// for overage sends on every response, so the reset has to survive
1307    /// that header to be reachable at all.
1308    #[tokio::test]
1309    async fn a_success_carrying_rejected_resets_the_429_streak() {
1310        let harness = harness(Duration::ZERO).await;
1311        pin_only(&harness, "OpenRouter");
1312        for _ in 0..4 {
1313            harness
1314                .script
1315                .lock()
1316                .push_back((429, vec![("retry-after".to_owned(), "30".to_owned())]));
1317            post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
1318        }
1319        // The healthy response lands between the streaks.
1320        harness.script.lock().push_back((
1321            StatusCode::OK.as_u16(),
1322            vec![("anthropic-ratelimit-unified-overage-status".to_owned(), "rejected".to_owned())],
1323        ));
1324        let response = post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
1325        assert_eq!(response.status(), StatusCode::OK);
1326        harness.script.lock().push_back((429, vec![("retry-after".to_owned(), "30".to_owned())]));
1327        post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
1328        assert_eq!(
1329            harness.gateway.bindings.binding_for("Busytools", "forge", "session-1"),
1330            Some(AccountKey("OpenRouter".to_owned())),
1331            "the streak counts consecutive 429s; one after a success does not rotate",
1332        );
1333    }
1334
1335    #[tokio::test]
1336    async fn a_rate_limit_report_cools_the_bound_account_until_the_reset() {
1337        let harness = harness(Duration::ZERO).await;
1338        harness.gateway.set_org_pins([(
1339            "Busytools".to_owned(),
1340            crate::selection::OrgPin {
1341                accounts: vec!["OpenRouter".to_owned()],
1342                fallback_accounts: Vec::new(),
1343            },
1344        )]);
1345        let reset = std::time::SystemTime::now() + Duration::from_secs(120);
1346        let reset_secs = reset
1347            .duration_since(std::time::SystemTime::UNIX_EPOCH)
1348            .expect("future reset")
1349            .as_secs();
1350        harness.gateway.report_rate_limit("Busytools", "forge", "session-1", Some(reset_secs));
1351        assert_eq!(
1352            harness.gateway.bindings.binding_for("Busytools", "forge", "session-1"),
1353            None,
1354            "the report drops the binding",
1355        );
1356        let retry = post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
1357        assert_eq!(
1358            retry.status(),
1359            StatusCode::SERVICE_UNAVAILABLE,
1360            "the account is cooling until the frame's reset time",
1361        );
1362        // An unbound session has nothing to rotate and must not panic.
1363        harness.gateway.report_rate_limit("Busytools", "forge", "ghost", None);
1364    }
1365
1366    #[tokio::test]
1367    async fn a_probe_report_cools_the_account_and_unbinds_every_session_on_it() {
1368        let harness = harness(Duration::ZERO).await;
1369        harness.gateway.set_org_pins([(
1370            "Busytools".to_owned(),
1371            crate::selection::OrgPin {
1372                accounts: vec!["OpenRouter".to_owned()],
1373                fallback_accounts: Vec::new(),
1374            },
1375        )]);
1376        // A second session on the same account, plus the harness's own.
1377        harness.gateway.bindings.bind(
1378            "Busytools",
1379            "forge",
1380            "session-2",
1381            AccountKey("OpenRouter".to_owned()),
1382        );
1383        let reset = std::time::SystemTime::now() + Duration::from_secs(300);
1384        let reset_secs = reset
1385            .duration_since(std::time::SystemTime::UNIX_EPOCH)
1386            .expect("future reset")
1387            .as_secs();
1388        harness.gateway.report_probe_limit(&AccountKey("OpenRouter".to_owned()), Some(reset_secs));
1389        assert_eq!(
1390            harness.gateway.bindings.binding_for("Busytools", "forge", "session-1"),
1391            None,
1392            "the probe's verdict rotates every session on the account",
1393        );
1394        assert_eq!(
1395            harness.gateway.bindings.binding_for("Busytools", "forge", "session-2"),
1396            None,
1397            "the probe's verdict rotates every session on the account",
1398        );
1399        let retry = post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
1400        assert_eq!(
1401            retry.status(),
1402            StatusCode::SERVICE_UNAVAILABLE,
1403            "the cooled account cannot be re-selected until its reset",
1404        );
1405    }
1406
1407    /// A usage snapshot with the five-hour window at the cap and a
1408    /// reset ahead: the probe-proven exhaustion shape.
1409    fn saturated_usage(reset_in: Duration) -> forge_primitives::usage::UsageSnapshot {
1410        forge_primitives::usage::UsageSnapshot {
1411            source: forge_primitives::usage::UsageSourceKind::Oauth,
1412            fetched_at: std::time::SystemTime::UNIX_EPOCH,
1413            five_hour: Some(forge_primitives::usage::UsageWindow {
1414                utilization: 100.0,
1415                resets_at: Some(SystemTime::now() + reset_in),
1416                reset_description: None,
1417                status: None,
1418            }),
1419            seven_day: None,
1420            seven_day_opus: None,
1421            seven_day_sonnet: None,
1422            extra_usage: None,
1423            spend: None,
1424            balance: None,
1425        }
1426    }
1427
1428    async fn post_model(url: &str, model: &str) -> reqwest::Response {
1429        reqwest::Client::new()
1430            .post(format!("{url}/v1/messages?beta=true"))
1431            .header(CONTENT_TYPE, "application/json")
1432            .body(format!(r#"{{"model":"{model}","messages":[]}}"#))
1433            .send()
1434            .await
1435            .expect("gateway responds")
1436    }
1437
1438    fn pin_only(harness: &Harness, account: &str) {
1439        harness.gateway.set_org_pins([(
1440            "Busytools".to_owned(),
1441            crate::selection::OrgPin {
1442                accounts: vec![account.to_owned()],
1443                fallback_accounts: Vec::new(),
1444            },
1445        )]);
1446    }
1447
1448    /// The forward leg reads a native account's credential from the
1449    /// variable the stamp put its dummy in - the same rule the stamper
1450    /// and the loader call, so the reader cannot drift to the other
1451    /// variable and 503 every turn.
1452    #[tokio::test]
1453    async fn the_forward_leg_reads_a_native_accounts_credential_variable() {
1454        let harness = harness(Duration::ZERO).await;
1455        pin_only(&harness, "Anthropic");
1456        harness
1457            .gateway
1458            .pool
1459            .set_loading(&AccountKey("Anthropic".to_owned()), crate::LoadingState::Ready);
1460        harness.gateway.bindings.bind(
1461            "Busytools",
1462            "forge",
1463            "session-1",
1464            AccountKey("Anthropic".to_owned()),
1465        );
1466
1467        let response = post_model(&harness.client_url, "claude-sonnet-5").await;
1468        assert_eq!(
1469            response.status(),
1470            StatusCode::OK,
1471            "the credential is found in the account's own variable; reading the other one 503s",
1472        );
1473        let requests = harness.requests.lock();
1474        assert_eq!(
1475            requests[0].first("authorization"),
1476            Some("Bearer real-oauth-token"),
1477            "the real credential is read back out of the account's own variable",
1478        );
1479    }
1480
1481    /// The account's declared slug replaces the canonical model name
1482    /// on the forwarded body - the upstream spelling is the account's
1483    /// business, invisible to the CLI.
1484    #[tokio::test]
1485    async fn a_declared_slug_replaces_the_canonical_model_upstream() {
1486        let harness = harness(Duration::ZERO).await;
1487        pin_only(&harness, "OpenRouter");
1488        harness
1489            .gateway
1490            .pool
1491            .set_loading(&AccountKey("OpenRouter".to_owned()), crate::LoadingState::Ready);
1492        // Give the account a slug for the canonical model: the forward
1493        // must splice the slug in, not the name the CLI sent.
1494        harness.gateway.pool.set_model_slug(
1495            &AccountKey("OpenRouter".to_owned()),
1496            "glm-5.3-flash",
1497            "zai-org/glm-5.3-flash",
1498        );
1499        let ghost_url = harness.client_url.replacen("session-1", "ghost", 1);
1500        let response = post_model(&ghost_url, "glm-5.3-flash").await;
1501        assert_eq!(response.status(), StatusCode::OK);
1502        let recorded = harness.requests.lock().first().expect("the upstream saw a request").clone();
1503        // Parsed, not a substring: a re-introduced double-quote around
1504        // the value would satisfy a contains check and still be
1505        // invalid JSON (the exact bug this test guards).
1506        let body: serde_json::Value =
1507            serde_json::from_str(&recorded.body).expect("the forwarded body parses as JSON");
1508        assert_eq!(
1509            body.get("model").and_then(|m| m.as_str()),
1510            Some("zai-org/glm-5.3-flash"),
1511            "the forwarded body's model is exactly the upstream slug",
1512        );
1513    }
1514
1515    #[tokio::test]
1516    async fn the_budget_failure_names_the_accounts_tried_and_the_soonest_reset() {
1517        let harness = harness(Duration::ZERO).await;
1518        pin_only(&harness, "OpenRouter");
1519        for _ in 0..5 {
1520            harness
1521                .script
1522                .lock()
1523                .push_back((429, vec![("retry-after".to_owned(), "30".to_owned())]));
1524            post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
1525        }
1526        // The bound account rotated off on the streak; a fresh session
1527        // finds the whole walk cooling, which is budget exhaustion, not
1528        // a declared-model miss.
1529        let ghost_url = harness.client_url.replacen("session-1", "ghost", 1);
1530        let response = post_model(&ghost_url, "glm-5.3-flash").await;
1531        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
1532        let body = response.text().await.expect("body");
1533        assert!(
1534            body.contains("tried 1 account(s)"),
1535            "the failure names how many exhausted accounts were tried, got: {body}",
1536        );
1537        assert!(
1538            body.contains("soonest reset in "),
1539            "the failure names the soonest reset, got: {body}",
1540        );
1541    }
1542
1543    /// **The reset a budget failure names is the one this org is waiting
1544    /// on.** The walk emptied because the caller's own accounts are
1545    /// cooling, so another org's account clearing sooner is not a time
1546    /// its retry can land - and a cron deferral prints this value as a
1547    /// wall clock, which is what turns a global minimum into a wrong
1548    /// time rather than only an imprecise one.
1549    #[tokio::test]
1550    async fn the_budget_failure_names_this_orgs_soonest_reset_not_another_orgs() {
1551        let harness = harness(Duration::ZERO).await;
1552        harness.gateway.set_org_pins([
1553            (
1554                "Busytools".to_owned(),
1555                crate::selection::OrgPin {
1556                    accounts: vec!["OpenRouter".to_owned()],
1557                    fallback_accounts: Vec::new(),
1558                },
1559            ),
1560            (
1561                "Other".to_owned(),
1562                crate::selection::OrgPin {
1563                    accounts: vec!["Anthropic".to_owned()],
1564                    fallback_accounts: Vec::new(),
1565                },
1566            ),
1567        ]);
1568        let now_secs = SystemTime::now()
1569            .duration_since(SystemTime::UNIX_EPOCH)
1570            .expect("clock after the epoch")
1571            .as_secs();
1572        harness
1573            .gateway
1574            .report_probe_limit(&AccountKey("OpenRouter".to_owned()), Some(now_secs + 600));
1575        harness
1576            .gateway
1577            .report_probe_limit(&AccountKey("Anthropic".to_owned()), Some(now_secs + 30));
1578
1579        let Err(failure) = harness.gateway.select_for("Busytools", "glm-5.3-flash") else {
1580            panic!("the org's only account is cooling");
1581        };
1582        let SelectFailure::BudgetExhausted { tried, org, model, reset_in } = failure else {
1583            panic!("a cooling walk is budget exhaustion, got {failure:?}");
1584        };
1585        assert_eq!((tried, org.as_str(), model.as_str()), (1, "Busytools", "glm-5.3-flash"));
1586        assert!(
1587            reset_in >= Duration::from_secs(590) && reset_in <= Duration::from_secs(600),
1588            "the reset is this org's account, not Other's sooner one: {reset_in:?}",
1589        );
1590    }
1591
1592    #[tokio::test]
1593    async fn a_declared_model_miss_stays_a_declared_model_miss_with_an_unrelated_cooldown_live() {
1594        let harness = harness(Duration::ZERO).await;
1595        pin_only(&harness, "Anthropic");
1596        harness
1597            .gateway
1598            .pool
1599            .set_loading(&AccountKey("Anthropic".to_owned()), crate::LoadingState::Ready);
1600        // An account OUTSIDE the walk is cooling; the walk itself holds
1601        // nothing cooling, so the walk emptied on the declared-model
1602        // gate and the failure must keep saying so.
1603        let reset = SystemTime::now() + Duration::from_secs(600);
1604        let reset_secs =
1605            reset.duration_since(SystemTime::UNIX_EPOCH).expect("future reset").as_secs();
1606        harness.gateway.report_probe_limit(&AccountKey("OpenRouter".to_owned()), Some(reset_secs));
1607        let ghost_url = harness.client_url.replacen("session-1", "ghost", 1);
1608        let response = post_model(&ghost_url, "glm-5.3-flash").await;
1609        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
1610        let body = response.text().await.expect("body");
1611        assert!(
1612            body.contains("no account in org 'Busytools' serves model 'glm-5.3-flash'"),
1613            "a declared-model miss keeps the declared-model message even with an unrelated cooldown live, got: {body}",
1614        );
1615    }
1616
1617    #[tokio::test]
1618    async fn a_zero_retry_after_falls_through_to_the_probe_reset() {
1619        let harness = harness(Duration::ZERO).await;
1620        pin_only(&harness, "OpenRouter");
1621        // The account's own probe reports a reset an hour out; the
1622        // response carries no usable Retry-After.
1623        harness.gateway.pool.set_usage(
1624            &AccountKey("OpenRouter".to_owned()),
1625            saturated_usage(Duration::from_secs(3600)),
1626        );
1627        for _ in 0..5 {
1628            harness
1629                .script
1630                .lock()
1631                .push_back((429, vec![("retry-after".to_owned(), "0".to_owned())]));
1632            post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
1633        }
1634        let ghost_url = harness.client_url.replacen("session-1", "ghost", 1);
1635        let response = post_model(&ghost_url, "glm-5.3-flash").await;
1636        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
1637        let body = response.text().await.expect("body");
1638        assert!(
1639            body.contains("3599") || body.contains("3600"),
1640            "the cooldown holds until the probe's reset, not the 60s default, got: {body}",
1641        );
1642    }
1643
1644    #[tokio::test]
1645    async fn a_rejected_overage_status_header_rotates_on_the_first_response() {
1646        let harness = harness(Duration::ZERO).await;
1647        pin_only(&harness, "OpenRouter");
1648        harness.script.lock().push_back((
1649            StatusCode::PAYMENT_REQUIRED.as_u16(),
1650            vec![("anthropic-ratelimit-unified-overage-status".to_owned(), "rejected".to_owned())],
1651        ));
1652        let response = post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
1653        assert_eq!(response.status(), StatusCode::PAYMENT_REQUIRED);
1654        assert_eq!(
1655            harness.gateway.bindings.binding_for("Busytools", "forge", "session-1"),
1656            None,
1657            "the overage arm is proven exhaustion, same as the unified arm",
1658        );
1659    }
1660
1661    #[tokio::test]
1662    async fn a_zero_reset_in_a_rate_limit_report_still_cools_the_account() {
1663        let harness = harness(Duration::ZERO).await;
1664        pin_only(&harness, "OpenRouter");
1665        harness.gateway.report_rate_limit("Busytools", "forge", "session-1", Some(0));
1666        let ghost_url = harness.client_url.replacen("session-1", "ghost", 1);
1667        let response = post_model(&ghost_url, "glm-5.3-flash").await;
1668        assert_eq!(
1669            response.status(),
1670            StatusCode::SERVICE_UNAVAILABLE,
1671            "a zero reset reads as absent, so the default cooldown still applies"
1672        );
1673    }
1674
1675    /// A session bound to an account the file removed keeps forwarding on
1676    /// it: retirement is not unbinding.
1677    #[tokio::test]
1678    async fn a_removed_account_still_serves_its_bound_session() {
1679        let harness = harness(Duration::ZERO).await;
1680        harness.gateway.pool.replace_accounts(&[]);
1681
1682        let response = post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
1683        assert_eq!(response.status(), StatusCode::OK);
1684        let requests = harness.requests.lock();
1685        assert_eq!(requests.len(), 1, "the removed account still forwards");
1686        assert_eq!(
1687            requests[0].first("authorization"),
1688            Some("Bearer real-openrouter-key"),
1689            "the retired account keeps the credential its session was bound with",
1690        );
1691    }
1692
1693    /// A token re-minted for an account the file still holds reaches the
1694    /// session bound to it on its next request, so rotating a credential
1695    /// needs neither a restart nor a new session.
1696    #[tokio::test]
1697    async fn a_reminted_token_reaches_a_bound_session() {
1698        let harness = harness(Duration::ZERO).await;
1699        let rotated = HashMap::from([
1700            ("ANTHROPIC_BASE_URL".to_owned(), harness.upstream_url.clone()),
1701            ("ANTHROPIC_AUTH_TOKEN".to_owned(), "rotated-openrouter-key".to_owned()),
1702            ("ANTHROPIC_API_KEY".to_owned(), String::new()),
1703        ]);
1704        harness.gateway.pool.replace_accounts(&[forge_primitives::account::LoadedAccount {
1705            display_name: "OpenRouter".to_owned(),
1706            provider: forge_primitives::account::Provider::Openrouter,
1707            base_url: None,
1708            models: vec!["glm-5.3-flash".to_owned()],
1709            model_aliases: std::collections::HashMap::new(),
1710            model_slugs: std::collections::HashMap::new(),
1711            env: rotated,
1712        }]);
1713
1714        let response = post_as_cli(&format!("{}/v1/messages?beta=true", harness.client_url)).await;
1715        assert_eq!(response.status(), StatusCode::OK);
1716        let requests = harness.requests.lock();
1717        assert_eq!(requests.len(), 1, "exactly one upstream request");
1718        assert_eq!(
1719            requests[0].first("authorization"),
1720            Some("Bearer rotated-openrouter-key"),
1721            "the re-minted token rides the next request on the bound session",
1722        );
1723    }
1724
1725    #[tokio::test]
1726    async fn a_model_change_keeps_the_bound_account() {
1727        let harness = harness(Duration::ZERO).await;
1728        pin_only(&harness, "Anthropic");
1729        harness
1730            .gateway
1731            .pool
1732            .set_loading(&AccountKey("Anthropic".to_owned()), crate::LoadingState::Ready);
1733        // session-1 is bound to OpenRouter; the body now asks for a
1734        // claude model. The binding stays: a running session keeps the
1735        // account it spawned under.
1736        let response = post_model(&harness.client_url, "claude-sonnet-5").await;
1737        assert_eq!(response.status(), StatusCode::OK);
1738        assert_eq!(
1739            harness.gateway.bindings.binding_for("Busytools", "forge", "session-1"),
1740            Some(AccountKey("OpenRouter".to_owned())),
1741            "the model change leaves the binding where it was",
1742        );
1743        assert_eq!(harness.requests.lock().len(), 1, "the request was forwarded once");
1744    }
1745
1746    #[tokio::test]
1747    async fn a_model_change_with_no_serving_account_still_uses_the_binding() {
1748        let harness = harness(Duration::ZERO).await;
1749        pin_only(&harness, "OpenRouter");
1750        harness
1751            .gateway
1752            .pool
1753            .set_loading(&AccountKey("OpenRouter".to_owned()), crate::LoadingState::Ready);
1754        // Bound to OpenRouter and asking for claude: the bound account is
1755        // still the session's account, so the request is forwarded rather
1756        // than failing on a selection walk nobody asked for.
1757        let response = post_model(&harness.client_url, "claude-sonnet-5").await;
1758        assert_eq!(response.status(), StatusCode::OK);
1759        assert_eq!(
1760            harness.gateway.bindings.binding_for("Busytools", "forge", "session-1"),
1761            Some(AccountKey("OpenRouter".to_owned())),
1762            "the binding is kept",
1763        );
1764        assert_eq!(harness.requests.lock().len(), 1, "the request is forwarded once");
1765    }
1766
1767    #[tokio::test]
1768    async fn a_non_claude_model_on_an_anthropic_binding_sticks() {
1769        let harness = harness(Duration::ZERO).await;
1770        pin_only(&harness, "OpenRouter");
1771        harness
1772            .gateway
1773            .pool
1774            .set_loading(&AccountKey("OpenRouter".to_owned()), crate::LoadingState::Ready);
1775        harness.gateway.bindings.bind(
1776            "Busytools",
1777            "forge",
1778            "session-2",
1779            AccountKey("Anthropic".to_owned()),
1780        );
1781        let session_2_url = harness.client_url.replacen("session-1", "session-2", 1);
1782        let response = post_model(&session_2_url, "glm-5.3-flash").await;
1783        assert_eq!(response.status(), StatusCode::OK);
1784        assert_eq!(
1785            harness.gateway.bindings.binding_for("Busytools", "forge", "session-2"),
1786            Some(AccountKey("Anthropic".to_owned())),
1787            "a bound session is not moved by the model in a request body",
1788        );
1789    }
1790
1791    /// `select_for` answers the walk for an org and model WITHOUT
1792    /// binding: the spawn calls it and registers its own binding, and
1793    /// the launchpad reads it to show what a spawn would do.
1794    #[test]
1795    fn select_for_walks_without_binding() {
1796        let account = |name: &str| forge_primitives::account::LoadedAccount {
1797            display_name: name.to_owned(),
1798            provider: forge_primitives::account::Provider::Anthropic,
1799            base_url: None,
1800            models: vec!["claude-sonnet-5".to_owned()],
1801            model_aliases: std::collections::HashMap::new(),
1802            model_slugs: std::collections::HashMap::new(),
1803            env: std::collections::HashMap::new(),
1804        };
1805        let pool = Arc::new(crate::AccountPool::new(&[account("Cooling"), account("Ready")]));
1806        for name in ["Cooling", "Ready"] {
1807            pool.set_loading(&AccountKey(name.to_owned()), crate::LoadingState::Ready);
1808        }
1809        let gateway = Gateway::new(Arc::clone(&pool), reqwest::Client::new());
1810        gateway.set_org_pins([(
1811            "Default".to_owned(),
1812            crate::selection::OrgPin {
1813                accounts: vec!["Cooling".to_owned(), "Ready".to_owned()],
1814                fallback_accounts: Vec::new(),
1815            },
1816        )]);
1817        // The first account in walk order is inside a rotation cooldown.
1818        gateway.rotation.lock().cool_down(
1819            &AccountKey("Cooling".to_owned()),
1820            SystemTime::now() + Duration::from_secs(60),
1821        );
1822
1823        let selected = gateway
1824            .select_for("Default", "claude-sonnet-5")
1825            .expect("the ready account serves the model");
1826        assert_eq!(selected, AccountKey("Ready".to_owned()), "the walk skips a cooling account");
1827        assert!(
1828            pool.provider(&selected).is_some(),
1829            "the walk only ever returns an account the pool holds. A spawn's registration comes \
1830             from `provider` on this key, so this is what keeps a pooled session's registration \
1831             `Some` - and with it the bridge's id slot non-empty",
1832        );
1833        assert_eq!(
1834            gateway.bindings.binding_for("Default", "project", "session"),
1835            None,
1836            "selection writes no binding; the caller owns that",
1837        );
1838    }
1839
1840    /// **The read surface's walk answers while the account is cooling.** A
1841    /// chip must name the account a spawn would land on once the window
1842    /// lifts, and the selecting walk keeps refusing exactly as it did.
1843    fn cooling_account(name: &str) -> forge_primitives::account::LoadedAccount {
1844        forge_primitives::account::LoadedAccount {
1845            display_name: name.to_owned(),
1846            provider: forge_primitives::account::Provider::Anthropic,
1847            base_url: None,
1848            models: vec!["claude-sonnet-5".to_owned()],
1849            model_aliases: std::collections::HashMap::new(),
1850            model_slugs: std::collections::HashMap::new(),
1851            env: std::collections::HashMap::new(),
1852        }
1853    }
1854
1855    /// **The read names the account a spawn lands on first, not the
1856    /// first-pinned one.** With two cooling accounts the walk thaws the one
1857    /// whose reset is soonest, so pin order alone names the wrong account.
1858    #[test]
1859    fn cooling_account_serving_names_the_account_that_thaws_first() {
1860        let pool = Arc::new(crate::AccountPool::new(&[
1861            cooling_account("First"),
1862            cooling_account("Second"),
1863        ]));
1864        for name in ["First", "Second"] {
1865            pool.set_loading(&AccountKey(name.to_owned()), crate::LoadingState::Ready);
1866        }
1867        let gateway = Gateway::new(Arc::clone(&pool), reqwest::Client::new());
1868        gateway.set_org_pins([(
1869            "Default".to_owned(),
1870            crate::selection::OrgPin {
1871                accounts: vec!["First".to_owned(), "Second".to_owned()],
1872                fallback_accounts: Vec::new(),
1873            },
1874        )]);
1875        let now = SystemTime::now();
1876        let first_until = now + Duration::from_secs(2 * 3600);
1877        let second_until = now + Duration::from_secs(10 * 60);
1878        gateway.rotation.lock().cool_down(&AccountKey("First".to_owned()), first_until);
1879        gateway.rotation.lock().cool_down(&AccountKey("Second".to_owned()), second_until);
1880
1881        assert!(
1882            matches!(
1883                gateway.select_for("Default", "claude-sonnet-5"),
1884                Err(SelectFailure::BudgetExhausted { .. }),
1885            ),
1886            "selection keeps refusing a cooling walk",
1887        );
1888        assert_eq!(
1889            gateway.cooling_account_serving("Default", "claude-sonnet-5"),
1890            Some(AccountKey("Second".to_owned())),
1891            "the read names the account whose cooldown ends first",
1892        );
1893        assert_eq!(
1894            gateway.bindings.binding_for("Default", "project", "session"),
1895            None,
1896            "the read writes no binding",
1897        );
1898
1899        // The claim itself: thaw the second account and the walk takes it.
1900        gateway
1901            .rotation
1902            .lock()
1903            .cool_down(&AccountKey("Second".to_owned()), now - Duration::from_secs(1));
1904        assert_eq!(
1905            gateway
1906                .select_for("Default", "claude-sonnet-5")
1907                .expect("the thawed account is selectable"),
1908            AccountKey("Second".to_owned()),
1909            "the first spawn after the read lands on the account it named",
1910        );
1911    }
1912
1913    /// One cooling account declaring the model is the read's answer, and a
1914    /// cooling account the model does not name is not.
1915    #[test]
1916    fn cooling_account_serving_respects_the_declared_model() {
1917        let mut elsewhere = cooling_account("Elsewhere");
1918        elsewhere.models = vec!["claude-opus-5".to_owned()];
1919        let pool = Arc::new(crate::AccountPool::new(&[cooling_account("Serving"), elsewhere]));
1920        for name in ["Serving", "Elsewhere"] {
1921            pool.set_loading(&AccountKey(name.to_owned()), crate::LoadingState::Ready);
1922        }
1923        let gateway = Gateway::new(Arc::clone(&pool), reqwest::Client::new());
1924        gateway.set_org_pins([(
1925            "Default".to_owned(),
1926            crate::selection::OrgPin {
1927                accounts: vec!["Elsewhere".to_owned(), "Serving".to_owned()],
1928                fallback_accounts: Vec::new(),
1929            },
1930        )]);
1931        let now = SystemTime::now();
1932        gateway
1933            .rotation
1934            .lock()
1935            .cool_down(&AccountKey("Serving".to_owned()), now + Duration::from_secs(3600));
1936
1937        assert_eq!(
1938            gateway.cooling_account_serving("Default", "claude-sonnet-5"),
1939            Some(AccountKey("Serving".to_owned())),
1940            "only the account declaring the model answers the read",
1941        );
1942    }
1943
1944    /// The published walk order is readable, so a view can render the
1945    /// orgs the gateway actually holds. Sorted by org name for a stable
1946    /// render: the store underneath is a HashMap.
1947    #[test]
1948    fn published_org_pins_come_back_sorted_with_their_lists_intact() {
1949        let gateway =
1950            Gateway::new(Arc::new(crate::AccountPool::empty_for_test()), reqwest::Client::new());
1951        let pin = |accounts: &[&str], fallbacks: &[&str]| crate::selection::OrgPin {
1952            accounts: accounts.iter().map(|name| (*name).to_owned()).collect(),
1953            fallback_accounts: fallbacks.iter().map(|name| (*name).to_owned()).collect(),
1954        };
1955        gateway.set_org_pins([
1956            ("Zed".to_owned(), pin(&["B"], &["C"])),
1957            ("Alpha".to_owned(), pin(&["A"], &[])),
1958            ("Mike".to_owned(), pin(&["M"], &[])),
1959            ("Bravo".to_owned(), pin(&["N"], &["O", "P"])),
1960        ]);
1961
1962        let pins = gateway.org_pins();
1963        let names: Vec<&str> = pins.iter().map(|(org, _)| org.as_str()).collect();
1964        assert_eq!(
1965            names,
1966            vec!["Alpha", "Bravo", "Mike", "Zed"],
1967            "every published org comes back in name order",
1968        );
1969        assert_eq!(pins[1].1.accounts, vec!["N".to_owned()], "a primary list survives");
1970        assert_eq!(
1971            pins[1].1.fallback_accounts,
1972            vec!["O".to_owned(), "P".to_owned()],
1973            "a fallback list survives in order",
1974        );
1975    }
1976
1977    /// **A cooldown write is published, with the end a reader sleeps to.** A
1978    /// project row's bindable answer changes on the cool and again on the
1979    /// clock-driven thaw; the gateway owns both moments and the reader
1980    /// outside this crate can only learn the first from a write.
1981    #[tokio::test]
1982    async fn a_cooldown_write_is_published_with_its_end() {
1983        let pool = Arc::new(crate::AccountPool::empty_for_test());
1984        let gateway = Gateway::new(pool, reqwest::Client::new());
1985        let watch = gateway.rotation_watch();
1986        let key = AccountKey("Cooling".to_owned());
1987        let now = SystemTime::now();
1988        assert_eq!(gateway.soonest_cooldown_end(now), None, "nothing is cooling yet");
1989        assert!(!watch.has_changed().expect("this test holds the watch"), "nothing published yet");
1990
1991        let reset_at =
1992            now.duration_since(std::time::UNIX_EPOCH).expect("after the epoch").as_secs() + 60;
1993        gateway.report_probe_limit(&key, Some(reset_at));
1994
1995        assert!(
1996            watch.has_changed().expect("this test holds the watch"),
1997            "the cooldown write is published, or the roster never learns of it",
1998        );
1999        assert_eq!(
2000            gateway.soonest_cooldown_end(SystemTime::now()),
2001            Some(crate::rotation::reset_instant(reset_at)),
2002            "the published end is the one the reader sleeps to",
2003        );
2004    }
2005}