myco_model/
lib.rs

1//! Backend-independent messages and one-attempt streaming model drivers.
2//!
3//! Construct a driver with [`new`] and [`GenerativeModelConfig`], then pass a
4//! complete conversation to [`GenerativeModel::generate`]. The library accepts
5//! resolved settings and credentials; it does not read application config files.
6//!
7//! # Streaming and policy
8//!
9//! Every call performs one attempt. Consume [`GenerationEvent`] directly for
10//! incremental output and failure metadata, feeding parts to [`MessageAccumulator`].
11//! Use [`GenerateOutput::from_generation`] when only the completed output is needed.
12//! Dropping a stream cancels its request. Retry, tool execution, and persistence
13//! belong to the caller; `myco-agent` supplies the model/tool loop and retry policy.
14//!
15//! # History contract
16//!
17//! [`Message::ToolResults`] must immediately follow its assistant's tool calls,
18//! with one result per call in the same order. Wire IDs are generated by the
19//! drivers. Preserve these pairs when saving or slicing history.
20//!
21//! See the [inference guide](https://tsnl.github.io/myco/developers/inference.html)
22//! and the `inference` example for a complete streaming client. The example
23//! requires an endpoint; ordinary library tests use local fixtures.
24
25use std::{pin::pin, sync::Arc};
26
27use futures::{Stream, StreamExt};
28
29pub type AsyncStream<T> = std::pin::Pin<Box<dyn Stream<Item = T> + Send>>;
30
31#[cfg(test)]
32mod test_support;
33
34mod anthropic;
35pub use anthropic::AnthropicBackendConfig;
36
37mod accumulator;
38pub use accumulator::MessageAccumulator;
39
40mod driver_core;
41
42mod openai_common;
43pub use openai_common::OpenAIBackendConfig;
44
45mod openai_completions;
46mod openai_responses;
47
48mod sse_parser;
49use sse_parser::SseParser;
50
51pub trait GenerativeModel: Send + Sync {
52    /// One attempt. Failure ends the stream; dropping it cancels the request.
53    ///
54    /// A successful stream starts with [`MessagePart::MessageStart`] and includes
55    /// a [`MessagePart::TurnEndReason`]. A custom model should emit parts that
56    /// [`MessageAccumulator`] can validate. Calls do not execute tools or retry.
57    fn generate(&self, input: &[Message]) -> AsyncStream<GenerationEvent>;
58}
59
60/// Wire protocol a model is served over.
61///
62/// Serde strings are the config.toml `protocol` values.
63#[derive(
64    Debug,
65    Clone,
66    Copy,
67    PartialEq,
68    Eq,
69    Hash,
70    serde::Serialize,
71    serde::Deserialize,
72    schemars::JsonSchema,
73)]
74pub enum Protocol {
75    #[serde(rename = "anthropic-messages")]
76    AnthropicMessages,
77    #[serde(rename = "openai-responses")]
78    OpenAIResponses,
79    #[serde(rename = "openai-completions")]
80    OpenAICompletions,
81}
82
83impl std::fmt::Display for Protocol {
84    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
85        match self {
86            Protocol::AnthropicMessages => f.write_str("anthropic-messages"),
87            Protocol::OpenAIResponses => f.write_str("openai-responses"),
88            Protocol::OpenAICompletions => f.write_str("openai-completions"),
89        }
90    }
91}
92
93/// How thinking/reasoning is requested for a model.
94///
95/// Serde strings are the config.toml `thinking` values. Compatibility is
96/// per-protocol (validated at catalog resolution): Anthropic Messages takes
97/// `adaptive` | `budget` | `none`; OpenAI Responses takes `effort` | `none`.
98#[derive(
99    Debug,
100    Clone,
101    Copy,
102    PartialEq,
103    Eq,
104    Hash,
105    serde::Serialize,
106    serde::Deserialize,
107    schemars::JsonSchema,
108)]
109#[serde(rename_all = "lowercase")]
110pub enum ThinkingMode {
111    /// Anthropic `thinking.type: "adaptive"` + `output_config.effort`
112    /// (frontier models; older models reject it).
113    Adaptive,
114    /// Anthropic `thinking.type: "enabled"` + a `budget_tokens` mapped from
115    /// [`Effort`] (e.g. Haiku 4.5).
116    Budget,
117    /// OpenAI-style `reasoning.effort`.
118    Effort,
119    /// Do not request thinking.
120    None,
121}
122
123impl std::fmt::Display for ThinkingMode {
124    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
125        f.write_str(match self {
126            ThinkingMode::Adaptive => "adaptive",
127            ThinkingMode::Budget => "budget",
128            ThinkingMode::Effort => "effort",
129            ThinkingMode::None => "none",
130        })
131    }
132}
133
134impl ThinkingMode {
135    /// Default mode when a catalog entry does not set `thinking`.
136    pub fn default_for(protocol: Protocol) -> Self {
137        match protocol {
138            Protocol::AnthropicMessages => ThinkingMode::Adaptive,
139            Protocol::OpenAIResponses | Protocol::OpenAICompletions => ThinkingMode::Effort,
140        }
141    }
142
143    /// Whether this mode is servable over `protocol`.
144    pub fn compatible_with(self, protocol: Protocol) -> bool {
145        match protocol {
146            Protocol::AnthropicMessages => {
147                matches!(
148                    self,
149                    ThinkingMode::Adaptive | ThinkingMode::Budget | ThinkingMode::None
150                )
151            }
152            Protocol::OpenAIResponses | Protocol::OpenAICompletions => {
153                matches!(self, ThinkingMode::Effort | ThinkingMode::None)
154            }
155        }
156    }
157}
158
159/// Agent policy for retrying transient failures before any response parts arrive.
160/// Resolved from `[gateways.NAME.retry]` or `[models.KEY.retry]`.
161#[derive(Debug, Clone, Copy, PartialEq, serde::Serialize, serde::Deserialize)]
162pub struct RetryPolicy {
163    /// Total attempts including the first. `1` disables retry.
164    pub max_attempts: u32,
165    pub initial_backoff: std::time::Duration,
166    /// Ceiling on one wait, applied to a provider's `Retry-After` too, so a
167    /// hostile or mistaken header cannot park an unattended run for hours.
168    pub max_backoff: std::time::Duration,
169    pub backoff_multiplier: f64,
170}
171
172impl Default for RetryPolicy {
173    fn default() -> Self {
174        Self {
175            max_attempts: 3,
176            initial_backoff: std::time::Duration::from_millis(500),
177            max_backoff: std::time::Duration::from_secs(30),
178            backoff_multiplier: 2.0,
179        }
180    }
181}
182
183impl RetryPolicy {
184    /// Wait before `attempt` (1-based). Attempt 1 is the original send and
185    /// never waits; `retry_after` is the provider's ask, honoured when it
186    /// exceeds the computed backoff and still capped by [`Self::max_backoff`].
187    ///
188    /// No jitter: myco is one client per user, so there is no fleet to
189    /// de-synchronise, and a deterministic schedule is one less thing to
190    /// reason about when reading an overnight log.
191    pub fn backoff(
192        &self,
193        attempt: u32,
194        retry_after: Option<std::time::Duration>,
195    ) -> std::time::Duration {
196        if attempt <= 1 {
197            return std::time::Duration::ZERO;
198        }
199        let steps = attempt.saturating_sub(2);
200        let factor = self.backoff_multiplier.max(1.0).powi(steps.min(32) as i32);
201        let millis = (self.initial_backoff.as_millis() as f64 * factor).min(u64::MAX as f64);
202        let computed = std::time::Duration::from_millis(millis as u64);
203        retry_after
204            .unwrap_or(std::time::Duration::ZERO)
205            .max(computed)
206            .min(self.max_backoff)
207    }
208}
209
210/// A resolved model: everything the protocol drivers need, minus credentials
211/// (those live in [`BackendConfig`]). Built by the application configuration from the
212/// `[models]` / `[gateways]` catalog in config.toml — myco ships no built-in
213/// models.
214#[derive(Debug, Clone, PartialEq, Eq)]
215pub struct ModelSpec {
216    /// Catalog key: what the user types after `--model` and what sessions
217    /// record. Distinct from `api_id` so one wire model can appear under
218    /// several keys (e.g. routed via different gateways).
219    pub key: String,
220    /// Wire id sent to the provider (the request `model` field).
221    pub api_id: String,
222    pub protocol: Protocol,
223    pub thinking: ThinkingMode,
224    /// Context window for UX (`USER n/m`) and auto-compact heuristics.
225    pub context_window_tokens: u64,
226    /// Largest image this model accepts, measured on the base64 payload.
227    /// Enforced locally by `view_image` and by REPL `@path` attachments so an
228    /// oversized image fails with a clear message instead of a provider 400.
229    /// Always concrete: config resolution applies the `max_image_base64_bytes`
230    /// entry or its default, and callers downstream take this value.
231    pub max_image_base64_bytes: u64,
232    /// How many consecutive `max_tokens` truncations one turn resumes through
233    /// before handing control back (`0` never resumes). Resolved from the
234    /// model's `max_truncated_resumes` or its default; the agent takes this
235    /// value via the agent's continuation policy.
236    pub max_truncated_resumes: u32,
237    /// Prompt size at which the REPL compacts without being asked. `None` =
238    /// no auto-compaction (the default; `/compact` still works).
239    ///
240    /// Resolved from the model's `auto_compact_at` *fraction* against
241    /// `context_window_tokens` so the comparison downstream is a plain token
242    /// count, and the fraction is validated once, at startup.
243    pub auto_compact_at_tokens: Option<u64>,
244}
245
246impl std::fmt::Display for ModelSpec {
247    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
248        f.write_str(&self.key)
249    }
250}
251
252/// One usable catalog entry: spec plus the backend (gateway + credentials)
253/// that serves it.
254#[derive(Debug, Clone)]
255pub struct CatalogModel {
256    pub spec: ModelSpec,
257    pub backend: BackendConfig,
258    /// Set when the auth source did not resolve (env var unset, auth file
259    /// unreadable). Reported by [`ModelCatalog::get`] when the model is
260    /// actually used — configuring a model without its credential is fine
261    /// until then.
262    pub auth_error: Option<String>,
263}
264
265/// Key → model catalog resolved from config.toml. Empty when the user has not
266/// configured any models.
267#[derive(Debug, Clone, Default)]
268pub struct ModelCatalog {
269    entries: std::collections::BTreeMap<String, CatalogModel>,
270}
271
272impl ModelCatalog {
273    pub fn new(entries: std::collections::BTreeMap<String, CatalogModel>) -> Self {
274        Self { entries }
275    }
276
277    /// Look up a usable model. Errors are user-actionable: unknown keys list
278    /// the configured catalog; entries with unresolved credentials report the
279    /// failing auth source (env var / file).
280    pub fn get(&self, key: &str) -> Result<&CatalogModel, String> {
281        let Some(entry) = self.entries.get(key) else {
282            if self.entries.is_empty() {
283                return Err(format!(
284                    "unknown model {key:?}: no models configured — define [models] \
285                     (and [gateways]) in config.toml"
286                ));
287            }
288            return Err(format!(
289                "unknown model {key:?}; configured models: [{}]",
290                self.keys().join(", ")
291            ));
292        };
293        if let Some(err) = &entry.auth_error {
294            return Err(err.clone());
295        }
296        Ok(entry)
297    }
298
299    /// Key exists (regardless of whether its credential resolved).
300    pub fn contains(&self, key: &str) -> bool {
301        self.entries.contains_key(key)
302    }
303
304    /// Spec for `key`, ignoring credential state. For settings that must be
305    /// read while merely *configuring* a run (the image cap host workers are
306    /// spawned with) rather than using the model — [`Self::get`] is still the
307    /// gate for that.
308    pub fn spec(&self, key: &str) -> Option<&ModelSpec> {
309        self.entries.get(key).map(|entry| &entry.spec)
310    }
311
312    pub fn keys(&self) -> Vec<&str> {
313        self.entries.keys().map(String::as_str).collect()
314    }
315
316    pub fn is_empty(&self) -> bool {
317        self.entries.is_empty()
318    }
319}
320
321/// Reasoning / extended-thinking effort level sent to providers.
322///
323/// Anthropic adaptive models map this to `output_config.effort`; Haiku-style models
324/// map it onto a `thinking.budget_tokens` value. OpenAI/xAI gateways receive it as
325/// `reasoning.effort`.
326#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
327#[serde(rename_all = "lowercase")]
328pub enum Effort {
329    Low,
330    Medium,
331    High,
332    Max,
333}
334
335impl Effort {
336    /// Wire string used by Anthropic `output_config.effort` and OpenAI `reasoning.effort`.
337    pub fn as_str(self) -> &'static str {
338        match self {
339            Effort::Low => "low",
340            Effort::Medium => "medium",
341            Effort::High => "high",
342            Effort::Max => "max",
343        }
344    }
345
346    /// Approximate Anthropic extended-thinking token budget for non-adaptive models.
347    pub fn budget_tokens(self) -> u32 {
348        match self {
349            Effort::Low => 1_024,
350            Effort::Medium => 4_096,
351            Effort::High => 16_000,
352            Effort::Max => 64_000,
353        }
354    }
355
356    /// Sensible default for interactive agent sessions.
357    pub const DEFAULT: Effort = Effort::High;
358}
359
360impl std::fmt::Display for Effort {
361    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
362        f.write_str(self.as_str())
363    }
364}
365
366impl std::str::FromStr for Effort {
367    type Err = String;
368
369    fn from_str(s: &str) -> Result<Self, Self::Err> {
370        match s.trim().to_ascii_lowercase().as_str() {
371            "low" | "l" => Ok(Effort::Low),
372            "medium" | "med" | "m" => Ok(Effort::Medium),
373            "high" | "h" => Ok(Effort::High),
374            "max" | "x" => Ok(Effort::Max),
375            other => Err(format!(
376                "unknown effort {other:?}; expected low|medium|high|max"
377            )),
378        }
379    }
380}
381
382/// Provider backend settings: gateway base URL, credential, per-request knobs.
383#[derive(Debug, Clone)]
384pub enum BackendConfig {
385    Anthropic(AnthropicBackendConfig),
386    /// Responses API (`{base_url}/responses`).
387    OpenAIResponses(OpenAIBackendConfig),
388    /// Chat Completions API (`{base_url}/chat/completions`).
389    OpenAICompletions(OpenAIBackendConfig),
390}
391
392impl BackendConfig {
393    pub fn retry_policy(&self) -> RetryPolicy {
394        match self {
395            Self::Anthropic(config) => config.retry,
396            Self::OpenAIResponses(config) | Self::OpenAICompletions(config) => config.retry,
397        }
398    }
399
400    pub fn protocol(&self) -> Protocol {
401        match self {
402            BackendConfig::Anthropic(_) => Protocol::AnthropicMessages,
403            BackendConfig::OpenAIResponses(_) => Protocol::OpenAIResponses,
404            BackendConfig::OpenAICompletions(_) => Protocol::OpenAICompletions,
405        }
406    }
407}
408
409/// Fully resolved driver inputs. Tool schemas advertise capabilities; the caller
410/// remains responsible for executing calls and validating their arguments.
411pub struct GenerativeModelConfig {
412    pub model: ModelSpec,
413    pub tools: Vec<ToolSpec>,
414    pub system_prompt: String,
415    pub backend_config: BackendConfig,
416}
417
418/// Construct a driver, rejecting a protocol/backend mismatch before any request.
419/// No network request is made until the returned model is asked to generate.
420pub fn new(config: GenerativeModelConfig) -> Result<Arc<dyn GenerativeModel>, ModelCreationError> {
421    if config.backend_config.protocol() != config.model.protocol {
422        return Err(ModelCreationError::BadConfig(format!(
423            "model `{}` speaks {} but the backend config is for {}",
424            config.model,
425            config.model.protocol,
426            config.backend_config.protocol()
427        )));
428    }
429    match config.backend_config.clone() {
430        BackendConfig::Anthropic(backend) => {
431            let model = anthropic::AnthropicGenerativeModel::new(config, backend)?;
432            Ok(model as Arc<dyn GenerativeModel>)
433        }
434        BackendConfig::OpenAIResponses(backend) => {
435            let model = openai_responses::OpenAIResponsesGenerativeModel::new(config, backend)?;
436            Ok(model as Arc<dyn GenerativeModel>)
437        }
438        BackendConfig::OpenAICompletions(backend) => {
439            let model = openai_completions::OpenAICompletionsGenerativeModel::new(config, backend)?;
440            Ok(model as Arc<dyn GenerativeModel>)
441        }
442    }
443}
444
445#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
446pub enum Message {
447    UserMessage {
448        content: Vec<Content>,
449    },
450    ToolResults {
451        tool_use_results: Vec<ToolResult>,
452    },
453    AssistantMessage {
454        content: Vec<Content>,
455        tool_uses: Vec<ToolUse>,
456        turn_end_reason: Option<TurnEndReason>,
457    },
458}
459
460impl Message {
461    pub fn content(&self) -> impl Iterator<Item = &Content> {
462        let (content, results): (&[Content], &[ToolResult]) = match self {
463            Self::UserMessage { content } | Self::AssistantMessage { content, .. } => {
464                (content, &[])
465            }
466            Self::ToolResults { tool_use_results } => (&[], tool_use_results),
467        };
468        content
469            .iter()
470            .chain(results.iter().flat_map(|result| &result.content))
471    }
472    /// Runtime-only input does not represent a human submission.
473    pub fn is_user_turn(&self) -> bool {
474        matches!(self, Self::UserMessage { content }
475            if content.is_empty() || content.iter().any(|part| !matches!(part, Content::System { .. })))
476    }
477}
478#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
479pub enum TurnEndReason {
480    EndTurn,
481    MaxTokens,
482    ToolUse,
483    /// Provider-specific / unknown stop reason (owned so sessions can serialize cleanly).
484    Other(String),
485}
486
487#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
488pub struct ToolSpec {
489    pub name: String,
490    pub description: String,
491    pub input_schema: serde_json::Value,
492}
493
494/// A tool call in an assistant turn. Carries no id: a call is identified by
495/// its position (message index + ordinal), and the `j`-th entry of the next
496/// message's `tool_use_results` answers it. Providers that need ids on the
497/// wire get minted ones from the driver.
498#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
499pub struct ToolUse {
500    pub name: String,
501    pub input: serde_json::Value,
502}
503
504#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
505pub struct ToolResult {
506    pub content: Vec<Content>,
507    pub is_error: bool,
508    /// Brief tool-authored outcome for transcripts, independent of model narration.
509    #[serde(default, skip_serializing_if = "Option::is_none")]
510    pub status: Option<String>,
511}
512
513impl ToolResult {
514    pub fn ok(content: Vec<Content>) -> Self {
515        Self {
516            content,
517            is_error: false,
518            status: None,
519        }
520    }
521
522    pub fn text(text: impl Into<String>) -> Self {
523        Self {
524            content: vec![Content::Text { text: text.into() }],
525            is_error: false,
526            status: None,
527        }
528    }
529
530    pub fn err(text: impl Into<String>) -> Self {
531        Self {
532            content: vec![Content::Text { text: text.into() }],
533            is_error: true,
534            status: None,
535        }
536    }
537
538    pub fn with_status(mut self, status: impl Into<String>) -> Self {
539        self.status = Some(status.into());
540        self
541    }
542}
543
544#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
545pub enum Content {
546    Text {
547        text: String,
548    },
549    /// Runtime-authored context. Providers receive `text` at this position in
550    /// the message; transcript renderers omit the entire part. `data` retains
551    /// structured lifecycle facts independently of their model-facing wording.
552    System {
553        kind: String,
554        text: String,
555        data: serde_json::Value,
556    },
557    Image {
558        /// Data URL, remote URL, or legacy base64. Application-local
559        /// `myco-image:` references must be resolved before provider dispatch.
560        source: String,
561    },
562    /// Model thinking text (session history + live UI).
563    ///
564    /// Stored in agent/session history for resume, but **stripped when backends
565    /// compose the next API request** (not echoed as CoT). Prefer provider
566    /// summary channels over raw reasoning text.
567    Thinking {
568        text: String,
569        /// Opaque provider signature (Anthropic). Not re-sent on subsequent turns.
570        signature: Option<String>,
571        /// True for redacted/encrypted thinking placeholders with no plaintext.
572        redacted: bool,
573    },
574}
575
576/// Clone only answer blocks (`Text` / `Image`), dropping thinking.
577pub fn answer_content(content: &[Content]) -> Vec<Content> {
578    content
579        .iter()
580        .filter(|c| matches!(c, Content::Text { .. } | Content::Image { .. }))
581        .cloned()
582        .collect()
583}
584
585/// Wire ids for every tool call in `input`: `out[i][j]` is the id a driver
586/// sends for the `j`-th tool_use of the assistant message at index `i`, and
587/// equally for the `j`-th result of the `ToolResults` message answering it.
588///
589/// History carries no tool ids — a call and its result pair positionally:
590/// `tool_use_results[j]` answers `tool_uses[j]` of the immediately preceding
591/// assistant message, the order the agent loop writes and compaction
592/// preserves (tails start at a user message and never split a pair).
593/// Providers, however, require an id on the wire to make that same pairing
594/// inside one request, and each has its own dialect ([`mint_tool_id`]) — so
595/// drivers mint one per position here. Provider-minted ids from responses are
596/// discarded at ingestion; storing and echoing them is what used to wedge a
597/// session the moment it resumed on a different provider.
598///
599/// Ids derive from (message index, ordinal), so history growth never changes
600/// the ids of earlier messages and the request prefix stays byte-identical
601/// across turns for provider prompt caching. A history whose result count
602/// disagrees with the preceding assistant's tool calls fails here, before any
603/// request is sent: guessing the pairing would corrupt the conversation
604/// silently.
605pub(crate) fn wire_tool_ids(input: &[Message]) -> Result<Vec<Vec<String>>, GenerateError> {
606    if input.iter().flat_map(Message::content).any(|part| {
607        matches!(part,
608        Content::Image { source } if source.starts_with("myco-image:"))
609    }) {
610        return Err(GenerateError::ExecutionError(
611            "resolve local image references before calling a provider".into(),
612        ));
613    }
614    let mut out: Vec<Vec<String>> = Vec::with_capacity(input.len());
615    for (i, message) in input.iter().enumerate() {
616        let ids = match message {
617            Message::UserMessage { .. } => Vec::new(),
618            Message::AssistantMessage { tool_uses, .. } => {
619                (0..tool_uses.len()).map(|j| mint_tool_id(i, j)).collect()
620            }
621            Message::ToolResults { tool_use_results } => {
622                let preceding_uses = match i.checked_sub(1).map(|p| &input[p]) {
623                    Some(Message::AssistantMessage { tool_uses, .. }) => tool_uses.len(),
624                    _ => 0,
625                };
626                if preceding_uses != tool_use_results.len() {
627                    return Err(GenerateError::ExecutionError(format!(
628                        "history is malformed: message {i} carries {} tool results but the \
629                         message before it has {preceding_uses} tool calls",
630                        tool_use_results.len()
631                    )));
632                }
633                out[i - 1].clone()
634            }
635        };
636        out.push(ids);
637    }
638    Ok(out)
639}
640
641/// Nine alphanumeric chars: `t`, then message index and tool ordinal as four
642/// base36 digits each — the intersection of every id dialect the drivers
643/// target:
644/// - Anthropic Messages: must match `^[a-zA-Z0-9_-]+$`
645/// - OpenAI: at most 40 chars
646/// - Mistral-style OpenAI-compatible backends: exactly nine alphanumerics
647///   (the binding constraint — it fixes both the length and the charset)
648fn mint_tool_id(message_index: usize, ordinal: usize) -> String {
649    const CAP: usize = 36 * 36 * 36 * 36;
650    // A history long enough to overflow four digits (1.6M messages) exceeds
651    // the request size cap long before it gets here.
652    assert!(message_index < CAP && ordinal < CAP);
653    let mut id = String::with_capacity(9);
654    id.push('t');
655    for n in [message_index, ordinal] {
656        for place in [36 * 36 * 36, 36 * 36, 36, 1] {
657            id.push(char::from_digit(((n / place) % 36) as u32, 36).unwrap());
658        }
659    }
660    id
661}
662
663/// One generation attempt emits parts, or ends with a failure.
664#[derive(Debug, Clone)]
665pub enum GenerationEvent {
666    Part(MessagePart),
667    Failure(GenerationFailure),
668}
669
670impl GenerationEvent {
671    pub fn into_result(self) -> Result<MessagePart, GenerateError> {
672        match self {
673            Self::Part(part) => Ok(part),
674            Self::Failure(failure) => Err(failure.cause),
675        }
676    }
677}
678
679#[derive(Debug, Clone)]
680pub struct GenerationFailure {
681    pub cause: GenerateError,
682    /// A transient cause; the caller must also ensure no response parts were emitted.
683    pub retryable: bool,
684    pub retry_after: Option<std::time::Duration>,
685}
686
687impl GenerationFailure {
688    pub fn terminal(cause: GenerateError) -> Self {
689        Self {
690            cause,
691            retryable: false,
692            retry_after: None,
693        }
694    }
695
696    pub fn transient(cause: GenerateError, retry_after: Option<std::time::Duration>) -> Self {
697        Self {
698            cause,
699            retryable: true,
700            retry_after,
701        }
702    }
703}
704
705#[derive(Debug, Clone)]
706pub enum MessagePart {
707    MessageStart,
708    ContentStart(ContentStart),
709    ContentDelta(ContentDelta),
710    ToolUseStart(ToolUseStart),
711    ToolUseDelta(ToolUseDelta),
712    TurnEndReason(TurnEndReason),
713    /// Provider token usage for this generate call (may appear mid-stream or at end).
714    Usage(TokenUsage),
715}
716
717/// Token counts for one generate call. `cached_input_tokens` is a subset of
718/// `input_tokens`.
719#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
720pub struct TokenUsage {
721    #[serde(default)]
722    pub input_tokens: u64,
723    #[serde(default)]
724    pub output_tokens: u64,
725    #[serde(default)]
726    pub cached_input_tokens: u64,
727}
728
729impl TokenUsage {
730    /// Context occupied by the prompt = total input tokens (cached input is a
731    /// subset, already included).
732    pub fn context_tokens(self) -> u64 {
733        self.input_tokens
734    }
735
736    /// Fold a later usage report into this one, keeping known fields when the
737    /// later report omits them (providers split usage across stream events).
738    pub fn merge(self, next: TokenUsage) -> TokenUsage {
739        fn pick(prev: u64, next: u64) -> u64 {
740            if next != 0 { next } else { prev }
741        }
742        TokenUsage {
743            input_tokens: pick(self.input_tokens, next.input_tokens),
744            output_tokens: pick(self.output_tokens, next.output_tokens),
745            cached_input_tokens: pick(self.cached_input_tokens, next.cached_input_tokens),
746        }
747    }
748}
749
750#[derive(Debug, Clone)]
751pub enum ContentStart {
752    Text {
753        index: usize,
754    },
755    Image {
756        index: usize,
757    },
758    Thinking {
759        index: usize,
760        signature: Option<String>,
761        redacted: bool,
762    },
763}
764
765#[derive(Debug, Clone)]
766pub enum ContentDelta {
767    Text { index: usize, delta: String },
768    Image { index: usize, delta: String },
769    Thinking { index: usize, delta: String },
770}
771
772#[derive(Debug, Clone)]
773pub struct ToolUseStart {
774    pub index: usize,
775    pub name: String,
776}
777
778#[derive(Debug, Clone)]
779pub struct ToolUseDelta {
780    pub index: usize,
781    pub input_json_delta: String,
782}
783
784//
785// GenerateOutput: accumulate a stream of MessageParts into a finished assistant turn
786//
787
788#[derive(Debug, Clone)]
789pub struct GenerateOutput {
790    pub content: Vec<Content>,
791    pub tool_uses: Vec<ToolUse>,
792    pub turn_end_reason: TurnEndReason,
793    /// Last usage observed on the stream, if the provider reported any.
794    pub usage: Option<TokenUsage>,
795}
796
797impl GenerateOutput {
798    /// Consume and validate one attempt without applying retry policy.
799    ///
800    /// Failures return their cause; consume [`GenerationEvent`] directly when
801    /// the transient classification or provider retry delay is needed.
802    pub async fn from_generation(
803        stream: impl Stream<Item = GenerationEvent>,
804    ) -> Result<Self, GenerateError> {
805        Self::from_stream(stream.map(GenerationEvent::into_result)).await
806    }
807
808    pub async fn from_stream(
809        stream: impl Stream<Item = Result<MessagePart, GenerateError>>,
810    ) -> Result<Self, GenerateError> {
811        let mut accumulator = MessageAccumulator::default();
812        let mut stream = pin!(stream);
813        while let Some(part) = stream.next().await {
814            accumulator.push(&part?)?;
815        }
816        accumulator.finish()
817    }
818}
819
820#[derive(thiserror::Error, Debug)]
821pub enum ModelCreationError {
822    #[error("Invalid configuration parameters supplied: {0}")]
823    BadConfig(String),
824
825    #[error("Uncategorized error occurred: {0}")]
826    Uncategorized(String),
827}
828
829#[cfg(test)]
830mod tests {
831    use super::*;
832    use crate::test_support::{assistant_tool, tool_results, user};
833
834    /// Every minted id must satisfy the strictest provider dialect at once:
835    /// exactly nine chars, alphanumeric only (covers Anthropic's
836    /// `^[a-zA-Z0-9_-]+$`, OpenAI's 40-char cap, and Mistral-style
837    /// nine-alphanumeric backends).
838    #[test]
839    fn wire_ids_are_nine_alphanumerics() {
840        for (i, j) in [(0, 0), (1, 3), (255, 7), (46_655, 12)] {
841            let id = mint_tool_id(i, j);
842            assert_eq!(id.len(), 9, "{id:?}");
843            assert!(id.chars().all(|c| c.is_ascii_alphanumeric()), "{id:?}");
844        }
845    }
846
847    /// A tool_result must carry the same wire id as the tool_use it answers.
848    #[test]
849    fn wire_ids_pair_results_with_uses_positionally() {
850        let input = [
851            user("hi"),
852            assistant_tool(None, "bash", serde_json::json!({})),
853            tool_results(&["ok"]),
854        ];
855        let ids = wire_tool_ids(&input).unwrap();
856        assert!(ids[0].is_empty());
857        assert_eq!(ids[1].len(), 1);
858        assert_eq!(ids[1], ids[2]);
859    }
860
861    /// Appending messages must not change earlier ids: the request prefix has
862    /// to stay byte-identical across turns or provider prompt caching breaks.
863    #[test]
864    fn wire_ids_stable_as_history_grows() {
865        let mut input = vec![
866            user("hi"),
867            assistant_tool(None, "bash", serde_json::json!({})),
868            tool_results(&["ok"]),
869        ];
870        let before = wire_tool_ids(&input).unwrap();
871        input.push(user("more"));
872        input.push(assistant_tool(None, "bash", serde_json::json!({})));
873        input.push(tool_results(&["ok"]));
874        let after = wire_tool_ids(&input).unwrap();
875        assert_eq!(before[..], after[..3]);
876        assert_ne!(after[1], after[4], "distinct calls must get distinct ids");
877    }
878
879    /// Positional pairing must refuse to guess: a results message whose count
880    /// disagrees with the preceding assistant's tool calls fails before any
881    /// request is sent, instead of silently mispairing.
882    #[test]
883    fn wire_ids_fail_loud_on_broken_pairing() {
884        let orphaned = [user("hi"), tool_results(&["ok"])];
885        assert!(wire_tool_ids(&orphaned).is_err());
886
887        let miscounted = [
888            assistant_tool(None, "bash", serde_json::json!({})),
889            tool_results(&["ok", "ok"]),
890        ];
891        assert!(wire_tool_ids(&miscounted).is_err());
892    }
893
894    #[tokio::test]
895    async fn accumulate_thinking_then_text() {
896        use futures::stream;
897
898        let parts = vec![
899            Ok(MessagePart::MessageStart),
900            Ok(MessagePart::ContentStart(ContentStart::Thinking {
901                index: 0,
902                signature: Some("sig".into()),
903                redacted: false,
904            })),
905            Ok(MessagePart::ContentDelta(ContentDelta::Thinking {
906                index: 0,
907                delta: "reason".into(),
908            })),
909            Ok(MessagePart::ContentStart(ContentStart::Text { index: 1 })),
910            Ok(MessagePart::ContentDelta(ContentDelta::Text {
911                index: 1,
912                delta: "answer".into(),
913            })),
914            Ok(MessagePart::TurnEndReason(TurnEndReason::EndTurn)),
915        ];
916        let output = GenerateOutput::from_stream(stream::iter(parts))
917            .await
918            .expect("accumulate");
919        assert_eq!(output.content.len(), 2);
920        match &output.content[0] {
921            Content::Thinking {
922                text,
923                signature,
924                redacted,
925            } => {
926                assert_eq!(text, "reason");
927                assert_eq!(signature.as_deref(), Some("sig"));
928                assert!(!*redacted);
929            }
930            other => panic!("expected thinking, got {other:?}"),
931        }
932        match &output.content[1] {
933            Content::Text { text } => assert_eq!(text, "answer"),
934            other => panic!("expected text, got {other:?}"),
935        }
936        assert_eq!(answer_content(&output.content).len(), 1);
937    }
938
939    #[test]
940    fn token_usage_merge_prefers_known_fields() {
941        let start = TokenUsage {
942            input_tokens: 2195,
943            output_tokens: 1,
944            cached_input_tokens: 2000,
945        };
946        let delta = TokenUsage {
947            input_tokens: 0,
948            output_tokens: 89,
949            cached_input_tokens: 0,
950        };
951        let merged = start.merge(delta);
952        assert_eq!(merged.input_tokens, 2195);
953        assert_eq!(merged.output_tokens, 89);
954        assert_eq!(merged.cached_input_tokens, 2000);
955        assert_eq!(merged.context_tokens(), 2195);
956    }
957
958    #[tokio::test]
959    async fn accumulate_merges_split_usage() {
960        use futures::stream;
961
962        let parts = vec![
963            Ok(MessagePart::MessageStart),
964            Ok(MessagePart::Usage(TokenUsage {
965                input_tokens: 2195,
966                output_tokens: 1,
967                cached_input_tokens: 2000,
968            })),
969            Ok(MessagePart::ContentStart(ContentStart::Text { index: 0 })),
970            Ok(MessagePart::ContentDelta(ContentDelta::Text {
971                index: 0,
972                delta: "hi".into(),
973            })),
974            Ok(MessagePart::Usage(TokenUsage {
975                input_tokens: 0,
976                output_tokens: 89,
977                cached_input_tokens: 0,
978            })),
979            Ok(MessagePart::TurnEndReason(TurnEndReason::EndTurn)),
980        ];
981        let output = GenerateOutput::from_stream(stream::iter(parts))
982            .await
983            .expect("accumulate");
984        let usage = output.usage.expect("usage present");
985        assert_eq!(usage.input_tokens, 2195);
986        assert_eq!(usage.output_tokens, 89);
987        assert_eq!(usage.cached_input_tokens, 2000);
988        assert_eq!(usage.context_tokens(), 2195);
989    }
990
991    /// Backoff doubles from `initial_backoff`, stops at `max_backoff`, and a
992    /// provider's `Retry-After` wins when it asks for longer — still under the
993    /// cap, so a mistaken header cannot park an unattended run for hours.
994    #[test]
995    fn retry_backoff_grows_caps_and_honours_retry_after() {
996        use std::time::Duration;
997        let policy = RetryPolicy {
998            max_attempts: 6,
999            initial_backoff: Duration::from_millis(100),
1000            max_backoff: Duration::from_millis(1000),
1001            backoff_multiplier: 2.0,
1002        };
1003
1004        // Attempt 1 is the original send; it never waits.
1005        assert_eq!(policy.backoff(1, None), Duration::ZERO);
1006        assert_eq!(policy.backoff(2, None), Duration::from_millis(100));
1007        assert_eq!(policy.backoff(3, None), Duration::from_millis(200));
1008        assert_eq!(policy.backoff(4, None), Duration::from_millis(400));
1009        // Growth stops at the cap rather than running away.
1010        assert_eq!(policy.backoff(9, None), Duration::from_millis(1000));
1011
1012        // A longer Retry-After wins over the computed wait...
1013        assert_eq!(
1014            policy.backoff(2, Some(Duration::from_millis(500))),
1015            Duration::from_millis(500)
1016        );
1017        // ...but is still capped.
1018        assert_eq!(
1019            policy.backoff(2, Some(Duration::from_secs(3600))),
1020            Duration::from_millis(1000)
1021        );
1022        // A shorter one does not shrink the backoff.
1023        assert_eq!(
1024            policy.backoff(3, Some(Duration::from_millis(1))),
1025            Duration::from_millis(200)
1026        );
1027    }
1028
1029    fn spec(key: &str, protocol: Protocol) -> ModelSpec {
1030        ModelSpec {
1031            key: key.into(),
1032            api_id: key.into(),
1033            protocol,
1034            thinking: ThinkingMode::default_for(protocol),
1035            context_window_tokens: 1_000_000,
1036            max_image_base64_bytes: 5 * 1024 * 1024,
1037            max_truncated_resumes: 3,
1038            auto_compact_at_tokens: None,
1039        }
1040    }
1041
1042    #[test]
1043    fn thinking_defaults_and_protocol_compatibility() {
1044        assert_eq!(
1045            ThinkingMode::default_for(Protocol::AnthropicMessages),
1046            ThinkingMode::Adaptive
1047        );
1048        assert_eq!(
1049            ThinkingMode::default_for(Protocol::OpenAIResponses),
1050            ThinkingMode::Effort
1051        );
1052        assert!(ThinkingMode::Budget.compatible_with(Protocol::AnthropicMessages));
1053        assert!(ThinkingMode::None.compatible_with(Protocol::AnthropicMessages));
1054        assert!(!ThinkingMode::Effort.compatible_with(Protocol::AnthropicMessages));
1055        assert!(ThinkingMode::None.compatible_with(Protocol::OpenAIResponses));
1056        // Both OpenAI dialects take the same effort-shaped thinking.
1057        assert_eq!(
1058            ThinkingMode::default_for(Protocol::OpenAICompletions),
1059            ThinkingMode::Effort
1060        );
1061        assert!(ThinkingMode::None.compatible_with(Protocol::OpenAICompletions));
1062        assert!(!ThinkingMode::Adaptive.compatible_with(Protocol::OpenAICompletions));
1063        assert!(!ThinkingMode::Adaptive.compatible_with(Protocol::OpenAIResponses));
1064        assert!(!ThinkingMode::Budget.compatible_with(Protocol::OpenAIResponses));
1065    }
1066
1067    #[test]
1068    fn protocol_serde_uses_config_strings() {
1069        assert_eq!(
1070            serde_json::to_value(Protocol::AnthropicMessages).unwrap(),
1071            serde_json::json!("anthropic-messages")
1072        );
1073        assert_eq!(
1074            serde_json::from_value::<Protocol>(serde_json::json!("openai-responses")).unwrap(),
1075            Protocol::OpenAIResponses
1076        );
1077        assert_eq!(
1078            serde_json::from_value::<Protocol>(serde_json::json!("openai-completions")).unwrap(),
1079            Protocol::OpenAICompletions
1080        );
1081    }
1082
1083    #[test]
1084    fn empty_catalog_get_says_no_models_configured() {
1085        let catalog = ModelCatalog::default();
1086        assert!(catalog.is_empty());
1087        let err = catalog.get("kimi-k3").unwrap_err();
1088        assert!(err.contains("no models configured"), "{err}");
1089        assert!(err.contains("[models]"), "{err}");
1090    }
1091
1092    #[test]
1093    fn catalog_get_unknown_key_lists_configured_models() {
1094        let entry = CatalogModel {
1095            spec: spec("opus", Protocol::AnthropicMessages),
1096            backend: BackendConfig::Anthropic(AnthropicBackendConfig::default()),
1097            auth_error: None,
1098        };
1099        let catalog = ModelCatalog::new([("opus".to_string(), entry)].into());
1100        let err = catalog.get("opsu").unwrap_err();
1101        assert!(err.contains("unknown model \"opsu\""), "{err}");
1102        assert!(err.contains("[opus]"), "{err}");
1103        assert!(catalog.get("opus").is_ok());
1104    }
1105
1106    #[test]
1107    fn catalog_get_reports_deferred_auth_error() {
1108        let entry = CatalogModel {
1109            spec: spec("kimi", Protocol::OpenAIResponses),
1110            backend: BackendConfig::OpenAIResponses(OpenAIBackendConfig::default()),
1111            auth_error: Some("model `kimi`: auth env:OPENROUTER_API_KEY is unset".into()),
1112        };
1113        let catalog = ModelCatalog::new([("kimi".to_string(), entry)].into());
1114        let err = catalog.get("kimi").unwrap_err();
1115        assert!(err.contains("OPENROUTER_API_KEY"), "{err}");
1116    }
1117
1118    #[test]
1119    fn new_rejects_protocol_mismatch() {
1120        let result = new(GenerativeModelConfig {
1121            model: spec("grok", Protocol::OpenAIResponses),
1122            tools: vec![],
1123            system_prompt: String::new(),
1124            backend_config: BackendConfig::Anthropic(AnthropicBackendConfig {
1125                anthropic_auth_token: "dummy".into(),
1126                ..Default::default()
1127            }),
1128        });
1129        let err = match result {
1130            Ok(_) => panic!("expected mismatch"),
1131            Err(e) => e,
1132        };
1133        assert!(err.to_string().contains("speaks openai-responses"), "{err}");
1134    }
1135
1136    /// The preflight is the whole point of the rewind path: a request that is
1137    /// too big must be refused *before* upload, and must say so in a way the
1138    /// top level can act on.
1139    #[test]
1140    fn oversized_request_is_refused_before_upload() {
1141        for limit in [1024, MAX_REQUEST_BYTES, 60_000_000] {
1142            assert!(check_request_size(limit, limit, "Anthropic").is_ok());
1143            let err = check_request_size(limit + 1, limit, "Anthropic").unwrap_err();
1144            assert!(
1145                matches!(err, GenerateError::RequestTooLargeError(_)),
1146                "{err:?}"
1147            );
1148            assert_eq!(err.recovery(), Recovery::OmitLastMessage);
1149            assert!(
1150                err.to_string().contains(&format!("limit is {limit} bytes")),
1151                "{err}"
1152            );
1153        }
1154    }
1155
1156    #[test]
1157    fn older_backend_configs_default_to_a_thirty_megabyte_request_cap() {
1158        let mut anthropic = serde_json::to_value(AnthropicBackendConfig::default()).unwrap();
1159        let mut openai = serde_json::to_value(OpenAIBackendConfig::default()).unwrap();
1160        for config in [&mut anthropic, &mut openai] {
1161            assert_eq!(config["max_request_bytes"], 30_000_000);
1162            config.as_object_mut().unwrap().remove("max_request_bytes");
1163        }
1164        assert_eq!(
1165            serde_json::from_value::<AnthropicBackendConfig>(anthropic)
1166                .unwrap()
1167                .max_request_bytes,
1168            30_000_000
1169        );
1170        assert_eq!(
1171            serde_json::from_value::<OpenAIBackendConfig>(openai)
1172                .unwrap()
1173                .max_request_bytes,
1174            30_000_000
1175        );
1176    }
1177
1178    /// A provider that rejects the size itself lands on the same variant, so
1179    /// the caller rewinds whether the cap was caught locally or remotely.
1180    #[test]
1181    fn provider_size_rejections_map_to_the_same_recovery() {
1182        let too_large = http_error(
1183            reqwest::StatusCode::PAYLOAD_TOO_LARGE,
1184            "HTTP 413: too big".into(),
1185        );
1186        assert_eq!(too_large.recovery(), Recovery::OmitLastMessage);
1187
1188        // Anthropic reports it as a 400 whose body names the error type.
1189        let named = http_error(
1190            reqwest::StatusCode::BAD_REQUEST,
1191            r#"HTTP 400: {"error":{"type":"request_too_large"}}"#.into(),
1192        );
1193        assert_eq!(named.recovery(), Recovery::OmitLastMessage);
1194
1195        // Others name no type and only describe the size in prose. Read as a
1196        // generic failure this is resent unchanged on every later turn.
1197        let described = http_error(
1198            reqwest::StatusCode::BAD_REQUEST,
1199            r#"HTTP 400: {"error":{"code":400,"message":"The message size (31271377 bytes) \
1200               exceeds 30.000MB limit.","status":"FAILED_PRECONDITION"}}"#
1201                .into(),
1202        );
1203        assert_eq!(described.recovery(), Recovery::OmitLastMessage);
1204
1205        let extent = http_error(
1206            reqwest::StatusCode::BAD_REQUEST,
1207            r#"{"type":"error","error":{"type":"invalid_request_error","message":"messages.9.content.13.image.source.base64.data: At least one of the image dimensions exceed max allowed size for many-image requests: 2576 pixels"}}"#.into(),
1208        );
1209        assert_eq!(extent.recovery(), Recovery::OmitLastMessage);
1210
1211        let unrelated = http_error(
1212            reqwest::StatusCode::INTERNAL_SERVER_ERROR,
1213            "HTTP 500: overloaded".into(),
1214        );
1215        assert_eq!(unrelated.recovery(), Recovery::Retry);
1216    }
1217
1218    /// Reading the body is how a size rejection is recognized, so an ordinary
1219    /// failure must not be mistaken for one: rewinding drops a user message,
1220    /// which is destructive when the request was never too big.
1221    #[test]
1222    fn ordinary_failures_are_not_read_as_size_rejections() {
1223        for body in [
1224            r#"HTTP 400: {"error":{"message":"max_tokens exceeds the model's limit"}}"#,
1225            r#"HTTP 400: {"error":{"message":"messages: unexpected role"}}"#,
1226            r#"HTTP 401: {"error":{"message":"invalid x-api-key"}}"#,
1227        ] {
1228            let err = http_error(reqwest::StatusCode::BAD_REQUEST, body.into());
1229            assert_eq!(err.recovery(), Recovery::Retry, "{body}");
1230        }
1231    }
1232
1233    #[test]
1234    fn message_types_serde_roundtrip() {
1235        let messages = vec![
1236            Message::UserMessage {
1237                content: vec![
1238                    Content::Text { text: "hi".into() },
1239                    Content::Image {
1240                        source: "data".into(),
1241                    },
1242                ],
1243            },
1244            Message::AssistantMessage {
1245                content: vec![Content::Text { text: "ok".into() }],
1246                tool_uses: vec![ToolUse {
1247                    name: "bash".into(),
1248                    input: serde_json::json!({"command": "true"}),
1249                }],
1250                turn_end_reason: Some(TurnEndReason::ToolUse),
1251            },
1252            Message::ToolResults {
1253                tool_use_results: vec![ToolResult {
1254                    content: vec![Content::Text {
1255                        text: "done".into(),
1256                    }],
1257                    is_error: false,
1258                    status: None,
1259                }],
1260            },
1261            Message::AssistantMessage {
1262                content: vec![],
1263                tool_uses: vec![],
1264                turn_end_reason: Some(TurnEndReason::Other("Anthropic::PauseTurn".into())),
1265            },
1266        ];
1267        let json = serde_json::to_string(&messages).expect("serialize");
1268        let back: Vec<Message> = serde_json::from_str(&json).expect("deserialize");
1269        assert_eq!(
1270            serde_json::to_value(&back).unwrap(),
1271            serde_json::to_value(&messages).unwrap()
1272        );
1273    }
1274}
1275
1276/// What a caller can do about a failed turn.
1277///
1278/// Failures that are a property of the *history* cannot be fixed by trying
1279/// again — every later turn resends that history and fails the same way, which
1280/// wedges the session. This is the top-level signal for those: it says whether
1281/// the last user message has to come back out (see
1282/// the chat adapter's rewind operation) before the conversation
1283/// can continue.
1284#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1285pub enum Recovery {
1286    /// Nothing about the history is known to be at fault; resubmitting as-is
1287    /// may work (provider blip, refusal, malformed stream).
1288    Retry,
1289    /// The request is too big for the provider. Retrying unchanged fails
1290    /// identically — drop the last user message (typically the one carrying an
1291    /// oversized attachment) and the session can go on.
1292    OmitLastMessage,
1293    /// An execution contract was violated. Resolve it before issuing more work.
1294    Stop,
1295}
1296
1297/// Default ceiling on one serialized API request body: 30 MB (decimal).
1298///
1299/// Each backend can override this with `max_request_bytes`. The check includes
1300/// images accumulated in history and JSON overhead, and refuses oversized bodies
1301/// before upload with [`Recovery::OmitLastMessage`].
1302pub const MAX_REQUEST_BYTES: usize = 30_000_000;
1303
1304fn default_max_request_bytes() -> usize {
1305    MAX_REQUEST_BYTES
1306}
1307
1308#[derive(thiserror::Error, Debug, Clone)]
1309pub enum GenerateError {
1310    #[error("Something went wrong while generating a response: {0}")]
1311    ExecutionError(String),
1312
1313    #[error("Generation succeeded, but the model refused to comply: {0}")]
1314    RefusalError(String),
1315
1316    #[error("Generation succeeded, but the output was malformed or corrupted: {0}")]
1317    MalformedResponseError(String),
1318
1319    /// Request exceeds the provider's size limit — locally detected, or the
1320    /// provider's own 413. Not retryable without shrinking the history.
1321    #[error("The request is too large to send: {0}")]
1322    RequestTooLargeError(String),
1323}
1324
1325impl GenerateError {
1326    pub fn recovery(&self) -> Recovery {
1327        match self {
1328            GenerateError::RequestTooLargeError(_) => Recovery::OmitLastMessage,
1329            GenerateError::ExecutionError(_)
1330            | GenerateError::RefusalError(_)
1331            | GenerateError::MalformedResponseError(_) => Recovery::Retry,
1332        }
1333    }
1334}
1335
1336/// Refuse a composed request body over the configured endpoint limit.
1337///
1338/// Takes the already-serialized length so the checked size is exactly the size
1339/// uploaded — no second serialization pass over a multi-megabyte body.
1340pub(crate) fn check_request_size(
1341    len: usize,
1342    limit: usize,
1343    provider: &str,
1344) -> Result<(), GenerateError> {
1345    if len > limit {
1346        return Err(GenerateError::RequestTooLargeError(format!(
1347            "the {provider} request is {len} bytes; the configured limit is {limit} bytes \
1348             (max_request_bytes). Images accumulate in the conversation; reduce \
1349             attachments or compact the session before retrying. Raise the gateway's \
1350             max_request_bytes only if it supports larger requests",
1351        )));
1352    }
1353    Ok(())
1354}
1355
1356/// Map a provider HTTP status to the right error variant: a size rejection is
1357/// not a generic failure.
1358///
1359/// 413 is unambiguous. Everything else has to be read out of the body, because
1360/// providers disagree on how they report an oversized request: some name a
1361/// machine-readable type (`request_too_large`), others return a 400 whose body
1362/// only describes the size in prose. Getting this wrong is expensive — the
1363/// identical body is resent on every later turn, so a size failure classified
1364/// as [`Recovery::Retry`] never triggers the rewind and the session fails
1365/// forever.
1366pub(crate) fn http_error(status: reqwest::StatusCode, message: String) -> GenerateError {
1367    if status == reqwest::StatusCode::PAYLOAD_TOO_LARGE || describes_a_size_rejection(&message) {
1368        GenerateError::RequestTooLargeError(message)
1369    } else {
1370        GenerateError::ExecutionError(message)
1371    }
1372}
1373
1374/// Does this provider error body say the request was too big?
1375///
1376/// Matching prose is unavoidable, so it is kept to phrasings only a size
1377/// rejection produces: "too large" however the provider spells it, or a size
1378/// that "exceeds" a stated limit.
1379fn describes_a_size_rejection(message: &str) -> bool {
1380    let message = message.to_ascii_lowercase();
1381    message.contains("too_large")
1382        || message.contains("too large")
1383        || (message.contains("size") && message.contains("exceeds"))
1384        || (message.contains("image dimensions")
1385            && message.contains("exceed")
1386            && message.contains("max allowed size"))
1387}
myco_model/lib.rs

myco_model/
lib.rs
