Skip to content

provider

LLMProvider

Bases: Protocol

Protocol for LLM Providers.

Source code in wintermute/ai/provider.py 
50
51
52
53
54
55
56
57
58
59
60
61
62
class LLMProvider(Protocol):
    """Protocol for LLM Providers."""

    @property
    def name(self) -> str: ...
    @property
    def description(self) -> str: ...
    def list_models(self) -> list[ModelInfo]: ...
    def chat(self, req: ChatRequest) -> ChatResponse: ...
    def embed(
        self, texts: Iterable[str], model: Optional[str] = None
    ) -> list[list[float]]: ...
    def count_tokens(self, text: str, model: Optional[str] = None) -> int: ...

LLMRegistry

Registry for LLM Providers.

Source code in wintermute/ai/provider.py 
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
class LLMRegistry:
    """Registry for LLM Providers."""

    def __init__(self) -> None:
        self._providers: dict[str, LLMProvider] = {}

    def register(self, provider: LLMProvider) -> None:
        self._providers[provider.name] = provider

    def get(self, name: str) -> LLMProvider:
        return self._providers[name]

    def providers(self) -> list[str]:
        return list(self._providers.keys())

    def get_provider_descriptions(self) -> dict[str, str]:
        """Return a mapping of provider names to their descriptions."""
        return {name: p.description for name, p in self._providers.items()}

get_provider_descriptions()

Return a mapping of provider names to their descriptions.

Source code in wintermute/ai/provider.py 
80
81
82
def get_provider_descriptions(self) -> dict[str, str]:
    """Return a mapping of provider names to their descriptions."""
    return {name: p.description for name, p in self._providers.items()}

ModelInfo dataclass

Information about a language model.

Source code in wintermute/ai/provider.py 
34
35
36
37
38
39
40
41
42
43
44
@dataclass(frozen=True)
class ModelInfo:
    """Information about a language model."""

    name: str
    family: str
    context_window: int
    supports_tools: bool
    supports_json: bool
    supports_stream: bool
    supports_vision: bool = False

Router

Pluggable routing policy. Default: stick to default provider/model.

Example

from wintermute.ai import llms from wintermute.ai.providers.router import Router router = Router(default_provider="openai", default_model="gpt-4.1-mini") req = ChatRequest(messages=[Message(role="user", content="Hello!")]) provider, routed_req = router.choose(req) response = provider.chat(routed_req)

Parameters:

Name Type Description Default
default_provider str

Default provider name to use.

 required
default_model Optional[str]

Default model name to use.

 None
Source code in wintermute/ai/provider.py 
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
class Router:
    """Pluggable routing policy. Default: stick to default provider/model.

    Example:
        >>> from wintermute.ai import llms
        >>> from wintermute.ai.providers.router import Router
        >>> router = Router(default_provider="openai", default_model="gpt-4.1-mini")
        >>> req = ChatRequest(messages=[Message(role="user", content="Hello!")])
        >>> provider, routed_req = router.choose(req)
        >>> response = provider.chat(routed_req)

    Args:
        default_provider (str): Default provider name to use.
        default_model (Optional[str]): Default model name to use.
    """

    def __init__(
        self, default_provider: str, default_model: Optional[str] = None
    ) -> None:
        self.default_provider = default_provider
        self.default_model = default_model

    def choose(self, req: ChatRequest) -> tuple[LLMProvider, ChatRequest]:
        """Choose the provider and possibly modify the request based on routing policy."""
        provider = llms.get(self.default_provider)
        model = req.model or self.default_model
        # Example heuristic: cheap tasks
        if req.task_tag and "cheap" in req.task_tag:
            for p in llms.providers():
                if p.startswith("groq"):
                    provider = llms.get(p)
                    break
        new_req = ChatRequest(**{**req.__dict__, "model": model})
        return provider, new_req

    def set_default(
        self, *, provider: Optional[str] = None, model: Optional[str] = None
    ) -> None:
        if provider is not None:
            self.default_provider = provider
        if model is not None:
            self.default_model = model

choose(req)

Choose the provider and possibly modify the request based on routing policy.

Source code in wintermute/ai/provider.py 
110
111
112
113
114
115
116
117
118
119
120
121
def choose(self, req: ChatRequest) -> tuple[LLMProvider, ChatRequest]:
    """Choose the provider and possibly modify the request based on routing policy."""
    provider = llms.get(self.default_provider)
    model = req.model or self.default_model
    # Example heuristic: cheap tasks
    if req.task_tag and "cheap" in req.task_tag:
        for p in llms.providers():
            if p.startswith("groq"):
                provider = llms.get(p)
                break
    new_req = ChatRequest(**{**req.__dict__, "model": model})
    return provider, new_req