51 lines
1.5 KiB
Python
51 lines
1.5 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ChatParams:
|
|
max_tokens: int = 512
|
|
temperature: float = 0.7
|
|
top_p: float = 0.95
|
|
top_k: int = 40
|
|
min_p: float = 0.05
|
|
repeat_penalty: float = 1.05
|
|
seed: int | None = None
|
|
stop: list[str] = field(default_factory=list)
|
|
enable_thinking: bool | None = None
|
|
extra: dict[str, Any] = field(default_factory=dict)
|
|
response_format: str | dict[str, Any] | None = None
|
|
tools: list[dict[str, Any]] = field(default_factory=list)
|
|
tool_choice: Any = None
|
|
keep_alive: str | int | float | None = None
|
|
|
|
def as_generation_kwargs(self) -> dict[str, Any]:
|
|
result: dict[str, Any] = {
|
|
"max_tokens": self.max_tokens,
|
|
"temperature": self.temperature,
|
|
"top_p": self.top_p,
|
|
"top_k": self.top_k,
|
|
"min_p": self.min_p,
|
|
"repeat_penalty": self.repeat_penalty,
|
|
}
|
|
if self.seed is not None:
|
|
result["seed"] = self.seed
|
|
if self.stop:
|
|
result["stop"] = self.stop
|
|
return result
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ChatChunk:
|
|
text: str = ""
|
|
done: bool = False
|
|
finish_reason: str | None = None
|
|
metadata: dict[str, Any] = field(default_factory=dict)
|
|
delta: dict[str, Any] = field(default_factory=dict)
|
|
|
|
|
|
class BackendError(RuntimeError):
|
|
"""An expected model backend failure with a user-facing message."""
|