128 lines
5.0 KiB
Python
128 lines
5.0 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from collections.abc import AsyncIterator
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from ..config import ModelConfig, RuntimeConfig
|
|
from ..types import BackendError, ChatChunk, ChatParams
|
|
from .base import BaseBackend
|
|
|
|
|
|
class OllamaBackend(BaseBackend):
|
|
"""Compatibility provider for models already managed by a local Ollama daemon."""
|
|
|
|
def __init__(self, model: ModelConfig, runtime: RuntimeConfig) -> None:
|
|
super().__init__(model, runtime)
|
|
self.base_url = runtime.ollama_url.rstrip("/")
|
|
self.remote_model = model.remote_model or model.id
|
|
|
|
async def load(self) -> None:
|
|
if self.loaded:
|
|
return
|
|
try:
|
|
async with httpx.AsyncClient(timeout=5.0) as client:
|
|
response = await client.get(f"{self.base_url}/api/version")
|
|
response.raise_for_status()
|
|
except httpx.HTTPError as exc:
|
|
raise BackendError(f"无法连接本地 Ollama: {self.base_url}: {exc}") from exc
|
|
self.loaded = True
|
|
|
|
async def unload(self) -> None:
|
|
self.loaded = False
|
|
|
|
async def chat(
|
|
self,
|
|
messages: list[dict[str, Any]],
|
|
params: ChatParams,
|
|
stream: bool = False,
|
|
) -> AsyncIterator[ChatChunk]:
|
|
if not self.loaded:
|
|
await self.load()
|
|
options: dict[str, Any] = {
|
|
"num_predict": params.max_tokens,
|
|
"temperature": params.temperature,
|
|
"top_p": params.top_p,
|
|
"top_k": params.top_k,
|
|
"min_p": params.min_p,
|
|
"repeat_penalty": params.repeat_penalty,
|
|
}
|
|
if params.seed is not None:
|
|
options["seed"] = params.seed
|
|
if params.stop:
|
|
options["stop"] = params.stop
|
|
options.update(params.extra)
|
|
payload: dict[str, Any] = {
|
|
"model": self.remote_model,
|
|
"messages": messages,
|
|
"stream": stream,
|
|
"options": options,
|
|
"keep_alive": params.keep_alive if params.keep_alive is not None else "5m",
|
|
}
|
|
if params.response_format is not None:
|
|
payload["format"] = params.response_format
|
|
if params.enable_thinking is not None:
|
|
payload["think"] = params.enable_thinking
|
|
if params.tools:
|
|
payload["tools"] = params.tools
|
|
async with httpx.AsyncClient(timeout=httpx.Timeout(600.0, connect=10.0)) as client:
|
|
if stream:
|
|
async with client.stream("POST", f"{self.base_url}/api/chat", json=payload) as response:
|
|
if response.status_code >= 400:
|
|
detail = (await response.aread()).decode("utf-8", errors="replace")
|
|
raise BackendError(f"Ollama 请求失败 {response.status_code}: {detail}")
|
|
async for line in response.aiter_lines():
|
|
if not line.strip():
|
|
continue
|
|
try:
|
|
data = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
message = data.get("message") or {}
|
|
text = message.get("content") or ""
|
|
if data.get("done"):
|
|
yield ChatChunk(done=True, finish_reason=data.get("done_reason") or "stop", metadata=data)
|
|
return
|
|
if text or len(message) > 1:
|
|
yield ChatChunk(text=text, delta=message, metadata=data)
|
|
yield ChatChunk(done=True, finish_reason="stop")
|
|
return
|
|
|
|
payload["stream"] = False
|
|
response = await client.post(f"{self.base_url}/api/chat", json=payload)
|
|
if response.status_code >= 400:
|
|
raise BackendError(f"Ollama 请求失败 {response.status_code}: {response.text}")
|
|
data = response.json()
|
|
message = data.get("message") or {}
|
|
yield ChatChunk(
|
|
text=message.get("content") or "",
|
|
delta=message,
|
|
done=True,
|
|
finish_reason=data.get("done_reason") or "stop",
|
|
metadata=data,
|
|
)
|
|
|
|
async def embed(self, inputs: list[str]) -> list[list[float]]:
|
|
if not self.loaded:
|
|
await self.load()
|
|
async with httpx.AsyncClient(timeout=120.0) as client:
|
|
response = await client.post(
|
|
f"{self.base_url}/api/embed",
|
|
json={"model": self.remote_model, "input": inputs},
|
|
)
|
|
if response.status_code >= 400:
|
|
raise BackendError(f"Ollama embeddings 失败 {response.status_code}: {response.text}")
|
|
data = response.json()
|
|
return data.get("embeddings", [])
|
|
|
|
def info(self) -> dict[str, Any]:
|
|
return {
|
|
"id": self.model.id,
|
|
"kind": "ollama",
|
|
"remote_model": self.remote_model,
|
|
"loaded": self.loaded,
|
|
"endpoint": self.base_url,
|
|
}
|