40 lines
1.0 KiB
Python
40 lines
1.0 KiB
Python
from __future__ import annotations
|
|
|
|
from abc import ABC, abstractmethod
|
|
from collections.abc import AsyncIterator
|
|
from typing import Any
|
|
|
|
from ..config import ModelConfig, RuntimeConfig
|
|
from ..types import BackendError, ChatChunk, ChatParams
|
|
|
|
|
|
class BaseBackend(ABC):
|
|
def __init__(self, model: ModelConfig, runtime: RuntimeConfig) -> None:
|
|
self.model = model
|
|
self.runtime = runtime
|
|
self.loaded = False
|
|
|
|
@abstractmethod
|
|
async def load(self) -> None:
|
|
raise NotImplementedError
|
|
|
|
@abstractmethod
|
|
async def unload(self) -> None:
|
|
raise NotImplementedError
|
|
|
|
@abstractmethod
|
|
async def chat(
|
|
self,
|
|
messages: list[dict[str, Any]],
|
|
params: ChatParams,
|
|
stream: bool = False,
|
|
) -> AsyncIterator[ChatChunk]:
|
|
raise NotImplementedError
|
|
|
|
@abstractmethod
|
|
def info(self) -> dict[str, Any]:
|
|
raise NotImplementedError
|
|
|
|
async def embed(self, inputs: list[str]) -> list[list[float]]:
|
|
raise BackendError(f"后端 {self.model.id} 不支持 embeddings")
|