79 lines
2.0 KiB
JSON
79 lines
2.0 KiB
JSON
{
|
|
"runtime": {
|
|
"llama_server": "E:\\ollama\\模型\\llama-b8642-bin-win-cuda-13.1-x64\\llama-server.exe",
|
|
"ollama_url": "http://127.0.0.1:11434",
|
|
"host": "127.0.0.1",
|
|
"port": 8787,
|
|
"default_model": "apex-qwen3",
|
|
"model_dir": "E:\\model",
|
|
"runtime_dir": "E:\\llm-backend\\runtime",
|
|
"cache_dir": "E:\\llm-backend\\cache",
|
|
"n_gpu_layers": "auto",
|
|
"ctx_size": 32768,
|
|
"batch_size": 2048,
|
|
"ubatch_size": 512,
|
|
"flash_attn": "auto",
|
|
"fit_vram": true,
|
|
"auto_load": false,
|
|
"api_key": null,
|
|
"max_loaded_models": 1,
|
|
"keep_alive_seconds": 300
|
|
},
|
|
"models": [
|
|
{
|
|
"id": "apex-qwen3",
|
|
"kind": "transformers",
|
|
"path": "E:\\ollama\\Apex_AI-catmodel",
|
|
"enabled": true,
|
|
"description": "本机 Qwen3 safetensors 模型",
|
|
"options": {
|
|
"torch_dtype": "float16",
|
|
"device": "cuda",
|
|
"device_map": "auto",
|
|
"load_in_4bit": false,
|
|
"attn_implementation": "sdpa"
|
|
}
|
|
},
|
|
{
|
|
"id": "gguf-local",
|
|
"kind": "gguf",
|
|
"path": "E:\\model\\example.gguf",
|
|
"enabled": false,
|
|
"description": "将 path 改成真实 GGUF 文件后启用",
|
|
"options": {}
|
|
},
|
|
{
|
|
"id": "onnx-local",
|
|
"kind": "onnx",
|
|
"path": "E:\\model\\example.onnx",
|
|
"enabled": false,
|
|
"description": "ONNX Runtime/Optimum 扩展入口",
|
|
"options": {}
|
|
},
|
|
{
|
|
"id": "mistral-nemo-ollama",
|
|
"kind": "ollama",
|
|
"remote_model": "mistral-nemo-12b:latest",
|
|
"enabled": false,
|
|
"description": "兼容本机 Ollama 模型仓库；不复制权重"
|
|
},
|
|
{
|
|
"id": "deepseek-cloud",
|
|
"kind": "cloud",
|
|
"cloud_profile": "deepseek",
|
|
"remote_model": "deepseek-chat",
|
|
"enabled": false,
|
|
"description": "OpenAI 兼容云端模型"
|
|
}
|
|
],
|
|
"cloud_profiles": [
|
|
{
|
|
"id": "deepseek",
|
|
"base_url": "https://api.deepseek.com",
|
|
"api_key_env": "DEEPSEEK_API_KEY",
|
|
"default_model": "deepseek-chat",
|
|
"timeout": 120
|
|
}
|
|
]
|
|
}
|