{
  "runtime": {
    "llama_server": "E:\\path\\to\\llama-server.exe",
    "ollama_url": "http://127.0.0.1:11434",
    "host": "127.0.0.1",
    "port": 8787,
    "default_model": "my-model",
    "runtime_dir": "E:\\llm-backend\\runtime",
    "cache_dir": "E:\\llm-backend\\cache",
    "n_gpu_layers": "auto",
    "ctx_size": 32768,
    "flash_attn": "auto",
    "fit_vram": true,
    "auto_load": false,
    "max_loaded_models": 1,
    "keep_alive_seconds": 300,
    "api_key": null
  },
  "models": [
    {
      "id": "my-gguf",
      "kind": "gguf",
      "path": "E:\\model\\my-model.Q4_K_M.gguf",
      "enabled": true,
      "options": {}
    },
    {
      "id": "my-safetensors",
      "kind": "transformers",
      "path": "E:\\model\\my-model",
      "enabled": true,
      "options": {
        "torch_dtype": "float16",
        "device": "cuda",
        "device_map": "auto",
        "quantization": "none",
        "load_in_4bit": false,
        "dynamic_int8_cpu": false
      }
    },
    {
      "id": "my-ollama-model",
      "kind": "ollama",
      "remote_model": "mistral-nemo:latest",
      "enabled": false
    }
  ],
  "cloud_profiles": [
    {
      "id": "openai-compatible",
      "base_url": "https://api.openai.com/v1",
      "api_key_env": "OPENAI_API_KEY",
      "default_model": "gpt-4o-mini"
    }
  ]
}