shawn
14cf8332a2
- start-llama-server.bat: fix LLAMA_DIR to bin\llama (was bin\ollama), fix model to qwen2.5-coder-1.5b (was Llama-3.2-1B), use correct GPU (llama-server.exe) and CPU (cpu\llama-server-cpu.exe) binaries - start-windows-mcp.bat: remove dead python313\ tier, use single embedded Python 3.13 at python\python.exe - run-hermes.bat: use Scripts\hermes.exe instead of -m hermes_cli (no __main__.py in package), fix offline mode to use single config.yaml instead of non-existent config.offline.yaml - run-hermes.ps1: same hermes_cli → hermes.exe fix - README.txt: update to match actual USB state (Python 3.13, 142 wheels, Qwen2.5-Coder-1.5B 1117MB model, correct dir structure)
246 lines
4.3 KiB
Batchfile
246 lines
4.3 KiB
Batchfile
@echo off
|
|
|
|
title Hermes Portable - Offline LLM Server with GPU Detection
|
|
|
|
:: ==============================================================================
|
|
|
|
:: Hermes Portable Rescue - llama-server launcher with multi-brand GPU detection
|
|
|
|
:: ==============================================================================
|
|
|
|
:: Detects GPU brand (NVIDIA, AMD, Intel, integrated) and launches the
|
|
|
|
:: appropriate llama-server build (Vulkan for GPU, CPU-only fallback).
|
|
|
|
:: ==============================================================================
|
|
|
|
setlocal enabledelayedexpansion
|
|
|
|
|
|
|
|
cd /d "%~dp0"
|
|
|
|
|
|
|
|
set LLAMA_DIR=%~dp0bin\llama
|
|
|
|
set OLLAMA_DIR=%~dp0bin\ollama
|
|
|
|
set MODEL=%OLLAMA_DIR%\qwen2.5-coder-1.5b-q4_k_m.gguf
|
|
|
|
set PORT=11434
|
|
|
|
set GPU_FOUND=0
|
|
|
|
|
|
|
|
echo.
|
|
|
|
echo ============================================
|
|
|
|
echo Hermes LLM Server - GPU Detection
|
|
|
|
echo ============================================
|
|
|
|
echo.
|
|
|
|
|
|
|
|
:: --- Detection 1: nvidia-smi (NVIDIA) --------------------------------------
|
|
|
|
nvidia-smi >nul 2>&1
|
|
|
|
if !errorlevel! equ 0 (
|
|
|
|
for /f "tokens=*" %%a in ('nvidia-smi --query-gpu=name --format=csv,noheader 2^>nul') do (
|
|
|
|
set GPU_NAME=%%a
|
|
|
|
)
|
|
|
|
echo [OK] NVIDIA GPU detected: !GPU_NAME!
|
|
|
|
set GPU_FOUND=1
|
|
|
|
)
|
|
|
|
|
|
|
|
:: --- Detection 2: WMIC - check for AMD/Intel GPUs ---------------------------
|
|
|
|
if !GPU_FOUND! equ 0 (
|
|
|
|
wmic path win32_videocontroller get name 2>nul | findstr /i "nvidia" >nul
|
|
|
|
if !errorlevel! equ 0 (
|
|
|
|
echo [OK] NVIDIA GPU detected via WMI
|
|
|
|
set GPU_FOUND=1
|
|
|
|
) else (
|
|
|
|
wmic path win32_videocontroller get name 2>nul | findstr /i "amd radeon" >nul
|
|
|
|
if !errorlevel! equ 0 (
|
|
|
|
echo [OK] AMD GPU detected via WMI
|
|
|
|
set GPU_FOUND=1
|
|
|
|
) else (
|
|
|
|
wmic path win32_videocontroller get name 2>nul | findstr /i "intel arc" >nul
|
|
|
|
if !errorlevel! equ 0 (
|
|
|
|
echo [OK] Intel GPU detected via WMI
|
|
|
|
set GPU_FOUND=1
|
|
|
|
) else (
|
|
|
|
wmic path win32_videocontroller get name 2>nul | findstr /i "intel hd intel iris intel uhd" >nul
|
|
|
|
if !errorlevel! equ 0 (
|
|
|
|
echo [OK] Intel integrated graphics detected via WMI
|
|
|
|
set GPU_FOUND=1
|
|
|
|
)
|
|
|
|
)
|
|
|
|
)
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
:: --- Detection 3: vulkaninfo (direct Vulkan probe) ---------------------------
|
|
|
|
if !GPU_FOUND! equ 0 (
|
|
|
|
where vulkaninfo >nul 2>&1
|
|
|
|
if !errorlevel! equ 0 (
|
|
|
|
vulkaninfo --summary 2>nul | findstr /c:"Vulkan Instance" >nul
|
|
|
|
if !errorlevel! equ 0 (
|
|
|
|
echo [OK] Vulkan-capable device detected
|
|
|
|
set GPU_FOUND=1
|
|
|
|
)
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
:: --- Check model exists ----------------------------------------------------
|
|
|
|
if not exist "!MODEL!" (
|
|
|
|
echo [ERROR] Model not found: !MODEL!
|
|
|
|
echo.
|
|
|
|
echo Please download a GGUF model and place it at:
|
|
|
|
echo !MODEL!
|
|
|
|
echo.
|
|
|
|
pause
|
|
|
|
exit /b 1
|
|
|
|
)
|
|
|
|
|
|
|
|
:: --- Launch ----------------------------------------------------------------
|
|
|
|
if !GPU_FOUND! equ 1 (
|
|
|
|
echo.
|
|
|
|
echo [*] Starting llama-server with GPU acceleration (Vulkan)...
|
|
|
|
echo [*] Model: Qwen2.5-Coder-1.5B (Q4_K_M)
|
|
|
|
echo [*] Server: http://127.0.0.1:!PORT!/v1
|
|
|
|
echo [*] Flags: --n-gpu-layers 99 --mlock --cont-batching -c 4096
|
|
|
|
echo.
|
|
|
|
echo [*] Press Ctrl+C to stop the server.
|
|
|
|
echo.
|
|
|
|
|
|
|
|
"!LLAMA_DIR!\llama-server.exe" ^
|
|
|
|
--host 127.0.0.1 --port !PORT! ^
|
|
|
|
-m "!MODEL!" ^
|
|
|
|
--n-gpu-layers 99 ^
|
|
|
|
--mlock ^
|
|
|
|
--cont-batching ^
|
|
|
|
-c 4096
|
|
|
|
) else (
|
|
|
|
echo.
|
|
|
|
echo [*] No compatible GPU found. Starting CPU-only mode.
|
|
|
|
echo [*] Model: Qwen2.5-Coder-1.5B (Q4_K_M)
|
|
|
|
echo [*] Server: http://127.0.0.1:!PORT!/v1
|
|
|
|
echo [*] Flags: --mlock --cont-batching -c 4096
|
|
|
|
echo.
|
|
|
|
echo [*] Press Ctrl+C to stop the server.
|
|
|
|
echo.
|
|
|
|
|
|
|
|
"!LLAMA_DIR!\cpu\llama-server-cpu.exe" ^
|
|
|
|
--host 127.0.0.1 --port !PORT! ^
|
|
|
|
-m "!MODEL!" ^
|
|
|
|
--mlock ^
|
|
|
|
--cont-batching ^
|
|
|
|
-c 4096
|
|
|
|
)
|
|
|
|
|
|
|
|
echo.
|
|
|
|
echo [*] Server exited with code !errorlevel!
|
|
|
|
pause
|