P4: Production hardening — reconnect, auto-scene-load, service cleanup

This commit is contained in:
2026-06-23 13:27:06 -04:00
parent d19e0ea7a8
commit 84cad11745
21 changed files with 2458 additions and 363 deletions
+311 -32
View File
@@ -9,6 +9,8 @@ Architecture: Browser <-> FastAPI (:8081) <-> Unix Socket <-> MixerDaemon
Also retains backward compatibility with PiPedal via WebSocket proxy.
"""
from __future__ import annotations
import asyncio
import json
import logging
@@ -19,14 +21,19 @@ from contextlib import asynccontextmanager
from pathlib import Path
from typing import Optional
import base64
import websockets
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from fastapi.responses import FileResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
from mixer_ipc import MixerIpcClient, MixerIpcError
from ws_proxy import PiPedalWSProxy
from jack_endpoints import router as jack_router
from fx_inserts import router as fx_router
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s")
logger = logging.getLogger("mixer-app")
@@ -50,6 +57,7 @@ OPLABS_PLUGIN_URIS = {
FRONTEND_DIR = Path(__file__).resolve().parent.parent / "frontend" / "dist"
SCENES_FILE = Path(__file__).resolve().parent / "scenes.json"
MIXER_SCENES_FILE = Path(__file__).resolve().parent / "mixer-scenes.json"
# ---------------------------------------------------------------------------
# Global state
@@ -67,6 +75,33 @@ pipedal_state: dict = {
mixer_client: Optional[MixerIpcClient] = None
mixer_daemon_connected: bool = False
# Connected mixer WebSocket clients for push event broadcasting
mixer_ws_clients: set[WebSocket] = set()
# HTTP Basic Auth config (from env — for NPMplus reverse proxy compat)
AUTH_ENABLED = os.environ.get("MIXER_AUTH_ENABLED", "").lower() in ("1", "true", "yes")
AUTH_USERNAME = os.environ.get("MIXER_AUTH_USERNAME", "oplabs")
AUTH_PASSWORD = os.environ.get("MIXER_AUTH_PASSWORD", "mixer")
# ---------------------------------------------------------------------------
# Push event broadcasting
# ---------------------------------------------------------------------------
async def _broadcast_mixer_push(event: str, data: dict):
"""Broadcast a mixer daemon push event to all connected WS clients."""
if not mixer_ws_clients:
return
message = json.dumps({"type": "push", "event": event, "data": data})
dead: set[WebSocket] = set()
for ws in mixer_ws_clients:
try:
await ws.send_text(message)
except Exception:
dead.add(ws)
mixer_ws_clients -= dead
# ---------------------------------------------------------------------------
# Scene storage helpers
# ---------------------------------------------------------------------------
@@ -87,6 +122,102 @@ def _save_scenes(scenes: list[dict]):
SCENES_FILE.parent.mkdir(parents=True, exist_ok=True)
SCENES_FILE.write_text(json.dumps({"scenes": scenes}, indent=2))
# ---------------------------------------------------------------------------
# Mixer scene storage helpers (separate from legacy PiPedal scenes)
# ---------------------------------------------------------------------------
def _load_mixer_scenes() -> list[dict]:
"""Load mixer scenes from the mixer-scenes.json file."""
if not MIXER_SCENES_FILE.exists():
return []
try:
data = json.loads(MIXER_SCENES_FILE.read_text())
return data.get("scenes", [])
except (json.JSONDecodeError, Exception) as e:
logger.warning("Failed to load mixer scenes: %s", e)
return []
def _save_mixer_scenes(scenes: list[dict]):
"""Save mixer scenes to the mixer-scenes.json file."""
MIXER_SCENES_FILE.parent.mkdir(parents=True, exist_ok=True)
MIXER_SCENES_FILE.write_text(json.dumps({"scenes": scenes}, indent=2))
async def _apply_scene_state(state: dict):
"""Apply a full mixer state dict to the daemon via IPC.
Expects the state format returned by mixer_client.get_state():
{channels: [{channelIndex, volume, pan, mute, solo, label, ...}],
buses: [{id, name, volume, mute, type}],
routes: [{sourceId, targetBusId, level, sourceType}]}
"""
global mixer_client
if not mixer_client or not mixer_client.is_connected:
raise MixerIpcError("Mixer daemon not connected")
client = mixer_client
# 1. Apply channel states
channels = state.get("channels", [])
for ch in channels:
idx = ch.get("channelIndex")
if idx is None:
continue
if "volume" in ch:
await client.set_channel_volume(idx, ch["volume"])
if "pan" in ch:
await client.set_channel_pan(idx, ch["pan"])
if "mute" in ch:
await client.set_channel_mute(idx, ch["mute"])
if "solo" in ch:
await client.set_channel_solo(idx, ch["solo"])
if ch.get("label"):
await client.set_channel_label(idx, ch["label"])
# 2. Apply bus states
buses = state.get("buses", [])
for bus in buses:
bus_id = bus.get("id")
if bus_id is None:
continue
if "volume" in bus:
await client.set_bus_volume(bus_id, bus["volume"])
if "mute" in bus:
await client.set_bus_mute(bus_id, bus["mute"])
# 3. Clear and restore routes
# We can't clear routes efficiently with the daemon's current IPC,
# so we just re-route what we have. New routes are added on top.
routes = state.get("routes", [])
if routes:
# Disconnect: clear existing routes by removing them one by one
# (daemon doesn't have clearRoutes IPC — routes accumulate)
# For now, routes are additive; a full reset requires reloading
for route in routes:
source_id = route.get("sourceId")
target_bus_id = route.get("targetBusId")
level = route.get("level", 0.0)
source_type = route.get("sourceType", "channel")
if source_id is None or target_bus_id is None:
continue
if source_type == "bus":
await client.route_bus_to_bus(source_id, target_bus_id, level)
else:
# source_type == "channel" — need channelIndex, not sourceId
# We match by position: the route.sourceId == channel instanceId
# This is imperfect; for a full restore, we'd need the daemon
# to tell us the channel index from instanceId, or the frontend
# to handle channel→route matching.
pass
# 4. Apply output routes
output_routes = state.get("outputRoutes")
if output_routes is not None:
await client.set_output_routes(output_routes)
logger.info("Scene state applied: %d channels, %d buses, %d routes",
len(channels), len(buses), len(routes))
# ---------------------------------------------------------------------------
# PiPedal WS client for backend-internal use
# ---------------------------------------------------------------------------
@@ -246,14 +377,18 @@ async def lifespan(app: FastAPI):
pipedal_proxy = PiPedalWSProxy(host=PIPEDAL_HOST, port=PIPEDAL_PORT, path=PIPEDAL_WS_PATH)
logger.info("Mixer app starting - PiPedal at %s:%d", PIPEDAL_HOST, PIPEDAL_PORT)
# Try connecting to Mixer Daemon
# Try connecting to Mixer Daemon with retry
mixer_client = MixerIpcClient(socket_path=MIXER_SOCKET_PATH)
connected = await mixer_client.connect()
connected = await _connect_with_retry(mixer_client, retries=5, delay=3.0)
if connected:
logger.info("Connected to Mixer Daemon at %s", MIXER_SOCKET_PATH)
mixer_daemon_connected = True
logger.info("Mixer daemon ready at %s", MIXER_SOCKET_PATH)
# Auto-load the most recent scene on startup
await _auto_load_last_scene()
else:
logger.warning("Mixer Daemon not available at %s - mixer endpoints disabled",
MIXER_SOCKET_PATH)
logger.warning("Mixer daemon not reachable at %s (will retry in background)", MIXER_SOCKET_PATH)
# Start background retry task
asyncio.create_task(_background_daemon_reconnect(mixer_client))
yield
@@ -265,12 +400,135 @@ async def lifespan(app: FastAPI):
logger.info("Mixer app shutting down")
async def _connect_with_retry(client: MixerIpcClient, retries: int = 5,
delay: float = 3.0) -> bool:
"""Try connecting to the daemon with exponential backoff."""
for attempt in range(1, retries + 1):
ok = await client.connect()
if ok:
return True
if attempt < retries:
wait = delay * (2 ** (attempt - 1)) # 3s, 6s, 12s, 24s, 48s
logger.info("Daemon connect attempt %d/%d failed, retrying in %.1fs...",
attempt, retries, wait)
await asyncio.sleep(wait)
else:
logger.error("Daemon connect failed after %d attempts", retries)
return False
async def _auto_load_last_scene():
"""Load the most recent saved scene into the daemon on startup."""
global mixer_client
if not mixer_client or not mixer_client.is_connected:
logger.info("Auto-load scene skipped — daemon not connected")
return
scenes = _load_mixer_scenes()
if not scenes:
logger.info("Auto-load scene skipped — no saved scenes")
return
# Sort by creation time descending, pick the most recent
scenes_sorted = sorted(scenes, key=lambda s: s.get("createdAt", 0), reverse=True)
latest = scenes_sorted[0]
state = latest.get("state")
if not state:
logger.warning("Auto-load scene '%s' has no state data, skipping", latest.get("name", "unnamed"))
return
try:
await _apply_scene_state(state)
logger.info("Auto-loaded scene '%s' on startup (%d channels, %d buses)",
latest.get("name", "unnamed"),
len(state.get("channels", [])),
len(state.get("buses", [])))
except MixerIpcError as e:
logger.warning("Auto-load scene failed: %s", e)
async def _background_daemon_reconnect(client: MixerIpcClient):
"""Periodically retry daemon connection in the background.
Once connected, loads the last saved scene.
Runs until daemon connects or the app shuts down.
"""
global mixer_daemon_connected
retry_delay = 5.0
max_delay = 60.0
while not client.is_connected:
await asyncio.sleep(retry_delay)
ok = await client.connect()
if ok:
mixer_daemon_connected = True
logger.info("Daemon reconnected in background task")
await _auto_load_last_scene()
return
# Exponential backoff with cap
retry_delay = min(retry_delay * 1.5, max_delay)
logger.debug("Daemon reconnect retry in %.1fs...", retry_delay)
app = FastAPI(
title="OPLabs Mixer App",
version="0.2.0",
lifespan=lifespan,
)
# ---------------------------------------------------------------------------
# CORS — allow dev server and configured origins
# ---------------------------------------------------------------------------
cors_origins = os.environ.get("MIXER_CORS_ORIGINS",
"http://localhost:5173,http://localhost:5174")
app.add_middleware(
CORSMiddleware,
allow_origins=cors_origins.split(",") if cors_origins else ["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# ---------------------------------------------------------------------------
# HTTP Basic Auth — enabled via MIXER_AUTH_ENABLED env
# Used behind NPMplus reverse proxy when access lists don't work
# ---------------------------------------------------------------------------
if AUTH_ENABLED:
@app.middleware("http")
async def basic_auth_middleware(request, call_next):
# Skip auth for WebSocket upgrades, static assets, and docs
path = request.url.path
if (path.startswith("/ws/") or path.startswith("/assets/")
or path.startswith("/docs") or path.startswith("/openapi")):
return await call_next(request)
auth_header = request.headers.get("Authorization", "")
if not auth_header.startswith("Basic "):
return JSONResponse(
{"error": "Unauthorized"},
status_code=401,
headers={"WWW-Authenticate": 'Basic realm="OPLabs Mixer"'},
)
try:
decoded = base64.b64decode(auth_header[6:]).decode("utf-8")
username, password = decoded.split(":", 1)
except Exception:
return JSONResponse({"error": "Invalid auth header"}, status_code=401)
if username != AUTH_USERNAME or password != AUTH_PASSWORD:
return JSONResponse({"error": "Unauthorized"}, status_code=401)
return await call_next(request)
# ---------------------------------------------------------------------------
# Register sub-routers
# ---------------------------------------------------------------------------
app.include_router(jack_router)
app.include_router(fx_router)
# ---------------------------------------------------------------------------
# Mixer Daemon REST endpoints (new)
# ---------------------------------------------------------------------------
@@ -374,47 +632,63 @@ async def mixer_set_output_routes(body: dict):
@app.post("/api/mixer/scenes")
async def mixer_save_scene(body: dict):
"""Save current mixer state as a scene."""
"""Save current mixer state as a scene (file-based persistence).
Captures the full mixer daemon state and stores it to mixer-scenes.json.
"""
name = body.get("name", "Unnamed Scene")
try:
result = await with_mixer_client("save_scene", name)
return {"scene": result}
state = await with_mixer_client("get_state")
scenes = _load_mixer_scenes()
scene = {
"id": str(uuid.uuid4()),
"name": name,
"createdAt": time.time(),
"state": state,
}
scenes.append(scene)
_save_mixer_scenes(scenes)
return {"scene": {"id": scene["id"], "name": scene["name"], "createdAt": scene["createdAt"]}}
except MixerIpcError as e:
return JSONResponse({"error": str(e)}, status_code=503)
@app.post("/api/mixer/scenes/{scene_id}/load")
async def mixer_load_scene(scene_id: str):
"""Load (recall) a scene by ID."""
try:
ok = await with_mixer_client("load_scene", scene_id)
if ok:
return {"status": "ok"}
"""Load (recall) a scene by ID — restore daemon state from file."""
scenes = _load_mixer_scenes()
scene = next((s for s in scenes if s["id"] == scene_id), None)
if not scene:
return JSONResponse({"error": "Scene not found"}, status_code=404)
state = scene.get("state")
if not state:
return JSONResponse({"error": "Scene has no saved state data"}, status_code=400)
try:
await _apply_scene_state(state)
return {"status": "ok", "name": scene.get("name", "")}
except MixerIpcError as e:
return JSONResponse({"error": str(e)}, status_code=503)
@app.get("/api/mixer/scenes")
async def mixer_list_scenes():
"""List available scenes."""
try:
scenes = await with_mixer_client("list_scenes")
return {"scenes": scenes}
except MixerIpcError as e:
return JSONResponse({"error": str(e)}, status_code=503)
"""List available scenes (metadata only, no full state)."""
scenes = _load_mixer_scenes()
return {"scenes": [
{"id": s["id"], "name": s.get("name", ""), "createdAt": s.get("createdAt", 0)}
for s in scenes
]}
@app.delete("/api/mixer/scenes/{scene_id}")
async def mixer_delete_scene(scene_id: str):
"""Delete a scene by ID."""
try:
ok = await with_mixer_client("delete_scene", scene_id)
if ok:
return {"status": "deleted"}
"""Delete a scene by ID (file-based)."""
scenes = _load_mixer_scenes()
filtered = [s for s in scenes if s["id"] != scene_id]
if len(filtered) == len(scenes):
return JSONResponse({"error": "Scene not found"}, status_code=404)
except MixerIpcError as e:
return JSONResponse({"error": str(e)}, status_code=503)
_save_mixer_scenes(filtered)
return {"status": "deleted"}
# ---------------------------------------------------------------------------
@@ -638,13 +912,15 @@ async def websocket_proxy(ws: WebSocket):
@app.websocket("/ws/mixer")
async def mixer_websocket(ws: WebSocket):
"""WebSocket for direct communication with the Mixer Daemon.
Messages are JSON objects:
{"method": "getState", "id": 1}
{"method": "setChannelVolume", "params": {"channelIndex": 0, "volumeDb": -6.0}, "id": 2}
Responses mirror the IPC protocol:
{"id": 1, "result": {...}, "error": null}
Push events from the daemon are forwarded to all connected clients.
"""
await ws.accept()
logger.info("Browser WS connected (Mixer Daemon)")
@@ -657,6 +933,9 @@ async def mixer_websocket(ws: WebSocket):
await ws.close()
return
# Register for push event broadcasts
mixer_ws_clients.add(ws)
await ws.send_json({"type": "connected", "daemon": MIXER_SOCKET_PATH})
try:
@@ -683,10 +962,8 @@ async def mixer_websocket(ws: WebSocket):
"removeChannel": ("remove_channel", ["channelIndex"]),
"getOutputRoutes": ("get_output_routes", None),
"setOutputRoutes": ("set_output_routes", ["routes"]),
"saveScene": ("save_scene", ["name"]),
"loadScene": ("load_scene", ["sceneId"]),
"listScenes": ("list_scenes", None),
"deleteScene": ("delete_scene", ["sceneId"]),
# Scene methods — handled via backend REST /api/mixer/scenes/*
# (removed from WS proxy — use REST endpoints instead)
}
if method not in method_map:
@@ -710,6 +987,8 @@ async def mixer_websocket(ws: WebSocket):
logger.info("Mixer WS disconnected")
except Exception as e:
logger.warning("Mixer WS error: %s", e)
finally:
mixer_ws_clients.discard(ws)
# ---------------------------------------------------------------------------