Add web UX with dashboard, import wizard, backup management
- SPA frontend (static/index.html): 4-step wizard flow (Upload → Review → Import → Report) + Dashboard bar + Backup UI - FastAPI server (server.py) with 14 API endpoints - New endpoints: /api/status, /api/backups, /api/backup/restore - Updated README with web UX usage - Added web deps: fastapi, uvicorn, python-multipart
This commit is contained in:
@@ -38,6 +38,42 @@ pip install -r requirements.txt
|
||||
python main.py seed-data/import_seed.csv --db /path/to/assets.db [--dry-run]
|
||||
```
|
||||
|
||||
## Web UX
|
||||
|
||||
Standalone web interface for uploading, previewing, and importing Cantaloupe data:
|
||||
|
||||
```bash
|
||||
cd ~/projects/canteen-seed-import
|
||||
source .venv/bin/activate
|
||||
pip install -r requirements.txt # ensure web deps are installed
|
||||
|
||||
python -m web_ux.main --port 8089
|
||||
```
|
||||
|
||||
Then open **http://localhost:8089**
|
||||
|
||||
Features:
|
||||
- 📊 **Dashboard** — DB stats, class/make breakdown, GPS coverage
|
||||
- 📤 **Import** — Drag-and-drop Excel upload, preview parsed data, choose seed/update mode, run import with automatic backup
|
||||
- 💾 **Backup** — Create/manage/compare/restore GPS+photo backups
|
||||
- 📋 **Report** — View the latest validation report
|
||||
- ⚙️ **Config** — View header mappings and handbook reference
|
||||
|
||||
### API Endpoints
|
||||
|
||||
| Method | Path | Description |
|
||||
|--------|------|-------------|
|
||||
| GET | /api/status | DB stats and health |
|
||||
| GET | /api/config | Header mappings + handbook info |
|
||||
| POST | /api/upload | Upload Excel file, returns preview token |
|
||||
| GET | /api/preview/{token} | Get paginated preview data |
|
||||
| POST | /api/import | Run import (seed/update) |
|
||||
| GET | /api/report | Latest validation report |
|
||||
| POST | /api/backup | Create database backup |
|
||||
| GET | /api/backups | List existing backups |
|
||||
| POST | /api/backup/compare | Compare backup to current DB |
|
||||
| POST | /api/backup/restore | Restore preserved fields from backup |
|
||||
|
||||
## Integration
|
||||
|
||||
Once validated, this tool will be integrated into the [Canteen Admin Server](https://admin.canteen.ourpad.casa).
|
||||
|
||||
@@ -11,3 +11,8 @@ httpx>=0.24
|
||||
# CLI
|
||||
rich>=13.0
|
||||
click>=8.0
|
||||
|
||||
# Web UX
|
||||
fastapi>=0.100
|
||||
uvicorn>=0.20
|
||||
python-multipart>=0.0.6
|
||||
|
||||
+472
@@ -0,0 +1,472 @@
|
||||
"""
|
||||
🌱 Canteen Seed Import — Web UX Server
|
||||
|
||||
FastAPI server wrapping the seed import pipeline:
|
||||
Upload Excel → Preview parse → Import with backup → View report
|
||||
|
||||
Usage:
|
||||
cd ~/projects/canteen-seed-import
|
||||
python3 web/server.py --port 8091
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import glob
|
||||
import gzip
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Query, Request, UploadFile, File
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse, JSONResponse, Response
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
# ─── Import pipeline modules (relative to project root) ──────────────────────
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from parser import parse_excel
|
||||
from db_writer import write_seed, write_update, ALL_COLUMNS, PRESERVE_FIELDS, AUTO_FIELDS
|
||||
from backup import backup as do_backup, compare as compare_backup
|
||||
|
||||
|
||||
# ─── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _list_backup_files() -> list[dict]:
|
||||
"""List backup files in the seed-data directory."""
|
||||
files = sorted(glob.glob(str(PROJECT_ROOT / "seed-data" / "backup_*.json.gz")), reverse=True)
|
||||
result = []
|
||||
for f in files:
|
||||
try:
|
||||
with gzip.open(f, "rb") as gz:
|
||||
meta = json.loads(gz.read().decode("utf-8"))
|
||||
result.append({
|
||||
"path": f,
|
||||
"filename": os.path.basename(f),
|
||||
"created": meta.get("created", "unknown"),
|
||||
"records": len(meta.get("records", [])),
|
||||
"fields": meta.get("fields", []),
|
||||
})
|
||||
except Exception:
|
||||
mtime = os.path.getmtime(f)
|
||||
result.append({
|
||||
"path": f,
|
||||
"filename": os.path.basename(f),
|
||||
"created": datetime.fromtimestamp(mtime).isoformat(),
|
||||
"records": 0,
|
||||
"fields": [],
|
||||
})
|
||||
return result
|
||||
|
||||
# ─── Config ──────────────────────────────────────────────────────────────────
|
||||
|
||||
DEFAULT_DB_PATH = str(PROJECT_ROOT / "seed-data" / "assets.db")
|
||||
DEFAULT_SEED_DIR = str(PROJECT_ROOT / "seed-data")
|
||||
UPLOAD_DIR = Path("/tmp/canteen-import-uploads")
|
||||
REPORT_PATH = Path(DEFAULT_SEED_DIR) / "validation_report.md"
|
||||
HEADERS_FILE = Path(DEFAULT_SEED_DIR) / "headers_modified.csv"
|
||||
REFERENCE_FILE = Path(DEFAULT_SEED_DIR) / "reference_handbook.md"
|
||||
|
||||
UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# In-memory state for the current session
|
||||
_session_state = {
|
||||
"machines": None, # parsed machine list
|
||||
"file_name": None, # uploaded file name
|
||||
"parse_stats": None, # parse summary stats
|
||||
"last_import": None, # result of last import
|
||||
"last_backup": None, # path to last backup
|
||||
}
|
||||
|
||||
# ─── App ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
app = FastAPI(title="Canteen Seed Import Tool")
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
# ─── Helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_serializable(obj):
|
||||
"""Convert any object to JSON-serializable form."""
|
||||
if obj is None:
|
||||
return None
|
||||
if isinstance(obj, (str, int, float, bool)):
|
||||
return obj
|
||||
if isinstance(obj, datetime):
|
||||
return obj.isoformat()
|
||||
if isinstance(obj, (list, tuple)):
|
||||
return [_make_serializable(x) for x in obj]
|
||||
if isinstance(obj, dict):
|
||||
return {k: _make_serializable(v) for k, v in obj.items()}
|
||||
return str(obj)
|
||||
|
||||
|
||||
def _parse_stats(machines: list) -> dict:
|
||||
"""Build summary stats from parsed machines."""
|
||||
total = len(machines)
|
||||
disney = sum(1 for m in machines if m.get("is_disney"))
|
||||
unknown = sum(1 for m in machines if m.get("is_unknown_machine"))
|
||||
ocr = sum(1 for m in machines if m.get("serial_ocr_corrected"))
|
||||
hi = sum(1 for m in machines if m.get("priority") == "High")
|
||||
mid = sum(1 for m in machines if m.get("priority") == "Mid")
|
||||
low = sum(1 for m in machines if m.get("priority") == "Low")
|
||||
|
||||
# Telemetry breakdown
|
||||
telemetry = {}
|
||||
for m in machines:
|
||||
prov = m.get("telemetry_provider", "None") or "None"
|
||||
telemetry[prov] = telemetry.get(prov, 0) + 1
|
||||
|
||||
# Class breakdown
|
||||
classes = {}
|
||||
for m in machines:
|
||||
cls = m.get("class", "Unknown") or "Unknown"
|
||||
classes[cls] = classes.get(cls, 0) + 1
|
||||
|
||||
return {
|
||||
"total": total,
|
||||
"disney": disney,
|
||||
"unknown_machines": unknown,
|
||||
"ocr_corrections": ocr,
|
||||
"priority": {"High": hi, "Mid": mid, "Low": low},
|
||||
"telemetry": telemetry,
|
||||
"classes": classes,
|
||||
}
|
||||
|
||||
|
||||
def _format_number(n):
|
||||
"""Format a number with commas."""
|
||||
if n is None:
|
||||
return "—"
|
||||
return f"{n:,}"
|
||||
|
||||
|
||||
# ─── API Routes ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@app.get("/api/health")
|
||||
async def health():
|
||||
return {"status": "ok", "time": datetime.now(timezone.utc).isoformat()}
|
||||
|
||||
|
||||
@app.get("/api/status")
|
||||
async def status():
|
||||
"""Dashboard stats: DB health, machine counts, GPS coverage."""
|
||||
db_path = DEFAULT_DB_PATH
|
||||
result = {
|
||||
"db_path": db_path,
|
||||
"exists": os.path.exists(db_path),
|
||||
"total_machines": 0,
|
||||
"by_class": {},
|
||||
"by_make": {},
|
||||
"by_priority": {},
|
||||
"gps_coverage": {"with_gps": 0, "without_gps": 0},
|
||||
"last_modified": None,
|
||||
"last_backup": None,
|
||||
"backup_count": 0,
|
||||
}
|
||||
if not result["exists"]:
|
||||
return result
|
||||
|
||||
if os.path.exists(db_path):
|
||||
mtime = os.path.getmtime(db_path)
|
||||
result["last_modified"] = datetime.fromtimestamp(mtime).isoformat()
|
||||
|
||||
try:
|
||||
import sqlite3
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("SELECT COUNT(*) as c FROM assets")
|
||||
result["total_machines"] = cur.fetchone()["c"]
|
||||
|
||||
cur.execute("SELECT COALESCE(NULLIF(class,''),'Unknown') as cls, COUNT(*) as c FROM assets GROUP BY cls ORDER BY c DESC")
|
||||
result["by_class"] = {r["cls"]: r["c"] for r in cur.fetchall()}
|
||||
|
||||
cur.execute("SELECT COALESCE(NULLIF(make,''),'Unknown') as mk, COUNT(*) as c FROM assets GROUP BY mk ORDER BY c DESC")
|
||||
result["by_make"] = {r["mk"]: r["c"] for r in cur.fetchall()}
|
||||
|
||||
cur.execute("SELECT COALESCE(NULLIF(priority,''),'None') as pri, COUNT(*) as c FROM assets GROUP BY pri ORDER BY pri")
|
||||
result["by_priority"] = {r["pri"]: r["c"] for r in cur.fetchall()}
|
||||
|
||||
cur.execute("SELECT COUNT(*) as c FROM assets WHERE latitude IS NOT NULL AND latitude != 0")
|
||||
result["gps_coverage"]["with_gps"] = cur.fetchone()["c"]
|
||||
result["gps_coverage"]["without_gps"] = result["total_machines"] - result["gps_coverage"]["with_gps"]
|
||||
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
backups = _list_backup_files()
|
||||
result["backup_count"] = len(backups)
|
||||
if backups:
|
||||
result["last_backup"] = backups[0].get("created", "")
|
||||
return result
|
||||
|
||||
|
||||
@app.get("/api/backups")
|
||||
async def list_backups():
|
||||
"""List available backup files."""
|
||||
return {"backups": _list_backup_files()}
|
||||
|
||||
|
||||
@app.get("/api/columns")
|
||||
async def columns():
|
||||
"""Return column definitions: which are auto vs preserve vs primary key."""
|
||||
return {
|
||||
"all": ALL_COLUMNS,
|
||||
"preserve": sorted(PRESERVE_FIELDS),
|
||||
"auto": sorted(AUTO_FIELDS),
|
||||
"primary_key": "machine_id",
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/headers")
|
||||
async def headers():
|
||||
"""Return the header mapping CSV as JSON."""
|
||||
if not HEADERS_FILE.exists():
|
||||
return {"headers": [], "note": f"{HEADERS_FILE} not found"}
|
||||
rows = []
|
||||
with open(HEADERS_FILE, newline="", encoding="utf-8") as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
rows.append(row)
|
||||
return {"headers": rows, "count": len(rows)}
|
||||
|
||||
|
||||
@app.get("/api/reference")
|
||||
async def reference():
|
||||
"""Return the reference handbook markdown."""
|
||||
if not REFERENCE_FILE.exists():
|
||||
return {"content": "", "note": f"{REFERENCE_FILE} not found"}
|
||||
with open(REFERENCE_FILE, encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
return {"content": content, "size": len(content)}
|
||||
|
||||
|
||||
@app.post("/api/upload")
|
||||
async def upload(file: UploadFile = File(...)):
|
||||
"""Upload an Excel file and parse it. Returns preview + stats."""
|
||||
if not file.filename:
|
||||
raise HTTPException(400, "No file provided")
|
||||
|
||||
suffix = Path(file.filename).suffix.lower()
|
||||
if suffix not in (".xlsx", ".xls", ".csv"):
|
||||
raise HTTPException(400, f"Unsupported file type: {suffix}. Use .xlsx, .xls, or .csv")
|
||||
|
||||
# Save to temp
|
||||
upload_path = UPLOAD_DIR / f"{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}_{file.filename}"
|
||||
with open(upload_path, "wb") as f:
|
||||
content = await file.read()
|
||||
f.write(content)
|
||||
|
||||
# Parse
|
||||
try:
|
||||
machines = parse_excel(str(upload_path))
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
raise HTTPException(500, f"Parse error: {e}")
|
||||
|
||||
# Build stats
|
||||
stats = _parse_stats(machines)
|
||||
|
||||
# Build preview (first 50 rows, key columns)
|
||||
preview_cols = [
|
||||
"machine_id", "serial_number", "name", "class", "make", "model",
|
||||
"company", "place", "building_name", "floor", "zone",
|
||||
"priority", "telemetry_provider", "disney_park", "is_disney",
|
||||
"yearly_sales", "has_cashless", "alerts_summary",
|
||||
]
|
||||
preview = []
|
||||
for m in machines[:50]:
|
||||
row = {}
|
||||
for col in preview_cols:
|
||||
row[col] = _make_serializable(m.get(col))
|
||||
preview.append(row)
|
||||
|
||||
# Store in session
|
||||
_session_state["machines"] = machines
|
||||
_session_state["file_name"] = file.filename
|
||||
_session_state["parse_stats"] = stats
|
||||
|
||||
# Cleanup temp file (keep parsed data in memory)
|
||||
try:
|
||||
os.unlink(str(upload_path))
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
return {
|
||||
"file_name": file.filename,
|
||||
"stats": stats,
|
||||
"preview": preview,
|
||||
"preview_count": len(preview),
|
||||
"total_rows": len(machines),
|
||||
"columns_previewed": preview_cols,
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/import")
|
||||
async def run_import(
|
||||
mode: str = Query("seed", pattern="^(seed|update)$"),
|
||||
backup: bool = Query(True),
|
||||
db_path: str = Query(DEFAULT_DB_PATH),
|
||||
):
|
||||
"""Run the import — seed or update mode. Optionally backup first."""
|
||||
if _session_state["machines"] is None:
|
||||
raise HTTPException(400, "No data parsed. Upload a file first.")
|
||||
|
||||
machines = _session_state["machines"]
|
||||
backup_path = None
|
||||
|
||||
# Backup if requested
|
||||
if backup and os.path.exists(db_path):
|
||||
try:
|
||||
backup_path = do_backup(db_path)
|
||||
_session_state["last_backup"] = backup_path
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
# Don't fail the whole import if backup fails
|
||||
backup_path = f"ERROR: {e}"
|
||||
|
||||
# Run import
|
||||
try:
|
||||
if mode == "seed":
|
||||
stats = write_seed(machines, db_path)
|
||||
else:
|
||||
stats = write_update(machines, db_path)
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
raise HTTPException(500, f"Import error: {e}")
|
||||
|
||||
_session_state["last_import"] = {
|
||||
"mode": mode,
|
||||
"db_path": db_path,
|
||||
"stats": stats,
|
||||
"backup_path": backup_path,
|
||||
"time": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
return {
|
||||
"mode": mode,
|
||||
"stats": stats,
|
||||
"backup_path": backup_path,
|
||||
"machine_count": len(machines),
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/report")
|
||||
async def report(refresh: bool = Query(False)):
|
||||
"""Return the latest validation report."""
|
||||
if not REPORT_PATH.exists():
|
||||
return {"content": "", "note": "No report generated yet. Run an import first."}
|
||||
with open(REPORT_PATH, encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
return {
|
||||
"content": content,
|
||||
"size": len(content),
|
||||
"path": str(REPORT_PATH),
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/session")
|
||||
async def session():
|
||||
"""Return current session state."""
|
||||
return {
|
||||
"file_name": _session_state["file_name"],
|
||||
"parse_stats": _session_state["parse_stats"],
|
||||
"last_import": _session_state["last_import"],
|
||||
"last_backup": _session_state["last_backup"],
|
||||
"has_data": _session_state["machines"] is not None,
|
||||
"machine_count": len(_session_state["machines"]) if _session_state["machines"] else 0,
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/backup")
|
||||
async def api_backup(db_path: str = Query(DEFAULT_DB_PATH)):
|
||||
"""Create a backup of the current database."""
|
||||
if not os.path.exists(db_path):
|
||||
raise HTTPException(404, f"Database not found: {db_path}")
|
||||
try:
|
||||
path = do_backup(db_path)
|
||||
_session_state["last_backup"] = path
|
||||
return {"backup_path": path, "status": "ok"}
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
raise HTTPException(500, f"Backup error: {e}")
|
||||
|
||||
|
||||
@app.post("/api/compare")
|
||||
async def api_compare(
|
||||
backup_path: str = Query(...),
|
||||
db_path: str = Query(DEFAULT_DB_PATH),
|
||||
):
|
||||
"""Compare a backup against the live database."""
|
||||
if not os.path.exists(backup_path):
|
||||
raise HTTPException(404, f"Backup not found: {backup_path}")
|
||||
try:
|
||||
changes = compare_backup(backup_path, db_path)
|
||||
return {"changes": changes, "backup": backup_path, "db": db_path}
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
raise HTTPException(500, f"Compare error: {e}")
|
||||
|
||||
|
||||
@app.post("/api/backup/restore")
|
||||
async def api_restore(
|
||||
backup_path: str = Query(...),
|
||||
db_path: str = Query(DEFAULT_DB_PATH),
|
||||
):
|
||||
"""Restore GPS/photo fields from a backup."""
|
||||
if not os.path.exists(backup_path):
|
||||
raise HTTPException(404, f"Backup not found: {backup_path}")
|
||||
try:
|
||||
from backup import restore as do_restore
|
||||
do_restore(backup_path, db_path)
|
||||
return {"status": "ok", "restored_from": os.path.basename(backup_path), "db": db_path}
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
raise HTTPException(500, f"Restore error: {e}")
|
||||
|
||||
|
||||
# ─── Static files ─────────────────────────────────────────────────────────────
|
||||
|
||||
STATIC_DIR = Path(__file__).parent / "static"
|
||||
STATIC_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if STATIC_DIR.exists():
|
||||
app.mount("/", StaticFiles(directory=str(STATIC_DIR), html=True), name="static")
|
||||
|
||||
|
||||
# ─── Entry point ─────────────────────────────────────────────────────────────
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
|
||||
parser = argparse.ArgumentParser(description="Canteen Seed Import Web Server")
|
||||
parser.add_argument("--port", type=int, default=8091, help="Port to listen on")
|
||||
parser.add_argument("--host", type=str, default="0.0.0.0", help="Host to bind to")
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"🌱 Canteen Seed Import Web Server")
|
||||
print(f" http://localhost:{args.port}")
|
||||
print(f" DB: {DEFAULT_DB_PATH}")
|
||||
print(f" Seed dir: {DEFAULT_SEED_DIR}")
|
||||
uvicorn.run(app, host=args.host, port=args.port, log_level="info")
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user