From b80ed4b48301bf2b4c95b200dd53d1c11899239c Mon Sep 17 00:00:00 2001 From: Shawn Date: Fri, 29 May 2026 08:59:53 -0400 Subject: [PATCH] feat: Ollama vision OCR via Windows PC for label photo matching - Add persistent SSH tunnel (systemd service) to Windows PC Ollama - Add _HAS_OLLAMA check at server startup (qwen2.5vl:3b) - Replace Tesseract-only OCR with Ollama-first strategy - Ollama (qwen2.5vl:3b) handles dark/complex labels Tesseract can't read - Keurig serial 2500.0100.0025534 now correctly matched as Asset #5144 - Add ocr_source field to /api/ocr response ('ollama' or 'tesseract') - Fix match_label_photo.py argparse bug (image_paths -> images) - Update match_label_photo.py CLI to read vision key from Hermes config --- scripts/match_label_photo.py | 2 +- server.py | 114 ++++++++++++++++++++++++++--------- 2 files changed, 88 insertions(+), 28 deletions(-) diff --git a/scripts/match_label_photo.py b/scripts/match_label_photo.py index 9f4a6bf..c0ae539 100644 --- a/scripts/match_label_photo.py +++ b/scripts/match_label_photo.py @@ -163,7 +163,7 @@ def main(): _process_text(text, db_path) return - for img_path in args.image_paths: + for img_path in args.images: if not Path(img_path).exists(): print(f"\n=== SKIP: {img_path} (not found) ===") continue diff --git a/server.py b/server.py index 4cd982a..e379edf 100644 --- a/server.py +++ b/server.py @@ -20,13 +20,30 @@ import uuid from contextlib import asynccontextmanager from datetime import date from pathlib import Path - +_HAS_TESSERACT: bool try: import pytesseract _HAS_TESSERACT = True except ImportError: - pytesseract = None + pytesseract = None # type: ignore _HAS_TESSERACT = False + +# Ollama vision model (Windows PC via SSH tunnel) +_OLLAMA_BASE = "http://127.0.0.1:11434" +_OLLAMA_VISION_MODEL = "qwen2.5vl:3b" +_HAS_OLLAMA: bool = False +try: + import urllib.request as _ollama_urllib + import json as _ollama_json + _req = _ollama_urllib.Request( + f"{_OLLAMA_BASE}/api/generate", + data=_ollama_json.dumps({"model": _OLLAMA_VISION_MODEL, "prompt": "ping", "stream": False}).encode(), + headers={"Content-Type": "application/json"}, + ) + _resp = _ollama_urllib.urlopen(_req, timeout=5) + _HAS_OLLAMA = True +except Exception: + _HAS_OLLAMA = False import piexif from PIL import Image as PILImage @@ -2186,38 +2203,81 @@ async def ocr_sticker(file: UploadFile = File(...), exif_data: str = Form(None)) ocr_path = temp_dir / f"{uuid.uuid4().hex}.{ext or 'jpg'}" ocr_path.write_bytes(contents) - try: - img = PILImage.open(ocr_path) - # Preprocess: convert to grayscale and increase contrast for better OCR - img_gray = img.convert("L") - if not _HAS_TESSERACT: - raise RuntimeError( - "Tesseract OCR library (pytesseract) is not installed. " - "Run: pip install pytesseract && apt-get install -y tesseract-ocr" - ) - text = pytesseract.image_to_string(img_gray, config="--psm 6") - except (RuntimeError, ImportError) as e: - if not saved_path: - ocr_path.unlink(missing_ok=True) - raise HTTPException( - status_code=503, - detail=f"OCR service unavailable: {str(e)}", - ) - except Exception as e: - if not saved_path: - ocr_path.unlink(missing_ok=True) - raise HTTPException( - status_code=500, - detail=f"OCR processing error: {str(e)}. Tesseract binary may not be installed system-wide. Run: apt-get install -y tesseract-ocr", - ) + text = "" + ocr_source = "none" + ollama_text = "" - # Clean up temp file (but keep permanent photos) + # Try Ollama vision model first (Windows PC) — most accurate + if _HAS_OLLAMA: + try: + import urllib.request as _ourl + import base64 as _b64 + img_vision = PILImage.open(ocr_path) + img_vision.thumbnail((640, 480)) + temp_vision = ocr_path.parent / f"vis_{ocr_path.name}" + img_vision.save(temp_vision) + with open(temp_vision, "rb") as _f: + _b64_data = _b64.b64encode(_f.read()).decode() + temp_vision.unlink(missing_ok=True) + + _vdata = _json.dumps({ + "model": _OLLAMA_VISION_MODEL, + "prompt": "Extract all text, numbers, serial numbers, and IDs visible on this sticker or label. Return ONLY the raw text content, one item per line. Do not describe the image.", + "images": [_b64_data], + "stream": False, + }).encode() + _vreq = _ourl.Request( + f"{_OLLAMA_BASE}/api/generate", + data=_vdata, + headers={"Content-Type": "application/json"}, + ) + _vresp = _ourl.urlopen(_vreq, timeout=120) + _vresult = _json.loads(_vresp.read().decode()) + ollama_text = _vresult.get("response", "").strip() + if ollama_text: + ocr_source = "ollama" + except Exception: + pass # Fall through to Tesseract + + # Try Tesseract if Ollama didn't yield usable text + if ocr_source == "none": + try: + img = PILImage.open(ocr_path) + img_gray = img.convert("L") + if _HAS_TESSERACT: + tess_text = pytesseract.image_to_string(img_gray, config="--psm 6") + tess_text = tess_text.strip() + clean_chars = sum(1 for c in tess_text if c.isalnum() or c in ' \n/-.') + if len(tess_text) > 0 and clean_chars >= 10: + ocr_source = "tesseract" + except Exception: + pass + + # Prefer Ollama text when available (much more accurate) + if ollama_text: + text = ollama_text + elif ocr_source == "tesseract": + text = tess_text + else: + text = "" + + # Clean up temp file if not saved_path: ocr_path.unlink(missing_ok=True) + if not text: + if _HAS_OLLAMA: + detail = "Could not read any text from the image via Ollama vision model." + elif _HAS_TESSERACT: + detail = "Could not read any text from the image via Tesseract. Try a clearer photo of the label." + else: + detail = "No OCR service available (Tesseract not installed, Ollama not connected)." + raise HTTPException(status_code=422, detail=detail) + # Build response — search for identifiers in the OCR text result: dict = { "raw_text": text.strip()[:1000], + "ocr_source": ocr_source, } # 1. Legacy pattern: XXXXX-XXXXXX (5 digits - 6+ digits = Connect ID)